Hi h01ger, On 03/21/18 22:53, Holger Levsen wrote: [snip] >> The patch attached is made against origin/master. > > cool cool. Just one thing before I'm willing to merge this: please also > patch piuparts.1.txt to explain this new option.
I have added doc for the new param and reworded some others :), please find the patches attached. I have pushed to the PR[0] as well because IMO it's easier for doing review there. It'd be awesome if you migrate the project to salsa, then I can close the PR and open a MR in there instead. Also, I tested a little bit more using no option, --schroot and the new one --docker-image. I'd be really happy if you do some testing on your side and give me some feedback. [0] https://github.com/h01ger/piuparts/pull/6 Cheers, -- TiN
From 873e3a30477dc5c32910b9c12ccebbaf43ab85f9 Mon Sep 17 00:00:00 2001
From: Agustin Henze <t...@aayy.com.ar>
Date: Thu, 22 Mar 2018 09:17:36 -0300
Subject: [PATCH 4/4] Rephrase `--schroot` param doc
---
piuparts.1.txt | 2 +-
piuparts.py | 3 +--
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/piuparts.1.txt b/piuparts.1.txt
index 76bba6c7..09f0018b 100644
--- a/piuparts.1.txt
+++ b/piuparts.1.txt
@@ -233,7 +233,7 @@ Note that file: addresses works if the directories are made accessible from with
Directory where are custom scripts are placed. By default, this is not set. For more information about this, read README_server.txt
*--schroot*='SCHROOT-NAME'::
- Use schroot session named SCHROOT-NAME for the chroot, instead of building a new one with debootstrap.
+ Use schroot session named SCHROOT-NAME for the testing environment.
*--docker-image='DOCKER-IMAGE'::
Use a container created from the docker image DOCKER-IMAGE for the testing environment.
diff --git a/piuparts.py b/piuparts.py
index fdf2644c..5e582e28 100644
--- a/piuparts.py
+++ b/piuparts.py
@@ -2770,8 +2770,7 @@ def parse_command_line():
"a new LVM snapshot (default: 1G)")
parser.add_option("--schroot", metavar="SCHROOT-NAME", action="store",
- help="Use schroot session named SCHROOT-NAME for the chroot, instead of building " +
- "a new one with debootstrap.")
+ help="Use schroot session named SCHROOT-NAME for the testing environment.")
parser.add_option("--docker-image", metavar="DOCKER-IMAGE", action="store",
help="Use a container created from the docker image "
--
2.16.2
From 1847812137e613fa32f848a27d47664a76cb5274 Mon Sep 17 00:00:00 2001
From: Agustin Henze <t...@aayy.com.ar>
Date: Thu, 22 Mar 2018 09:11:28 -0300
Subject: [PATCH 3/4] Rewrite `--keep-tmpdir` doc
---
piuparts.1.txt | 5 ++++-
piuparts.py | 4 ++--
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/piuparts.1.txt b/piuparts.1.txt
index 048e9202..76bba6c7 100644
--- a/piuparts.1.txt
+++ b/piuparts.1.txt
@@ -144,7 +144,10 @@ The tarball can be created with the '-s' option, or you can use one that *pbuild
Remove package after installation and reinstall. For testing installation in config-files-remaining state.
*-k*, *--keep-tmpdir*::
- Don't remove the temporary directory for the chroot when the program ends.
+ Depending on which option is passed, it keeps the environment used for testing after the program ends::
+ * By default it doesn't remove the temporary directory for the chroot
+ * if --schroot is used, the schroot session is not terminated
+ * or if --docker-image is used, the container created is not destroyed.
*-K*, *--keyring*='filename'::
Use FILE as the keyring to use with debootstrap when creating chroots.
diff --git a/piuparts.py b/piuparts.py
index 5a063c67..fdf2644c 100644
--- a/piuparts.py
+++ b/piuparts.py
@@ -2740,8 +2740,8 @@ def parse_command_line():
parser.add_option("-k", "--keep-tmpdir",
action="store_true", default=False,
- help="Don't remove the temporary directory for the " +
- "chroot when the program ends.")
+ help="It keeps the environment used for testing after "
+ "the program ends.")
parser.add_option("-K", "--keyring", action="store", metavar="FILE",
help="Use FILE as the keyring to use with debootstrap when creating chroots.")
--
2.16.2
From d09d98e8fc1aa5d4725e973a7c60554ce47e2d15 Mon Sep 17 00:00:00 2001 From: Agustin Henze <t...@aayy.com.ar> Date: Thu, 22 Mar 2018 09:06:16 -0300 Subject: [PATCH 2/4] Add `--docker-image` param doc in manpage --- piuparts.1.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/piuparts.1.txt b/piuparts.1.txt index 94403240..048e9202 100644 --- a/piuparts.1.txt +++ b/piuparts.1.txt @@ -232,6 +232,9 @@ Note that file: addresses works if the directories are made accessible from with *--schroot*='SCHROOT-NAME':: Use schroot session named SCHROOT-NAME for the chroot, instead of building a new one with debootstrap. +*--docker-image='DOCKER-IMAGE':: + Use a container created from the docker image DOCKER-IMAGE for the testing environment. + *--single-changes-list*:: When processing changes files, piuparts will process the packages in each individual changes file seperately. This option will set piuparts to scan the packages of all changes files together along with any individual package files that may have been given on the command line. -- 2.16.2
From e4b76c31d96a0c86c4931b7ec929b0078616b4cf Mon Sep 17 00:00:00 2001
From: Agustin Henze <t...@aayy.com.ar>
Date: Wed, 21 Mar 2018 16:54:50 -0300
Subject: [PATCH 1/4] Add docker support, new param is introduced
`--docker-image`
e.g. piuparts --docker-image debian:unstable package.deb
It only supports overlay2 for now and it uses the `MergedDir` layer
where piuparts can access, add, edit and remove files easily.
---
piuparts.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 58 insertions(+), 6 deletions(-)
diff --git a/piuparts.py b/piuparts.py
index 3daba797..5a063c67 100644
--- a/piuparts.py
+++ b/piuparts.py
@@ -47,6 +47,7 @@ import os
import tarfile
import stat
import re
+import json
import pickle
import subprocess
import traceback
@@ -191,6 +192,7 @@ class Settings:
self.skip_minimize = True
self.minimize = False
self.debfoster_options = None
+ self.docker_image = None
# tests and checks
self.no_install_purge_test = False
self.no_upgrade_test = False
@@ -769,7 +771,7 @@ class Chroot:
def create(self, temp_tgz=None):
"""Create a chroot according to user's wishes."""
self.panic_handler_id = do_on_panic(self.remove)
- if not settings.schroot:
+ if not settings.schroot and not settings.docker_image:
self.create_temp_dir()
if temp_tgz:
@@ -782,10 +784,12 @@ class Chroot:
self.setup_from_dir(settings.existing_chroot)
elif settings.schroot:
self.setup_from_schroot(settings.schroot)
+ elif settings.docker_image:
+ self.setup_from_docker(settings.docker_image)
else:
self.setup_minimal_chroot()
- if not settings.schroot:
+ if not settings.schroot and not settings.docker_image:
self.mount_proc()
self.configure_chroot()
@@ -807,7 +811,7 @@ class Chroot:
self.run_scripts("post_chroot_unpack")
self.run(["apt-get", "update"])
- if settings.basetgz or settings.schroot or settings.existing_chroot:
+ if settings.basetgz or settings.docker_image or settings.schroot or settings.existing_chroot:
self.run(["apt-get", "-yf", "dist-upgrade"])
self.minimize()
self.remember_available_md5()
@@ -832,7 +836,10 @@ class Chroot:
if settings.schroot:
logging.debug("Terminate schroot session '%s'" % self.name)
run(['schroot', '--end-session', '--chroot', "session:" + self.schroot_session])
- if not settings.schroot:
+ if settings.docker_image:
+ logging.debug("Destroy docker container '%s'" % self.docker_container)
+ run(['docker', 'rm', '-f', self.docker_container])
+ if not settings.schroot and not settings.docker_image:
run(['rm', '-rf', '--one-file-system', self.name])
if os.path.exists(self.name):
create_file(os.path.join(self.name, ".piuparts.tmpdir"), "removal failed")
@@ -840,6 +847,8 @@ class Chroot:
elif settings.keep_tmpdir:
if settings.schroot:
logging.debug("Keeping schroot session %s at %s" % (self.schroot_session, self.name))
+ elif settings.docker_image:
+ logging.debug("Keeping container %s" % self.docker_container)
else:
logging.debug("Keeping directory tree at %s" % self.name)
dont_do_on_panic(self.panic_handler_id)
@@ -892,6 +901,25 @@ class Chroot:
self.name = output.strip()
logging.info("New schroot session in '%s'" % self.name)
+ @staticmethod
+ def check_if_docker_storage_driver_is_supported():
+ ret_code, output = run(['docker', 'info'])
+ if 'overlay2' not in output:
+ logging.error('Only overlay2 storage driver is supported')
+ panic()
+
+ def setup_from_docker(self, docker_image):
+ self.check_if_docker_storage_driver_is_supported()
+ ret_code, output = run(['docker', 'run', '-d', '-it', docker_image, 'bash'])
+ if ret_code != 0:
+ logging.error("Couldn't start the container from '%s'" % docker_image)
+ panic()
+ self.docker_container = output.strip()
+ ret_code, output = run(['docker', 'inspect', self.docker_container])
+ container_data = json.loads(output)[0]
+ self.name = container_data['GraphDriver']['Data']['MergedDir']
+ logging.info("New container created '%s'" % self.docker_container)
+
def setup_from_lvm(self, lvm_volume):
"""Create a chroot by creating an LVM snapshot."""
self.lvm_base = os.path.dirname(lvm_volume)
@@ -938,6 +966,12 @@ class Chroot:
["schroot", "--preserve-environment", "--run-session", "--chroot", "session:" +
self.schroot_session, "--directory", "/", "-u", "root", "--"] + prefix + command,
ignore_errors=ignore_errors, timeout=settings.max_command_runtime)
+ elif settings.docker_image:
+ return run(
+ ['docker', 'exec', self.docker_container,] + prefix + command,
+ ignore_errors=ignore_errors,
+ timeout=settings.max_command_runtime,
+ )
else:
return run(["chroot", self.name] + prefix + command,
ignore_errors=ignore_errors, timeout=settings.max_command_runtime)
@@ -1042,6 +1076,9 @@ class Chroot:
def create_resolv_conf(self):
"""Update resolv.conf based on the current configuration in the host system. Strip comments and whitespace."""
+ if settings.docker_image:
+ # Do nothing, docker already takes care of this
+ return
full_name = self.relative("etc/resolv.conf")
resolvconf = ""
with open("/etc/resolv.conf", "r") as f:
@@ -1379,6 +1416,9 @@ class Chroot:
'broken-symlink',
]
ignored_tags = []
+ if not os.path.exists(self.name + '/dev/null'):
+ device = os.makedev(1, 3)
+ os.mknod(self.name + '/dev/null', 0o666, device)
(status, output) = run(["adequate", "--root", self.name] + packages, ignore_errors=True)
for tag in ignored_tags:
# ignore some tags
@@ -1624,8 +1664,12 @@ class Chroot:
def check_for_no_processes(self, fail=None):
"""Check there are no processes running inside the chroot."""
- (status, output) = run(["lsof", "-w", "+D", self.name], ignore_errors=True)
- count = len(output.split("\n")) - 1
+ if settings.docker_image:
+ (status, output) = run(["docker", "top", self.docker_container])
+ count = len(output.strip().split("\n")) - 2 # header + bash launched on container creation
+ else:
+ (status, output) = run(["lsof", "-w", "+D", self.name], ignore_errors=True)
+ count = len(output.split("\n")) - 1
if count > 0:
if fail is None:
fail = not settings.allow_database
@@ -1637,6 +1681,9 @@ class Chroot:
def terminate_running_processes(self):
"""Terminate all processes running in the chroot."""
+ if settings.docker_image:
+ # docker takes care of this
+ return
seen = []
while True:
p = subprocess.Popen(["lsof", "-t", "+D", self.name],
@@ -2726,6 +2773,10 @@ def parse_command_line():
help="Use schroot session named SCHROOT-NAME for the chroot, instead of building " +
"a new one with debootstrap.")
+ parser.add_option("--docker-image", metavar="DOCKER-IMAGE", action="store",
+ help="Use a container created from the docker image "
+ "DOCKER-IMAGE for the testing environment.")
+
parser.add_option("-m", "--mirror", action="append", metavar="URL",
default=[],
help="Which Debian mirror to use.")
@@ -2948,6 +2999,7 @@ def parse_command_line():
if settings.minimize:
settings.skip_minimize = False
settings.debfoster_options = opts.debfoster_options.split()
+ settings.docker_image = opts.docker_image
# tests and checks
settings.no_install_purge_test = opts.no_install_purge_test
settings.no_upgrade_test = opts.no_upgrade_test
--
2.16.2
signature.asc
Description: OpenPGP digital signature
