Skia has proposed merging ~hyask/autopkgtest-cloud:skia/git_clone_in_charms
into autopkgtest-cloud:master.
Requested reviews:
Canonical's Ubuntu QA (canonical-ubuntu-qa)
For more details, see:
https://code.launchpad.net/~hyask/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/471159
Change the way the code is run, to have the `git` repo available and ease the
way we handle cowboys.
--
Your team Canonical's Ubuntu QA is requested to review the proposed merge of
~hyask/autopkgtest-cloud:skia/git_clone_in_charms into autopkgtest-cloud:master.
diff --git a/charms/focal/autopkgtest-cloud-worker/actions/update-sources b/charms/focal/autopkgtest-cloud-worker/actions/update-sources
index 7bd2f9d..449d462 100755
--- a/charms/focal/autopkgtest-cloud-worker/actions/update-sources
+++ b/charms/focal/autopkgtest-cloud-worker/actions/update-sources
@@ -1,10 +1,12 @@
#!/usr/local/sbin/charm-env python3
-import sys
-
from reactive.autopkgtest_cloud_worker import (
AUTOPKGTEST_LOCATION,
+ AUTOPKGTEST_CLONE_BRANCH,
AUTODEP8_LOCATION,
+ AUTODEP8_CLONE_BRANCH,
+ AUTOPKGTEST_CLOUD_GIT_LOCATION,
+ AUTOPKGTEST_CLOUD_CLONE_BRANCH,
)
import pygit2
@@ -12,8 +14,12 @@ from utils import pull, install_autodep8, UnixUser
if __name__ == "__main__":
- for location in [AUTOPKGTEST_LOCATION, AUTODEP8_LOCATION]:
+ for location, branch in [
+ (AUTOPKGTEST_LOCATION, AUTOPKGTEST_CLONE_BRANCH),
+ (AUTODEP8_LOCATION, AUTODEP8_CLONE_BRANCH),
+ (AUTOPKGTEST_CLOUD_GIT_LOCATION, AUTOPKGTEST_CLOUD_CLONE_BRANCH),
+ ]:
repo = pygit2.Repository(location)
with UnixUser("ubuntu"):
- pull(repo)
+ pull(repo, branch)
install_autodep8(AUTODEP8_LOCATION)
diff --git a/charms/focal/autopkgtest-cloud-worker/config.yaml b/charms/focal/autopkgtest-cloud-worker/config.yaml
index 17dc182..6c61b2d 100644
--- a/charms/focal/autopkgtest-cloud-worker/config.yaml
+++ b/charms/focal/autopkgtest-cloud-worker/config.yaml
@@ -1,4 +1,8 @@
options:
+ package_status:
+ type: string
+ description: "APT layer package_status"
+ default: install
nova-rcs:
type: string
description: "base64 encoded tarball of nova OpenStack credential .rc files"
diff --git a/charms/focal/autopkgtest-cloud-worker/lib/utils.py b/charms/focal/autopkgtest-cloud-worker/lib/utils.py
index 29ae3f4..722aa37 100644
--- a/charms/focal/autopkgtest-cloud-worker/lib/utils.py
+++ b/charms/focal/autopkgtest-cloud-worker/lib/utils.py
@@ -46,19 +46,23 @@ def install_autodep8(location):
subprocess.check_call(["make", "install"])
-def pull(repository):
- """This will do a sort of git fetch origin && git reset --hard origin/master"""
+def pull(repository, branch):
+ """
+ This will do a sort of git fetch origin && git reset --hard origin/master
+ This raises pygit2.GitError whenever the tree is not clean, preventing to loose cowboys.
+ """
origin = [
remote for remote in repository.remotes if remote.name == "origin"
][0]
origin.fetch()
remote_master_id = repository.lookup_reference(
- "refs/remotes/origin/master"
+ f"refs/remotes/origin/{branch}"
).target
repository.checkout_tree(repository.get(remote_master_id))
- master_ref = repository.lookup_reference("refs/heads/master")
+ master_ref = repository.lookup_reference(f"refs/heads/{branch}")
master_ref.set_target(remote_master_id)
repository.head.set_target(remote_master_id)
+ log(f"Updated repository {repository} to {remote_master_id}", "INFO")
@total_ordering
diff --git a/charms/focal/autopkgtest-cloud-worker/reactive/autopkgtest_cloud_worker.py b/charms/focal/autopkgtest-cloud-worker/reactive/autopkgtest_cloud_worker.py
index f2a8ab7..bd6e4e4 100644
--- a/charms/focal/autopkgtest-cloud-worker/reactive/autopkgtest_cloud_worker.py
+++ b/charms/focal/autopkgtest-cloud-worker/reactive/autopkgtest_cloud_worker.py
@@ -9,13 +9,7 @@ from textwrap import dedent
import pygit2
import yaml
-from charmhelpers.core.hookenv import (
- charm_dir,
- config,
- log,
- storage_get,
- storage_list,
-)
+from charmhelpers.core.hookenv import config, log, storage_get, storage_list
from charms.layer import status
from charms.reactive import (
clear_flag,
@@ -30,27 +24,35 @@ from charms.reactive import (
)
from utils import UnixUser, install_autodep8
-AUTOPKGTEST_LOCATION = os.path.expanduser("~ubuntu/autopkgtest")
AUTOPKGTEST_CLOUD_LOCATION = os.path.expanduser("~ubuntu/autopkgtest-cloud")
-AUTODEP8_LOCATION = os.path.expanduser("~ubuntu/autodep8")
-AUTOPKGTEST_PER_PACKAGE_LOCATION = os.path.expanduser(
- "~ubuntu/autopkgtest-package-configs/"
+AUTOPKGTEST_CLOUD_GIT_LOCATION = os.path.expanduser(
+ "~ubuntu/autopkgtest-cloud.git"
+)
+AUTOPKGTEST_CLOUD_CLONE_LOCATION = (
+ "https://git.launchpad.net/autopkgtest-cloud";
)
+AUTOPKGTEST_CLOUD_CLONE_BRANCH = "master"
+AUTOPKGTEST_LOCATION = os.path.expanduser("~ubuntu/autopkgtest")
AUTOPKGTEST_CLONE_LOCATION = (
"https://salsa.debian.org/ubuntu-ci-team/autopkgtest.git";
)
-
AUTOPKGTEST_CLONE_BRANCH = "ubuntu/production"
+AUTODEP8_LOCATION = os.path.expanduser("~ubuntu/autodep8")
AUTODEP8_CLONE_LOCATION = (
"https://git.launchpad.net/~ubuntu-release/+git/autodep8";
)
+AUTODEP8_CLONE_BRANCH = "master"
+AUTOPKGTEST_PER_PACKAGE_LOCATION = os.path.expanduser(
+ "~ubuntu/autopkgtest-package-configs/"
+)
AUTOPKGTEST_PER_PACKAGE_CLONE_LOCATION = (
"https://git.launchpad.net/~ubuntu-release/autopkgtest";
+ "-cloud/+git/autopkgtest-package-configs"
)
+AUTOPKGTEST_PER_PACKAGE_CLONE_BRANCH = "main"
RABBITMQ_CRED_PATH = os.path.expanduser("~ubuntu/rabbitmq.cred")
@@ -72,18 +74,23 @@ match ExtraWhitespace /\\s\\+$\\| \\+\\ze\\t/ " define t
)
+@when("autopkgtest.autopkgtest-cloud_cloned")
@when_not("autopkgtest.autopkgtest_cloud_symlinked")
def symlink_autopkgtest_cloud():
status.maintenance("Creating symlink to charmed autopkgtest-cloud code")
with UnixUser("ubuntu"):
try:
- autopkgtest_cloud = os.path.join(charm_dir(), "autopkgtest-cloud")
+ autopkgtest_cloud = os.path.join(
+ AUTOPKGTEST_CLOUD_GIT_LOCATION,
+ "charms",
+ "focal",
+ "autopkgtest-cloud-worker",
+ "autopkgtest-cloud",
+ )
os.symlink(autopkgtest_cloud, AUTOPKGTEST_CLOUD_LOCATION)
except FileExistsError:
pass
- status.maintenance(
- "Done creating symlink to charmed autopkgtest-cloud code"
- )
+ status.maintenance("Done creating symlink to autopkgtest-cloud code")
set_flag("autopkgtest.autopkgtest_cloud_symlinked")
@@ -94,7 +101,11 @@ def clone_autodep8():
log("Cloning autodep8", "INFO")
with UnixUser("ubuntu"):
try:
- pygit2.clone_repository(AUTODEP8_CLONE_LOCATION, AUTODEP8_LOCATION)
+ pygit2.clone_repository(
+ AUTODEP8_CLONE_LOCATION,
+ AUTODEP8_LOCATION,
+ checkout_branch=AUTODEP8_CLONE_BRANCH,
+ )
log("autodep8 cloned", "INFO")
status.maintenance("autodep8 cloned")
set_flag("autopkgtest.autodep8_cloned")
@@ -112,6 +123,7 @@ def clone_autopkgtest_package_configs():
pygit2.clone_repository(
AUTOPKGTEST_PER_PACKAGE_CLONE_LOCATION,
AUTOPKGTEST_PER_PACKAGE_LOCATION,
+ checkout_branch=AUTOPKGTEST_PER_PACKAGE_CLONE_BRANCH,
)
log("autopkgtest-package-configs cloned", "INFO")
status.maintenance("autopkgtest-package-configs cloned")
@@ -149,6 +161,26 @@ def clone_autopkgtest():
log("autopkgtest already cloned", "INFO")
+@when("apt.installed.git")
+@when_not("autopkgtest.autopkgtest-cloud_cloned")
+def clone_autopkgtest_cloud():
+ status.maintenance("Cloning autopkgtest-cloud")
+ log("Cloning autopkgtest-cloud", "INFO")
+ with UnixUser("ubuntu"):
+ try:
+ pygit2.clone_repository(
+ AUTOPKGTEST_CLOUD_CLONE_LOCATION,
+ AUTOPKGTEST_CLOUD_GIT_LOCATION,
+ checkout_branch=AUTOPKGTEST_CLOUD_CLONE_BRANCH,
+ )
+ status.maintenance("autopkgtest-cloud cloned")
+ log("autopkgtest-cloud cloned", "INFO")
+ set_flag("autopkgtest.autopkgtest-cloud_cloned")
+ except ValueError:
+ # exists
+ log("autopkgtest-cloud already cloned", "INFO")
+
+
@when_all(
"autopkgtest.autopkgtest_cloud_symlinked",
"autopkgtest.rabbitmq-configured",
@@ -156,7 +188,16 @@ def clone_autopkgtest():
)
def set_up_systemd_units():
status.maintenance("Installing systemd units")
- for unit in glob.glob(os.path.join(charm_dir(), "units", "*")):
+ for unit in glob.glob(
+ os.path.join(
+ AUTOPKGTEST_CLOUD_GIT_LOCATION,
+ "charms",
+ "focal",
+ "autopkgtest-cloud-worker",
+ "units",
+ "*",
+ )
+ ):
base = os.path.basename(unit)
dest = os.path.join(os.path.sep, "etc", "systemd", "system", base)
diff --git a/charms/focal/autopkgtest-web/actions.yaml b/charms/focal/autopkgtest-web/actions.yaml
new file mode 100644
index 0000000..efe984d
--- /dev/null
+++ b/charms/focal/autopkgtest-web/actions.yaml
@@ -0,0 +1,2 @@
+update-sources:
+ description: Update (git pull) cloned repositories
diff --git a/charms/focal/autopkgtest-web/actions/update-sources b/charms/focal/autopkgtest-web/actions/update-sources
new file mode 100755
index 0000000..46a562e
--- /dev/null
+++ b/charms/focal/autopkgtest-web/actions/update-sources
@@ -0,0 +1,18 @@
+#!/usr/local/sbin/charm-env python3
+
+from reactive.autopkgtest_web import (
+ AUTOPKGTEST_CLOUD_GIT_LOCATION,
+ AUTOPKGTEST_CLOUD_CLONE_BRANCH,
+)
+
+import pygit2
+from utils import pull, UnixUser
+
+
+if __name__ == "__main__":
+ for location, branch in [
+ (AUTOPKGTEST_CLOUD_GIT_LOCATION, AUTOPKGTEST_CLOUD_CLONE_BRANCH),
+ ]:
+ repo = pygit2.Repository(location)
+ with UnixUser("ubuntu"):
+ pull(repo, branch)
diff --git a/charms/focal/autopkgtest-web/config.yaml b/charms/focal/autopkgtest-web/config.yaml
index 51df278..1cab2a2 100644
--- a/charms/focal/autopkgtest-web/config.yaml
+++ b/charms/focal/autopkgtest-web/config.yaml
@@ -1,4 +1,8 @@
options:
+ package_status:
+ type: string
+ description: "APT layer package_status"
+ default: install
storage-url-internal:
type: string
default:
diff --git a/charms/focal/autopkgtest-web/layer.yaml b/charms/focal/autopkgtest-web/layer.yaml
index d003a4a..d49c75e 100644
--- a/charms/focal/autopkgtest-web/layer.yaml
+++ b/charms/focal/autopkgtest-web/layer.yaml
@@ -20,6 +20,7 @@ options:
- python3-flask
- python3-flask-openid
- python3-influxdb
+ - python3-pygit2
- python3-swiftclient
- python3-werkzeug
- vim
diff --git a/charms/focal/autopkgtest-web/lib/utils.py b/charms/focal/autopkgtest-web/lib/utils.py
new file mode 100644
index 0000000..4f1316c
--- /dev/null
+++ b/charms/focal/autopkgtest-web/lib/utils.py
@@ -0,0 +1,55 @@
+# pylint: disable=missing-module-docstring, missing-class-docstring, missing-function-docstring
+import os
+import pwd
+
+from charmhelpers.core.hookenv import log
+
+
+# class UnixUser(object):
+class UnixUser:
+ def __init__(self, username):
+ self.username = username
+ pwnam = pwd.getpwnam(username)
+ self.uid = pwnam.pw_uid
+ self.gid = pwnam.pw_gid
+
+ def __enter__(self):
+ log(
+ "Raising UID to {uid} and GID to {gid}".format(
+ uid=self.uid, gid=self.gid
+ ),
+ "INFO",
+ )
+ os.setresgid(self.gid, self.gid, os.getgid())
+ os.setresuid(self.uid, self.uid, os.getuid())
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ _, _, suid = os.getresuid()
+ _, _, sgid = os.getresgid()
+ log(
+ "Restoring UID to {suid} and GID to {sgid}".format(
+ suid=suid, sgid=sgid
+ ),
+ "INFO",
+ )
+ os.setresuid(suid, suid, suid)
+ os.setresgid(sgid, sgid, sgid)
+
+
+def pull(repository, branch):
+ """
+ This will do a sort of git fetch origin && git reset --hard origin/master
+ This raises pygit2.GitError whenever the tree is not clean, preventing to loose cowboys.
+ """
+ origin = [
+ remote for remote in repository.remotes if remote.name == "origin"
+ ][0]
+ origin.fetch()
+ remote_master_id = repository.lookup_reference(
+ f"refs/remotes/origin/{branch}"
+ ).target
+ repository.checkout_tree(repository.get(remote_master_id))
+ master_ref = repository.lookup_reference(f"refs/heads/{branch}")
+ master_ref.set_target(remote_master_id)
+ repository.head.set_target(remote_master_id)
+ log(f"Updated repository {repository} to {remote_master_id}", "INFO")
diff --git a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
index ba091b7..df18d70 100644
--- a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
+++ b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
@@ -6,7 +6,8 @@ import shutil
import subprocess
from textwrap import dedent
-from charmhelpers.core.hookenv import charm_dir, config
+import pygit2
+from charmhelpers.core.hookenv import config, log
from charms.layer import status
from charms.reactive import (
clear_flag,
@@ -18,8 +19,18 @@ from charms.reactive import (
when_not,
when_not_all,
)
+from utils import UnixUser
AUTOPKGTEST_CLOUD_CONF = os.path.expanduser("~ubuntu/autopkgtest-cloud.conf")
+
+AUTOPKGTEST_CLOUD_GIT_LOCATION = os.path.expanduser(
+ "~ubuntu/autopkgtest-cloud.git"
+)
+AUTOPKGTEST_CLOUD_CLONE_LOCATION = (
+ "https://git.launchpad.net/autopkgtest-cloud";
+)
+AUTOPKGTEST_CLOUD_CLONE_BRANCH = "master"
+
GITHUB_SECRETS_PATH = os.path.expanduser("~ubuntu/github-secrets.json")
GITHUB_STATUS_CREDENTIALS_PATH = os.path.expanduser(
"~ubuntu/github-status-credentials.txt"
@@ -85,15 +96,22 @@ match ExtraWhitespace /\\s\\+$\\| \\+\\ze\\t/ " define t
)
+@when("autopkgtest.autopkgtest-cloud_cloned")
@when_not("autopkgtest-web.autopkgtest_web_symlinked")
def symlink_autopkgtest_cloud():
status.maintenance("Creating symlink to charmed autopkgtest-web code")
try:
- autopkgtest_cloud = os.path.join(charm_dir(), "webcontrol")
+ autopkgtest_cloud = os.path.join(
+ AUTOPKGTEST_CLOUD_GIT_LOCATION,
+ "charms",
+ "focal",
+ "autopkgtest-web",
+ "webcontrol",
+ )
os.symlink(autopkgtest_cloud, os.path.expanduser("~ubuntu/webcontrol"))
except FileExistsError:
pass
- status.maintenance("Done creating symlink to charmed autopkgtest-web code")
+ status.maintenance("Done creating symlink to autopkgtest-web code")
set_flag("autopkgtest-web.autopkgtest_web_symlinked")
@@ -144,6 +162,26 @@ def write_autopkgtest_cloud_conf(rabbitmq):
set_flag("autopkgtest-web.config-written")
+@when("apt.installed.git")
+@when_not("autopkgtest.autopkgtest-cloud_cloned")
+def clone_autopkgtest_cloud():
+ status.maintenance("Cloning autopkgtest-cloud")
+ log("Cloning autopkgtest-cloud", "INFO")
+ with UnixUser("ubuntu"):
+ try:
+ pygit2.clone_repository(
+ AUTOPKGTEST_CLOUD_CLONE_LOCATION,
+ AUTOPKGTEST_CLOUD_GIT_LOCATION,
+ checkout_branch=AUTOPKGTEST_CLOUD_CLONE_BRANCH,
+ )
+ status.maintenance("autopkgtest-cloud cloned")
+ log("autopkgtest-cloud cloned", "INFO")
+ set_flag("autopkgtest.autopkgtest-cloud_cloned")
+ except ValueError:
+ # exists
+ log("autopkgtest-cloud already cloned", "INFO")
+
+
@when_all(
"autopkgtest-web.autopkgtest_web_symlinked",
"autopkgtest-web.config-written",
@@ -151,7 +189,16 @@ def write_autopkgtest_cloud_conf(rabbitmq):
def set_up_systemd_units():
status.maintenance("Setting up systemd units")
any_changed = False
- for unit in glob.glob(os.path.join(charm_dir(), "units", "*")):
+ for unit in glob.glob(
+ os.path.join(
+ AUTOPKGTEST_CLOUD_GIT_LOCATION,
+ "charms",
+ "focal",
+ "autopkgtest-web",
+ "units",
+ "*",
+ )
+ ):
base = os.path.basename(unit)
try:
os.symlink(
@@ -202,7 +249,7 @@ def initially_configure_website(website):
)
def set_up_web_config(apache):
status.maintenance("Setting up web config (apache)")
- webcontrol_dir = os.path.join(charm_dir(), "webcontrol")
+ webcontrol_dir = os.path.expanduser("~ubuntu/webcontrol")
sn = config().get("hostname")
https_proxy = config().get("https-proxy")
no_proxy = config().get("no-proxy")
@@ -485,6 +532,7 @@ def make_runtime_tmpfiles():
set_flag("autopkgtest-web.runtime-dir-created")
+@when("autopkgtest-web.autopkgtest_web_symlinked")
@when_not("autopkgtest-web.running-json-symlinked")
def symlink_running():
status.maintenance("Creating symlink to running.json")
@@ -493,7 +541,12 @@ def symlink_running():
os.path.join(
os.path.sep, "run", "amqp-status-collector", "running.json"
),
- os.path.join(charm_dir(), "webcontrol", "static", "running.json"),
+ os.path.join(
+ os.path.expanduser("~ubuntu"),
+ "webcontrol",
+ "static",
+ "running.json",
+ ),
)
status.maintenance("Done creating symlink to running.json")
set_flag("autopkgtest-web.running-json-symlinked")
@@ -501,6 +554,7 @@ def symlink_running():
pass
+@when("autopkgtest-web.autopkgtest_web_symlinked")
@when_not_all(
"autopkgtest-web.public-db-symlinked",
"autopkgtest-web.public-db-sha256-symlinked",
@@ -523,7 +577,10 @@ def symlink_public_db():
os.symlink(
os.path.join(publicdir, symlink_file),
os.path.join(
- charm_dir(), "webcontrol", "static", symlink_file
+ os.path.expanduser("~ubuntu"),
+ "webcontrol",
+ "static",
+ symlink_file,
),
)
set_flag(symlink_flag)
--
Mailing list: https://launchpad.net/~canonical-ubuntu-qa
Post to : canonical-ubuntu-qa@lists.launchpad.net
Unsubscribe : https://launchpad.net/~canonical-ubuntu-qa
More help : https://help.launchpad.net/ListHelp
