Tim Andersson has proposed merging
~andersson123/autopkgtest-cloud:update-github-jobs-swiftclient-refactor into
autopkgtest-cloud:master.
Requested reviews:
Canonical's Ubuntu QA (canonical-ubuntu-qa)
For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/456202
--
Your team Canonical's Ubuntu QA is requested to review the proposed merge of
~andersson123/autopkgtest-cloud:update-github-jobs-swiftclient-refactor into
autopkgtest-cloud:master.
diff --git a/charms/focal/autopkgtest-web/config.yaml b/charms/focal/autopkgtest-web/config.yaml
index cc97824..9a3148a 100644
--- a/charms/focal/autopkgtest-web/config.yaml
+++ b/charms/focal/autopkgtest-web/config.yaml
@@ -34,3 +34,7 @@ options:
default:
description: "$no_proxy environment variable (for accessing sites \
other than github, like login.ubuntu.com and launchpad.net)"
+ openstack-creds:
+ type: string
+ default:
+ description: "creds for openstack swift storage for updating github jobs"
diff --git a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
index 842348b..5b9e6fd 100644
--- a/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
+++ b/charms/focal/autopkgtest-web/reactive/autopkgtest_web.py
@@ -24,6 +24,8 @@ SWIFT_WEB_CREDENTIALS_PATH = os.path.expanduser(
"~ubuntu/swift-web-credentials.conf"
)
+OPENSTACK_CREDS_PATH = os.path.expanduser("~ubuntu/openstack-creds")
+
@when_not("autopkgtest-web.autopkgtest_web_symlinked")
def symlink_autopkgtest_cloud():
@@ -259,6 +261,14 @@ def write_swift_web_credentials():
pass
+@when_all("config.changed.openstack-creds", "config.set.openstack-creds")
+def write_openstack_creds():
+ openstack_creds = config().get("openstack-creds")
+
+ with open(OPENSTACK_CREDS_PATH, "w") as f:
+ f.write(openstack_creds)
+
+
@when_not("config.set.swift-web-credentials")
def clear_swift_web_credentials():
try:
diff --git a/charms/focal/autopkgtest-web/units/update-github-jobs.service b/charms/focal/autopkgtest-web/units/update-github-jobs.service
index a52bac5..17fb673 100644
--- a/charms/focal/autopkgtest-web/units/update-github-jobs.service
+++ b/charms/focal/autopkgtest-web/units/update-github-jobs.service
@@ -5,6 +5,5 @@ Description=Update GitHub job status
Type=oneshot
User=www-data
Group=www-data
-TimeoutStartSec=1m
-EnvironmentFile=/etc/environment.d/*.conf
+EnvironmentFile=/home/ubuntu/openstack-creds
ExecStart=/home/ubuntu/webcontrol/update-github-jobs
diff --git a/charms/focal/autopkgtest-web/webcontrol/update-github-jobs b/charms/focal/autopkgtest-web/webcontrol/update-github-jobs
index 6362803..0223385 100755
--- a/charms/focal/autopkgtest-web/webcontrol/update-github-jobs
+++ b/charms/focal/autopkgtest-web/webcontrol/update-github-jobs
@@ -7,40 +7,22 @@ import logging
import os
import sys
import tarfile
-import time
-import urllib.parse
-import urllib.request
-from urllib.error import HTTPError
+import swiftclient
from request.submit import Submit
PENDING_DIR = "/run/autopkgtest_webcontrol/github-pending"
-swift_url = None
-external_url = None
-
-
-def result_matches_job(result_url, params):
- # download result.tar and get exit code and testinfo
- for _ in range(5):
- try:
- with urllib.request.urlopen(result_url + "/result.tar") as f:
- tar_bytes = io.BytesIO(f.read())
- break
- except IOError as e:
- logging.debug(
- "failed to download result %s: %s, retrying...", result_url, e
- )
- time.sleep(1)
- else:
- logging.error("failed to download result %s", result_url)
- return
+
+def result_matches_job(swift_conn, container, result_obj, params):
+ _, contents = swift_conn.get_object(container, result_obj)
+ tar_bytes = io.BytesIO(contents)
try:
with tarfile.open(None, "r", tar_bytes) as tar:
exitcode = int(tar.extractfile("exitcode").read().strip())
info = json.loads(tar.extractfile("testinfo.json").read().decode())
except (KeyError, ValueError, tarfile.TarError) as e:
- logging.error("broken result %s: %s", result_url, e)
+ logging.error("broken result %s: %s", result_obj, e)
return
try:
result_env = info["custom_environment"]
@@ -88,19 +70,25 @@ def finish_job(jobfile, params, code, log_url):
Submit.post_json(
statuses_url,
data,
- os.path.expanduser("~/github-status-credentials.txt"),
+ os.path.expanduser("/home/ubuntu/github-status-credentials.txt"),
params["package"],
)
- logging.debug("removing job file")
- os.unlink(jobfile)
+ logging.debug("removing job file %s" % jobfile)
+ try:
+ os.unlink(jobfile)
+ except FileNotFoundError as _:
+ logging.debug(
+ "jobfile %s not found, maybe it was already deleted?" % jobfile
+ )
+ return
+ logging.debug("job %s finished!" % jobfile)
-def process_job(jobfile):
+def process_job(jobfile, swift_conn, ext_url):
try:
with open(jobfile) as f:
params = json.load(f)
- mtime = os.fstat(f.fileno()).st_mtime
except json.decoder.JSONDecodeError as e:
logging.error("couldn't read %s, skipping: %s", jobfile, e)
return
@@ -117,47 +105,17 @@ def process_job(jobfile):
container += "-" + params["ppas"][-1].replace("/", "-")
except (KeyError, IndexError):
pass
- container_url = os.path.join(swift_url, container)
- package = params["package"]
- pkghash = package.startswith("lib") and package[:4] or package[0]
- timestamp = time.strftime("%Y%m%d_%H%M%S", time.gmtime(mtime))
-
- args = {
- "format": "plain",
- "delimiter": "@",
- "prefix": "%s/%s/%s/%s/"
- % (params["release"], params["arch"], pkghash, package),
- }
- args["marker"] = args["prefix"] + timestamp
- query_url = container_url + "?" + urllib.parse.urlencode(args)
- logging.debug("Swift URL query: %s", query_url)
- try:
- with urllib.request.urlopen(query_url) as f:
- for result in f:
- result_url = os.path.join(
- container_url, result.strip().decode()
- )
- logging.debug(
- "checking result %s for job %s",
- result_url,
- os.path.basename(jobfile),
- )
- code = result_matches_job(result_url, params)
- if code is not None:
- finish_job(
- jobfile,
- params,
- code,
- result_url.replace(swift_url, external_url)
- + "/log.gz",
- )
- break
- except HTTPError as e:
- logging.error(
- "job %s URL %s failed: %s", os.path.basename(jobfile), query_url, e
- )
- if e.code == 404:
- os.unlink(jobfile)
+
+ _, object_list = swift_conn.get_container(container, full_listing=True)
+
+ for obj in object_list:
+ if "result.tar" in obj["name"]:
+ code = result_matches_job(
+ swift_conn, container, obj["name"], params
+ )
+ if code is not None:
+ log_url = "/".join([ext_url, container, obj["name"]])
+ finish_job(jobfile, params, code, log_url)
if __name__ == "__main__":
@@ -168,12 +126,23 @@ if __name__ == "__main__":
sys.exit(0)
config = configparser.ConfigParser()
- config.read(os.path.expanduser("~ubuntu/autopkgtest-cloud.conf"))
- swift_url = config["web"]["SwiftURL"]
- try:
- external_url = config["web"]["ExternalURL"]
- except KeyError:
- external_url = swift_url
+ config.read(os.path.expanduser("/home/ubuntu/autopkgtest-cloud.conf"))
+ external_url = config["web"]["ExternalURL"]
+
+ # setup_swift_conn
+ swift_creds = {
+ "authurl": os.environ["OS_AUTH_URL"],
+ "user": os.environ["OS_USERNAME"],
+ "key": os.environ["OS_PASSWORD"],
+ "os_options": {
+ "region_name": os.environ["OS_REGION_NAME"],
+ "project_domain_name": os.environ["OS_PROJECT_DOMAIN_NAME"],
+ "project_name": os.environ["OS_PROJECT_NAME"],
+ "user_domain_name": os.environ["OS_USER_DOMAIN_NAME"],
+ },
+ "auth_version": 3,
+ }
+ swift_conn = swiftclient.Connection(**swift_creds)
jobs = sys.argv[1:]
@@ -181,4 +150,4 @@ if __name__ == "__main__":
jobs = os.listdir(PENDING_DIR)
for job in jobs:
- process_job(os.path.join(PENDING_DIR, job))
+ process_job(os.path.join(PENDING_DIR, job), swift_conn, external_url)
diff --git a/docs/deploying.rst b/docs/deploying.rst
index 85d853f..17102c0 100644
--- a/docs/deploying.rst
+++ b/docs/deploying.rst
@@ -76,6 +76,10 @@ These are all that are required for a devel deployment. For production there
are more secrets required, see the ``service-bundle`` and charm
``config.yaml`` for more information.
+``openstack-creds`` is also a required secrets file for production. Run ``load_creds openstack``
+and then ``env | grep OS_``. The variables set here need to be placed into the secrets directory
+with the filename ``openstack-creds``.
+
Configure
---------
diff --git a/mojo/service-bundle b/mojo/service-bundle
index 26b74c0..df2ffe7 100644
--- a/mojo/service-bundle
+++ b/mojo/service-bundle
@@ -189,6 +189,7 @@ applications:
github-secrets: include-file://{{local_dir}}/github-secrets.json
github-status-credentials: include-file://{{local_dir}}/github-status-credentials.txt
swift-web-credentials: include-file://{{local_dir}}/swift-web-credentials.conf
+ openstack-creds: include-file://{{local_dir}}/openstack-creds
https-proxy: {{ https_proxy }}
no-proxy: {{ no_proxy }}
{%- endif %}
--
Mailing list: https://launchpad.net/~canonical-ubuntu-qa
Post to : canonical-ubuntu-qa@lists.launchpad.net
Unsubscribe : https://launchpad.net/~canonical-ubuntu-qa
More help : https://help.launchpad.net/ListHelp
