Tim Andersson has proposed merging
~andersson123/autopkgtest-cloud:swift-cleanup into autopkgtest-cloud:master.
Requested reviews:
Canonical's Ubuntu QA (canonical-ubuntu-qa)
For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/463593
--
Your team Canonical's Ubuntu QA is requested to review the proposed merge of
~andersson123/autopkgtest-cloud:swift-cleanup into autopkgtest-cloud:master.
diff --git a/charms/focal/autopkgtest-web/webcontrol/swift-cleanup b/charms/focal/autopkgtest-web/webcontrol/swift-cleanup
new file mode 100755
index 0000000..3b5b5f0
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/swift-cleanup
@@ -0,0 +1,170 @@
+#!/usr/bin/python3
+
+
+import configparser
+import io
+import itertools
+import json
+import logging
+import os
+import sys
+import tarfile
+import tempfile
+
+import swiftclient
+from distro_info import UbuntuDistroInfo
+
+LOGGER = logging.getLogger(__name__)
+SWIFT_CREDS_FILE = "/home/ubuntu/public-swift-creds"
+
+config = None
+
+
+def fix_broken_testinfo_json(
+ object_contents, swift_conn, container_name, object_name
+):
+ tar_bytes = io.BytesIO(object_contents)
+ temp_dir = tempfile.TemporaryDirectory()
+ tmp_dir = temp_dir.name
+ result_dir = tmp_dir + "/result"
+ os.makedirs(result_dir, exist_ok=True)
+ files_dict = {}
+ with tarfile.open(None, "r", tar_bytes) as tar:
+ # amend testinfo.json
+ try:
+ testinfo = json.loads(
+ tar.extractfile("testinfo.json").read().decode()
+ )
+ except Exception as _:
+ testinfo = {}
+ files = tar.getnames()
+ for f in files:
+ files_dict[f] = tar.extractfile(f).read().decode().strip()
+ files_dict["testinfo.json"] = testinfo
+ # write the files to new directory for tar'ing
+ for file_name, values in files_dict.items():
+ with open(f"{result_dir}/{file_name}", "w") as f:
+ if file_name.endswith(".json"):
+ json.dump(values, f)
+ else:
+ f.write(values)
+ # tar up the directory
+ output_filename = tmp_dir + "/result.tar"
+ with tarfile.open(output_filename, "w") as tar:
+ for fn in os.listdir(result_dir):
+ p = os.path.join(result_dir, fn)
+ tar.add(p, arcname=fn)
+ # now need to upload the result.tar to swift
+ logging.info("Deleting old object...")
+ try:
+ swift_conn.delete_object(container_name, object_name)
+ except Exception as e:
+ logging.error(f"swift delete failed with {e}")
+ return
+ logging.info("Uploading new object...")
+ with open(output_filename, "rb") as f:
+ swift_conn.put_object(
+ container_name,
+ object_name,
+ contents=f,
+ content_type=None,
+ headers=None,
+ content_length=os.path.getsize(output_filename),
+ )
+ logging.info("New object uploaded.")
+ temp_dir.cleanup()
+
+
+def check_object_integrity(object_contents, object_name):
+ tar_bytes = io.BytesIO(object_contents)
+ try:
+ with tarfile.open(None, "r", tar_bytes) as tar:
+ _ = json.loads(tar.extractfile("testinfo.json").read().decode())
+ except (KeyError, ValueError, tarfile.TarError) as e:
+ # here, I should amend testinfo.json and reupload
+ logging.info("%s is damaged, ignoring: %s" % (object_name, str(e)))
+ return False
+ return True
+
+
+def fetch_container(release, swift_conn):
+ """Download new results from a swift container"""
+ container_name = "autopkgtest-" + release
+ logging.info(f"Fetching objects in container: {container_name}")
+
+ try:
+ _, objects = swift_conn.get_container(
+ container_name, full_listing=True
+ )
+ logging.info("Objects fetched...")
+ num_objects = len(objects)
+ ctr = 0
+ increment = 10
+ for obj in objects:
+ ctr += 1
+ percent = (ctr / num_objects) * 100
+ if percent > increment or ctr == 1:
+ logging.info("%f percent" % ((ctr / num_objects) * 100))
+ increment += 10
+ if obj["name"].endswith("result.tar"):
+ _, contents = swift_conn.get_object(
+ container_name, obj["name"]
+ )
+ integrity = check_object_integrity(contents, obj["name"])
+ if not integrity:
+ fix_broken_testinfo_json(
+ contents, swift_conn, container_name, obj["name"]
+ )
+ sys.stdout.flush()
+ except swiftclient.ClientException as e:
+ LOGGER.error(
+ "Something went wrong accessing container %s\nTraceback: %s"
+ % (container_name, str(e))
+ )
+ raise
+
+
+def get_swift_con():
+ swift_cfg = configparser.ConfigParser()
+ with open(SWIFT_CREDS_FILE) as fp:
+ swift_cfg.read_file(
+ itertools.chain(["[swift]"], fp), source=SWIFT_CREDS_FILE
+ )
+ swift_creds = {
+ "authurl": swift_cfg["swift"]["OS_AUTH_URL"],
+ "user": swift_cfg["swift"]["OS_USERNAME"],
+ "key": swift_cfg["swift"]["OS_PASSWORD"],
+ "os_options": {
+ "region_name": swift_cfg["swift"]["OS_REGION_NAME"],
+ "project_domain_name": swift_cfg["swift"][
+ "OS_PROJECT_DOMAIN_NAME"
+ ],
+ "project_name": swift_cfg["swift"]["OS_PROJECT_NAME"],
+ "user_domain_name": swift_cfg["swift"]["OS_USER_DOMAIN_NAME"],
+ },
+ "auth_version": 3,
+ }
+ return swiftclient.Connection(**swift_creds)
+
+
+def get_releases():
+ releases = list(
+ set(
+ UbuntuDistroInfo().supported() + UbuntuDistroInfo().supported_esm()
+ )
+ )
+ releases.sort(key=UbuntuDistroInfo().all.index, reverse=True)
+ return releases
+
+
+def main():
+ logging.basicConfig(level=logging.INFO)
+ logging.info("Setting up swift connection")
+ swift_conn = get_swift_con()
+ releases = get_releases()
+ for release in releases:
+ fetch_container(release, swift_conn)
+
+
+if __name__ == "__main__":
+ main()
--
Mailing list: https://launchpad.net/~canonical-ubuntu-qa
Post to : canonical-ubuntu-qa@lists.launchpad.net
Unsubscribe : https://launchpad.net/~canonical-ubuntu-qa
More help : https://help.launchpad.net/ListHelp
