Tim Andersson has proposed merging
~andersson123/autopkgtest-cloud:swift-cleanup into autopkgtest-cloud:master.
Requested reviews:
Canonical's Ubuntu QA (canonical-ubuntu-qa)
For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/463593
--
Your team Canonical's Ubuntu QA is requested to review the proposed merge of
~andersson123/autopkgtest-cloud:swift-cleanup into autopkgtest-cloud:master.
diff --git a/charms/focal/autopkgtest-web/webcontrol/swift-cleanup b/charms/focal/autopkgtest-web/webcontrol/swift-cleanup
new file mode 100644
index 0000000..ff7a43e
--- /dev/null
+++ b/charms/focal/autopkgtest-web/webcontrol/swift-cleanup
@@ -0,0 +1,126 @@
+#!/usr/bin/python3
+
+
+import configparser
+import datetime
+import io
+import itertools
+import json
+import logging
+import os
+import sqlite3
+import sys
+import tarfile
+import time
+import urllib.parse
+
+import amqplib.client_0_8 as amqp
+import swiftclient
+from distro_info import UbuntuDistroInfo
+from helpers.utils import SqliteWriterConfig, get_autopkgtest_cloud_conf
+
+
+LOGGER = logging.getLogger(__name__)
+SWIFT_CREDS_FILE = "/home/ubuntu/public-swift-creds"
+
+config = None
+
+
+def check_object_integrity(object_contents, object_name):
+ (_, _, _, src, _, _) = object_name.split("/")
+ tar_bytes = io.BytesIO(object_contents)
+ try:
+ with tarfile.open(None, "r", tar_bytes) as tar:
+ exitcode = int(tar.extractfile("exitcode").read().strip())
+ try:
+ srcver = (
+ tar.extractfile("testpkg-version").read().decode().strip()
+ )
+ except KeyError:
+ # not found
+ if exitcode in (4, 12, 20):
+ # repair it
+ srcver = "%s unknown" % (src)
+ else:
+ raise
+ (_, _) = srcver.split()
+ testinfo = json.loads(
+ tar.extractfile("testinfo.json").read().decode()
+ )
+ _ = testinfo.get("uuid", "")
+ _ = int(tar.extractfile("duration").read().strip())
+ # KeyError means the file is not there, i.e. there isn't a human
+ # requester
+ try:
+ _ = (
+ tar.extractfile("requester").read().decode().strip()
+ )
+ except KeyError:
+ _ = ""
+ except (KeyError, ValueError, tarfile.TarError) as e:
+ LOGGER.debug("%s is damaged, ignoring: %s" % (object_name, str(e)))
+ return False
+ return True
+
+
+def fetch_container(release, swift_conn):
+ """Download new results from a swift container"""
+ container_name = "autopkgtest-" + release
+
+ try:
+ _, objects = swift_conn.get_container(container_name, full_listing=True)
+ for obj in objects:
+ _, contents = swift_conn.get_object(obj["name"])
+ integrity = check_object_integrity(contents, obj["name"])
+ if not integrity:
+ print(obj["name"])
+ except swiftclient.ClientException as e:
+ LOGGER.error(
+ "Something went wrong accessing container %s\nTraceback: %s"
+ % (container_name, str(e))
+ )
+ raise
+
+
+def get_swift_con():
+ swift_cfg = configparser.ConfigParser()
+ with open(SWIFT_CREDS_FILE) as fp:
+ swift_cfg.read_file(
+ itertools.chain(["[swift]"], fp), source=SWIFT_CREDS_FILE
+ )
+ swift_creds = {
+ "authurl": swift_cfg["swift"]["OS_AUTH_URL"],
+ "user": swift_cfg["swift"]["OS_USERNAME"],
+ "key": swift_cfg["swift"]["OS_PASSWORD"],
+ "os_options": {
+ "region_name": swift_cfg["swift"]["OS_REGION_NAME"],
+ "project_domain_name": swift_cfg["swift"][
+ "OS_PROJECT_DOMAIN_NAME"
+ ],
+ "project_name": swift_cfg["swift"]["OS_PROJECT_NAME"],
+ "user_domain_name": swift_cfg["swift"]["OS_USER_DOMAIN_NAME"],
+ },
+ "auth_version": 3,
+ }
+ return swiftclient.Connection(**swift_creds)
+
+
+def get_releases():
+ releases = list(
+ set(
+ UbuntuDistroInfo().supported() + UbuntuDistroInfo().supported_esm()
+ )
+ )
+ releases.sort(key=UbuntuDistroInfo().all.index, reverse=True)
+ return releases
+
+
+def main():
+ swift_conn = get_swift_con()
+ releases = get_releases()
+ for release in releases:
+ fetch_container(release, swift_conn)
+
+
+if __name__ == "__main__":
+ main()
--
Mailing list: https://launchpad.net/~canonical-ubuntu-qa
Post to : canonical-ubuntu-qa@lists.launchpad.net
Unsubscribe : https://launchpad.net/~canonical-ubuntu-qa
More help : https://help.launchpad.net/ListHelp
