Tim Andersson has proposed merging
~andersson123/autopkgtest-cloud:cleanup-old-systemd-upstream-jobs into
autopkgtest-cloud:master.
Requested reviews:
Canonical's Ubuntu QA (canonical-ubuntu-qa)
For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/471073
Remove results from systemd-upstream containers that are older than 30 days.
--
Your team Canonical's Ubuntu QA is requested to review the proposed merge of
~andersson123/autopkgtest-cloud:cleanup-old-systemd-upstream-jobs into
autopkgtest-cloud:master.
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/cleanup-old-upstream-results b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/cleanup-old-upstream-results
new file mode 100755
index 0000000..74f7730
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/cleanup-old-upstream-results
@@ -0,0 +1,92 @@
+#!/usr/bin/python3
+
+import argparse
+import datetime
+import logging
+import os
+import re
+
+import swiftclient
+
+UPSTREAM_RESULTS_RETENTION_TIME = 30 # days
+SWIFT_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+SWIFT_RETRIES = 5
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ description="Script for removing upstream results older than 30 days."
+ )
+ parser.add_argument(
+ "--debug", "-d", action="store_true", help="debug mode"
+ )
+ return parser.parse_args()
+
+
+def connect_swift():
+ return swiftclient.Connection(
+ authurl=os.environ["SWIFT_AUTH_URL"],
+ user=os.environ["SWIFT_USERNAME"],
+ key=os.environ["SWIFT_PASSWORD"],
+ tenant_name=os.environ["SWIFT_PROJECT_NAME"],
+ os_options={"region_name": os.environ["SWIFT_REGION"]},
+ auth_version="3.0",
+ retries=10,
+ starting_backoff=10,
+ )
+
+
+def main():
+ args = parse_args()
+ if args.debug:
+ logging.basicConfig(level=logging.DEBUG)
+ else:
+ logging.basicConfig(level=logging.INFO)
+ swift_con = connect_swift()
+ _, container_list = swift_con.get_account()
+ current_time = datetime.datetime.now()
+ upstream_regex = re.compile("^autopkgtest-[a-z]*-upstream-", re.DOTALL)
+ for container in container_list:
+ if not upstream_regex.search(container["name"]):
+ continue
+ logging.info(f"Examining container {container['name']}")
+ _, objects = swift_con.get_container(
+ container["name"], full_listing=True
+ )
+ num_deleted = 0
+ for object in objects:
+ object_datetime = datetime.datetime.strptime(
+ object["last_modified"], SWIFT_TIME_FORMAT
+ )
+ if (current_time - object_datetime) > datetime.timedelta(
+ days=UPSTREAM_RESULTS_RETENTION_TIME
+ ):
+ logging.debug(
+ (
+ f"Deleting object {object['name']} as it's older "
+ f"than {UPSTREAM_RESULTS_RETENTION_TIME} days"
+ )
+ )
+ for retry in range(SWIFT_RETRIES):
+ try:
+ swift_con.delete_object(
+ container["name"], object["name"]
+ )
+ num_deleted += 1
+ break
+ except swiftclient.ClientException as e:
+ logging.info(
+ (
+ f"Deleting object failed with {e} - "
+ f"retrying {SWIFT_RETRIES - retry} more times"
+ )
+ )
+ swift_con = connect_swift()
+ logging.info(
+ f"Deleted {num_deleted} of {len(objects)} from container {container['name']}"
+ )
+ logging.info("Done cleaning up old upstream swift results")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/charms/focal/autopkgtest-cloud-worker/units/cleanup-old-upstream-results.service b/charms/focal/autopkgtest-cloud-worker/units/cleanup-old-upstream-results.service
new file mode 100644
index 0000000..44aea91
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/units/cleanup-old-upstream-results.service
@@ -0,0 +1,9 @@
+[Unit]
+Description=Delete old upstream results from swift
+
+[Service]
+Type=oneshot
+User=ubuntu
+Group=ubuntu
+EnvironmentFile=/home/ubuntu/swift-password.cred
+ExecStart=/home/ubuntu/autopkgtest-cloud/tools/cleanup-old-upstream-results
diff --git a/charms/focal/autopkgtest-cloud-worker/units/cleanup-old-upstream-results.timer b/charms/focal/autopkgtest-cloud-worker/units/cleanup-old-upstream-results.timer
new file mode 100644
index 0000000..3c72c87
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/units/cleanup-old-upstream-results.timer
@@ -0,0 +1,8 @@
+[Unit]
+Description=Delete old upstream results from swift (timer)
+
+[Timer]
+OnCalendar=daily
+
+[Install]
+WantedBy=autopkgtest.target
--
Mailing list: https://launchpad.net/~canonical-ubuntu-qa
Post to : canonical-ubuntu-qa@lists.launchpad.net
Unsubscribe : https://launchpad.net/~canonical-ubuntu-qa
More help : https://help.launchpad.net/ListHelp
