Tim Andersson has proposed merging ~andersson123/autopkgtest-cloud:auto-queue-cleanup into autopkgtest-cloud:master.
Requested reviews: Canonical's Ubuntu QA (canonical-ubuntu-qa) For more details, see: https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/463146 -- Your team Canonical's Ubuntu QA is requested to review the proposed merge of ~andersson123/autopkgtest-cloud:auto-queue-cleanup into autopkgtest-cloud:master.
diff --git a/charms/focal/autopkgtest-web/webcontrol/queue-merger b/charms/focal/autopkgtest-web/webcontrol/queue-merger new file mode 100755 index 0000000..f1f9f81 --- /dev/null +++ b/charms/focal/autopkgtest-web/webcontrol/queue-merger @@ -0,0 +1,111 @@ +#!/usr/bin/python3 + +# import argparse +import apt_pkg +import json +import itertools +import os +import apt.progress.base +# import urllib + +# steal from here: +# https://gist.github.com/julian-klode/2785b07e76abaed78d34242a39d9bf0b + + +""" +I think maybe this has to run on the cloud worker to run filter-amqp? Needs some more thought. + +contexts = ["huge", "ppa", "upstream", "ubuntu"] + +Steps: +for release in releases: + for arch in arches: + for queue in queue_names: + get queue + get proposed packages + calculate merged queue + empty context/release/arch queue + re-queue the merged queue + +make sure to differentiate between huge and non-huge queues +remember to add logging!!! +""" + +def get_queue(release, arch, context, all_queues): + """Build a single queue of package, args tuples.""" + queue = [ + (i.split()[0], json.loads(i.split(None, 1)[1])) + for i in all_queues["queues"][context][release][arch]["requests"] + ] + return queue + + +def get_proposed_packages(): + """Get all valid triggers in proposed, returns set of package/version strings.""" + tmpdir = "merge-requests.tmp" + os.makedirs(os.path.join(tmpdir, "etc/apt"), exist_ok=True) + os.makedirs(os.path.join(tmpdir, "var/lib/apt"), exist_ok=True) + with open(os.path.join(tmpdir, "etc/apt/sources.list"), "w") as sources: + print( + "deb-src [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] https://snapshot.ubuntu.com/ubuntu noble-proposed main universe restricted multiverse", + file=sources, + ) + + apt_pkg.config["Dir"] = tmpdir + apt_pkg.init() + sl = apt_pkg.SourceList() + sl.read_main_list() + cache = apt_pkg.Cache(None) + cache.update(apt.progress.base.AcquireProgress(), sl) + + srcrecords = apt_pkg.SourceRecords() + + proposed = set() + while srcrecords.step(): + proposed.add(srcrecords.package + "/" + srcrecords.version) + return proposed + + +def merge_queue(queue, proposed): + """Calculate the merged queue""" + out = [] + for package, runs in itertools.groupby( + sorted(queue, key=lambda i: i[0]), lambda i: i[0] + ): + triggers = set() + for package, args in runs: + for trigger in args["triggers"]: + if trigger in proposed or trigger == "migration-reference/0": + triggers.add(trigger) + else: + print( + "Dropping obsolete trigger", + trigger, + "for", + package, + file=sys.stderr, + ) + + if "migration-reference/0" in triggers: + out.append((package, {"triggers": ["migration-reference/0"]})) + triggers.remove("migration-reference/0") + if triggers: + out.append((package, {"all-proposed": "1", "triggers": sorted(triggers)})) + return out + + +def main(): + with open("/var/lib/cache-amqp/queued.json", "r") as f: + all_queue_info = json.load(f) + contexts = ["huge", "ppa", "upstream", "ubuntu"] + arches = all_queue_info.get("arches") + releases = all_queue_info.get("releases") + proposed_packages = get_proposed_packages() + for context in contexts: + for release in releases: + for arch in arches: + queue_name = f"debci-{context}-{release}-{arch}" if context != "ubuntu" else f"debci-{release}-{arch}" + individual_queue = get_queue(release, arch, context, all_queue_info) + merged_queue = merge_queue(individual_queue, proposed_packages) + for pkg, run_args in merged_queue: + print(run_args)
-- Mailing list: https://launchpad.net/~canonical-ubuntu-qa Post to : canonical-ubuntu-qa@lists.launchpad.net Unsubscribe : https://launchpad.net/~canonical-ubuntu-qa More help : https://help.launchpad.net/ListHelp
