Tim Andersson has proposed merging
~andersson123/autopkgtest-cloud:d-a-r-why-keep-dying into
autopkgtest-cloud:master.
Requested reviews:
Canonical's Ubuntu QA (canonical-ubuntu-qa)
For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/471633
Fix download-all-results OOM'ing by iterating through each container in batches
--
Your team Canonical's Ubuntu QA is requested to review the proposed merge of
~andersson123/autopkgtest-cloud:d-a-r-why-keep-dying into
autopkgtest-cloud:master.
diff --git a/charms/focal/autopkgtest-web/webcontrol/download-all-results b/charms/focal/autopkgtest-web/webcontrol/download-all-results
index 0798695..4d6a807 100755
--- a/charms/focal/autopkgtest-web/webcontrol/download-all-results
+++ b/charms/focal/autopkgtest-web/webcontrol/download-all-results
@@ -53,10 +53,12 @@ def amqp_connect():
return amqp_con
-def list_remote_container(container_name, swift_conn):
+def list_remote_container(container_name, swift_conn, marker, limit=1000):
LOGGER.debug("Listing container %s", container_name)
_, list_of_test_results = swift_conn.get_container(
- container_name, full_listing=True
+ container_name,
+ marker=marker,
+ limit=limit,
)
ret_me = {}
for result in list_of_test_results:
@@ -190,27 +192,35 @@ def fetch_one_result(container_name, object_name, swift_conn):
def fetch_container(release, swift_conn):
"""Download new results from a swift container"""
container_name = "autopkgtest-" + release
+ marker = None
- try:
- our_results = list_our_results(release)
- known_results = list_remote_container(container_name, swift_conn)
+ while True:
+ try:
+ last_marker = marker
+ our_results = list_our_results(release)
+ known_results = list_remote_container(
+ container_name, swift_conn, marker
+ )
+ marker = known_results[list(known_results.keys())[-1]]
+ if last_marker == marker:
+ return
- need_to_fetch = set(known_results.keys()) - our_results
+ need_to_fetch = set(known_results.keys()) - our_results
- LOGGER.debug("Need to download %d items", len(need_to_fetch))
+ LOGGER.debug("Need to download %d items", len(need_to_fetch))
- for run_id in need_to_fetch:
- fetch_one_result(
- container_name=container_name,
- object_name=known_results[run_id],
- swift_conn=swift_conn,
+ for run_id in need_to_fetch:
+ fetch_one_result(
+ container_name=container_name,
+ object_name=known_results[run_id],
+ swift_conn=swift_conn,
+ )
+ except swiftclient.ClientException as e:
+ LOGGER.error(
+ "Something went wrong accessing container %s\nTraceback: %s"
+ % (container_name, str(e))
)
- except swiftclient.ClientException as e:
- LOGGER.error(
- "Something went wrong accessing container %s\nTraceback: %s"
- % (container_name, str(e))
- )
- raise
+ raise
if __name__ == "__main__":
--
Mailing list: https://launchpad.net/~canonical-ubuntu-qa
Post to : canonical-ubuntu-qa@lists.launchpad.net
Unsubscribe : https://launchpad.net/~canonical-ubuntu-qa
More help : https://help.launchpad.net/ListHelp
