Tim Andersson has proposed merging
~andersson123/autopkgtest-cloud:make-killing-tests-less-painful into
autopkgtest-cloud:master.
Requested reviews:
Canonical's Ubuntu QA (canonical-ubuntu-qa)
For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/464740
--
Your team Canonical's Ubuntu QA is requested to review the proposed merge of
~andersson123/autopkgtest-cloud:make-killing-tests-less-painful into
autopkgtest-cloud:master.
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
index bfa35e7..40cb8a3 100755
--- a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/worker/worker
@@ -29,10 +29,14 @@ from urllib.error import HTTPError
import amqplib.client_0_8 as amqp
import distro_info
+import novaclient.client
+import novaclient.exceptions
import swiftclient
import systemd.journal
from influxdb import InfluxDBClient
from influxdb.exceptions import InfluxDBClientError
+from keystoneauth1 import session
+from keystoneauth1.identity import v2, v3
ALL_RELEASES = distro_info.UbuntuDistroInfo().get_all(result="object")
@@ -621,6 +625,36 @@ def cleanup_and_sleep(out_dir):
time.sleep(300)
+def kill_openstack_server(test_uuid: str):
+ if int(os.environ.get("OS_IDENTITY_API_VERSION")) == 3:
+ auth = v3.Password(
+ auth_url=os.environ["OS_AUTH_URL"],
+ username=os.environ["OS_USERNAME"],
+ password=os.environ["OS_PASSWORD"],
+ project_name=os.environ["OS_PROJECT_NAME"],
+ user_domain_name=os.environ["OS_USER_DOMAIN_NAME"],
+ project_domain_name=os.environ["OS_PROJECT_DOMAIN_NAME"],
+ )
+ else:
+ auth = v2.Password(
+ auth_url=os.environ["OS_AUTH_URL"],
+ username=os.environ["OS_USERNAME"],
+ password=os.environ["OS_PASSWORD"],
+ tenant_name=os.environ["OS_TENANT_NAME"],
+ )
+ sess = session.Session(auth=auth)
+ nova = novaclient.client.Client(
+ "2",
+ session=sess,
+ region_name=os.environ["OS_REGION_NAME"],
+ )
+ for instance in nova.servers.list():
+ if test_uuid in instance.name:
+ instance.delete()
+ return instance.name
+ return None
+
+
def request(msg):
"""Callback for AMQP queue request"""
@@ -1118,6 +1152,8 @@ def request(msg):
test_uuid,
private,
)
+ if code == -10:
+ exit_requested = 99
is_failure = code in FAIL_CODES
files = set(os.listdir(out_dir))
is_unknown_version = "testpkg-version" not in files
@@ -1174,12 +1210,38 @@ def request(msg):
elif code == 16 or code < 0:
contents = log_contents(out_dir)
if exit_requested is not None:
- logging.warning(
- "Testbed failure and exit %i requested. Log follows:",
- exit_requested,
- )
- logging.error(contents)
- sys.exit(exit_requested)
+ # exit_requested is set to 99 when the test is requested to be killed
+ if exit_requested != 99:
+ logging.warning(
+ "Testbed failure and exit %i requested. Log follows:",
+ exit_requested,
+ )
+ logging.error(contents)
+ sys.exit(exit_requested)
+ else:
+ # Test has been requested to be killed
+ logging.info(
+ "Test has been killed by test-killer, exiting."
+ )
+ running_test = False
+ # ack the message so it doesn't go back in the queue
+ msg.channel.basic_ack(msg.delivery_tag)
+ # make this a function
+ logging.info(
+ "Killing openstack server with uuid %s",
+ test_uuid,
+ )
+ server_name = kill_openstack_server(test_uuid)
+ if server_name is not None:
+ logging.info(
+ "Deleted test server: %s", server_name
+ )
+ else:
+ logging.info(
+ "Failed to delete openstack server: %s"
+ % server_name
+ )
+ return
# Get the package-specific string for triggers too, since they might have broken the run
trigs = [
t.split("/", 1)[0] for t in params.get("triggers", [])
diff --git a/charms/focal/autopkgtest-web/webcontrol/browse.cgi b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
index 309fb82..85024bf 100755
--- a/charms/focal/autopkgtest-web/webcontrol/browse.cgi
+++ b/charms/focal/autopkgtest-web/webcontrol/browse.cgi
@@ -16,24 +16,13 @@ from helpers.utils import (
get_all_releases,
get_autopkgtest_cloud_conf,
get_supported_releases,
- setup_key,
+ initialise_app,
)
-from werkzeug.middleware.proxy_fix import ProxyFix
# Initialize app
-PATH = os.path.join(
- os.path.sep, os.getenv("XDG_RUNTIME_DIR", "/run"), "autopkgtest_webcontrol"
-)
-os.makedirs(PATH, exist_ok=True)
-app = flask.Flask("browse")
-# we don't want a long cache, as we only serve files that are regularly updated
+PATH, app, secret_path, _ = initialise_app("browse")
app.config["SEND_FILE_MAX_AGE_DEFAULT"] = 60
-app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1)
-
-secret_path = os.path.join(PATH, "secret_key")
-setup_key(app, secret_path)
-
db_con = None
swift_container_url = None
diff --git a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
index 4e26eb8..a92cc51 100644
--- a/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
+++ b/charms/focal/autopkgtest-web/webcontrol/helpers/utils.py
@@ -14,12 +14,37 @@ import typing
# introduced in python3.7, we use 3.8
from dataclasses import dataclass
+from html import escape as _escape
import distro_info
+from flask import Flask
+from flask_openid import OpenID
+from werkzeug.middleware.proxy_fix import ProxyFix
sqlite3.paramstyle = "named"
+def initialise_app(app_name):
+ PATH = os.path.join(
+ os.path.sep,
+ os.getenv("XDG_RUNTIME_DIR", "/run"),
+ "autopkgtest_webcontrol",
+ )
+ os.makedirs(PATH, exist_ok=True)
+ app = Flask(app_name)
+ app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1)
+ # keep secret persistent between CGI invocations
+ secret_path = os.path.join(PATH, "secret_key")
+ setup_key(app, secret_path)
+ oid = OpenID(app, os.path.join(PATH, "openid"), safe_roots=[])
+ return PATH, app, secret_path, oid
+
+
+def maybe_escape(value):
+ """Escape the value if it is True-ish"""
+ return _escape(value) if value else value
+
+
@dataclass
class SqliteWriterConfig:
writer_exchange_name = "sqlite-write-me.fanout"
@@ -220,3 +245,16 @@ def get_test_id(db_con, release, arch, src):
get_test_id._cache = {}
+
+HTML = """
+<!doctype html>
+<html>
+<head>
+<meta charset="utf-8">
+<title>Autopkgtest Test Request</title>
+</head>
+<body>
+{}
+</body>
+</html>
+"""
diff --git a/charms/focal/autopkgtest-web/webcontrol/request/app.py b/charms/focal/autopkgtest-web/webcontrol/request/app.py
index 4fca679..8ee33d4 100644
--- a/charms/focal/autopkgtest-web/webcontrol/request/app.py
+++ b/charms/focal/autopkgtest-web/webcontrol/request/app.py
@@ -5,33 +5,17 @@ import logging
import os
import pathlib
from collections import ChainMap
-from html import escape as _escape
-from flask import Flask, redirect, request, session
-from flask_openid import OpenID
+from flask import redirect, request, session
from helpers.exceptions import WebControlException
-from helpers.utils import setup_key
+from helpers.utils import HTML, initialise_app, maybe_escape
from request.submit import Submit
-from werkzeug.middleware.proxy_fix import ProxyFix
# map multiple GET vars to AMQP JSON request parameter list
MULTI_ARGS = {"trigger": "triggers", "ppa": "ppas", "env": "env"}
EMPTY = ""
-HTML = """
-<!doctype html>
-<html>
-<head>
-<meta charset="utf-8">
-<title>Autopkgtest Test Request</title>
-</head>
-<body>
-{}
-</body>
-</html>
-"""
-
LOGIN = """
<form action="/login" method="post">
<input type="submit" value="Log in with Ubuntu SSO">
@@ -106,11 +90,6 @@ def invalid(inv_exception, code=400):
return HTML.format(html), code
-def maybe_escape(value):
- """Escape the value if it is True-ish"""
- return _escape(value) if value else value
-
-
def get_api_keys():
"""
API keys is a json file like this:
@@ -132,17 +111,7 @@ def get_api_keys():
# Initialize app
-PATH = os.path.join(
- os.path.sep, os.getenv("XDG_RUNTIME_DIR", "/run"), "autopkgtest_webcontrol"
-)
-os.makedirs(PATH, exist_ok=True)
-app = Flask("request")
-app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1)
-# keep secret persistent between CGI invocations
-secret_path = os.path.join(PATH, "secret_key")
-setup_key(app, secret_path)
-oid = OpenID(app, os.path.join(PATH, "openid"), safe_roots=[])
-
+PATH, app, secret_path, oid = initialise_app("request")
#
# Flask routes
--
Mailing list: https://launchpad.net/~canonical-ubuntu-qa
Post to : canonical-ubuntu-qa@lists.launchpad.net
Unsubscribe : https://launchpad.net/~canonical-ubuntu-qa
More help : https://help.launchpad.net/ListHelp
