Tim Andersson has proposed merging
~andersson123/autopkgtest-cloud:create-instances-refactor into
autopkgtest-cloud:master.
Requested reviews:
Canonical's Ubuntu QA (canonical-ubuntu-qa)
For more details, see:
https://code.launchpad.net/~andersson123/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/445759
Create common functions for spawning openstack instances and refactor scripts
that'd benefit from these common functions
--
Your team Canonical's Ubuntu QA is requested to review the proposed merge of
~andersson123/autopkgtest-cloud:create-instances-refactor into
autopkgtest-cloud:master.
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/create-instances-common.sh b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/create-instances-common.sh
new file mode 100644
index 0000000..2046ad8
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/create-instances-common.sh
@@ -0,0 +1,154 @@
+#!/bin/bash
+
+# Common functions shared by scripts in this directory which create openstack instances
+
+get_random_string(){
+ ran=$(echo $RANDOM | md5sum | head -c 20)
+ printf "%s" "${ran}"
+}
+
+
+make_and_load_key(){
+ # $1 rc file
+ # $2 name prefix
+ key_id="${2}-key-$(get_random_string)"
+ if ! openstack keypair list | grep "${key_id}" &>/dev/null; then
+ if ! openstack keypair create --public-key ~/.ssh/id_rsa.pub "${key_id}" &> /dev/null; then
+ printf "keypair creation failed for %s, skipping" "${1}"
+ exit 1
+ fi
+ fi
+ while ! openstack keypair list | grep "${key_id}" &>/dev/null; do
+ ctr=$((ctr+1))
+ if [ $ctr -gt 9 ]; then
+ printf "key not loading for %s, exiting" "${1}"
+ exit 1
+ fi
+ done
+ printf "%s" "${key_id}"
+}
+
+get_arch(){
+ # $1 rc file
+ arch=$(echo "${1}" | awk -F[-.] '{print $2}')
+ if [ -z "$arch" ]; then
+ arch="amd64"
+ elif [[ "${arch}" == "rc" ]]; then
+ arch="amd64"
+ fi
+ printf "%s" "${arch}"
+}
+
+get_image(){
+ # $1 release
+ # $2 arch
+ image=$(openstack image list | grep "adt/ubuntu-${1}-${2}" | cut -d' ' -f2)
+ printf "%s" "${image}"
+}
+
+get_netid(){
+ net=""
+ if [[ "${OS_USERNAME}" =~ .*"stg".* ]]; then
+ if openstack network list | grep 'net_stg_proposed-migration' &> /dev/null; then
+ net=$(openstack network list | grep 'net_stg_proposed-migration' | cut -d' ' -f2)
+ elif openstack network list | grep 'net_stg-proposed-migration' &> /dev/null; then
+ net=$(openstack network list | grep 'net_stg-proposed-migration' | cut -d' ' -f2)
+ else
+ echo "Network finding failure, skipping ${FILE}..."
+ exit 1
+ fi
+ elif [[ "${OS_USERNAME}" =~ .*"prod".* ]]; then
+ net=$(openstack network list | grep 'net_prod-proposed-migration' | cut -d' ' -f2)
+ fi
+ printf "%s" "${net}"
+}
+
+wait_for_server_to_build(){
+ # $1 server name
+ # $2 key
+ while ! openstack server show "${1}" | grep status | grep ACTIVE &>/dev/null; do
+ # echo "waiting for server to come up..."
+ if openstack server show "${1}" | grep status | grep ERROR; then
+ openstack keypair delete "${key}"
+ echo "Server building failed, see below..."
+ openstack server show "${1}"
+ exit 1
+ fi
+ sleep 5
+ done
+}
+
+wait_for_server_connectivity(){
+ # $1 server_name
+ ip_address=$(openstack server list | grep "${1}" | cut -d'|' -f5 | cut -d'=' -f2)
+ sleep 10
+ while ! ssh -o StrictHostKeyChecking=no ubuntu@"${ip_address}" : &>/dev/null; do
+ # echo "waiting for server to have connectivity..."
+ sleep 3
+ done
+ printf "%s" "${ip_address}"
+}
+
+create_server_and_wait_for_connectivity(){
+ # maybe this changes and we just pass the file or something
+ # need name prefix
+ # new args
+ # $1 release
+ # $2 name_prefix
+ # $3 rc file
+ # $4 flavor (default is autopkgtest)
+ # shellcheck disable=SC1090
+ if ! . "${3}"; then
+ printf "Sourcing file: %s failed, exiting...\n" "${3}"
+ exit 1
+ fi
+
+ key=$(make_and_load_key "${3}" "${2}")
+ # printf "Key loaded: %s\n" "${key}"
+ server_name="${2}-$(get_random_string)"
+ # printf "Booting up instance %s for %s\n" "${server_name}" "${3}"
+ if openstack server list | grep "${server_name}" &> /dev/null; then
+ printf "Server %s already exists, exiting" "${server_name}"
+ exit 1
+ fi
+ arch=$(get_arch "${3}")
+ # printf "Architecture: %s\n" "${arch}"
+ img=$(get_image "${1}" "${arch}")
+ # printf "Using image: %s\n" "${img}"
+ flavor="autopkgtest"
+ if [ $# -gt 3 ]; then
+ flavor="${4}"
+ fi
+ netid=$(get_netid)
+ openstack server create --image "${img}" --flavor "${flavor}" --nic net-id="${netid}" -- "${server_name}" &> /dev/null
+ wait_for_server_to_build "${server_name}" "${key}"
+ # printf "Server built: %s\n" "${server_name}"
+ ip=$(wait_for_server_connectivity "${server_name}")
+ # printf "Server has connectivity: %s\n" "${server_name}"
+ printf "%s|%s|%s" "${server_name}" "${key}" "${ip}"
+}
+
+show_server_details(){
+ # $1 server details
+ name=$(printf "%s" "${1}" | cut -d'|' -f1)
+ key=$(printf "%s" "${1}" | cut -d'|' -f2)
+ ip=$(printf "%s" "${1}" | cut -d'|' -f3)
+ printf "%s\n%s\n%s" "${name}" "${key}" "${ip}"
+}
+
+get_server_ip(){
+ # $1 server details
+ printf "%s" "${1}" | cut -d'|' -f3
+}
+
+kill_server(){
+ # $1 server details
+ # $2 file
+ # shellcheck disable=SC1090
+ source "${2}"
+ printf "Killing server: \n%s" "$(show_server_details "${1}")"
+ server_name=$(printf "%s" "${1}" | cut -d'|' -f1)
+ key=$(printf "%s" "${1}" | cut -d'|' -f2)
+ openstack server delete "${server_name}"
+ openstack keypair delete "${key}"
+}
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/create-instances-test-mirrors b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/create-instances-test-mirrors
new file mode 100755
index 0000000..03fcbd1
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/create-instances-test-mirrors
@@ -0,0 +1,36 @@
+#!/bin/bash
+# Spawns openstack servers for each environment in cloudrc's dir, tests
+# different archives for each environment and tests the speed of each archive,
+# and reports the results.
+
+set -e
+
+CLOUDRC_DIR="${1}"
+SERVERNAME="mirror_speed_tester"
+RELEASE="jammy"
+
+results_file=$(mktemp)
+
+# shellcheck disable=1091
+source ./create-instances-common.sh
+
+for FILE in "${CLOUDRC_DIR}"*; do
+ if [[ ! "${FILE}" =~ .rc ]]; then
+ printf "%s has incorrect file type, skipping" "${FILE}"
+ continue
+ fi
+ (
+ printf "===================================================================\n"
+ printf "Booting up instance\n"
+ server_details=$(create_server_and_wait_for_connectivity "${RELEASE}" "${SERVERNAME}" "${FILE}")
+ printf "Instance booted:\n%s\n" "$(show_server_details "${server_details}")"
+ ip=$(get_server_ip "${server_details}")
+ best_archive=$(ssh -o StrictHostKeyChecking=no ubuntu@"${ip}" 'sudo python3' < test_mirrors | tail -1)
+ printf "Best archive for %s is %s" "${FILE}" "${best_archive}\n" >> "${results_file}"
+ kill_server "${server_details}" "${FILE}"
+ echo "==================================================================="
+ )
+done
+
+cat "${results_file}"
+rm "${results_file}"
diff --git a/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/test_mirrors b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/test_mirrors
new file mode 100755
index 0000000..9fd12af
--- /dev/null
+++ b/charms/focal/autopkgtest-cloud-worker/autopkgtest-cloud/tools/test_mirrors
@@ -0,0 +1,96 @@
+#!/usr/bin/python3
+'''
+Script to be run on a spawned openstack instance to test mirror speed for
+a particular datacentre.
+'''
+# pylint: disable=invalid-name
+import datetime
+import os
+import sys
+import tempfile
+
+import apt
+
+
+def get_nvidia_package():
+ '''
+ iterate backward through the cache to
+ find the latest nvidia driver
+ '''
+ cache = apt.Cache()
+ for key in cache.keys()[::-1]:
+ if "libnvidia-common" in key and "server" not in key:
+ return key
+ return None
+
+
+SOURCES = """deb http://$ARCHIVE/ubuntu $RELEASE main restricted
+deb-src http://$ARCHIVE/ubuntu $RELEASE main restricted
+
+deb http://$ARCHIVE/ubuntu $RELEASE-updates main restricted
+deb-src http://$ARCHIVE/ubuntu $RELEASE-updates main restricted
+
+deb http://$ARCHIVE/ubuntu $RELEASE universe
+deb-src http://$ARCHIVE/ubuntu $RELEASE universe
+deb http://$ARCHIVE/ubuntu $RELEASE-updates universe
+deb-src http://$ARCHIVE/ubuntu $RELEASE-updates universe
+
+deb http://$ARCHIVE/ubuntu $RELEASE multiverse
+deb-src http://$ARCHIVE/ubuntu $RELEASE multiverse
+deb http://$ARCHIVE/ubuntu $RELEASE-updates multiverse
+deb-src http://$ARCHIVE/ubuntu $RELEASE-updates multiverse
+
+deb http://$ARCHIVE/ubuntu $RELEASE-backports main restricted universe multiverse
+deb-src http://$ARCHIVE/ubuntu $RELEASE-backports main restricted universe multiverse
+
+deb http://$ARCHIVE/ubuntu $RELEASE-security main restricted
+deb http://$ARCHIVE/ubuntu $RELEASE-security universe
+deb http://$ARCHIVE/ubuntu $RELEASE-security multiverse
+"""
+
+ARCHIVES = [
+ "gb.archive.ubuntu.com",
+ "us.archive.ubuntu.com",
+ "nova.clouds.ports.ubuntu.com",
+ "ftpmaster.internal"
+]
+
+results = []
+quickest = 1000000000000000
+quickest_archive = ""
+release = "focal"
+os.system("apt update")
+package = get_nvidia_package()
+if package is None:
+ print("No valid source package found.")
+ sys.exit()
+
+for archive in ARCHIVES:
+ sources = SOURCES.replace("$ARCHIVE", archive)
+ sources = sources.replace("$RELEASE", release)
+ arch = os.popen('dpkg --print-architecture').read().replace("\n", "")
+ if arch != "x86_64":
+ sources = sources.replace("archive", "ports")
+ if "ftpmaster" not in archive:
+ sources = sources.replace("/ubuntu", "/ubuntu-ports")
+ with open("/etc/apt/sources.list", 'w', encoding='utf-8') as f:
+ f.write(sources)
+ os.system("apt update")
+ # pylint: disable=consider-using-with
+ temp_dir = tempfile.TemporaryDirectory()
+ os.chdir(temp_dir.name)
+ start = datetime.datetime.now()
+ print(type(package))
+ x = os.system("apt source " + package)
+ if x != 0:
+ continue
+ length = (datetime.datetime.now() - start).total_seconds()
+ if length < quickest:
+ quickest = length
+ quickest_archive = archive
+ os.chdir("/tmp/")
+ temp_dir.cleanup()
+ results.append((archive, length))
+
+print(results)
+print(quickest_archive + ": " + str(quickest))
--
Mailing list: https://launchpad.net/~canonical-ubuntu-qa
Post to : canonical-ubuntu-qa@lists.launchpad.net
Unsubscribe : https://launchpad.net/~canonical-ubuntu-qa
More help : https://help.launchpad.net/ListHelp
