Skia has proposed merging ~hyask/autopkgtest-cloud:skia/stats_include_flavor
into autopkgtest-cloud:master.
Requested reviews:
Canonical's Ubuntu QA (canonical-ubuntu-qa)
For more details, see:
https://code.launchpad.net/~hyask/autopkgtest-cloud/+git/autopkgtest-cloud/+merge/476862
Describe how to compute the quota we need, and back that with some
`stats.ipynb` data analysis.
--
Your team Canonical's Ubuntu QA is requested to review the proposed merge of
~hyask/autopkgtest-cloud:skia/stats_include_flavor into
autopkgtest-cloud:master.
diff --git a/dev-tools/stats.ipynb b/dev-tools/stats.ipynb
index fd898f2..90d513c 100644
--- a/dev-tools/stats.ipynb
+++ b/dev-tools/stats.ipynb
@@ -44,7 +44,7 @@
"import matplotlib.pyplot as plt\n",
"\n",
"# Update this path with the corresponding path to the database you want to analyze\n",
- "db_path = \"./autopkgtest_2024-06-18 15:20:42.817741_with_stats.db\"\n",
+ "db_path = \"./autopkgtest_2024-11-20 11:25:43.562429_with_stats.db\"\n",
"\n",
"db = sqlite3.connect(f\"file:{db_path}?mode=ro\")\n",
"sqlite3.paramstyle = \"named\"\n",
@@ -253,6 +253,75 @@
" plt.show()\n",
" print(df)\n"
]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b9a948b3",
+ "metadata": {},
+ "source": [
+ "## Flavors distribution for a single datacenter-arch\n",
+ "\n",
+ "When a quota issue arises, it's useful to have a look at the proportion of regular tests vs `big_packages` ones. It's datacenter-arch specific for easier readability, but it's sometimes useful to remove that restriction.\n",
+ "\n",
+ "Remember that this cells requires you to set the `datacenter` and `arch` at the beginning."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6eba9dcd",
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "with db as db_con:\n",
+ " db_con.row_factory = sqlite3.Row\n",
+ "\n",
+ " datacenter = \"bos03\"\n",
+ " arch = \"amd64\"\n",
+ " \n",
+ " query = f\"\"\"\n",
+ " SELECT flavor, substr(result.run_id, 1, 8) as date\n",
+ " FROM tests_stats\n",
+ " JOIN result ON result.uuid=tests_stats.uuid\n",
+ " JOIN test ON test.id=result.test_id\n",
+ " WHERE arch = '{arch}' AND datacenter = '{datacenter}'\n",
+ " AND flavor IS NOT NULL\n",
+ " ORDER BY date\n",
+ " \"\"\"\n",
+ " df = pd.read_sql_query(query, db_con)\n",
+ " # Get the date as datetime object\n",
+ " df[\"date\"] = pd.to_datetime(df.date)\n",
+ " \n",
+ " # Display data as a graph\n",
+ " plt.figure(figsize=(14, 5))\n",
+ "\n",
+ " # Plot point for each dc-arch over time\n",
+ " for flavor in sorted(df['flavor'].unique()):\n",
+ " flavor_data = df[df['flavor'] == flavor]\n",
+ " flavor_data = flavor_data.groupby(\"date\").count()\n",
+ " print(flavor, flavor_data)\n",
+ " plt.plot(flavor_data, 'o-', drawstyle='steps-post', label=flavor)\n",
+ "\n",
+ " # Add some title and labels\n",
+ " plt.title(f'Flavor counts over time')\n",
+ " plt.xlabel('Date')\n",
+ " plt.ylabel('Flavor counts')\n",
+ " plt.legend()\n",
+ "\n",
+ " # Plot the graph\n",
+ " plt.show()\n",
+ " print(df)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8d8413a8",
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
@@ -271,7 +340,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.12.3"
+ "version": "3.12.7"
}
},
"nbformat": 4,
diff --git a/dev-tools/stats.py b/dev-tools/stats.py
index 82f6f43..7edb897 100755
--- a/dev-tools/stats.py
+++ b/dev-tools/stats.py
@@ -40,6 +40,7 @@ class Run:
self.lxd_image = None
self.first_boot_time = None
self.boot_attempts = None
+ self.flavor = None
def __str__(self):
return (
@@ -54,6 +55,7 @@ class Run:
"host": self.host,
"datacenter": self.datacenter,
"lxd_image": self.lxd_image,
+ "flavor": self.flavor,
}
def log_url(self):
@@ -99,6 +101,7 @@ class Run:
self.extract_datacenter()
self.extract_first_boot_time()
self.extract_boot_attempts()
+ self.extract_flavor()
host_pattern = re.compile(
r"host juju-7f2275-prod-proposed-migration-environment-(\d);"
@@ -173,6 +176,18 @@ class Run:
except Exception as e:
logging.debug(e)
+ flavor_pattern = re.compile(r"--flavor (\S*)")
+
+ def extract_flavor(self):
+ try:
+ lines = [l for l in self.log_lines if "--flavor" in l]
+ if lines:
+ m = self.flavor_pattern.search(lines[0])
+ if m:
+ self.flavor = m.group(1)
+ except Exception as e:
+ logging.debug(e)
+
def get_stats(db_con, since_days_ago, until_days_ago, limit):
try:
@@ -184,6 +199,7 @@ def get_stats(db_con, since_days_ago, until_days_ago, limit):
" datacenter CHAR[10],"
" lxd_image CHAR[10],"
" host CHAR[10],"
+ " flavor CHAR[32],"
" PRIMARY KEY(uuid))"
)
logging.info("Created tests_stats table")
@@ -228,7 +244,8 @@ def get_stats(db_con, since_days_ago, until_days_ago, limit):
boot_attempts,
host,
datacenter,
- lxd_image
+ lxd_image,
+ flavor
)
VALUES (
:uuid,
@@ -236,7 +253,8 @@ def get_stats(db_con, since_days_ago, until_days_ago, limit):
:boot_attempts,
:host,
:datacenter,
- :lxd_image
+ :lxd_image,
+ :flavor
)""",
r.as_row_dict(),
)
diff --git a/docs/deploying.rst b/docs/deploying.rst
index 8a6dd64..7df78cf 100644
--- a/docs/deploying.rst
+++ b/docs/deploying.rst
@@ -131,6 +131,35 @@ Making configuration changes
Edit the ``service-bundle`` file as above, and run ``mojo run`` again.
+About cloud environments quotas
+-------------------------------
+
+Each OpenStack environment has a quota, meaning there is a limit to the number
+of instances, cpu cores, RAM, disk, etc, amount that can be spawned at the same
+time. This quota can be seen with the command `nova limits`.
+
+Changing the quota requires IS approval, and how to do that depends on the cloud
+environments, so it's out of scope of this doc.
+Computing the required quota however fits right in here, so let's details a bit what we want.
+
+Let's say we want 100 autopkgtest runners.
+Regular flavor is ``cpu2-ram4-disk20``, so:
+
+ 100*2 = 200 cores
+ 100*4 = 400GB of RAM
+ 100*20 = 2000GB of disk
+
+We take a 10% margin to accommodate "big_packages", that run on
+``cpu4-ram8-disk100``, which gives us:
+
+ 220 cores
+ 440GB RAM
+ 2200GB disk
+
+A 10% margin means we can concretely run 90 regular jobs, and 10 big_packages.
+That should be enough for most cases, as in average, we're more around 2-4% of
+big_packages tests (observed today, 2024-11-20, with ``stats.ipynb``).
+
autopkgtest-cloud Storage
----------------------------
--
Mailing list: https://launchpad.net/~canonical-ubuntu-qa
Post to : canonical-ubuntu-qa@lists.launchpad.net
Unsubscribe : https://launchpad.net/~canonical-ubuntu-qa
More help : https://help.launchpad.net/ListHelp
