On Wed, Oct 19, 2022 at 8:29 AM Ani Sinha <a...@anisinha.ca> wrote:
>
> This introduces QEMU acpi/smbios biosbits avocado test which is run
> from within the python virtual environment. When the bits tests are run, bits
> binaries are downloaded from an external repo/location, bios bits iso is
> regenerated containing the acpi/smbios bits tests that are maintained as a
> part
> of the QEMU source under tests/avocado/acpi-bits/bits-test . When the VM is
> spawned with the iso, it runs the tests in batch mode and the results are
> pushed
> out from the VM to the test machine where they are analyzed by this script and
> pass/fail results are reported.
>
> Cc: Daniel P. Berrangé <berra...@redhat.com>
> Cc: Paolo Bonzini <pbonz...@redhat.com>
> Cc: Maydell Peter <peter.mayd...@linaro.org>
> Cc: John Snow <js...@redhat.com>
> Cc: Thomas Huth <th...@redhat.com>
> Cc: Alex Bennée <alex.ben...@linaro.org>
> Cc: Igor Mammedov <imamm...@redhat.com>
> Cc: Michael Tsirkin <m...@redhat.com>
> Signed-off-by: Ani Sinha <a...@anisinha.ca>
> ---
> tests/avocado/acpi-bits.py | 363 +++++++++++++++++++++++++++++++++++++
> 1 file changed, 363 insertions(+)
> create mode 100644 tests/avocado/acpi-bits.py
>
> diff --git a/tests/avocado/acpi-bits.py b/tests/avocado/acpi-bits.py
> new file mode 100644
> index 0000000000..4365537fa8
> --- /dev/null
> +++ b/tests/avocado/acpi-bits.py
> @@ -0,0 +1,363 @@
> +#!/usr/bin/env python3
> +# group: rw quick
> +# Exercize QEMU generated ACPI/SMBIOS tables using biosbits,
> +# https://biosbits.org/
> +#
> +# This program is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation; either version 2 of the License, or
> +# (at your option) any later version.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License
> +# along with this program. If not, see <http://www.gnu.org/licenses/>.
> +#
> +#
> +# Author:
> +# Ani Sinha <a...@anisinha.ca>
> +
> +# pylint: disable=invalid-name
> +# pylint: disable=consider-using-f-string
> +
> +"""
> +This is QEMU ACPI/SMBIOS avocado tests using biosbits.
> +Biosbits is available originally at https://biosbits.org/.
> +This test uses a fork of the upstream bits and has numerous fixes
> +including an upgraded acpica. The fork is located here:
> +https://gitlab.com/qemu-project/biosbits-bits .
> +"""
> +
> +import logging
> +import os
> +import re
> +import shutil
> +import subprocess
> +import tarfile
> +import tempfile
> +import time
> +import zipfile
> +from typing import (
> + List,
> + Optional,
> + Sequence,
> +)
> +from qemu.machine import QEMUMachine
> +from avocado import skipIf
> +from avocado_qemu import QemuBaseTest
> +
> +def _print_log(log):
> + print('\nlogs from biosbits follows:')
> + print('==========================================\n')
> + print(log)
> + print('==========================================\n')
> +
> +class QEMUBitsMachine(QEMUMachine): # pylint: disable=too-few-public-methods
> + """
> + A QEMU VM, with isa-debugcon enabled and bits iso passed
> + using -cdrom to QEMU commandline.
> +
> + """
> + def __init__(self,
> + binary: str,
> + args: Sequence[str] = (),
> + wrapper: Sequence[str] = (),
> + name: Optional[str] = None,
> + base_temp_dir: str = "/var/tmp",
> + debugcon_log: str = "debugcon-log.txt",
> + debugcon_addr: str = "0x403",
> + sock_dir: Optional[str] = None,
> + qmp_timer: Optional[float] = None):
> + # pylint: disable=too-many-arguments
> +
> + if name is None:
> + name = "qemu-bits-%d" % os.getpid()
> + if sock_dir is None:
> + sock_dir = base_temp_dir
> + super().__init__(binary, args, wrapper=wrapper, name=name,
> + base_temp_dir=base_temp_dir,
> + sock_dir=sock_dir, qmp_timer=qmp_timer)
> + self.debugcon_log = debugcon_log
> + self.debugcon_addr = debugcon_addr
> + self.base_temp_dir = base_temp_dir
> +
> + @property
> + def _base_args(self) -> List[str]:
> + args = super()._base_args
> + args.extend([
> + '-chardev',
> + 'file,path=%s,id=debugcon' %os.path.join(self.base_temp_dir,
> + self.debugcon_log),
> + '-device',
> + 'isa-debugcon,iobase=%s,chardev=debugcon' %self.debugcon_addr,
> + ])
> + return args
> +
> + def base_args(self):
> + """return the base argument to QEMU binary"""
> + return self._base_args
> +
> +@skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab')
> +class AcpiBitsTest(QemuBaseTest): #pylint:
> disable=too-many-instance-attributes
> + """
> + ACPI and SMBIOS tests using biosbits.
> +
> + :avocado: tags=arch:x86_64
> + :avocado: tags=acpi
> +
> + """
> + def __init__(self, *args, **kwargs):
> + super().__init__(*args, **kwargs)
> + self._vm = None
> + self._workDir = None
> + self._baseDir = None
> +
> + # following are some standard configuration constants
> + self._bitsInternalVer = 2020
> + self._bitsCommitHash = 'b48b88ff' # commit hash must match
> + # the artifact tag below
> + self._bitsTag = "qemu-bits-10182022" # this is the latest bits
> + # release as of today.
> + self._bitsArtSHA1Hash = 'b04790ac9b99b5662d0416392c73b97580641fe5'
> + self._bitsArtURL = ("https://gitlab.com/qemu-project/";
> + "biosbits-bits/-/jobs/artifacts/%s/"
> + "download?job=qemu-bits-build" %self._bitsTag)
> + self._debugcon_addr = '0x403'
> + self._debugcon_log = 'debugcon-log.txt'
> + logging.basicConfig(level=logging.INFO)
> +
> + def copy_bits_config(self):
> + """ copies the bios bits config file into bits.
> + """
> + config_file = 'bits-cfg.txt'
> + bits_config_dir = os.path.join(self._baseDir, 'acpi-bits',
> + 'bits-config')
> + target_config_dir = os.path.join(self._workDir,
> + 'bits-%d' %self._bitsInternalVer,
> + 'boot')
> + self.assertTrue(os.path.exists(bits_config_dir))
> + self.assertTrue(os.path.exists(target_config_dir))
> + self.assertTrue(os.access(os.path.join(bits_config_dir,
> + config_file), os.R_OK))
> + shutil.copy2(os.path.join(bits_config_dir, config_file),
> + target_config_dir)
> + logging.info('copied config file %s to %s',
> + config_file, target_config_dir)
> +
> + def copy_test_scripts(self):
> + """copies the python test scripts into bits. """
> +
> + bits_test_dir = os.path.join(self._baseDir, 'acpi-bits',
> + 'bits-tests')
> + target_test_dir = os.path.join(self._workDir,
> + 'bits-%d' %self._bitsInternalVer,
> + 'boot', 'python')
> +
> + self.assertTrue(os.path.exists(bits_test_dir))
> + self.assertTrue(os.path.exists(target_test_dir))
> +
> + for filename in os.listdir(bits_test_dir):
> + if os.path.isfile(os.path.join(bits_test_dir, filename)) and \
> + filename.endswith('.py2'):
> + # all test scripts are named with extension .py2 so that
> + # avocado does not try to load them. These scripts are
> + # written for python 2.7 not python 3 and hence if avocado
> + # loaded them, it would complain about python 3 specific
> + # syntaxes.
> + newfilename = os.path.splitext(filename)[0] + '.py'
> + shutil.copy2(os.path.join(bits_test_dir, filename),
> + os.path.join(target_test_dir, newfilename))
> + logging.info('copied test file %s to %s',
> + filename, target_test_dir)
> +
> + # now remove the pyc test file if it exists, otherwise the
> + # changes in the python test script won't be executed.
> + testfile_pyc = os.path.splitext(filename)[0] + '.pyc'
> + if os.access(os.path.join(target_test_dir, testfile_pyc),
> + os.F_OK):
> + os.remove(os.path.join(target_test_dir, testfile_pyc))
> + logging.info('removed compiled file %s',
> + os.path.join(target_test_dir, testfile_pyc))
> +
> + def fix_mkrescue(self, mkrescue):
> + """ grub-mkrescue is a bash script with two variables, 'prefix' and
> + 'libdir'. They must be pointed to the right location so that the
> + iso can be generated appropriately. We point the two variables to
> + the directory where we have extracted our pre-built bits grub
> + tarball.
> + """
> + grub_x86_64_mods = os.path.join(self._workDir,
> 'grub-inst-x86_64-efi')
> + grub_i386_mods = os.path.join(self._workDir, 'grub-inst')
> +
> + self.assertTrue(os.path.exists(grub_x86_64_mods))
> + self.assertTrue(os.path.exists(grub_i386_mods))
> +
> + new_script = ""
> + with open(mkrescue, 'r', encoding='utf-8') as filehandle:
> + orig_script = filehandle.read()
> + new_script = re.sub('(^prefix=)(.*)',
> + r'\1"%s"' %grub_x86_64_mods,
> + orig_script, flags=re.M)
> + new_script = re.sub('(^libdir=)(.*)', r'\1"%s/lib"'
> %grub_i386_mods,
> + new_script, flags=re.M)
> +
> + with open(mkrescue, 'w', encoding='utf-8') as filehandle:
> + filehandle.write(new_script)
> +
> + def generate_bits_iso(self):
> + """ Uses grub-mkrescue to generate a fresh bits iso with the python
> + test scripts
> + """
> + bits_dir = os.path.join(self._workDir,
> + 'bits-%d' %self._bitsInternalVer)
> + iso_file = os.path.join(self._workDir,
> + 'bits-%d.iso' %self._bitsInternalVer)
> + mkrescue_script = os.path.join(self._workDir,
> + 'grub-inst-x86_64-efi', 'bin',
> + 'grub-mkrescue')
> +
> + self.assertTrue(os.access(mkrescue_script,
> + os.R_OK | os.W_OK | os.X_OK))
> +
> + self.fix_mkrescue(mkrescue_script)
> +
> + logging.info('calling grub-mkrescue to generate the biosbits iso
> ...')
> +
> + try:
> + if os.getenv('V'):
> + subprocess.check_call([mkrescue_script, '-o', iso_file,
> + bits_dir], stderr=subprocess.STDOUT)
> + else:
> + subprocess.check_call([mkrescue_script, '-o',
> + iso_file, bits_dir],
> + stderr=subprocess.DEVNULL,
> + stdout=subprocess.DEVNULL)
> + except Exception as e: # pylint: disable=broad-except
> + self.skipTest("Error while generating the bits iso. "
> + "Pass V=1 in the environment to get more details. "
> + + str(e))
> +
> + self.assertTrue(os.access(iso_file, os.R_OK))
> +
> + logging.info('iso file %s successfully generated.', iso_file)
> +
> + def setUp(self): # pylint: disable=arguments-differ
> + super().setUp('qemu-system-')
> +
> + if shutil.which('xorriso') is None:
> + logging.error('xorriso is required to run this test.')
> + self.skipTest("xorriso is not installed. Please install it.")
This would result in output like this when xorriso is not found:
$ which xorriso
/usr/bin/which: no xorriso in
(/home/anisinha/.local/bin:/home/anisinha/bin:/usr/share/Modules/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin)
[anisinha@centos8 build]$ ./tests/venv/bin/avocado run -t acpi tests/avocado
Fetching asset from
tests/avocado/acpi-bits.py:AcpiBitsTest.test_acpi_smbios_bits
JOB ID : 95aba043201755ed888ef7d1598402c555118df4
JOB LOG :
/home/anisinha/avocado/job-results/job-2022-10-19T02.27-95aba04/job.log
(1/1) tests/avocado/acpi-bits.py:AcpiBitsTest.test_acpi_smbios_bits:
ERROR: xorriso is not installed. Please install it. (0.00 s)
RESULTS : PASS 0 | ERROR 1 | FAIL 0 | SKIP 0 | WARN 0 | INTERRUPT 0
| CANCEL 0
JOB TIME : 0.61 s
> +
> + self._baseDir = os.getenv('AVOCADO_TEST_BASEDIR')
> +
> + # workdir could also be avocado's own workdir in self.workdir.
> + # At present, I prefer to maintain my own temporary working
> + # directory. It gives us more control over the generated bits
> + # log files and also for debugging, we may chose not to remove
> + # this working directory so that the logs and iso can be
> + # inspected manually and archived if needed.
> + self._workDir = tempfile.mkdtemp(prefix='acpi-bits-',
> + suffix='.tmp')
> + logging.info('working dir: %s', self._workDir)
> +
> + prebuiltDir = os.path.join(self._workDir, 'prebuilt')
> + if not os.path.isdir(prebuiltDir):
> + os.mkdir(prebuiltDir, mode=0o775)
> +
> + bits_zip_file = os.path.join(prebuiltDir, 'bits-%d-%s.zip'
> + %(self._bitsInternalVer,
> + self._bitsCommitHash))
> + grub_tar_file = os.path.join(prebuiltDir,
> + 'bits-%d-%s-grub.tar.gz'
> + %(self._bitsInternalVer,
> + self._bitsCommitHash))
> +
> + bitsLocalArtLoc = self.fetch_asset(self._bitsArtURL,
> + asset_hash=self._bitsArtSHA1Hash)
> + logging.info("downloaded bits artifacts to %s", bitsLocalArtLoc)
> +
> + # extract the bits artifact in the temp working directory
> + with zipfile.ZipFile(bitsLocalArtLoc, 'r') as zref:
> + zref.extractall(prebuiltDir)
> +
> + # extract the bits software in the temp working directory
> + with zipfile.ZipFile(bits_zip_file, 'r') as zref:
> + zref.extractall(self._workDir)
> +
> + with tarfile.open(grub_tar_file, 'r', encoding='utf-8') as tarball:
> + tarball.extractall(self._workDir)
> +
> + self.copy_test_scripts()
> + self.copy_bits_config()
> + self.generate_bits_iso()
> +
> + def parse_log(self):
> + """parse the log generated by running bits tests and
> + check for failures.
> + """
> + debugconf = os.path.join(self._workDir, self._debugcon_log)
> + log = ""
> + with open(debugconf, 'r', encoding='utf-8') as filehandle:
> + log = filehandle.read()
> +
> + matchiter = re.finditer(r'(.*Summary: )(\d+ passed), (\d+ failed).*',
> + log)
> + for match in matchiter:
> + # verify that no test cases failed.
> + try:
> + self.assertEqual(match.group(3).split()[0], '0',
> + 'Some bits tests seems to have failed. ' \
> + 'Please check the test logs for more info.')
> + except AssertionError as e:
> + _print_log(log)
> + raise e
> + else:
> + if os.getenv('V'):
> + _print_log(log)
> +
> + def tearDown(self):
> + """
> + Lets do some cleanups.
> + """
> + if self._vm:
> + self.assertFalse(not self._vm.is_running)
> + logging.info('removing the work directory %s', self._workDir)
> + shutil.rmtree(self._workDir)
> + super().tearDown()
> +
> + def test_acpi_smbios_bits(self):
> + """The main test case implementaion."""
> +
> + iso_file = os.path.join(self._workDir,
> + 'bits-%d.iso' %self._bitsInternalVer)
> +
> + self.assertTrue(os.access(iso_file, os.R_OK))
> +
> + self._vm = QEMUBitsMachine(binary=self.qemu_bin,
> + base_temp_dir=self._workDir,
> + debugcon_log=self._debugcon_log,
> + debugcon_addr=self._debugcon_addr)
> +
> + self._vm.add_args('-cdrom', '%s' %iso_file)
> +
> + args = " ".join(str(arg) for arg in self._vm.base_args()) + \
> + " " + " ".join(str(arg) for arg in self._vm.args)
> +
> + logging.info("launching QEMU vm with the following arguments: %s",
> + args)
> +
> + self._vm.launch()
> + # biosbits has been configured to run all the specified test suites
> + # in batch mode and then automatically initiate a vm shutdown.
> + # sleep for maximum of one minute
> + max_sleep_time = time.monotonic() + 60
> + while self._vm.is_running() and time.monotonic() < max_sleep_time:
> + time.sleep(1)
> +
> + self.assertFalse(time.monotonic() > max_sleep_time,
> + 'The VM seems to have failed to shutdown in time')
> +
> + self.parse_log()
> --
> 2.34.1
>
