[PULL 03/54] acpi: fdc-isa: replace ISADeviceClass::build_aml with AcpiDevAmlIfClass:build_dev_aml
From: Igor Mammedov
Signed-off-by: Igor Mammedov
Acked-by: Gerd Hoffmann
Message-Id: <20220608135340.3304695-4-imamm...@redhat.com>
Reviewed-by: Michael S. Tsirkin
Signed-off-by: Michael S. Tsirkin
---
hw/block/fdc-isa.c | 16 ++--
hw/i386/acpi-build.c | 1 -
2 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/hw/block/fdc-isa.c b/hw/block/fdc-isa.c
index fa20450747..fee1ca68a8 100644
--- a/hw/block/fdc-isa.c
+++ b/hw/block/fdc-isa.c
@@ -32,7 +32,7 @@
#include "qapi/error.h"
#include "qemu/error-report.h"
#include "qemu/timer.h"
-#include "hw/acpi/aml-build.h"
+#include "hw/acpi/acpi_aml_interface.h"
#include "hw/irq.h"
#include "hw/isa/isa.h"
#include "hw/qdev-properties.h"
@@ -214,9 +214,9 @@ int cmos_get_fd_drive_type(FloppyDriveType fd0)
return val;
}
-static void fdc_isa_build_aml(ISADevice *isadev, Aml *scope)
+static void build_fdc_aml(AcpiDevAmlIf *adev, Aml *scope)
{
-FDCtrlISABus *isa = ISA_FDC(isadev);
+FDCtrlISABus *isa = ISA_FDC(adev);
Aml *dev;
Aml *crs;
int i;
@@ -241,7 +241,7 @@ static void fdc_isa_build_aml(ISADevice *isadev, Aml *scope)
aml_append(dev, aml_name_decl("_CRS", crs));
for (i = 0; i < MIN(MAX_FD, ACPI_FDE_MAX_FD); i++) {
-FloppyDriveType type = isa_fdc_get_drive_type(isadev, i);
+FloppyDriveType type = isa_fdc_get_drive_type(ISA_DEVICE(adev), i);
if (type < FLOPPY_DRIVE_TYPE_NONE) {
fde_buf[i] = cpu_to_le32(1); /* drive present */
@@ -283,14 +283,14 @@ static Property isa_fdc_properties[] = {
static void isabus_fdc_class_init(ObjectClass *klass, void *data)
{
DeviceClass *dc = DEVICE_CLASS(klass);
-ISADeviceClass *isa = ISA_DEVICE_CLASS(klass);
+AcpiDevAmlIfClass *adevc = ACPI_DEV_AML_IF_CLASS(klass);
dc->desc = "virtual floppy controller";
dc->realize = isabus_fdc_realize;
dc->fw_name = "fdc";
dc->reset = fdctrl_external_reset_isa;
dc->vmsd = &vmstate_isa_fdc;
-isa->build_aml = fdc_isa_build_aml;
+adevc->build_dev_aml = build_fdc_aml;
device_class_set_props(dc, isa_fdc_properties);
set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
}
@@ -313,6 +313,10 @@ static const TypeInfo isa_fdc_info = {
.instance_size = sizeof(FDCtrlISABus),
.class_init= isabus_fdc_class_init,
.instance_init = isabus_fdc_instance_init,
+.interfaces = (InterfaceInfo[]) {
+{ TYPE_ACPI_DEV_AML_IF },
+{ },
+},
};
static void isa_fdc_register_types(void)
diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c
index c125939ed6..1449832aa9 100644
--- a/hw/i386/acpi-build.c
+++ b/hw/i386/acpi-build.c
@@ -40,7 +40,6 @@
#include "hw/acpi/bios-linker-loader.h"
#include "hw/isa/isa.h"
#include "hw/input/i8042.h"
-#include "hw/block/fdc.h"
#include "hw/acpi/memory_hotplug.h"
#include "sysemu/tpm.h"
#include "hw/acpi/tpm.h"
--
MST
[PULL 53/54] hw/vhost-user-scsi|blk: set `supports_config` flag correctly
From: Changpeng Liu
Currently vhost-user-scsi driver doesn't allow to change
the configuration space of virtio_scsi, while vhost-user-blk
support that, so here we set the flag in vhost-user-blk driver
and unset it in vhost-user-scsi.
Signed-off-by: Changpeng Liu
Message-Id: <20220525125540.50979-2-changpeng@intel.com>
Reviewed-by: Michael S. Tsirkin
Signed-off-by: Michael S. Tsirkin
Signed-off-by: Changpeng Liu
Signed-off-by: Changpeng Liu
---
hw/block/vhost-user-blk.c | 1 +
hw/scsi/vhost-user-scsi.c | 1 -
2 files changed, 1 insertion(+), 1 deletion(-)
diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c
index 5dca4eab09..9117222456 100644
--- a/hw/block/vhost-user-blk.c
+++ b/hw/block/vhost-user-blk.c
@@ -337,6 +337,7 @@ static int vhost_user_blk_connect(DeviceState *dev, Error
**errp)
vhost_dev_set_config_notifier(&s->dev, &blk_ops);
+s->vhost_user.supports_config = true;
ret = vhost_dev_init(&s->dev, &s->vhost_user, VHOST_BACKEND_TYPE_USER, 0,
errp);
if (ret < 0) {
diff --git a/hw/scsi/vhost-user-scsi.c b/hw/scsi/vhost-user-scsi.c
index 9be21d07ee..1b2f7eed98 100644
--- a/hw/scsi/vhost-user-scsi.c
+++ b/hw/scsi/vhost-user-scsi.c
@@ -121,7 +121,6 @@ static void vhost_user_scsi_realize(DeviceState *dev, Error
**errp)
vsc->dev.backend_features = 0;
vqs = vsc->dev.vqs;
-s->vhost_user.supports_config = true;
ret = vhost_dev_init(&vsc->dev, &s->vhost_user,
VHOST_BACKEND_TYPE_USER, 0, errp);
if (ret < 0) {
--
MST
Re: [PATCH] tests/qtest: Reduce npcm7xx_sdhci test image size
On Thu, 9 Jun 2022 at 22:41, Hao Wu wrote: > > Creating 1GB image for a simple qtest is unnecessary > and could lead to failures. We reduce the image size > to 1MB to reduce the test overhead. > > Signed-off-by: Hao Wu > --- > tests/qtest/npcm7xx_sdhci-test.c | 2 +- > 1 file changed, 1 insertion(+), 1 deletion(-) Applied to target-arm.next, thanks. -- PMM
Re: [PATCH] tests/qtest: Reduce npcm7xx_sdhci test image size
On 9/6/22 23:41, Hao Wu wrote: Creating 1GB image for a simple qtest is unnecessary and could lead to failures. We reduce the image size to 1MB to reduce the test overhead. Signed-off-by: Hao Wu --- tests/qtest/npcm7xx_sdhci-test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) Reviewed-by: Philippe Mathieu-Daudé
Re: [PATCH] qsd: Do not use error_report() before monitor_init
On 9/6/22 14:28, Hanna Reitz wrote:
error_report() only works once monitor_init_globals_core() has been
called, which is not the case when parsing the --daemonize option. Use
fprintf(stderr, ...) instead.
Fixes: 2525edd85fec53e23fda98974a15e3b3c8957596 ("qsd: Add --daemonize")
Signed-off-by: Hanna Reitz
---
storage-daemon/qemu-storage-daemon.c | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
Reviewed-by: Philippe Mathieu-Daudé
Re: [PATCH] qsd: Do not use error_report() before monitor_init
On Thu, 9 Jun 2022 at 15:26, Hanna Reitz wrote:
>
> error_report() only works once monitor_init_globals_core() has been
> called, which is not the case when parsing the --daemonize option. Use
> fprintf(stderr, ...) instead.
>
> Fixes: 2525edd85fec53e23fda98974a15e3b3c8957596 ("qsd: Add --daemonize")
> Signed-off-by: Hanna Reitz
> ---
> storage-daemon/qemu-storage-daemon.c | 6 +-
> 1 file changed, 5 insertions(+), 1 deletion(-)
>
> diff --git a/storage-daemon/qemu-storage-daemon.c
> b/storage-daemon/qemu-storage-daemon.c
> index c104817cdd..0890495c40 100644
> --- a/storage-daemon/qemu-storage-daemon.c
> +++ b/storage-daemon/qemu-storage-daemon.c
> @@ -286,7 +286,11 @@ static void process_options(int argc, char *argv[], bool
> pre_init_pass)
> }
> case OPTION_DAEMONIZE:
> if (os_set_daemonize(true) < 0) {
> -error_report("--daemonize not supported in this build");
> +/*
> + * --daemonize is parsed before monitor_init_globals_core(),
> so
> + * error_report() does not work yet
> + */
> +fprintf(stderr, "--daemonize not supported in this build\n");
> exit(EXIT_FAILURE);
> }
> break;
Is it worth making error_report() have code for "called in early startup
before monitor was initialized" that falls back to "just print to stderr"?
Having our standard error reporting function not be usable everywhere is
kind of surprising...
-- PMM
[RFC PATCH v2 0/7] tests: run python tests under a venv
Hi, here's another RFC for bringing external Python dependencies to the QEMU test suite. This patchset is not without some problems that need to be solved, but I've been sitting on these long enough and they need to see the light of day. Problems I am aware of, and there's a few: - Ubuntu 18.04 ships with a version of pip that is too old to support setup.cfg-based installations. We are allowed to drop support for 18.04 by now, but we need a suitable 32bit debian VM configuration to replace it. - Multiple VM tests are still failing for me; but they fail with or without my patches as far as I can tell. I'm having problems with Haiku and CentOS, primarily -- which I think fail even without my patches. I'll have more info after the weekend, these tests are SLOW. - This version of the patch series does not itself enforce any offline-only behavior for venv creation, but downstreams can modify any call to 'mkvenv' to pass '--offline'. A more flexible approach might be to allow an environment variable to be passed that toggles the switch on. - iotests will now actually never run mypy or pylint tests by default anymore, because the bootstrapper won't select those packages by default, and the virtual environment won't utilize the system packages -- so iotest 297 will just "skip" all of the time now. The reason we don't want to install these packages by default is because we don't want to add dependencies on mypy and pylint for downstream builds. With these patches, 297 would still work if you manually opened up the testing venv and installed suitable mypy/pylint packages there. I could also add a new optional dependency group, and one could theoretically invoke a once-per-build-dir command of 'make check-venv-pylint' to help make the process only semi-manual, but it's still annoying. Ideally, the python checks in qemu.git/python/ can handle the same tests as 297 does -- but we need to give a shorthand invocation like "make check-python" that is excluded from the default "make check" to allow block developers to quickly opt-in to the same tests. I've covered some of the problems here on-list before: https://lists.gnu.org/archive/html/qemu-devel/2022-05/msg03661.html ...But I haven't quite solved them yet. That's all for now. Paolo, can we chat about build system integration next? I want to know how you envision the integration at this point -- adding different test-invocation styles (online, offline, etc) may help solve the iotest 297 problem and the iotest self-bootstrap problem. --js John Snow (7): tests: create optional tests/venv dependency groups tests: pythonize test venv creation tests: Remove spurious pip warnings on Ubuntu20.04 tests/vm: add venv pre-requisites to VM building recipes tests: add 'check-venv' as a dependency of 'make check' iotests: use tests/venv for running tests iotests: self-bootstrap testing venv tests/Makefile.include| 32 +++--- tests/mkvenv.py | 187 ++ tests/qemu-iotests/testenv.py | 25 +++-- tests/requirements.txt| 6 -- tests/setup.cfg | 20 tests/setup.py| 16 +++ tests/vm/netbsd | 1 + tests/vm/ubuntu.i386 | 9 +- 8 files changed, 268 insertions(+), 28 deletions(-) create mode 100644 tests/mkvenv.py delete mode 100644 tests/requirements.txt create mode 100644 tests/setup.cfg create mode 100644 tests/setup.py -- 2.34.3
[RFC PATCH v2 7/7] iotests: self-bootstrap testing venv
When iotests are invoked manually from
e.g. $build/tests/qemu-iotests/check, it is not necessarily guaranteed
that we'll have run 'check-venv' yet.
With this patch, teach testenv.py how to create its own environment.
Note: this self-bootstrapping is fairly rudimentary and will miss
certain triggers to refresh the venv. It will miss when new dependencies
are added to either python/setup.cfg or tests/setup.cfg. It can be
coaxed into updating by running 'make check', 'make check-block', 'make
check-venv', etc.
Signed-off-by: John Snow
---
tests/qemu-iotests/testenv.py | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py
index 29404ac94be..e985eaf3e97 100644
--- a/tests/qemu-iotests/testenv.py
+++ b/tests/qemu-iotests/testenv.py
@@ -112,10 +112,10 @@ def init_directories(self) -> None:
"""
venv_path = Path(self.build_root, 'tests/venv/')
if not venv_path.exists():
-raise FileNotFoundError(
-f"Virtual environment \"{venv_path!s}\" isn't found."
-" (Maybe you need to run 'make check-venv'"
-" from the build dir?)"
+mkvenv = Path(self.source_iotests, '../mkvenv.py')
+subprocess.run(
+(sys.executable, str(mkvenv), str(venv_path)),
+check=True,
)
self.virtual_env: str = str(venv_path)
--
2.34.3
[RFC PATCH v2 4/7] tests/vm: add venv pre-requisites to VM building recipes
Ubuntu needs "python3-venv" in order to create virtual environments, and
NetBSD needs "py37-pip" in order to do the same.
Signed-off-by: John Snow
---
tests/vm/netbsd | 1 +
tests/vm/ubuntu.i386 | 9 ++---
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/tests/vm/netbsd b/tests/vm/netbsd
index 45aa9a7fda7..53fe508e487 100755
--- a/tests/vm/netbsd
+++ b/tests/vm/netbsd
@@ -31,6 +31,7 @@ class NetBSDVM(basevm.BaseVM):
"pkgconf",
"xz",
"python37",
+"py37-pip",
"ninja-build",
# gnu tools
diff --git a/tests/vm/ubuntu.i386 b/tests/vm/ubuntu.i386
index 47681b6f87d..40fd5ec86da 100755
--- a/tests/vm/ubuntu.i386
+++ b/tests/vm/ubuntu.i386
@@ -16,9 +16,12 @@ import basevm
import ubuntuvm
DEFAULT_CONFIG = {
-'install_cmds' : "apt-get update,"\
- "apt-get build-dep -y qemu,"\
- "apt-get install -y libfdt-dev language-pack-en
ninja-build",
+'install_cmds' : (
+"apt-get update,"
+"apt-get build-dep -y qemu,"
+"apt-get install -y libfdt-dev language-pack-en ninja-build,"
+"apt-get install -y python3-venv"
+),
}
class UbuntuX86VM(ubuntuvm.UbuntuVM):
--
2.34.3
[RFC PATCH v2 5/7] tests: add 'check-venv' as a dependency of 'make check'
This patch adds the 'check-venv' target as a requisite of all meson driven check-* targets. As of this commit, it will only install the "qemu" namespace package from the source tree, and nothing else. In the future, the "qemu" namespace package in qemu.git will begin to require an external qemu.qmp package, and this would be installed into this environment as well. The avocado test dependencies will *not* be pulled into this venv by default, but they may be added in at a later point in time by running 'make check-avocado' or, without running the tests, 'make check-venv-avocado'. Signed-off-by: John Snow --- tests/Makefile.include | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/Makefile.include b/tests/Makefile.include index d8af6a38112..d484a335be5 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -155,6 +155,9 @@ check-acceptance-deprecated-warning: check-acceptance: check-acceptance-deprecated-warning | check-avocado +# The do-meson-check and do-meson-bench targets are defined in Makefile.mtest +do-meson-check do-meson-bench: check-venv + # Consolidated targets .PHONY: check check-clean get-vm-images -- 2.34.3
[RFC PATCH v2 6/7] iotests: use tests/venv for running tests
Essentially, the changes to testenv.py here mimic the changes that occur when
you "source venv/bin/activate.fish" or similar.
(1) update iotest's internal notion of which python binary to use,
(2) export the VIRTUAL_ENV variable,
(3) front-load the venv/bin directory to PATH.
If the venv directory isn't found, raise a friendly exception that tries
to give the human operator a friendly clue as to what's gone wrong. The
subsequent commit attempts to address this shortcoming by teaching
iotests how to invoke the venv bootstrapper in this circumstance
instead.
Signed-off-by: John Snow
---
tests/qemu-iotests/testenv.py | 25 +++--
1 file changed, 19 insertions(+), 6 deletions(-)
diff --git a/tests/qemu-iotests/testenv.py b/tests/qemu-iotests/testenv.py
index a864c74b123..29404ac94be 100644
--- a/tests/qemu-iotests/testenv.py
+++ b/tests/qemu-iotests/testenv.py
@@ -65,8 +65,9 @@ class TestEnv(ContextManager['TestEnv']):
# lot of them. Silence pylint:
# pylint: disable=too-many-instance-attributes
-env_variables = ['PYTHONPATH', 'TEST_DIR', 'SOCK_DIR', 'SAMPLE_IMG_DIR',
- 'PYTHON', 'QEMU_PROG', 'QEMU_IMG_PROG',
+env_variables = ['PYTHONPATH', 'VIRTUAL_ENV', 'PYTHON', 'PATH',
+ 'TEST_DIR', 'SOCK_DIR', 'SAMPLE_IMG_DIR',
+ 'QEMU_PROG', 'QEMU_IMG_PROG',
'QEMU_IO_PROG', 'QEMU_NBD_PROG', 'QSD_PROG',
'QEMU_OPTIONS', 'QEMU_IMG_OPTIONS',
'QEMU_IO_OPTIONS', 'QEMU_IO_OPTIONS_NO_FMT',
@@ -102,18 +103,29 @@ def get_env(self) -> Dict[str, str]:
def init_directories(self) -> None:
"""Init directory variables:
+ VIRTUAL_ENV
+ PATH
PYTHONPATH
TEST_DIR
SOCK_DIR
SAMPLE_IMG_DIR
"""
+venv_path = Path(self.build_root, 'tests/venv/')
+if not venv_path.exists():
+raise FileNotFoundError(
+f"Virtual environment \"{venv_path!s}\" isn't found."
+" (Maybe you need to run 'make check-venv'"
+" from the build dir?)"
+)
+self.virtual_env: str = str(venv_path)
-# Path where qemu goodies live in this source tree.
-qemu_srctree_path = Path(__file__, '../../../python').resolve()
+self.path = os.pathsep.join((
+str(venv_path / 'bin'),
+os.environ['PATH'],
+))
self.pythonpath = os.pathsep.join(filter(None, (
self.source_iotests,
-str(qemu_srctree_path),
os.getenv('PYTHONPATH'),
)))
@@ -138,7 +150,7 @@ def init_binaries(self) -> None:
PYTHON (for bash tests)
QEMU_PROG, QEMU_IMG_PROG, QEMU_IO_PROG, QEMU_NBD_PROG, QSD_PROG
"""
-self.python = sys.executable
+self.python: str = os.path.join(self.virtual_env, 'bin', 'python3')
def root(*names: str) -> str:
return os.path.join(self.build_root, *names)
@@ -300,6 +312,7 @@ def print_env(self, prefix: str = '') -> None:
{prefix}GDB_OPTIONS -- {GDB_OPTIONS}
{prefix}VALGRIND_QEMU -- {VALGRIND_QEMU}
{prefix}PRINT_QEMU_OUTPUT -- {PRINT_QEMU}
+{prefix}VIRTUAL_ENV -- {VIRTUAL_ENV}
{prefix}"""
args = collections.defaultdict(str, self.get_env())
--
2.34.3
[RFC PATCH v2 3/7] tests: Remove spurious pip warnings on Ubuntu20.04
The version of pip ("20.0.2") that ships with Ubuntu 20.04 has a bug
where it will try to attempt building a wheel even if the "wheel" python
package that enables it to do so is not installed. Even though pip
continues gracefully from source, The result is a lot of irrelevant
failure output.
Upstream pip 20.0.2 does not have this problem, and pip 20.1 introduces
a new info message that informs a user that wheel building is being skipped.
On this version, the output can be silenced by passing --no-binary to
coax pip into skipping that step to begin with. Note, this error does
not seem to show up for the "qemu" package because we install that
package in editable mode. (I think, but did not test, that installing an
empty package in editable mode caused more problems than it fixed.)
Signed-off-by: John Snow
---
tests/mkvenv.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/tests/mkvenv.py b/tests/mkvenv.py
index 0667106d6aa..78f8d0382e0 100644
--- a/tests/mkvenv.py
+++ b/tests/mkvenv.py
@@ -144,7 +144,8 @@ def make_qemu_venv(
with enter_venv(venv_path):
if do_initialize:
install("-e", str(pysrc_path), offline=offline)
-install(str(test_src_path), offline=offline)
+install("--no-binary", "qemu.dummy-tests",
+str(test_src_path), offline=offline)
venv_path.touch()
for option in options:
--
2.34.3
[RFC PATCH v2 2/7] tests: pythonize test venv creation
This splits the venv creation logic out of the Makefile and into a
Python script.
One reason for doing this is to be able to call the venv bootstrapper
from places outside of the Makefile, e.g. configure and iotests. Another
reason is to be able to add "offline" logic to modify the behavior of
the script in certain circumstances.
RFC: I don't like how I have an entire "enter_venv" function here, and
by the end of the series, completely separate logic for entering a venv
in testenv.py -- but they both operate in different ways: this patch
offers a method that alters the current ENV immediately, whereas the
testenv.py method alters a copied ENV that is explicitly passed to
subprocesses.
Still, there's a bit more boilerplate than I'd like, but it's probably
fine and it probably won't need to be updated much, but... less code is
usually better.
But, even if I did write it more generically here; iotests wouldn't be
able to use it anyway, because importing it would mean more sys.path
hacking. So... eh?
Signed-off-by: John Snow
---
tests/Makefile.include | 16 ++--
tests/mkvenv.py| 186 +
2 files changed, 192 insertions(+), 10 deletions(-)
create mode 100644 tests/mkvenv.py
diff --git a/tests/Makefile.include b/tests/Makefile.include
index 82c697230e0..d8af6a38112 100644
--- a/tests/Makefile.include
+++ b/tests/Makefile.include
@@ -104,21 +104,17 @@ else
AVOCADO_CMDLINE_TAGS=$(addprefix -t , $(AVOCADO_TAGS))
endif
-quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \
-$(TESTS_PYTHON) -m pip -q --disable-pip-version-check $1, \
-"VENVPIP","$1")
-
# Core dependencies for tests/venv
$(TESTS_VENV_DIR): $(SRC_PATH)/tests/setup.cfg $(SRC_PATH)/python/setup.cfg
- $(call quiet-command, $(PYTHON) -m venv $@, VENV, $@)
- $(call quiet-venv-pip,install -e "$(SRC_PATH)/python/")
- $(call quiet-venv-pip,install "$(SRC_PATH)/tests/")
- $(call quiet-command, touch $@)
+ $(call quiet-command, \
+$(PYTHON) "$(SRC_PATH)/tests/mkvenv.py" "$@", \
+MKVENV, $@)
# Optional avocado dependencies for tests/venv
$(TESTS_VENV_DIR)/avocado: $(TESTS_VENV_DIR)
- $(call quiet-venv-pip,install "$(SRC_PATH)/tests/[avocado]")
- $(call quiet-command, touch $@)
+ $(call quiet-command, \
+$(PYTHON) "$(SRC_PATH)/tests/mkvenv.py" --opt avocado "$<", \
+MKVENV, $@)
$(TESTS_RESULTS_DIR):
$(call quiet-command, mkdir -p $@, \
diff --git a/tests/mkvenv.py b/tests/mkvenv.py
new file mode 100644
index 000..0667106d6aa
--- /dev/null
+++ b/tests/mkvenv.py
@@ -0,0 +1,186 @@
+"""
+mkvenv - QEMU venv bootstrapping utility
+
+usage: mkvenv.py [-h] [--offline] [--opt OPT] target
+
+Bootstrap QEMU testing venv.
+
+positional arguments:
+ target Target directory to install virtual environment into.
+
+optional arguments:
+ -h, --help show this help message and exit
+ --offline Use system packages and work entirely offline.
+ --opt OPT Install an optional dependency group.
+"""
+
+# Copyright (C) 2022 Red Hat, Inc.
+#
+# Authors:
+# John Snow
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or
+# later. See the COPYING file in the top-level directory.
+
+# Do not add any dependencies from outside the stdlib:
+# This script must be usable on its own!
+
+import argparse
+from contextlib import contextmanager
+import logging
+import os
+from pathlib import Path
+import subprocess
+import sys
+from typing import Iterable, Iterator, Union
+import venv
+
+
+def make_venv(
+venv_path: Union[str, Path],
+system_site_packages: bool = False
+) -> None:
+"""
+Create a venv using the python3 'venv' module.
+
+Identical to:
+``python3 -m venv --clear [--system-site-packages] {venv_path}``
+
+:param venv_path: The target directory
+:param system_site_packages: If True, allow system packages in venv.
+"""
+logging.debug("%s: make_venv(venv_path=%s, system_site_packages=%s)",
+ __file__, str(venv_path), system_site_packages)
+venv.create(
+str(venv_path),
+clear=True,
+symlinks=os.name != "nt", # Same as venv CLI's default.
+with_pip=True,
+system_site_packages=system_site_packages,
+)
+
+
+@contextmanager
+def enter_venv(venv_dir: Union[str, Path]) -> Iterator[None]:
+"""Scoped activation of an existing venv."""
+venv_keys = ('PATH', 'PYTHONHOME', 'VIRTUAL_ENV')
+old = {k: v for k, v in os.environ.items() if k in venv_keys}
+vdir = Path(venv_dir).resolve()
+
+def _delete_keys() -> None:
+for k in venv_keys:
+try:
+del os.environ[k]
+except KeyError:
+pass
+
+try:
+_delete_keys()
+
+os.environ['VIRTUAL_ENV'] = str(vdir)
+os.environ['PATH'] = os.pathsep.join([
+str(vdir / "bin/"),
+old['PATH'],
+])
+
[RFC PATCH v2 1/7] tests: create optional tests/venv dependency groups
This patch uses a dummy package and setup.cfg/setup.py files to manage optional dependency groups for the test venv specification. Now, there's a core set of dependencies which for now includes just "qemu" (but soon, qemu.qmp) and a separate, optional 'avocado' group that includes avocado-framework and pycdlib. Practical upshot: We install only a minimum of things for the majority of check-* targets, but allow optional add-ons to be processed when running avocado tests. This will spare downstreams from having to add more dependencies than is necessary as a build dependencies when invoking "make check". (We also keep both sets of dependencies in one file, which is helpful for review to ensure that different option groups don't conflict with one another.) NOTE: There is a non-fatal caveat introduced by this patch on Ubuntu 20.04 systems; see the subsequent commit "tests: Remove spurious pip warnings on Ubuntu20.04" for more information. Signed-off-by: John Snow --- tests/Makefile.include | 21 ++--- tests/requirements.txt | 6 -- tests/setup.cfg| 20 tests/setup.py | 16 4 files changed, 50 insertions(+), 13 deletions(-) delete mode 100644 tests/requirements.txt create mode 100644 tests/setup.cfg create mode 100644 tests/setup.py diff --git a/tests/Makefile.include b/tests/Makefile.include index 3accb83b132..82c697230e0 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -81,13 +81,13 @@ clean-tcg: $(CLEAN_TCG_TARGET_RULES) # Python venv for running tests -.PHONY: check-venv check-avocado check-acceptance check-acceptance-deprecated-warning +.PHONY: check-venv check-venv-avocado check-avocado check-acceptance \ +check-acceptance-deprecated-warning # Build up our target list from the filtered list of ninja targets TARGETS=$(patsubst libqemu-%.fa, %, $(filter libqemu-%.fa, $(ninja-targets))) TESTS_VENV_DIR=$(BUILD_DIR)/tests/venv -TESTS_VENV_REQ=$(SRC_PATH)/tests/requirements.txt TESTS_RESULTS_DIR=$(BUILD_DIR)/tests/results TESTS_PYTHON=$(TESTS_VENV_DIR)/bin/python3 ifndef AVOCADO_TESTS @@ -108,10 +108,16 @@ quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \ $(TESTS_PYTHON) -m pip -q --disable-pip-version-check $1, \ "VENVPIP","$1") -$(TESTS_VENV_DIR): $(TESTS_VENV_REQ) +# Core dependencies for tests/venv +$(TESTS_VENV_DIR): $(SRC_PATH)/tests/setup.cfg $(SRC_PATH)/python/setup.cfg $(call quiet-command, $(PYTHON) -m venv $@, VENV, $@) $(call quiet-venv-pip,install -e "$(SRC_PATH)/python/") - $(call quiet-venv-pip,install -r $(TESTS_VENV_REQ)) + $(call quiet-venv-pip,install "$(SRC_PATH)/tests/") + $(call quiet-command, touch $@) + +# Optional avocado dependencies for tests/venv +$(TESTS_VENV_DIR)/avocado: $(TESTS_VENV_DIR) + $(call quiet-venv-pip,install "$(SRC_PATH)/tests/[avocado]") $(call quiet-command, touch $@) $(TESTS_RESULTS_DIR): @@ -119,6 +125,7 @@ $(TESTS_RESULTS_DIR): MKDIR, $@) check-venv: $(TESTS_VENV_DIR) +check-venv-avocado: $(TESTS_VENV_DIR)/avocado FEDORA_31_ARCHES_TARGETS=$(patsubst %-softmmu,%, $(filter %-softmmu,$(TARGETS))) FEDORA_31_ARCHES_CANDIDATES=$(patsubst ppc64,ppc64le,$(FEDORA_31_ARCHES_TARGETS)) @@ -126,16 +133,16 @@ FEDORA_31_ARCHES := x86_64 aarch64 ppc64le s390x FEDORA_31_DOWNLOAD=$(filter $(FEDORA_31_ARCHES),$(FEDORA_31_ARCHES_CANDIDATES)) # download one specific Fedora 31 image -get-vm-image-fedora-31-%: check-venv +get-vm-image-fedora-31-%: check-venv-avocado $(call quiet-command, \ $(TESTS_PYTHON) -m avocado vmimage get \ --distro=fedora --distro-version=31 --arch=$*, \ "AVOCADO", "Downloading avocado tests VM image for $*") # download all vm images, according to defined targets -get-vm-images: check-venv $(patsubst %,get-vm-image-fedora-31-%, $(FEDORA_31_DOWNLOAD)) +get-vm-images: check-venv-avocado $(patsubst %,get-vm-image-fedora-31-%, $(FEDORA_31_DOWNLOAD)) -check-avocado: check-venv $(TESTS_RESULTS_DIR) get-vm-images +check-avocado: check-venv-avocado $(TESTS_RESULTS_DIR) get-vm-images $(call quiet-command, \ $(TESTS_PYTHON) -m avocado \ --show=$(AVOCADO_SHOW) run --job-results-dir=$(TESTS_RESULTS_DIR) \ diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index 0ba561b6bdf..000 --- a/tests/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Add Python module requirements, one per line, to be installed -# in the tests/venv Python virtual environment. For more info, -# refer to: https://pip.pypa.io/en/stable/user_guide/#id1 -# Note that qemu.git/python/ is always implicitly installed. -avocado-framework==88.1 -pycdlib==1.11.0 diff --git a/tests/setup.cfg b/tests/setup.cfg new file mode 100644 index 000..263a5de01af --- /dev/null +++ b/tests/setup.cfg @@ -0,0 +1,20 @@ +# This file represents a "dummy" package that expresses +# t
