On Fri, 2023-02-24 at 17:45 +0100, Alexis Lothoré via
lists.openembedded.org wrote:
> From: Alexis Lothoré <alexis.loth...@bootlin.com>
>
> Introduce new tests for the metadata-based filtering added for oeselftest
> results
>
> Signed-off-by: Alexis Lothoré <alexis.loth...@bootlin.com>
> ---
> .../oeqa/selftest/cases/resulttooltests.py | 137 ++++++++++++++++++
> 1 file changed, 137 insertions(+)
>
> diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py
> b/meta/lib/oeqa/selftest/cases/resulttooltests.py
> index efdfd98af3c..75d406c122d 100644
> --- a/meta/lib/oeqa/selftest/cases/resulttooltests.py
> +++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py
> @@ -98,3 +98,140 @@ class ResultToolTests(OESelftestTestCase):
> resultutils.append_resultsdata(results,
> ResultToolTests.target_results_data, configmap=resultutils.flatten_map)
> self.assertEqual(len(results[''].keys()), 5, msg="Flattened results
> not correct %s" % str(results))
>
> + def test_results_without_metadata_can_be_compared(self):
> + base_configuration = {"TEST_TYPE": "oeselftest",
> + "TESTSERIES": "series1",
> + "IMAGE_BASENAME": "image",
> + "IMAGE_PKGTYPE": "ipk",
> + "DISTRO": "mydistro",
> + "MACHINE": "qemux86"}
> + target_configuration = {"TEST_TYPE": "oeselftest",
> + "TESTSERIES": "series1",
> + "IMAGE_BASENAME": "image",
> + "IMAGE_PKGTYPE": "ipk",
> + "DISTRO": "mydistro",
> + "MACHINE": "qemux86"}
> + self.assertTrue(regression.can_be_compared(base_configuration,
> target_configuration),
> + msg="incorrect metadata filtering, tests without
> metadata should be compared")
> +
> + def test_target_result_with_missing_metadata_can_not_be_compared(self):
> + base_configuration = {"TEST_TYPE": "oeselftest",
> + "TESTSERIES": "series1",
> + "IMAGE_BASENAME": "image",
> + "IMAGE_PKGTYPE": "ipk",
> + "DISTRO": "mydistro",
> + "MACHINE": "qemux86",
> + "OESELFTEST_METADATA": {"run_all_tests": True,
> + "run_tests": None,
> + "skips": None,
> + "machine": None,
> + "select_tags":
> ["toolchain-user", "toolchain-system"],
> + "exclude_tags": None}}
> + target_configuration = {"TEST_TYPE": "oeselftest",
> + "TESTSERIES": "series1",
> + "IMAGE_BASENAME": "image",
> + "IMAGE_PKGTYPE": "ipk",
> + "DISTRO": "mydistro",
> + "MACHINE": "qemux86"}
> + self.assertFalse(regression.can_be_compared(base_configuration,
> target_configuration),
> + msg="incorrect metadata filtering, tests should not
> be compared")
> +
> + def test_results_with_matching_metadata_can_be_compared(self):
> + base_configuration = {"TEST_TYPE": "oeselftest",
> + "TESTSERIES": "series1",
> + "IMAGE_BASENAME": "image",
> + "IMAGE_PKGTYPE": "ipk",
> + "DISTRO": "mydistro",
> + "MACHINE": "qemux86",
> + "OESELFTEST_METADATA": {"run_all_tests": True,
> + "run_tests": None,
> + "skips": None,
> + "machine": None,
> + "select_tags":
> ["toolchain-user", "toolchain-system"],
> + "exclude_tags": None}}
> + target_configuration = {"TEST_TYPE": "oeselftest",
> + "TESTSERIES": "series1",
> + "IMAGE_BASENAME": "image",
> + "IMAGE_PKGTYPE": "ipk",
> + "DISTRO": "mydistro",
> + "MACHINE": "qemux86",
> + "OESELFTEST_METADATA": {"run_all_tests":
> True,
> + "run_tests": None,
> + "skips": None,
> + "machine": None,
> + "select_tags":
> ["toolchain-user", "toolchain-system"],
> + "exclude_tags":
> None}}
> + self.assertTrue(regression.can_be_compared(base_configuration,
> target_configuration),
> + msg="incorrect metadata filtering, tests with
> matching metadata should be compared")
> +
> + def test_results_with_mismatching_metadata_can_not_be_compared(self):
> + base_configuration = {"TEST_TYPE": "oeselftest",
> + "TESTSERIES": "series1",
> + "IMAGE_BASENAME": "image",
> + "IMAGE_PKGTYPE": "ipk",
> + "DISTRO": "mydistro",
> + "MACHINE": "qemux86",
> + "OESELFTEST_METADATA": {"run_all_tests": True,
> + "run_tests": None,
> + "skips": None,
> + "machine": None,
> + "select_tags":
> ["toolchain-user", "toolchain-system"],
> + "exclude_tags": None}}
> + target_configuration = {"TEST_TYPE": "oeselftest",
> + "TESTSERIES": "series1",
> + "IMAGE_BASENAME": "image",
> + "IMAGE_PKGTYPE": "ipk",
> + "DISTRO": "mydistro",
> + "MACHINE": "qemux86",
> + "OESELFTEST_METADATA": {"run_all_tests":
> True,
> + "run_tests": None,
> + "skips": None,
> + "machine": None,
> + "select_tags":
> ["machine"],
> + "exclude_tags":
> None}}
> + self.assertFalse(regression.can_be_compared(base_configuration,
> target_configuration),
> + msg="incorrect metadata filtering, tests with
> mismatching metadata should not be compared")
> +
> + def test_metadata_matching_is_only_checked_for_relevant_test_type(self):
> + base_configuration = {"TEST_TYPE": "runtime",
> + "TESTSERIES": "series1",
> + "IMAGE_BASENAME": "image",
> + "IMAGE_PKGTYPE": "ipk",
> + "DISTRO": "mydistro",
> + "MACHINE": "qemux86",
> + "OESELFTEST_METADATA": {"run_all_tests": True,
> + "run_tests": None,
> + "skips": None,
> + "machine": None,
> + "select_tags":
> ["toolchain-user", "toolchain-system"],
> + "exclude_tags": None}}
> + target_configuration = {"TEST_TYPE": "runtime",
> + "TESTSERIES": "series1",
> + "IMAGE_BASENAME": "image",
> + "IMAGE_PKGTYPE": "ipk",
> + "DISTRO": "mydistro",
> + "MACHINE": "qemux86",
> + "OESELFTEST_METADATA": {"run_all_tests":
> True,
> + "run_tests": None,
> + "skips": None,
> + "machine": None,
> + "select_tags":
> ["machine"],
> + "exclude_tags":
> None}}
> + self.assertTrue(regression.can_be_compared(base_configuration,
> target_configuration),
> + msg="incorrect metadata filtering, %s tests should
> be compared" % base_configuration['TEST_TYPE'])
> +
> + def test_machine_matches(self):
> + base_configuration = {"TEST_TYPE": "runtime",
> + "MACHINE": "qemux86"}
> + target_configuration = {"TEST_TYPE": "runtime",
> + "MACHINE": "qemux86"}
> + self.assertTrue(regression.can_be_compared(base_configuration,
> target_configuration),
> + msg="incorrect machine filtering, identical machine
> tests should be compared")
> +
> + def test_machine_mismatches(self):
> + base_configuration = {"TEST_TYPE": "runtime",
> + "MACHINE": "qemux86"}
> + target_configuration = {"TEST_TYPE": "runtime",
> + "MACHINE": "qemux86_64"}
> + self.assertFalse(regression.can_be_compared(base_configuration,
> target_configuration),
> + msg="incorrect machine filtering, mismatching
> machine tests should not be compared")
I love the fact this has tests but they don't work:
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/4854
then with the obvious error fixed to add self.logger:
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/4858/steps/14/logs/stdio
Cheers,
Richard
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#177722):
https://lists.openembedded.org/g/openembedded-core/message/177722
Mute This Topic: https://lists.openembedded.org/mt/97209736/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
