In case of a failure during execution, build target or suite setup
the test case results will be recorded as blocked. We also unify
methods add_<result> to add_child_result to be more consistently.
Now we store the corresponding config in each result with child
configs and parent result.
Signed-off-by: Juraj Linkeš <juraj.lin...@pantheon.tech>
---
dts/framework/runner.py | 12 +-
dts/framework/test_result.py | 361 ++++++++++++++++++++---------------
2 files changed, 216 insertions(+), 157 deletions(-)
diff --git a/dts/framework/runner.py b/dts/framework/runner.py
index acc3342f0c..28570d4a1c 100644
--- a/dts/framework/runner.py
+++ b/dts/framework/runner.py
@@ -74,7 +74,7 @@ class DTSRunner:
def __init__(self, configuration: Configuration):
self._logger = getLogger("DTSRunner")
- self._result = DTSResult(self._logger)
+ self._result = DTSResult(configuration, self._logger)
self._executions = create_executions(configuration.executions)
def run(self):
@@ -150,7 +150,7 @@ def _run_execution(
"Running execution with SUT "
f"'{execution.config.system_under_test_node.name}'."
)
- execution_result = self._result.add_execution(sut_node.config)
+ execution_result =
self._result.add_child_result(execution.processed_config())
execution_result.add_sut_info(sut_node.node_info)
try:
@@ -190,7 +190,7 @@ def _run_build_target(
Run the given build target.
"""
self._logger.info(f"Running build target '{build_target.name}'.")
- build_target_result = execution_result.add_build_target(build_target)
+ build_target_result = execution_result.add_child_result(build_target)
try:
sut_node.set_up_build_target(build_target)
@@ -265,7 +265,9 @@ def _run_test_suite(
"""
test_suite = test_suite_setup.test_suite(sut_node, tg_node)
test_suite_name = test_suite_setup.test_suite.__name__
- test_suite_result = build_target_result.add_test_suite(test_suite_name)
+ test_suite_result = build_target_result.add_child_result(
+ test_suite_setup.processed_config()
+ )
try:
self._logger.info(f"Starting test suite setup: {test_suite_name}")
test_suite.set_up_suite()
@@ -308,7 +310,7 @@ def _execute_test_suite(
"""
for test_case_method in test_cases:
test_case_name = test_case_method.__name__
- test_case_result = test_suite_result.add_test_case(test_case_name)
+ test_case_result =
test_suite_result.add_child_result(test_case_name)
all_attempts = SETTINGS.re_run + 1
attempt_nr = 1
self._run_test_case(test_case_method, test_suite, test_case_result)
diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py
index 4c2e7e2418..dba2c55d36 100644
--- a/dts/framework/test_result.py
+++ b/dts/framework/test_result.py
@@ -9,6 +9,7 @@
import os.path
from collections.abc import MutableSequence
from enum import Enum, auto
+from typing import Any, Union
from .config import (
OS,
@@ -16,9 +17,11 @@
BuildTargetConfiguration,
BuildTargetInfo,
Compiler,
+ Configuration,
CPUType,
- NodeConfiguration,
+ ExecutionConfiguration,
NodeInfo,
+ TestSuiteConfig,
)
from .exception import DTSError, ErrorSeverity
from .logger import DTSLOG
@@ -35,6 +38,7 @@ class Result(Enum):
FAIL = auto()
ERROR = auto()
SKIP = auto()
+ BLOCK = auto()
def __bool__(self) -> bool:
return self is self.PASS
@@ -63,42 +67,6 @@ def __bool__(self) -> bool:
return bool(self.result)
-class Statistics(dict):
- """
- A helper class used to store the number of test cases by its result
- along a few other basic information.
- Using a dict provides a convenient way to format the data.
- """
-
- def __init__(self, dpdk_version: str | None):
- super(Statistics, self).__init__()
- for result in Result:
- self[result.name] = 0
- self["PASS RATE"] = 0.0
- self["DPDK VERSION"] = dpdk_version
-
- def __iadd__(self, other: Result) -> "Statistics":
- """
- Add a Result to the final count.
- """
- self[other.name] += 1
- self["PASS RATE"] = (
- float(self[Result.PASS.name]) * 100 / sum(self[result.name] for
result in Result)
- )
- return self
-
- def __str__(self) -> str:
- """
- Provide a string representation of the data.
- """
- stats_str = ""
- for key, value in self.items():
- stats_str += f"{key:<12} = {value}\n"
- # according to docs, we should use \n when writing to text files
- # on all platforms
- return stats_str
-
-
class BaseResult(object):
"""
The Base class for all results. Stores the results of
@@ -109,6 +77,12 @@ class BaseResult(object):
setup_result: FixtureResult
teardown_result: FixtureResult
_inner_results: MutableSequence["BaseResult"]
+ _child_configs: Union[
+ list[ExecutionConfiguration],
+ list[BuildTargetConfiguration],
+ list[TestSuiteConfig],
+ list[str],
+ ]
def __init__(self):
self.setup_result = FixtureResult()
@@ -119,6 +93,23 @@ def update_setup(self, result: Result, error: Exception |
None = None) -> None:
self.setup_result.result = result
self.setup_result.error = error
+ if result in [Result.BLOCK, Result.ERROR, Result.FAIL]:
+ for child_config in self._child_configs:
+ child_result = self.add_child_result(child_config)
+ child_result.block()
+
+ def add_child_result(self, config: Any) -> "BaseResult":
+ """
+ Adding corresponding result for each classes.
+ """
+
+ def block(self):
+ """
+ Mark the result as block on corresponding classes.
+ """
+ self.update_setup(Result.BLOCK)
+ self.update_teardown(Result.BLOCK)
+
def update_teardown(self, result: Result, error: Exception | None = None)
-> None:
self.teardown_result.result = result
self.teardown_result.error = error
@@ -139,119 +130,11 @@ def _get_inner_errors(self) -> list[Exception]:
def get_errors(self) -> list[Exception]:
return self._get_setup_teardown_errors() + self._get_inner_errors()
- def add_stats(self, statistics: Statistics) -> None:
+ def add_stats(self, statistics: "Statistics") -> None:
for inner_result in self._inner_results:
inner_result.add_stats(statistics)
-class TestCaseResult(BaseResult, FixtureResult):
- """
- The test case specific result.
- Stores the result of the actual test case.
- Also stores the test case name.
- """
-
- test_case_name: str
-
- def __init__(self, test_case_name: str):
- super(TestCaseResult, self).__init__()
- self.test_case_name = test_case_name
-
- def update(self, result: Result, error: Exception | None = None) -> None:
- self.result = result
- self.error = error
-
- def _get_inner_errors(self) -> list[Exception]:
- if self.error:
- return [self.error]
- return []
-
- def add_stats(self, statistics: Statistics) -> None:
- statistics += self.result
-
- def __bool__(self) -> bool:
- return bool(self.setup_result) and bool(self.teardown_result) and
bool(self.result)
-
-
-class TestSuiteResult(BaseResult):
- """
- The test suite specific result.
- The _inner_results list stores results of test cases in a given test suite.
- Also stores the test suite name.
- """
-
- suite_name: str
-
- def __init__(self, suite_name: str):
- super(TestSuiteResult, self).__init__()
- self.suite_name = suite_name
-
- def add_test_case(self, test_case_name: str) -> TestCaseResult:
- test_case_result = TestCaseResult(test_case_name)
- self._inner_results.append(test_case_result)
- return test_case_result
-
-
-class BuildTargetResult(BaseResult):
- """
- The build target specific result.
- The _inner_results list stores results of test suites in a given build
target.
- Also stores build target specifics, such as compiler used to build DPDK.
- """
-
- arch: Architecture
- os: OS
- cpu: CPUType
- compiler: Compiler
- compiler_version: str | None
- dpdk_version: str | None
-
- def __init__(self, build_target: BuildTargetConfiguration):
- super(BuildTargetResult, self).__init__()
- self.arch = build_target.arch
- self.os = build_target.os
- self.cpu = build_target.cpu
- self.compiler = build_target.compiler
- self.compiler_version = None
- self.dpdk_version = None
-
- def add_build_target_info(self, versions: BuildTargetInfo) -> None:
- self.compiler_version = versions.compiler_version
- self.dpdk_version = versions.dpdk_version
-
- def add_test_suite(self, test_suite_name: str) -> TestSuiteResult:
- test_suite_result = TestSuiteResult(test_suite_name)
- self._inner_results.append(test_suite_result)
- return test_suite_result
-
-
-class ExecutionResult(BaseResult):
- """
- The execution specific result.
- The _inner_results list stores results of build targets in a given
execution.
- Also stores the SUT node configuration.
- """
-
- sut_node: NodeConfiguration
- sut_os_name: str
- sut_os_version: str
- sut_kernel_version: str
-
- def __init__(self, sut_node: NodeConfiguration):
- super(ExecutionResult, self).__init__()
- self.sut_node = sut_node
-
- def add_build_target(self, build_target: BuildTargetConfiguration) ->
BuildTargetResult:
- build_target_result = BuildTargetResult(build_target)
- self._inner_results.append(build_target_result)
- return build_target_result
-
- def add_sut_info(self, sut_info: NodeInfo):
- self.sut_os_name = sut_info.os_name
- self.sut_os_version = sut_info.os_version
- self.sut_kernel_version = sut_info.kernel_version
-
-
class DTSResult(BaseResult):
"""
Stores environment information and test results from a DTS run, which are:
@@ -269,25 +152,27 @@ class DTSResult(BaseResult):
"""
dpdk_version: str | None
+ _child_configs: list[ExecutionConfiguration]
_logger: DTSLOG
_errors: list[Exception]
_return_code: ErrorSeverity
- _stats_result: Statistics | None
+ _stats_result: Union["Statistics", None]
_stats_filename: str
- def __init__(self, logger: DTSLOG):
+ def __init__(self, configuration: Configuration, logger: DTSLOG):
super(DTSResult, self).__init__()
self.dpdk_version = None
+ self._child_configs = configuration.executions
self._logger = logger
self._errors = []
self._return_code = ErrorSeverity.NO_ERR
self._stats_result = None
self._stats_filename = os.path.join(SETTINGS.output_dir,
"statistics.txt")
- def add_execution(self, sut_node: NodeConfiguration) -> ExecutionResult:
- execution_result = ExecutionResult(sut_node)
- self._inner_results.append(execution_result)
- return execution_result
+ def add_child_result(self, config: ExecutionConfiguration) ->
"ExecutionResult":
+ result = ExecutionResult(config, self)
+ self._inner_results.append(result)
+ return result
def add_error(self, error) -> None:
self._errors.append(error)
@@ -325,3 +210,175 @@ def get_return_code(self) -> int:
self._return_code = error_return_code
return int(self._return_code)
+
+
+class ExecutionResult(BaseResult):
+ """
+ The execution specific result.
+ The _inner_results list stores results of build targets in a given
execution.
+ Also stores the SUT node configuration.
+ """
+
+ sut_os_name: str
+ sut_os_version: str
+ sut_kernel_version: str
+ _config: ExecutionConfiguration
+ _parent_result: DTSResult
+ _child_configs: list[BuildTargetConfiguration]
+
+ def __init__(self, config: ExecutionConfiguration, parent_result:
DTSResult):
+ super(ExecutionResult, self).__init__()
+ self._config = config
+ self._parent_result = parent_result
+ self._child_configs = config.build_targets
+
+ def add_sut_info(self, sut_info: NodeInfo):
+ self.sut_os_name = sut_info.os_name
+ self.sut_os_version = sut_info.os_version
+ self.sut_kernel_version = sut_info.kernel_version
+
+ def add_child_result(self, config: BuildTargetConfiguration) ->
"BuildTargetResult":
+ result = BuildTargetResult(config, self)
+ self._inner_results.append(result)
+ return result
+
+
+class BuildTargetResult(BaseResult):
+ """
+ The build target specific result.
+ The _inner_results list stores results of test suites in a given build
target.
+ Also stores build target specifics, such as compiler used to build DPDK.
+ """
+
+ arch: Architecture
+ os: OS
+ cpu: CPUType
+ compiler: Compiler
+ compiler_version: str | None
+ dpdk_version: str | None
+ _config: BuildTargetConfiguration
+ _parent_result: ExecutionResult
+ _child_configs: list[TestSuiteConfig]
+
+ def __init__(
+ self, config: BuildTargetConfiguration, parent_result: ExecutionResult
+ ):
+ super(BuildTargetResult, self).__init__()
+ self.arch = config.arch
+ self.os = config.os
+ self.cpu = config.cpu
+ self.compiler = config.compiler
+ self.compiler_version = None
+ self.dpdk_version = None
+ self._config = config
+ self._parent_result = parent_result
+ self._child_configs = parent_result._config.test_suites
+
+ def add_build_target_info(self, versions: BuildTargetInfo) -> None:
+ self.compiler_version = versions.compiler_version
+ self.dpdk_version = versions.dpdk_version
+
+ def add_child_result(
+ self,
+ config: TestSuiteConfig,
+ ) -> "TestSuiteResult":
+ result = TestSuiteResult(config, self)
+ self._inner_results.append(result)
+ return result
+
+
+class TestSuiteResult(BaseResult):
+ """
+ The test suite specific result.
+ The _inner_results list stores results of test cases in a given test suite.
+ Also stores the test suite name.
+ """
+
+ _config: TestSuiteConfig
+ _parent_result: BuildTargetResult
+ _child_configs: list[str]
+
+ def __init__(self, config: TestSuiteConfig, parent_result:
BuildTargetResult):
+ super(TestSuiteResult, self).__init__()
+ self._config = config
+ self._parent_result = parent_result
+ self._child_configs = config.test_cases
+
+ def add_child_result(self, config: str) -> "TestCaseResult":
+ result = TestCaseResult(config, self)
+ self._inner_results.append(result)
+ return result
+
+
+class TestCaseResult(BaseResult, FixtureResult):
+ """
+ The test case specific result.
+ Stores the result of the actual test case.
+ Also stores the test case name.
+ """
+
+ _config: str
+ _parent_result: TestSuiteResult
+
+ def __init__(self, config: str, parent_result: TestSuiteResult):
+ super(TestCaseResult, self).__init__()
+ self._config = config
+ self._parent_result = parent_result
+
+ def block(self):
+ self.update(Result.BLOCK)
+
+ def update(self, result: Result, error: Exception | None = None) -> None:
+ self.result = result
+ self.error = error
+
+ def _get_inner_errors(self) -> list[Exception]:
+ if self.error:
+ return [self.error]
+ return []
+
+ def add_stats(self, statistics: "Statistics") -> None:
+ statistics += self.result
+
+ def __bool__(self) -> bool:
+ return (
+ bool(self.setup_result) and bool(self.teardown_result) and
bool(self.result)
+ )
+
+
+class Statistics(dict):
+ """
+ A helper class used to store the number of test cases by its result
+ along a few other basic information.
+ Using a dict provides a convenient way to format the data.
+ """
+
+ def __init__(self, dpdk_version: str | None):
+ super(Statistics, self).__init__()
+ for result in Result:
+ self[result.name] = 0
+ self["PASS RATE"] = 0.0
+ self["DPDK VERSION"] = dpdk_version
+
+ def __iadd__(self, other: Result) -> "Statistics":
+ """
+ Add a Result to the final count.
+ """
+ self[other.name] += 1
+ self["PASS RATE"] = (
+ float(self[Result.PASS.name])
+ * 100
+ / sum(self[result.name] for result in Result)
+ )
+ return self
+
+ def __str__(self) -> str:
+ """
+ Provide a string representation of the data.
+ """
+ stats_str = ""
+ for key, value in self.items():
+ stats_str += f"{key:<12} = {value}\n"
+ # according to docs, we should use \n when writing to text files
+ # on all platforms
+ return stats_str
--
2.34.1
