The single core forward performance test suite can run a variety of combinations of buffer sizes and number of descriptors. It is beneficial if any possible combination of these parameters could be tested against a specific delta tolerance; Rather than a globally defined tolerance for all testing. Along with this, adjusting traffic duration and test repetitions helps users define a more accurate test run if they wish to get a more refined average. Furthermore, the cores allotted to TREX can vary among systems and should be defined by the user.
Signed-off-by: Andrew Bailey <[email protected]> --- dts/configurations/test_run.example.yaml | 1 + dts/framework/config/test_run.py | 1 + .../testbed_model/traffic_generator/trex.py | 3 +- .../TestSuite_single_core_forward_perf.py | 52 ++++++++++++++----- 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/dts/configurations/test_run.example.yaml b/dts/configurations/test_run.example.yaml index 51f9ab8237..c6eb086211 100644 --- a/dts/configurations/test_run.example.yaml +++ b/dts/configurations/test_run.example.yaml @@ -52,6 +52,7 @@ func_traffic_generator: # type: TREX # remote_path: "/opt/trex/v3.03" # The remote path of the traffic generator application. # config: "/opt/trex_config/trex_config.yaml" # Additional configuration files. (Leave blank if not required) +# cores: 10 # The number of cores to run TRex with. perf: false # disable performance testing func: true # enable functional testing crypto: false # disable cryptographic testing diff --git a/dts/framework/config/test_run.py b/dts/framework/config/test_run.py index 81630df77d..1ffb6b2365 100644 --- a/dts/framework/config/test_run.py +++ b/dts/framework/config/test_run.py @@ -420,6 +420,7 @@ class TrexTrafficGeneratorConfig(TrafficGeneratorConfig): type: Literal[TrafficGeneratorType.TREX] remote_path: PurePath config: PurePath + cores: int #: A union type discriminating traffic generators by the `type` field. diff --git a/dts/framework/testbed_model/traffic_generator/trex.py b/dts/framework/testbed_model/traffic_generator/trex.py index d53791b0a6..2b52cfcbd1 100644 --- a/dts/framework/testbed_model/traffic_generator/trex.py +++ b/dts/framework/testbed_model/traffic_generator/trex.py @@ -82,7 +82,7 @@ class TrexTrafficGenerator(PerformanceTrafficGenerator): _streaming_mode: TrexStatelessTXModes = TrexStatelessTXModes.STLTXCont - _tg_cores: int = 10 + _tg_cores: int _trex_app: BlockingApp @@ -102,6 +102,7 @@ def __init__(self, tg_node: Node, config: TrexTrafficGeneratorConfig) -> None: super().__init__(tg_node=tg_node, config=config) self._tg_node_config = tg_node.config self._tg_config = config + self._tg_cores = config.cores self._os_session = create_session(self._tg_node.config, "TRex", self._logger) diff --git a/dts/tests/TestSuite_single_core_forward_perf.py b/dts/tests/TestSuite_single_core_forward_perf.py index 57aecdf7a4..a966d44ebd 100644 --- a/dts/tests/TestSuite_single_core_forward_perf.py +++ b/dts/tests/TestSuite_single_core_forward_perf.py @@ -32,14 +32,40 @@ class Config(BaseConfig): """Performance test metrics.""" test_parameters: list[dict[str, int | float]] = [ - {"frame_size": 64, "num_descriptors": 1024, "expected_mpps": 1.00}, - {"frame_size": 128, "num_descriptors": 1024, "expected_mpps": 1.00}, - {"frame_size": 256, "num_descriptors": 1024, "expected_mpps": 1.00}, - {"frame_size": 512, "num_descriptors": 1024, "expected_mpps": 1.00}, - {"frame_size": 1024, "num_descriptors": 1024, "expected_mpps": 1.00}, - {"frame_size": 1518, "num_descriptors": 1024, "expected_mpps": 1.00}, + {"frame_size": 64, "num_descriptors": 1024, "expected_mpps": 1.00, "delta_tolerance": 0.05}, + { + "frame_size": 128, + "num_descriptors": 1024, + "expected_mpps": 1.00, + "delta_tolerance": 0.05, + }, + { + "frame_size": 256, + "num_descriptors": 1024, + "expected_mpps": 1.00, + "delta_tolerance": 0.05, + }, + { + "frame_size": 512, + "num_descriptors": 1024, + "expected_mpps": 1.00, + "delta_tolerance": 0.05, + }, + { + "frame_size": 1024, + "num_descriptors": 1024, + "expected_mpps": 1.00, + "delta_tolerance": 0.05, + }, + { + "frame_size": 1518, + "num_descriptors": 1024, + "expected_mpps": 1.00, + "delta_tolerance": 0.05, + }, ] - delta_tolerance: float = 0.05 + traffic_duration: int = 5 + test_repetitions: int = 5 @requires_link_topology(LinkTopology.TWO_LINKS) @@ -51,7 +77,8 @@ class TestSingleCoreForwardPerf(TestSuite): def set_up_suite(self): """Set up the test suite.""" self.test_parameters = self.config.test_parameters - self.delta_tolerance = self.config.delta_tolerance + self.traffic_duration = self.config.traffic_duration + self.test_repetitions = self.config.test_repetitions def _transmit(self, testpmd: TestPmd, frame_size: int, repetitions: int = 1) -> float: """Create a testpmd session with every rule in the given list, verify jump behavior. @@ -76,8 +103,7 @@ def _transmit(self, testpmd: TestPmd, frame_size: int, repetitions: int = 1) -> rx_avg = 0.0 for _ in range(repetitions): - # Transmit for 5 seconds. - stats = assess_performance_by_packet(packet=packet, duration=5) + stats = assess_performance_by_packet(packet=packet, duration=self.traffic_duration) rx_avg += stats.rx_pps return rx_avg / (repetitions * 1_000_000) @@ -134,21 +160,21 @@ def single_core_forward_perf(self) -> None: **driver_specific_testpmd_args, ) as testpmd: params["measured_mpps"] = round( - self._transmit(testpmd, frame_size, repetitions=5), 3 + self._transmit(testpmd, frame_size, repetitions=self.test_repetitions), 3 ) params["performance_delta"] = round( (float(params["measured_mpps"]) - float(params["expected_mpps"])) / float(params["expected_mpps"]), 3, ) - params["pass"] = float(params["performance_delta"]) >= -self.delta_tolerance + params["pass"] = float(params["performance_delta"]) >= -params["delta_tolerance"] self._produce_stats_table(self.test_parameters) for params in self.test_parameters: verify( params["pass"] is True, - f"""Packets forwarded is less than {(1 - self.delta_tolerance) * 100}% + f"""Packets forwarded is less than {(1 - params["delta_tolerance"]) * 100}% of the expected baseline. Measured MPPS = {params["measured_mpps"]} Expected MPPS = {params["expected_mpps"]}""", -- 2.55.0
