The previous implementation configures and allocates hugepage sizes
based on a system default. This can lead to two problems: overallocation of
hugepages (which may crash the remote host), and configuration of hugepage
sizes that are not recommended during runtime. This new implementation
allows only 2MB hugepage allocation during runtime; any other unique
hugepage size must be configured by the end-user for initializing DTS.
If the amount of 2MB hugepages requested exceeds the amount of 2MB
hugepages already configured on the system, then the system will remount
hugepages to cover the difference. If the amount of hugepages requested is
either less than or equal to the amount already configured on the system,
then nothing is done.
Bugzilla ID: 1370
Signed-off-by: Nicholas Pratte <npra...@iol.unh.edu>
Reviewed-by: Jeremy Spewock <jspew...@iol.unh.edu>
---
doc/guides/tools/dts.rst | 7 +++++-
dts/conf.yaml | 4 ++--
dts/framework/config/__init__.py | 4 ++--
dts/framework/config/conf_yaml_schema.json | 6 ++---
dts/framework/config/types.py | 2 +-
dts/framework/testbed_model/linux_session.py | 24 +++++++++++---------
dts/framework/testbed_model/node.py | 8 ++++++-
dts/framework/testbed_model/os_session.py | 4 +++-
8 files changed, 37 insertions(+), 22 deletions(-)
diff --git a/doc/guides/tools/dts.rst b/doc/guides/tools/dts.rst
index 47b218b2c6..1103db0faa 100644
--- a/doc/guides/tools/dts.rst
+++ b/doc/guides/tools/dts.rst
@@ -131,7 +131,12 @@ There are two areas that need to be set up on a System
Under Test:
You may specify the optional hugepage configuration in the DTS config
file.
If you do, DTS will take care of configuring hugepages,
- overwriting your current SUT hugepage configuration.
+ overwriting your current SUT hugepage configuration. Configuration of
hugepages via DTS
+ allows only for configuration of 2MB hugepages. Thus, if your needs
require hugepage
+ sizes not equal to 2MB, then you these configurations must be done
outside of the DTS
+ framework; moreover, if you do not desire the use of 2MB hugepages, and
instead perfer
+ other sizes (e.g 1GB), then we assume that hugepages have been manually
configued before
+ runtime.
* System under test configuration
diff --git a/dts/conf.yaml b/dts/conf.yaml
index 8068345dd5..56c3ae6f4c 100644
--- a/dts/conf.yaml
+++ b/dts/conf.yaml
@@ -35,7 +35,7 @@ nodes:
lcores: "" # use all the available logical cores
use_first_core: false # tells DPDK to use any physical core
memory_channels: 4 # tells DPDK to use 4 memory channels
- hugepages: # optional; if removed, will use system hugepage configuration
+ hugepages_2mb: # optional; if removed, will use system hugepage
configuration
amount: 256
force_first_numa: false
ports:
@@ -71,7 +71,7 @@ nodes:
os_driver: rdma
peer_node: "SUT 1"
peer_pci: "0000:00:08.1"
- hugepages: # optional; if removed, will use system hugepage configuration
+ hugepages_2mb: # optional; if removed, will use system hugepage
configuration
amount: 256
force_first_numa: false
traffic_generator:
diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py
index 4cb5c74059..b6f820e39e 100644
--- a/dts/framework/config/__init__.py
+++ b/dts/framework/config/__init__.py
@@ -255,8 +255,8 @@ def from_dict(
Either an SUT or TG configuration instance.
"""
hugepage_config = None
- if "hugepages" in d:
- hugepage_config_dict = d["hugepages"]
+ if "hugepages_2mb" in d:
+ hugepage_config_dict = d["hugepages_2mb"]
if "force_first_numa" not in hugepage_config_dict:
hugepage_config_dict["force_first_numa"] = False
hugepage_config = HugepageConfiguration(**hugepage_config_dict)
diff --git a/dts/framework/config/conf_yaml_schema.json
b/dts/framework/config/conf_yaml_schema.json
index 4731f4511d..f4d7199523 100644
--- a/dts/framework/config/conf_yaml_schema.json
+++ b/dts/framework/config/conf_yaml_schema.json
@@ -146,7 +146,7 @@
"compiler"
]
},
- "hugepages": {
+ "hugepages_2mb": {
"type": "object",
"description": "Optional hugepage configuration. If not specified,
hugepages won't be configured and DTS will use system configuration.",
"properties": {
@@ -253,8 +253,8 @@
"type": "integer",
"description": "How many memory channels to use. Optional,
defaults to 1."
},
- "hugepages": {
- "$ref": "#/definitions/hugepages"
+ "hugepages_2mb": {
+ "$ref": "#/definitions/hugepages_2mb"
},
"ports": {
"type": "array",
diff --git a/dts/framework/config/types.py b/dts/framework/config/types.py
index 1927910d88..016e0c3dbd 100644
--- a/dts/framework/config/types.py
+++ b/dts/framework/config/types.py
@@ -46,7 +46,7 @@ class NodeConfigDict(TypedDict):
"""Allowed keys and values."""
#:
- hugepages: HugepageConfigurationDict
+ hugepages_2mb: HugepageConfigurationDict
#:
name: str
#:
diff --git a/dts/framework/testbed_model/linux_session.py
b/dts/framework/testbed_model/linux_session.py
index 5d24030c3d..d0f7cfa77c 100644
--- a/dts/framework/testbed_model/linux_session.py
+++ b/dts/framework/testbed_model/linux_session.py
@@ -15,7 +15,7 @@
from typing_extensions import NotRequired
-from framework.exception import RemoteCommandExecutionError
+from framework.exception import ConfigurationError, RemoteCommandExecutionError
from framework.utils import expand_range
from .cpu import LogicalCore
@@ -84,14 +84,20 @@ def get_dpdk_file_prefix(self, dpdk_prefix: str) -> str:
"""Overrides :meth:`~.os_session.OSSession.get_dpdk_file_prefix`."""
return dpdk_prefix
- def setup_hugepages(self, hugepage_count: int, force_first_numa: bool) ->
None:
+ def setup_hugepages(
+ self, hugepage_count: int, hugepage_size: int, force_first_numa: bool
+ ) -> None:
"""Overrides :meth:`~.os_session.OSSession.setup_hugepages`."""
self._logger.info("Getting Hugepage information.")
- hugepage_size = self._get_hugepage_size()
- hugepages_total = self._get_hugepages_total()
+ hugepages_total = self._get_hugepages_total(hugepage_size)
+ if (
+ f"hugepages-{hugepage_size}kB"
+ not in self.send_command("ls /sys/kernel/mm/hugepages").stdout
+ ):
+ raise ConfigurationError("hugepage size not supported by operating
system")
self._numa_nodes = self._get_numa_nodes()
- if force_first_numa or hugepages_total != hugepage_count:
+ if force_first_numa or hugepages_total < hugepage_count:
# when forcing numa, we need to clear existing hugepages regardless
# of size, so they can be moved to the first numa node
self._configure_huge_pages(hugepage_count, hugepage_size,
force_first_numa)
@@ -99,13 +105,9 @@ def setup_hugepages(self, hugepage_count: int,
force_first_numa: bool) -> None:
self._logger.info("Hugepages already configured.")
self._mount_huge_pages()
- def _get_hugepage_size(self) -> int:
- hugepage_size = self.send_command("awk '/Hugepagesize/ {print $2}'
/proc/meminfo").stdout
- return int(hugepage_size)
-
- def _get_hugepages_total(self) -> int:
+ def _get_hugepages_total(self, hugepage_size: int) -> int:
hugepages_total = self.send_command(
- "awk '/HugePages_Total/ { print $2 }' /proc/meminfo"
+ f"cat
/sys/kernel/mm/hugepages/hugepages-{hugepage_size}kB/nr_hugepages"
).stdout
return int(hugepages_total)
diff --git a/dts/framework/testbed_model/node.py
b/dts/framework/testbed_model/node.py
index 74061f6262..056a031ca0 100644
--- a/dts/framework/testbed_model/node.py
+++ b/dts/framework/testbed_model/node.py
@@ -97,6 +97,10 @@ def __init__(self, node_config: NodeConfiguration):
self.virtual_devices = []
self._init_ports()
+ @property
+ def _hugepage_default_size(self) -> int:
+ return 2048
+
def _init_ports(self) -> None:
self.ports = [Port(self.name, port_config) for port_config in
self.config.ports]
self.main_session.update_ports(self.ports)
@@ -266,7 +270,9 @@ def _setup_hugepages(self) -> None:
"""
if self.config.hugepages:
self.main_session.setup_hugepages(
- self.config.hugepages.amount,
self.config.hugepages.force_first_numa
+ self.config.hugepages.amount,
+ self._hugepage_default_size,
+ self.config.hugepages.force_first_numa,
)
def configure_port_state(self, port: Port, enable: bool = True) -> None:
diff --git a/dts/framework/testbed_model/os_session.py
b/dts/framework/testbed_model/os_session.py
index d5bf7e0401..69f9483f05 100644
--- a/dts/framework/testbed_model/os_session.py
+++ b/dts/framework/testbed_model/os_session.py
@@ -345,7 +345,9 @@ def get_dpdk_file_prefix(self, dpdk_prefix: str) -> str:
"""
@abstractmethod
- def setup_hugepages(self, hugepage_count: int, force_first_numa: bool) ->
None:
+ def setup_hugepages(
+ self, hugepage_count: int, hugepage_size: int, force_first_numa: bool
+ ) -> None:
"""Configure hugepages on the node.
Get the node's Hugepage Size, configure the specified count of
hugepages
--
2.44.0
