The Linux pressure monitoring system helps determine when system resources
are being overutilized by measuring how contended the CPU, IO and memory are.
This information can be found under /proc/pressure/ which contains 3 files -
cpu, memory and io. In each of the files, the format of the files is as follows:
some avg10=70.24 avg60=68.52 avg300=69.91 total=3559632828
full avg10=57.59 avg60=58.06 avg300=60.38 total=3300487258
The "some" state of a given resource represents when one or more tasks are
delayed
on that resource whereas the "full" state represents when all the tasks are
delayed. Currently, we only collect data from the "some" state but the
"full" data can simply be appended to the log files if neccessary.
The "avg10", "avg60" and "avg300" fields represent the average percentage
of time runnable tasks were delayed in the last 10, 60 or 300 seconds
respectively. The "total" field represents the total time, in microseconds,
that some runnable task was delayed on a resource.
More information can be found at:
https://www.kernel.org/doc/html/latest/accounting/psi.html
and in the source code under /kernel/sched/psi.c
This commit adds functionality to collect and log the "some" CPU, memory and IO
pressure. The "avg10", "avg60" and "avg300" fields are logged without change.
In place of the "total" field, the difference between the current "total" and
the previous sample's "total" is logged, allowing the measurement of pressure
in between each polling interval, as was done for /proc/stat data. The log files
are stored in
<build_name>/tmp/buildstats/<build_time>/reduced_proc_pressure/{cpu,io,memory},
mirroring the directory structure of /proc/pressure.
Signed-off-by: Aryaman Gupta <aryaman.gu...@windriver.com>
Signed-off-by: Randy MacLeod <randy.macl...@windriver.com>
---
meta/lib/buildstats.py | 42 ++++++++++++++++++++++++++++++++++++------
1 file changed, 36 insertions(+), 6 deletions(-)
diff --git a/meta/lib/buildstats.py b/meta/lib/buildstats.py
index c52b6c3b72..1875bac3c1 100644
--- a/meta/lib/buildstats.py
+++ b/meta/lib/buildstats.py
@@ -13,13 +13,20 @@ class SystemStats:
def __init__(self, d):
bn = d.getVar('BUILDNAME')
bsdir = os.path.join(d.getVar('BUILDSTATS_BASE'), bn)
- bb.utils.mkdirhier(bsdir)
+ pressuredir = os.path.join(bsdir, 'reduced_proc_pressure')
+ if os.path.exists("/proc/pressure/cpu"):
+ bb.utils.mkdirhier(pressuredir)
+ else:
+ bb.utils.mkdirhier(bsdir)
self.proc_files = []
for filename, handler in (
('diskstats', self._reduce_diskstats),
('meminfo', self._reduce_meminfo),
('stat', self._reduce_stat),
+ ('pressure/cpu', self._reduce_pressure),
+ ('pressure/io', self._reduce_pressure),
+ ('pressure/memory', self._reduce_pressure),
):
# The corresponding /proc files might not exist on the host.
# For example, /proc/diskstats is not available in virtualized
@@ -48,13 +55,15 @@ class SystemStats:
self.diskstats_ltime = None
self.diskstats_data = None
self.stat_ltimes = None
+ #Last time we sampled /proc/pressure. All resources stored in a single
dict with the key as filename
+ self.last_pressure = {"pressure/cpu": None, "pressure/io": None,
"pressure/memory": None}
def close(self):
self.monitor_disk.close()
for _, output, _ in self.proc_files:
output.close()
- def _reduce_meminfo(self, time, data):
+ def _reduce_meminfo(self, time, data, filename):
"""
Extracts 'MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal',
'SwapFree'
and writes their values into a single line, in that order.
@@ -75,7 +84,7 @@ class SystemStats:
disk = linetokens[2]
return self.diskstats_regex.match(disk)
- def _reduce_diskstats(self, time, data):
+ def _reduce_diskstats(self, time, data, filename):
relevant_tokens = filter(self._diskstats_is_relevant_line, map(lambda
x: x.split(), data.split(b'\n')))
diskdata = [0] * 3
reduced = None
@@ -104,10 +113,10 @@ class SystemStats:
return reduced
- def _reduce_nop(self, time, data):
+ def _reduce_nop(self, time, data, filename):
return (time, data)
- def _reduce_stat(self, time, data):
+ def _reduce_stat(self, time, data, filename):
if not data:
return None
# CPU times {user, nice, system, idle, io_wait, irq, softirq} from
first line
@@ -125,6 +134,27 @@ class SystemStats:
self.stat_ltimes = times
return reduced
+
+ def _reduce_pressure(self, time, data, filename):
+ """
+ Return reduced pressure: {avg10, avg60, avg300} and delta total
compared to the previous sample
+ for the cpu, io and memory resources. A common function is used for
all 3 resources since the
+ format of the /proc/pressure file is the same in each case.
+ """
+ if not data:
+ return None
+ tokens = data.split(b'\n', 1)[0].split()
+ avg10 = float(tokens[1].split(b'=')[1])
+ avg60 = float(tokens[2].split(b'=')[1])
+ avg300 = float(tokens[3].split(b'=')[1])
+ total = int(tokens[4].split(b'=')[1])
+
+ reduced = None
+ if self.last_pressure[filename]:
+ delta = total - self.last_pressure[filename]
+ reduced = (time, (avg10, avg60, avg300, delta))
+ self.last_pressure[filename] = total
+ return reduced
def sample(self, event, force):
now = time.time()
@@ -133,7 +163,7 @@ class SystemStats:
with open(os.path.join('/proc', filename), 'rb') as input:
data = input.read()
if handler:
- reduced = handler(now, data)
+ reduced = handler(now, data, filename)
else:
reduced = (now, data)
if reduced:
--
2.35.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#167009):
https://lists.openembedded.org/g/openembedded-core/message/167009
Mute This Topic: https://lists.openembedded.org/mt/91806994/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
