There have been errors seen when assembling root file system SPDX
documents where they will references files that don't exist in the
package SPDX.
The speculation is that this is caused by os.walk() ignoring errors when
walking, causing files to be omitted. Improve the code by adding an
error handler to os.walk() to report errors when they occur.
In addition, sort the files and directories while walking to ensure
consistent ordering of the file SPDX IDs.
Signed-off-by: Joshua Watt <jpewhac...@gmail.com>
---
meta/lib/oe/spdx30_tasks.py | 20 ++++++++++++++++----
1 file changed, 16 insertions(+), 4 deletions(-)
diff --git a/meta/lib/oe/spdx30_tasks.py b/meta/lib/oe/spdx30_tasks.py
index 6a39246fe1c..e3e5dbc7427 100644
--- a/meta/lib/oe/spdx30_tasks.py
+++ b/meta/lib/oe/spdx30_tasks.py
@@ -19,6 +19,10 @@ from datetime import datetime, timezone
from pathlib import Path
+def walk_error(err):
+ bb.error(f"ERROR walking {err.filename}: {err}")
+
+
def set_timestamp_now(d, o, prop):
if d.getVar("SPDX_INCLUDE_TIMESTAMPS") == "1":
setattr(o, prop, datetime.now(timezone.utc))
@@ -148,11 +152,13 @@ def add_package_files(
spdx_files = set()
file_counter = 1
- for subdir, dirs, files in os.walk(topdir):
+ for subdir, dirs, files in os.walk(topdir, onerror=walk_error):
dirs[:] = [d for d in dirs if d not in ignore_dirs]
if subdir == str(topdir):
dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs]
+ dirs.sort()
+ files.sort()
for file in files:
filepath = Path(subdir) / file
if filepath.is_symlink() or not filepath.is_file():
@@ -356,7 +362,9 @@ def add_download_files(d, objset):
if fd.type == "file":
if os.path.isdir(fd.localpath):
walk_idx = 1
- for root, dirs, files in os.walk(fd.localpath):
+ for root, dirs, files in os.walk(fd.localpath,
onerror=walk_error):
+ dirs.sort()
+ files.sort()
for f in files:
f_path = os.path.join(root, f)
if os.path.islink(f_path):
@@ -1046,7 +1054,9 @@ def create_rootfs_spdx(d):
collect_build_package_inputs(d, objset, rootfs_build, packages,
files_by_hash)
files = set()
- for dirpath, dirnames, filenames in os.walk(image_rootfs):
+ for dirpath, dirnames, filenames in os.walk(image_rootfs,
onerror=walk_error):
+ dirnames.sort()
+ filenames.sort()
for fn in filenames:
fpath = Path(dirpath) / fn
if not fpath.is_file() or fpath.is_symlink():
@@ -1282,7 +1292,9 @@ def create_sdk_sbom(d, sdk_deploydir, spdx_work_dir,
toolchain_outputname):
root_files = []
# NOTE: os.walk() doesn't return symlinks
- for dirpath, dirnames, filenames in os.walk(sdk_deploydir):
+ for dirpath, dirnames, filenames in os.walk(sdk_deploydir,
onerror=walk_error):
+ dirnames.sort()
+ filenames.sort()
for fn in filenames:
fpath = Path(dirpath) / fn
if not fpath.is_file() or fpath.is_symlink():
--
2.47.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#211361):
https://lists.openembedded.org/g/openembedded-core/message/211361
Mute This Topic: https://lists.openembedded.org/mt/111166555/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
