Convert the copydebugsources command pipelines to explicit Popen calls using argument lists. Use env= for LC_ALL, cwd= for the cpio working directory and glob.glob() for the externalsrc move.
The first pipeline keeps ignoring command failures as before since some inputs are expected to fail. The symlink fixup pipeline checks each stage so failures are reported directly. Skip the externalsrc mv when the glob has no matches and let the following empty-directory cleanup handle the empty tree. Signed-off-by: Anders Heimer <[email protected]> --- meta/lib/oe/package.py | 63 ++++++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py index c375acc124..ad4b7a2769 100644 --- a/meta/lib/oe/package.py +++ b/meta/lib/oe/package.py @@ -1017,26 +1017,50 @@ def copydebugsources(debugsrcdir, sources, d): bb.utils.mkdirhier(basepath) cpath.updatecache(basepath) - for pmap in prefixmap: + env = os.environ.copy() + env["LC_ALL"] = "C" + + for pmap, prefix in prefixmap.items(): + dstroot = dvar + prefix # Ignore files from the recipe sysroots (target and native) - cmd = "LC_ALL=C ; sort -z -u '%s' | egrep -v -z '((<internal>|<built-in>)$|/.*recipe-sysroot.*/)' | " % sourcefile + sort_p = subprocess.Popen(["sort", "-z", "-u", "--", sourcefile], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env) + egrep_p = subprocess.Popen(["egrep", "-v", "-z", "-e", r"((<internal>|<built-in>)$|/.*recipe-sysroot.*/)"], stdin=sort_p.stdout, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env) + sort_p.stdout.close() + # We need to ignore files that are not actually ours # we do this by only paying attention to items from this package - cmd += "fgrep -zw '%s' | " % prefixmap[pmap] + fgrep_p = subprocess.Popen(["fgrep", "-zw", "-e", prefix], stdin=egrep_p.stdout, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env) + egrep_p.stdout.close() + # Remove prefix in the source paths - cmd += "sed 's#%s/##g' | " % (prefixmap[pmap]) - cmd += "(cd '%s' ; cpio -pd0mlLu --no-preserve-owner '%s%s' 2>/dev/null)" % (pmap, dvar, prefixmap[pmap]) + sed_p = subprocess.Popen(["sed", "s#%s/##g" % prefix], stdin=fgrep_p.stdout, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env) + fgrep_p.stdout.close() + + cpio_p = subprocess.Popen(["cpio", "-pd0mlLu", "--no-preserve-owner", dstroot], stdin=sed_p.stdout, cwd=pmap, stderr=subprocess.DEVNULL, env=env) + sed_p.stdout.close() + + for proc in (cpio_p, sed_p, fgrep_p, egrep_p, sort_p): + proc.wait() - try: - subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError: - # Can "fail" if internal headers/transient sources are attempted - pass # cpio seems to have a bug with -lL together and symbolic links are just copied, not dereferenced. # Work around this by manually finding and copying any symbolic links that made it through. - cmd = "find %s%s -type l -print0 -delete | sed s#%s%s/##g | (cd '%s' ; cpio -pd0mL --no-preserve-owner '%s%s')" % \ - (dvar, prefixmap[pmap], dvar, prefixmap[pmap], pmap, dvar, prefixmap[pmap]) - subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + # The source copy pipeline above can fail without aborting, so there may be no copied tree to scan for symlinks. + if not os.path.exists(dstroot): + continue + + find_p = subprocess.Popen(["find", dstroot, "-type", "l", "-print0", "-delete"], stdout=subprocess.PIPE) + sed_p = subprocess.Popen(["sed", "s#%s/##g" % dstroot], stdin=find_p.stdout, stdout=subprocess.PIPE) + find_p.stdout.close() + + cpio_p = subprocess.Popen(["cpio", "-pd0mL", "--no-preserve-owner", dstroot], stdin=sed_p.stdout, stderr=subprocess.DEVNULL, cwd=pmap) + sed_p.stdout.close() + + procs = (cpio_p, sed_p, find_p) + for proc in procs: + proc.wait() + for proc in procs: + if proc.returncode: + raise subprocess.CalledProcessError(proc.returncode, proc.args) # debugsources.list may be polluted from the host if we used externalsrc, # cpio uses copy-pass and may have just created a directory structure @@ -1046,13 +1070,16 @@ def copydebugsources(debugsrcdir, sources, d): # Same check as above for externalsrc if workdir not in sdir: - if os.path.exists(dvar + debugsrcdir + sdir): - cmd = "mv %s%s%s/* %s%s" % (dvar, debugsrcdir, sdir, dvar,debugsrcdir) - subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + srcdir = dvar + debugsrcdir + sdir + dstdir = dvar + debugsrcdir + if os.path.exists(srcdir): + entries = glob.glob(os.path.join(glob.escape(srcdir), "*")) + if entries: + subprocess.check_output(["mv", "--"] + entries + [dstdir], stderr=subprocess.STDOUT) # The copy by cpio may have resulted in some empty directories! Remove these - cmd = "find %s%s -empty -type d -delete" % (dvar, debugsrcdir) - subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + cmd = ["find", dvar + debugsrcdir, "-empty", "-type", "d", "-delete"] + subprocess.check_output(cmd, stderr=subprocess.STDOUT) # Also remove debugsrcdir if its empty for p in nosuchdir[::-1]:
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#238878): https://lists.openembedded.org/g/openembedded-core/message/238878 Mute This Topic: https://lists.openembedded.org/mt/119830169/21656 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
