commit: d2ffee09c8470e1e84b5b07442536757f17130f5
Author: Kerin Millar <kfm <AT> plushkava <DOT> net>
AuthorDate: Tue Jun 24 10:45:07 2025 +0000
Commit: Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Sat Jun 28 02:28:40 2025 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=d2ffee09
isolated-functions.sh: introduce ___parallel() to replace ___parallel_xargs()
Presently, the "isolated-functions.sh" unit provides the
___parallel_xargs() function, whose purpose is to dispatch a simple
command multiple times in parallel, with each invocation taking one
additional parameter. However, in order to be able to issue commands in
parallel, it requires for the xargs(1) utility to support the wholly
non-standard --max-procs option, which only the GNU implementation
provides. This is in spite of the fact that the -P option is commonly
implemented by the current releases of popular unix-like operating
systems, though that, too, is a non-standard option.
This commit introduces a new function, named ___parallel(). It acts in a
way that is very similar to GNU xargs(1), as accompanied by its -0, -L
and -P options. Yet it requires no particular implementation thereof.
Indeed, it does not use xargs(1) at all. Rather, it uses the native
process management features provided by bash. In particular, it takes
advantage of the -n option of the wait builtin, which was introduced by
the release of bash-4.4.
As the following sample commands demonstrate, it is trivial to port any
existing call sites.
# BEFORE
find . -type f -name '*.png' -print0 | ___parallel_xargs -0 pngfix
# AFTER
find . -type f -name '*.png' -print0 | ___parallel pngfix
Indeed, it transpires that ___parallel_xargs() is used only by the
following two units, both of which are ported by this commit.
- bin/ecompress
- bin/install-qa-check.d/60pngfix
Signed-off-by: Kerin Millar <kfm <AT> plushkava.net>
Signed-off-by: Sam James <sam <AT> gentoo.org>
bin/ecompress | 2 +-
bin/install-qa-check.d/60pngfix | 2 +-
bin/isolated-functions.sh | 49 +++++++++++++++++++++++++++++++++++++++++
3 files changed, 51 insertions(+), 2 deletions(-)
diff --git a/bin/ecompress b/bin/ecompress
index 2500bd5a2c..a59e699f07 100755
--- a/bin/ecompress
+++ b/bin/ecompress
@@ -248,7 +248,7 @@ export PORTAGE_COMPRESS_SUFFIX PORTAGE_COMPRESS_FLAGS
PORTAGE_COMPRESS
printf '%s\0' "${ED}" \
| find0 -name '*.ecompress' ! -path $'*\n*' -delete -print0 \
-| ___parallel_xargs -0 "${PORTAGE_BIN_PATH}"/ecompress-file
+| ___parallel "${PORTAGE_BIN_PATH}"/ecompress-file
all_compressed=$(( $? == 0 ))
if [[ -s ${T}/.ecompress_had_precompressed ]]; then
diff --git a/bin/install-qa-check.d/60pngfix b/bin/install-qa-check.d/60pngfix
index 0300ed2ac1..d00810e594 100644
--- a/bin/install-qa-check.d/60pngfix
+++ b/bin/install-qa-check.d/60pngfix
@@ -28,7 +28,7 @@ pngfix_check() {
fi
eqawarn " ${pngout[@]:7}: ${error}"
fi
- done < <(find "${ED}" -type f -name '*.png' -print0 |
___parallel_xargs -0 "${pngfix}")
+ done < <(find "${ED}" -type f -name '*.png' -print0 |
___parallel "${pngfix}")
fi
}
diff --git a/bin/isolated-functions.sh b/bin/isolated-functions.sh
index 88f1913fdf..d0f7d4d09b 100644
--- a/bin/isolated-functions.sh
+++ b/bin/isolated-functions.sh
@@ -509,6 +509,55 @@ ___makeopts_jobs() {
printf '%s\n' "${jobs}"
}
+# Considers the positional parameters as comprising a simple command, which
+# shall be executed for each null-terminated record read from the standard
+# input. For each record processed, its value shall be taken as an additional
+# parameter to append to the command. Commands shall be executed in parallel,
+# with the maximal degree of concurrency being determined by the output of the
+# ___makeopts_jobs function. Thus, the behaviour is quite similar to that of
+# xargs -0 -L1 -P"$(___makeopts_jobs)".
+#
+# If no records are read, or if all commands complete successfully, the return
+# value shall be 0. Otherwise, the return value shall be that of the failed
+# command that was last reaped by bash. Should any command fail, no further
+# records shall be consumed and the function shall attempt to return as soon as
+# possible. Hence, the caller should assume that not all records were processed
+# in the event of a non-zero return value. As a special case, the function
shall
+# return 127 if the first parameter cannot be resolved as a valid command name.
+___parallel() (
+ local max_procs retval arg i
+
+ if ! hash "$1" 2>/dev/null; then
+ printf >&2 '%s: %q: command not found\n' "$0" "$1"
+ return 127
+ fi
+
+ max_procs=$(___makeopts_jobs)
+ retval=0
+
+ while IFS= read -rd '' arg; do
+ if (( ++i > max_procs )); then
+ wait -n
+ case $? in
+ 0) ;;
+ *) retval=$?; break
+ esac
+ fi
+ "$@" "${arg}" &
+ done
+
+ while true; do
+ wait -n
+ case $? in
+ 127) break ;; # no more unwaited-for children left
+ 0) ;;
+ *) retval=$?
+ esac
+ done
+
+ return "${retval}"
+)
+
# Run ${XARGS} in parallel for detected number of CPUs, if supported.
# Passes all arguments to xargs, and returns its exit code
___parallel_xargs() {
