Re: [gentoo-dev] [PATCH] cargo.eclass: Optimize crate unpacking

[Florian Schmaus]

On 12/05/2024 04.26, Michał Górny wrote:

Unpack crates in parallel using xargs to utilize multicore systems
better.  Perform checksumming via a single sha256sum invocation.

For dev-python/watchfiles, this speeds up unpacking on my machine
from 2.6 s to 0.75 s (warm cache).

Signed-off-by: Michał Górny <mgo...@gentoo.org>
---
  eclass/cargo.eclass | 56 ++++++++++++++++++++++++++-------------------
  1 file changed, 33 insertions(+), 23 deletions(-)

diff --git a/eclass/cargo.eclass b/eclass/cargo.eclass
index 0f2da982f60c..5a16d3a30528 100644
--- a/eclass/cargo.eclass
+++ b/eclass/cargo.eclass
@@ -329,40 +329,50 @@ _cargo_gen_git_config() {
  cargo_src_unpack() {
        debug-print-function ${FUNCNAME} "$@"

-       mkdir -p "${S}" || die
+       mkdir -p "${ECARGO_VENDOR}" "${S}" || die

+       local crates=()
        for archive in ${A}; do
                case "${archive}" in
                        *.crate)
-                               # when called by pkgdiff-mg, do not unpack 
crates
-                               [[ ${PKGBUMPING} == ${PVR} ]] && continue
-
-                               ebegin "Loading ${archive} into Cargo registry"
-                               tar -xf "${DISTDIR}"/${archive} -C 
"${ECARGO_VENDOR}/" || die
-                               # generate sha256sum of the crate itself as 
cargo needs this
-                               shasum=$(sha256sum "${DISTDIR}"/${archive} | 
cut -d ' ' -f 1)
-                               pkg=$(basename ${archive} .crate)
-                               cat <<- EOF > 
${ECARGO_VENDOR}/${pkg}/.cargo-checksum.json
-                               {
-                                       "package": "${shasum}",
-                                       "files": {}
-                               }
-                               EOF
-                               # if this is our target package we need it in 
${WORKDIR} too
-                               # to make ${S} (and handle any revisions too)
-                               if [[ ${P} == ${pkg}* ]]; then
-                                       tar -xf "${DISTDIR}"/${archive} -C 
"${WORKDIR}" || die
-                               fi
-                               eend $?
+                               crates+=( "${archive}" )
                                ;;
                        *)
-                               unpack ${archive}
+                               unpack "${archive}"
                                ;;
                esac
        done

+               pushd "${DISTDIR}" >/dev/null || die
+
+               ebegin "Unpacking crates"
+               printf '%s\0' "${crates[@]}" |
+                       xargs -0 -P "$(makeopts_jobs)" -n 1 -- \

- mkdir -p "${ECARGO_VENDOR}" || die local archive shasum pkg + if [[ ${PKGBUMPING} != ${PVR} ]]; then

Consider using get_makeopts_jobs instead of makeopts_jobs, as it searches more variables for --jobs. N.B.: If this where asking for a load-average limit, then using get_makeopts_loadavg would be the ideal way to pick up portage's default wrt --load-average. Therefore we should IMHO encourage using the get_makeopts_* functions over the (legacy?) non-get_ variants.

+                               tar -x -C "${ECARGO_VENDOR}" -f
+               assert
+               eend $?
+
+               while read -d '' -r shasum archive; do
+                       pkg=${archive%.crate}
+                       cat <<- EOF > 
${ECARGO_VENDOR}/${pkg}/.cargo-checksum.json || die
+                       {
+                               "package": "${shasum}",
+                               "files": {}
+                       }
+                       EOF
+
+                       # if this is our target package we need it in 
${WORKDIR} too
+                       # to make ${S} (and handle any revisions too)
+                       if [[ ${P} == ${pkg}* ]]; then
+                               tar -xf "${archive}" -C "${WORKDIR}" || die
+                       fi
+               done < <(sha256sum -z "${crates[@]}" || die)
+
+               popd >/dev/null || die
+       fi
+
        cargo_gen_config
  }

OpenPGP_0x8CAC2A9678548E35.asc
Description: OpenPGP public key

OpenPGP_signature.asc
Description: OpenPGP digital signature