From: Stefan Herbrechtsmeier <stefan.herbrechtsme...@weidmueller.com>
Rework the npm class to use plain npm commands and remove the usage of
the npm cache to speed-up builds.
Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsme...@weidmueller.com>
---
meta/classes/npm.bbclass | 340 +++++++++++---------------------
scripts/lib/devtool/standard.py | 6 +-
2 files changed, 119 insertions(+), 227 deletions(-)
diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass
index ba50fcac20..cb22dc6998 100644
--- a/meta/classes/npm.bbclass
+++ b/meta/classes/npm.bbclass
@@ -1,7 +1,12 @@
# Copyright (C) 2020 Savoir-Faire Linux
#
+# Copyright (C) 2021 Weidmüller Interface GmbH & Co. KG
+# Author: Stefan Herbrechtsmeier <stefan.herbrechtsme...@weidmueller.com>
+#
# SPDX-License-Identifier: GPL-2.0-only
#
+# A bbclass to install an npm package and build its dependencies.
+#
# This bbclass builds and installs an npm package to the target. The package
# sources files should be fetched in the calling recipe by using the SRC_URI
# variable. The ${S} variable should be updated depending of your fetcher.
@@ -16,18 +21,29 @@
#
# NPM_INSTALL_DEV:
# Set to 1 to also install devDependencies.
+#
+# NPM_BUILD_PRUNE_FOR_PRODUCTION
+# Set to 0 to keep installed devDependencies.
inherit python3native
DEPENDS:prepend = "nodejs-native "
-RDEPENDS:${PN}:append:class-target = " nodejs"
+NPM_RDEPENDS = "nodejs"
+RDEPENDS:${PN}:append:class-target = " ${NPM_RDEPENDS}"
-EXTRA_OENPM = ""
+EXTRA_OENPM ?= ""
+EXTRA_OENPM_BUILD ?= ""
NPM_INSTALL_DEV ?= "0"
NPM_NODEDIR ?= "${RECIPE_SYSROOT_NATIVE}${prefix_native}"
+def npm_flag_dev(value):
+ if bb.utils.to_boolean(value, False):
+ return "--also=development"
+ else:
+ return "--only=production"
+
def npm_target_arch_map(target_arch):
"""Maps arch names to npm arch names"""
import re
@@ -43,247 +59,124 @@ def npm_target_arch_map(target_arch):
NPM_ARCH ?= "${@npm_target_arch_map(d.getVar("TARGET_ARCH"))}"
-NPM_PACKAGE = "${WORKDIR}/npm-package"
NPM_CACHE = "${WORKDIR}/npm-cache"
-NPM_BUILD = "${WORKDIR}/npm-build"
-
-def npm_global_configs(d):
- """Get the npm global configuration"""
- configs = []
- # Ensure no network access is done
- configs.append(("offline", "true"))
- configs.append(("proxy", "http://invalid";))
- # Configure the cache directory
- configs.append(("cache", d.getVar("NPM_CACHE")))
- return configs
-
-def npm_pack(env, srcdir, workdir):
- """Run 'npm pack' on a specified directory"""
- import shlex
- cmd = "npm pack %s" % shlex.quote(srcdir)
- args = [("ignore-scripts", "true")]
- tarball = env.run(cmd, args=args, workdir=workdir).strip("\n")
- return os.path.join(workdir, tarball)
-
-python npm_do_configure() {
- """
- Step one: configure the npm cache and the main npm package
-
- Every dependencies have been fetched and patched in the source directory.
- They have to be packed (this remove unneeded files) and added to the npm
- cache to be available for the next step.
-
- The main package and its associated manifest file and shrinkwrap file have
- to be configured to take into account these cached dependencies.
- """
- import base64
- import copy
- import json
- import re
- import shlex
- import tempfile
- from bb.fetch2.npm import NpmEnvironment
- from bb.fetch2.npm import npm_unpack
- from bb.fetch2.npmsw import foreach_dependencies
- from bb.progress import OutOfProgressHandler
-
- bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True)
- bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True)
-
- env = NpmEnvironment(d, configs=npm_global_configs(d))
-
- def _npm_cache_add(tarball):
- """Run 'npm cache add' for a specified tarball"""
- cmd = "npm cache add %s" % shlex.quote(tarball)
- env.run(cmd)
-
- def _npm_integrity(tarball):
- """Return the npm integrity of a specified tarball"""
- sha512 = bb.utils.sha512_file(tarball)
- return "sha512-" + base64.b64encode(bytes.fromhex(sha512)).decode()
-
- def _npm_version(tarball):
- """Return the version of a specified tarball"""
- regex = r"-(\d+\.\d+\.\d+(-.*)?(\+.*)?)\.tgz"
- return re.search(regex, tarball).group(1)
-
- def _npmsw_dependency_dict(orig, deptree):
- """
- Return the sub dictionary in the 'orig' dictionary corresponding to the
- 'deptree' dependency tree. This function follows the shrinkwrap file
- format.
- """
- ptr = orig
- for dep in deptree:
- if "dependencies" not in ptr:
- ptr["dependencies"] = {}
- ptr = ptr["dependencies"]
- if dep not in ptr:
- ptr[dep] = {}
- ptr = ptr[dep]
- return ptr
-
- # Manage the manifest file and shrinkwrap files
- orig_manifest_file = d.expand("${S}/package.json")
- orig_shrinkwrap_file = d.expand("${S}/npm-shrinkwrap.json")
- cached_manifest_file = d.expand("${NPM_PACKAGE}/package.json")
- cached_shrinkwrap_file = d.expand("${NPM_PACKAGE}/npm-shrinkwrap.json")
-
- with open(orig_manifest_file, "r") as f:
- orig_manifest = json.load(f)
-
- cached_manifest = copy.deepcopy(orig_manifest)
- cached_manifest.pop("dependencies", None)
- cached_manifest.pop("devDependencies", None)
-
- has_shrinkwrap_file = True
-
- try:
- with open(orig_shrinkwrap_file, "r") as f:
- orig_shrinkwrap = json.load(f)
- except IOError:
- has_shrinkwrap_file = False
-
- if has_shrinkwrap_file:
- cached_shrinkwrap = copy.deepcopy(orig_shrinkwrap)
- cached_shrinkwrap.pop("dependencies", None)
-
- # Manage the dependencies
- progress = OutOfProgressHandler(d, r"^(\d+)/(\d+)$")
- progress_total = 1 # also count the main package
- progress_done = 0
-
- def _count_dependency(name, params, deptree):
- nonlocal progress_total
- progress_total += 1
-
- def _cache_dependency(name, params, deptree):
- destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
- destsuffix = os.path.join(*destsubdirs)
- with tempfile.TemporaryDirectory() as tmpdir:
- # Add the dependency to the npm cache
- destdir = os.path.join(d.getVar("S"), destsuffix)
- tarball = npm_pack(env, destdir, tmpdir)
- _npm_cache_add(tarball)
- # Add its signature to the cached shrinkwrap
- dep = _npmsw_dependency_dict(cached_shrinkwrap, deptree)
- dep["version"] = _npm_version(tarball)
- dep["integrity"] = _npm_integrity(tarball)
- if params.get("dev", False):
- dep["dev"] = True
- # Display progress
- nonlocal progress_done
- progress_done += 1
- progress.write("%d/%d" % (progress_done, progress_total))
-
- dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
-
- if has_shrinkwrap_file:
- foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
- foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
-
- # Configure the main package
- with tempfile.TemporaryDirectory() as tmpdir:
- tarball = npm_pack(env, d.getVar("S"), tmpdir)
- npm_unpack(tarball, d.getVar("NPM_PACKAGE"), d)
-
- # Configure the cached manifest file and cached shrinkwrap file
- def _update_manifest(depkey):
- for name in orig_manifest.get(depkey, {}):
- version = cached_shrinkwrap["dependencies"][name]["version"]
- if depkey not in cached_manifest:
- cached_manifest[depkey] = {}
- cached_manifest[depkey][name] = version
-
- if has_shrinkwrap_file:
- _update_manifest("dependencies")
-
- if dev:
- if has_shrinkwrap_file:
- _update_manifest("devDependencies")
-
- with open(cached_manifest_file, "w") as f:
- json.dump(cached_manifest, f, indent=2)
-
- if has_shrinkwrap_file:
- with open(cached_shrinkwrap_file, "w") as f:
- json.dump(cached_shrinkwrap, f, indent=2)
-}
-
-python npm_do_compile() {
- """
- Step two: install the npm package
- Use the configured main package and the cached dependencies to run the
- installation process. The installation is done in a directory which is
- not the destination directory yet.
+NPM_PRUNE_FOR_PRODUCTION ?= "1"
- A combination of 'npm pack' and 'npm install' is used to ensure that the
- installed files are actual copies instead of symbolic links (which is the
- default npm behavior).
- """
- import shlex
- import tempfile
- from bb.fetch2.npm import NpmEnvironment
+B = "${WORKDIR}/build"
- bb.utils.remove(d.getVar("NPM_BUILD"), recurse=True)
+NPM_SOURCEPATH ?= "${S}"
- with tempfile.TemporaryDirectory() as tmpdir:
- args = []
- configs = npm_global_configs(d)
+export NPM_CONFIG_GLOBALCONFIG = "${WORKDIR}/npmrc"
+export NPM_CONFIG_USERCONFIG = "/dev/null"
- if bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False):
- configs.append(("also", "development"))
- else:
- configs.append(("only", "production"))
-
- # Report as many logs as possible for debugging purpose
- configs.append(("loglevel", "silly"))
+oe_runnpm_plain() {
+ HOME=${B}/root npm "$@"
+}
- # Configure the installation to be done globally in the build directory
- configs.append(("global", "true"))
- configs.append(("prefix", d.getVar("NPM_BUILD")))
+oe_runnpm() {
+ bbnote npm "$@"
+ oe_runnpm_plain "$@"
+}
- # Add node-gyp configuration
- configs.append(("arch", d.getVar("NPM_ARCH")))
- configs.append(("release", "true"))
- configs.append(("nodedir", d.getVar("NPM_NODEDIR")))
- configs.append(("python", d.getVar("PYTHON")))
+oe_runnpm_pack() {
+ oe_runnpm_plain pack --ignore-scripts --loglevel=warn "$@"
+}
- env = NpmEnvironment(d, configs)
+oe_runnpm_rebuild() {
+ arch=$1
+ shift
- # Add node-pre-gyp configuration
- args.append(("target_arch", d.getVar("NPM_ARCH")))
- args.append(("build-from-source", "true"))
+ # Create symlinks for package executables
+ # because rebuild doesn't respect the dependency tree
+ oe_runnpm rebuild --ignore-scripts
- # Pack and install the main package
- tarball = npm_pack(env, d.getVar("NPM_PACKAGE"), tmpdir)
- cmd = "npm install %s %s" % (shlex.quote(tarball),
d.getVar("EXTRA_OENPM"))
- env.run(cmd, args=args)
+ # Rebuild dependencies
+ oe_runnpm rebuild \
+ --arch=${arch} \
+ --target_arch=${arch} \
+ --build-from-source \
+ "$@"
}
-npm_do_install() {
- # Step three: final install
- #
- # The previous installation have to be filtered to remove some extra files.
+do_generate_npmrc() {
+ cat > ${NPM_CONFIG_GLOBALCONFIG} <<EOF
+cache=${NPM_CACHE}
+loglevel=silly
+offline=true
+package-lock=false
+proxy=http://invalid
+; node-gyp
+arch=${NPM_ARCH}
+nodedir=${NPM_NODEDIR}
+python=${PYTHON}
+release=true
+EOF
+}
+do_generate_npmrc[vardeps] += "NPM_ARCH NPM_CACHE NPM_CONFIG_GLOBALCONFIG
NPM_NODEDIR PYTHON"
+addtask generate_npmrc after do_patch before do_populate_cache
- rm -rf ${D}
+npm_do_configure() {
+ :
+}
- # Copy the entire lib and bin directories
- install -d ${D}/${nonarch_libdir}
- cp --no-preserve=ownership --recursive ${NPM_BUILD}/lib/.
${D}/${nonarch_libdir}
+npm_do_compile() {
+ # Create a tarball from main npm package
+ # to respect ignore files and files property of package.json
+ cd ${B}/pkg
+ tarball=$(oe_runnpm_pack ${NPM_SOURCEPATH})
+ [ -n "${tarball}" ] || exit 1
+
+ # Unpack main npm package
+ mkdir ${B}/lib/node_modules/${BPN}/
+ tar -xzf ${B}/pkg/$tarball --no-same-owner \
+ --strip-components=1 -C ${B}/lib/node_modules/${BPN}
+
+ # Copy npm dependencies from sources
+ if [ -d ${S}/node_modules ]; then
+ # Copy sources without preserve mode and ownership
+ # to prevent ugly file permissions
+ mkdir ${B}/lib/node_modules/${BPN}/node_modules
+ cp -a --no-preserve=mode,ownership \
+ ${S}/node_modules/. ${B}/lib/node_modules/${BPN}/node_modules/
+
+ # Prune dependencies for production
+ if [ "${NPM_INSTALL_DEV}" = "1" -a "${NPM_PRUNE_FOR_PRODUCTION}" = "1"
] ; then
+ oe_runnpm prune --only=production
+ fi
+
+ # Rebuild dependencies inside node_modules folder
+ # to create bin links in .bin folder
+ cd ${B}/lib/node_modules
+ oe_runnpm_rebuild ${NPM_ARCH} ${EXTRA_OENPM}
+ fi
+}
+do_compile[cleandirs] = "${B}/lib/node_modules ${B}/pkg"
- if [ -d "${NPM_BUILD}/bin" ]
- then
- install -d ${D}/${bindir}
- cp --no-preserve=ownership --recursive ${NPM_BUILD}/bin/.
${D}/${bindir}
+npm_do_install() {
+ # Copy the entire main node_modules directory
+ install -d ${D}${nonarch_libdir}
+ cp -a --no-preserve=ownership ${B}/lib/node_modules \
+ ${D}${nonarch_libdir}/
+
+ # Create symlinks for package executables
+ if [ -d ${D}${nonarch_libdir}/node_modules/.bin ]; then
+ install -d ${D}${bindir}
+ for f in ${D}${nonarch_libdir}/node_modules/.bin/*; do
+ name="$(basename $f)"
+ link="$(readlink $f)"
+ target="$(realpath -m --relative-to ${D}${bindir} \
+ ${D}${nonarch_libdir}/node_modules/.bin/$link)"
+ ln -s "$target" "${D}${bindir}/$name"
+ done
+ rm -rf ${D}${nonarch_libdir}/node_modules/.bin
fi
# If the package (or its dependencies) uses node-gyp to build native
addons,
# object files, static libraries or other temporary files can be hidden in
# the lib directory. To reduce the package size and to avoid QA issues
# (staticdev with static library files) these files must be removed.
- local GYP_REGEX=".*/build/Release/[^/]*.node"
+ local GYP_REGEX=".*/build.*/Release/[^/]*.node"
# Remove any node-gyp directory in ${D} to remove temporary build files
for GYP_D_FILE in $(find ${D} -regex "${GYP_REGEX}")
@@ -294,9 +187,9 @@ npm_do_install() {
done
# Copy only the node-gyp release files
- for GYP_B_FILE in $(find ${NPM_BUILD} -regex "${GYP_REGEX}")
+ for GYP_B_FILE in $(find ${B}/pkg -regex "${GYP_REGEX}")
do
- local GYP_D_FILE=${D}/${prefix}/${GYP_B_FILE#${NPM_BUILD}}
+ local GYP_D_FILE=${D}/${prefix}/${GYP_B_FILE#${B}/pkg}
install -d ${GYP_D_FILE%/*}
install -m 755 ${GYP_B_FILE} ${GYP_D_FILE}
@@ -310,6 +203,7 @@ npm_do_install() {
# using /usr/lib/node_modules as install directory. Let's make both happy.
ln -fs node_modules ${D}/${nonarch_libdir}/node
}
+do_install[cleandirs] = "${D}"
FILES:${PN} += " \
${bindir} \
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 01fb5ad96f..10dc2b1373 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -260,10 +260,8 @@ def add(args, config, basepath, workspace):
f.write('}\n')
if bb.data.inherits_class('npm', rd):
- f.write('python do_configure:append() {\n')
- f.write(' pkgdir = d.getVar("NPM_PACKAGE")\n')
- f.write(' lockfile = os.path.join(pkgdir,
"singletask.lock")\n')
- f.write(' bb.utils.remove(lockfile)\n')
+ f.write('do_configure:append() {\n')
+ f.write(' rm -f ${NPM_PACKAGE}/singletask.lock\n')
f.write('}\n')
# Check if the new layer provides recipes whose priorities have been
--
2.20.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#158697):
https://lists.openembedded.org/g/openembedded-core/message/158697
Mute This Topic: https://lists.openembedded.org/mt/87282275/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
