A description of the changes and how the new code works is missing. How is
the SRC_URI formed? How is the appropriate class selected?
Alex
On Wed, 24 Nov 2021 at 15:48, Stefan Herbrechtsmeier <
stefan.herbrechtsmeier-...@weidmueller.com> wrote:
> From: Stefan Herbrechtsmeier <stefan.herbrechtsme...@weidmueller.com>
>
> Signed-off-by: Stefan Herbrechtsmeier <
> stefan.herbrechtsme...@weidmueller.com>
>
> ---
>
> scripts/lib/recipetool/create_npm.py | 243 ++++++++++++++++++++++++---
> 1 file changed, 222 insertions(+), 21 deletions(-)
>
> diff --git a/scripts/lib/recipetool/create_npm.py
> b/scripts/lib/recipetool/create_npm.py
> index 3394a89970..296b84340e 100644
> --- a/scripts/lib/recipetool/create_npm.py
> +++ b/scripts/lib/recipetool/create_npm.py
> @@ -39,6 +39,14 @@ class NpmRecipeHandler(RecipeHandler):
> name = name.strip("-")
> return name
>
> + @staticmethod
> + def _node_recipe_name(name):
> + """Generate a OE friendly Node.js recipe name"""
> + name = NpmRecipeHandler._npm_name(name)
> + if not name.startswith("node-"):
> + name = "node-" + name
> + return name
> +
> @staticmethod
> def _get_registry(lines):
> """Get the registry value from the 'npm://registry' url"""
> @@ -54,6 +62,24 @@ class NpmRecipeHandler(RecipeHandler):
>
> return registry
>
> + @staticmethod
> + def _get_srcdir(lines):
> + """Get the source directory value from the url"""
> + srcdir = ""
> +
> + def _handle_srcdir(varname, origvalue, op, newlines):
> + nonlocal srcdir
> + if origvalue.startswith("${WORKDIR}"):
> + srcdir = origvalue[11:]
> + else:
> + srcdir = "${BP}"
> +
> + return origvalue, None, 0, True
> +
> + bb.utils.edit_metadata(lines, ["S"], _handle_srcdir)
> +
> + return srcdir
> +
> @staticmethod
> def _ensure_npm():
> """Check if the 'npm' command is available in the recipes"""
> @@ -116,6 +142,118 @@ class NpmRecipeHandler(RecipeHandler):
>
> return os.path.join(srctree, "npm-shrinkwrap.json")
>
> + def _process_shrinkwrap(self, srctree, shrinkwrap, srcdir):
> + """
> + Extract package urls from shrinkwrap dependencies
> + """
> +
> + urls = []
> +
> + def _populate_modules(name, params, deptree):
> + from bb.fetch2 import URI
> + from bb.fetch2.npm import npm_integrity
> + from bb.fetch2.npm import npm_localfile
> +
> + destsubdirs = [os.path.join("node_modules", dep) for dep in
> deptree]
> + destsuffix = os.path.join(srcdir, *destsubdirs)
> +
> + dev = params.get("dev", False)
> + integrity = params.get("integrity")
> + resolved = params.get("resolved")
> + version = params.get("version")
> + requires = params.get("requires", {})
> +
> + # Handle registry sources
> + if bb.utils.is_semver(version) and integrity:
> + # Skip dependencies without url
> + if not resolved:
> + return
> +
> + pkgv = version
> +
> + uri = URI(resolved)
> + uri.params["downloadfilename"] = npm_localfile(name,
> version)
> +
> + checksum_name, checksum_expected =
> npm_integrity(integrity)
> + uri.params[checksum_name] = checksum_expected
> +
> + uri.params["subdir"] = destsuffix
> + uri.params["striplevel"] = "1"
> +
> + url = str(uri)
> +
> + # Handle http tarball sources
> + elif version.startswith("http") and integrity:
> + checksum_name, checksum_expected =
> npm_integrity(integrity)
> +
> + pkgv = checksum_expected[:13]
> +
> + uri = URI(version)
> + uri.params["downloadfilename"] = npm_localfile(name, pkgv)
> +
> + uri.params[checksum_name] = checksum_expected
> +
> + uri.params["destsuffix"] = destsuffix
> + uri.params["striplevel"] = "1"
> +
> + url = str(uri)
> +
> + # Handle git sources
> + elif version.startswith("git"):
> + if version.startswith("github:"):
> + version = "git+https://github.com/"; +
> version[len("github:"):]
> + regex = re.compile(r"""
> + ^
> + git\+
> + (?P<protocol>[a-z]+)
> + ://
> + (?P<url>[^#]+)
> + \#
> + (?P<rev>[0-9a-f]+)
> + $
> + """, re.VERBOSE)
> +
> + match = regex.match(version)
> +
> + if not match:
> + raise Exception("Invalid git url: %s - %s" %
> (version, url))
> +
> + groups = match.groupdict()
> +
> + pkgv = str(groups["rev"])[:10]
> +
> + uri = URI("git://" + str(groups["url"]))
> + uri.params["destsuffix"] = destsuffix
> + uri.params["nobranch"] = "1"
> + uri.params["protocol"] = str(groups["protocol"])
> + uri.params["rev"] = str(groups["rev"])
> +
> + url = str(uri)
> +
> + else:
> + raise Exception("Unsupported dependency: %s - %s" %
> (name, version))
> +
> + # Set package version in shrinkwrap for dependency resolution
> + params["pkgv"] = pkgv
> +
> + urls.append(url)
> +
> + def _foreach_shrinkwrap_dependency(shrinkwrap, callback):
> + def _walk_dependencies(deps, deptree):
> + for name in deps:
> + subtree = [*deptree, name]
> + _walk_dependencies(deps[name].get("dependencies",
> {}), subtree)
> + if deps[name].get("bundled", False):
> + continue
> + callback(name, deps[name], subtree)
> +
> + _walk_dependencies(shrinkwrap.get("dependencies", {}), [])
> +
> + _foreach_shrinkwrap_dependency(shrinkwrap, _populate_modules)
> +
> + return urls
> +
> +
> def _handle_licenses(self, srctree, shrinkwrap_file, dev):
> """Return the extra license files and the list of packages"""
> licfiles = []
> @@ -173,7 +311,7 @@ class NpmRecipeHandler(RecipeHandler):
> if "name" not in data or "version" not in data:
> return False
>
> - extravalues["PN"] = self._npm_name(data["name"])
> + extravalues["PN"] = self._node_recipe_name(data["name"])
> extravalues["PV"] = data["version"]
>
> if "description" in data:
> @@ -184,6 +322,24 @@ class NpmRecipeHandler(RecipeHandler):
>
> dev = bb.utils.to_boolean(str(extravalues.get("NPM_INSTALL_DEV",
> "0")), False)
> registry = self._get_registry(lines_before)
> + srcdir = self._get_srcdir(lines_before)
> + # Replace reserved directories
> + if srcdir == "package":
> + srcdir = "npm"
> + def _handle_srcuri(varname, origvalue, op, newlines):
> + """Update the version value"""
> + values = "%s;subdir=%s;striplevel=1" % (origvalue, srcdir)
> + return values, None, 4, False
> +
> + (_, newlines) = bb.utils.edit_metadata(lines_before,
> ["SRC_URI"], _handle_srcuri)
> + lines_before[:] = [line.rstrip('\n') for line in newlines]
> +
> + def _handle_srcdir(varname, origvalue, op, newlines):
> + value = "${WORKDIR}/%s" % (srcdir)
> + return value, None, 0, True
> +
> + (_, newlines) = bb.utils.edit_metadata(lines_before, ["S"],
> _handle_srcdir)
> + lines_before[:] = [line.rstrip('\n') for line in newlines]
>
> bb.note("Checking if npm is available ...")
> # The native npm is used here (and not the host one) to ensure
> that the
> @@ -223,37 +379,65 @@ class NpmRecipeHandler(RecipeHandler):
> if os.path.exists(lock_copy):
> bb.utils.movefile(lock_copy, lock_file)
>
> - # Add the shrinkwrap file as 'extrafiles'
> - shrinkwrap_copy = shrinkwrap_file + ".copy"
> - bb.utils.copyfile(shrinkwrap_file, shrinkwrap_copy)
> - extravalues.setdefault("extrafiles", {})
> - extravalues["extrafiles"]["npm-shrinkwrap.json"] = shrinkwrap_copy
> -
> - url_local = "npmsw://%s" % shrinkwrap_file
> - url_recipe= "npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json"
> -
> - if dev:
> - url_local += ";dev=1"
> - url_recipe += ";dev=1"
> -
> - # Add the npmsw url in the SRC_URI of the generated recipe
> def _handle_srcuri(varname, origvalue, op, newlines):
> - """Update the version value and add the 'npmsw://' url"""
> + """Update the version value"""
> value = origvalue.replace("version=" + data["version"],
> "version=${PV}")
> value = value.replace("version=latest", "version=${PV}")
> values = [line.strip() for line in
> value.strip('\n').splitlines()]
> - if "dependencies" in shrinkwrap:
> - values.append(url_recipe)
> return values, None, 4, False
>
> (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"],
> _handle_srcuri)
> lines_before[:] = [line.rstrip('\n') for line in newlines]
>
> + urls = self._process_shrinkwrap(srctree, shrinkwrap, srcdir)
> +
> + # Add the package urls in the SRC_URI of the generated recipe
> + def _handle_srcuri(varname, origvalue, op, newlines):
> + """Add the package urls and git SRCREVs"""
> + values = None
> + # Handle SRCREVs
> + if varname == "SRCREV":
> + newlines.append('SRCREV_FORMAT = "main"')
> + newlines.append('SRCREV_main = "%s"' % origvalue)
> + # Handle urls
> + elif varname == "SRC_URI":
> + values = [line.strip() for line in origvalue.split()]
> + values = ["%s;name=main" % (v) if v.startswith(("git",
> "http")) else v for v in values]
> + values.extend(sorted(urls))
> + # Handle hashes
> + else:
> + newvarname = varname.replace("[", "[main.")
> + newlines.append('%s = "%s"' % (newvarname, origvalue))
> + return values, None, 4, False
> +
> + (_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI",
> "SRCREV", "SRC_URI\[\w*\]"], _handle_srcuri)
> + lines_before[:] = [line.rstrip('\n') for line in newlines]
> +
> +
> + # Add the package urls in the SRC_URI of the generated recipe
> + def _handle_sums(varname, origvalue, op, newlines):
> + """Add the package urls and git SRCREVs"""
> + values = None
> + if varname == "SRC_URI[md5sum]":
> + # Add the urls
> + values = [line.strip() for line in origvalue.split()]
> + values = [v + ";name=main" if v.startswith("git", "http")
> else v for v in values]
> + values.extend(sorted(urls))
> + else:
> + # Add git SRCREVs
> + newlines.append('SRCREV_FORMAT = "main"')
> + newlines.append('SRCREV_main = "%s"' % origvalue)
> + return values, None, 4, False
> +
> + (_, newlines) = bb.utils.edit_metadata(lines_before,
> ["SRC_URI[md5sum]", "SRCREV" ], _handle_sums)
> + lines_before[:] = [line.rstrip('\n') for line in newlines]
> +
> # In order to generate correct licence checksums in the recipe the
> - # dependencies have to be fetched again using the npmsw url
> + # dependencies have to be fetched agai n using the urls
> bb.note("Fetching npm dependencies ...")
> bb.utils.remove(os.path.join(srctree, "node_modules"),
> recurse=True)
> - fetcher = bb.fetch2.Fetch([url_local], d)
> + srcurls = [url.replace("=%s/node_modules" % srcdir,
> "=node_modules") for url in urls]
> + fetcher = bb.fetch2.Fetch(srcurls, d)
> fetcher.download()
> fetcher.unpack(srctree)
>
> @@ -289,7 +473,24 @@ class NpmRecipeHandler(RecipeHandler):
> (licenses, extravalues["LIC_FILES_CHKSUM"]) =
> _guess_odd_license(licfiles)
> split_pkg_licenses([*licenses, *guess_license(srctree, d)],
> packages, lines_after)
>
> - classes.append("npm")
> + # Add suitable npm class
> + if dev:
> + scripts = data.get("scripts", {})
> + if scripts.get("build"):
> + deps = data.get("devDependencies", {})
> + if deps.get("@angular/cli"):
> + classes.append("angular")
> + elif deps.get("karma"):
> + classes.append("karma")
> + elif scripts.get("test"):
> + classes.append("npm_test")
> + else:
> + classes.append("npm_build")
> + else:
> + classes.append("npm")
> + else:
> + classes.append("npm")
> +
> handled.append("buildsystem")
>
> return True
> --
> 2.20.1
>
>
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#158721):
https://lists.openembedded.org/g/openembedded-core/message/158721
Mute This Topic: https://lists.openembedded.org/mt/87282286/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
