On 19/10/2023 20:20:33+0200, Julien Stephan wrote:
> Le jeu. 19 oct. 2023 à 15:49, Alexandre Belloni
> <alexandre.bell...@bootlin.com> a écrit :
> >
> > Hello,
> >
> > On 19/10/2023 09:36:53+0200, Julien Stephan wrote:
> > > add support for PEP517 [1]
> > >
> > > if a pyproject.toml file is found, use it to create the recipe,
> > > otherwise fallback to the old setup.py method.
> > >
> > > [YOCTO #14737]
> > >
> > > [1]: https://peps.python.org/pep-0517/
> > >
> > > Signed-off-by: Julien Stephan <jstep...@baylibre.com>
> > > ---
> > > .../lib/recipetool/create_buildsys_python.py | 234 +++++++++++++++++-
> > > 1 file changed, 233 insertions(+), 1 deletion(-)
> > >
> > > diff --git a/scripts/lib/recipetool/create_buildsys_python.py
> > > b/scripts/lib/recipetool/create_buildsys_python.py
> > > index 69f6f5ca511..0b601d50a4b 100644
> > > --- a/scripts/lib/recipetool/create_buildsys_python.py
> > > +++ b/scripts/lib/recipetool/create_buildsys_python.py
> > > @@ -18,6 +18,7 @@ import os
> > > import re
> > > import sys
> > > import subprocess
> > > +import toml
> >
> > This fails on the autobuilders because we don't have the toml module
> > installed so I guess you need to add a dependency.
> >
>
> Hello,
>
> Sure I 'll do it. Just to confirm, I should add it here:
> https://docs.yoctoproject.org/ref-manual/system-requirements.html#required-packages-for-the-build-host
> ?
I guess the preferred way would be to depend on python3-toml-native
instead of requiring installation on the host.
>
> Cheers
> Julien
>
> > > from recipetool.create import RecipeHandler
> > >
> > > logger = logging.getLogger('recipetool')
> > > @@ -656,6 +657,235 @@ class
> > > PythonSetupPyRecipeHandler(PythonRecipeHandler):
> > >
> > > handled.append('buildsystem')
> > >
> > > +class PythonPyprojectTomlRecipeHandler(PythonRecipeHandler):
> > > + """Base class to support PEP517 and PEP518
> > > +
> > > + PEP517 https://peps.python.org/pep-0517/#source-trees
> > > + PEP518 https://peps.python.org/pep-0518/#build-system-table
> > > + """
> > > +
> > > + # PEP621:
> > > https://packaging.python.org/en/latest/specifications/declaring-project-metadata/
> > > + # add only the ones that map to a BB var
> > > + # potentially missing: optional-dependencies
> > > + bbvar_map = {
> > > + "name": "PN",
> > > + "version": "PV",
> > > + "Homepage": "HOMEPAGE",
> > > + "description": "SUMMARY",
> > > + "license": "LICENSE",
> > > + "dependencies": "RDEPENDS:${PN}",
> > > + "requires": "DEPENDS",
> > > + }
> > > +
> > > + replacements = [
> > > + ("license", r" +$", ""),
> > > + ("license", r"^ +", ""),
> > > + ("license", r" ", "-"),
> > > + ("license", r"^GNU-", ""),
> > > + ("license", r"-[Ll]icen[cs]e(,?-[Vv]ersion)?", ""),
> > > + ("license", r"^UNKNOWN$", ""),
> > > + # Remove currently unhandled version numbers from these variables
> > > + ("requires", r"\[[^\]]+\]$", ""),
> > > + ("requires", r"^([^><= ]+).*", r"\1"),
> > > + ("dependencies", r"\[[^\]]+\]$", ""),
> > > + ("dependencies", r"^([^><= ]+).*", r"\1"),
> > > + ]
> > > +
> > > + build_backend_map = {
> > > + "setuptools.build_meta": "python_setuptools_build_meta",
> > > + "poetry.core.masonry.api": "python_poetry_core",
> > > + "flit_core.buildapi": "python_flit_core",
> > > + }
> > > +
> > > + excluded_native_pkgdeps = [
> > > + # already provided by python_setuptools_build_meta.bbclass
> > > + "python3-setuptools-native",
> > > + "python3-wheel-native",
> > > + # already provided by python_poetry_core.bbclass
> > > + "python3-poetry-core-native",
> > > + # already provided by python_flit_core.bbclass
> > > + "python3-flit-core-native",
> > > + ]
> > > +
> > > + # add here a list of known and often used packages and the
> > > corresponding bitbake package
> > > + known_deps_map = {
> > > + "setuptools": "python3-setuptools",
> > > + "wheel": "python3-wheel",
> > > + "poetry-core": "python3-poetry-core",
> > > + "flit_core": "python3-flit-core",
> > > + "setuptools-scm": "python3-setuptools-scm",
> > > + }
> > > +
> > > + def __init__(self):
> > > + pass
> > > +
> > > + def process(self, srctree, classes, lines_before, lines_after,
> > > handled, extravalues):
> > > + info = {}
> > > +
> > > + if 'buildsystem' in handled:
> > > + return False
> > > +
> > > + # Check for non-zero size setup.py files
> > > + setupfiles = RecipeHandler.checkfiles(srctree,
> > > ["pyproject.toml"])
> > > + for fn in setupfiles:
> > > + if os.path.getsize(fn):
> > > + break
> > > + else:
> > > + return False
> > > +
> > > + setupscript = os.path.join(srctree, "pyproject.toml")
> > > +
> > > + try:
> > > + config = self.parse_pyproject_toml(setupscript)
> > > + build_backend = config["build-system"]["build-backend"]
> > > + if build_backend in self.build_backend_map:
> > > + classes.append(self.build_backend_map[build_backend])
> > > + else:
> > > + logger.error(
> > > + "Unsupported build-backend: %s, cannot use
> > > pyproject.toml. Will try to use legacy setup.py"
> > > + % build_backend
> > > + )
> > > + return False
> > > +
> > > + licfile = ""
> > > + if "project" in config:
> > > + for field, values in config["project"].items():
> > > + if field == "license":
> > > + value = values.get("text", "")
> > > + if not value:
> > > + licfile = values.get("file", "")
> > > + elif isinstance(values, dict):
> > > + for k, v in values.items():
> > > + info[k] = v
> > > + continue
> > > + else:
> > > + value = values
> > > +
> > > + info[field] = value
> > > +
> > > + # Grab the license value before applying replacements
> > > + license_str = info.get("license", "").strip()
> > > +
> > > + if license_str:
> > > + for i, line in enumerate(lines_before):
> > > + if line.startswith("##LICENSE_PLACEHOLDER##"):
> > > + lines_before.insert(
> > > + i, "# NOTE: License in pyproject.toml is:
> > > %s" % license_str
> > > + )
> > > + break
> > > +
> > > + info["requires"] = config["build-system"]["requires"]
> > > +
> > > + self.apply_info_replacements(info)
> > > +
> > > + if "classifiers" in info:
> > > + license = self.handle_classifier_license(
> > > + info["classifiers"], info.get("license", "")
> > > + )
> > > + if license:
> > > + if licfile:
> > > + lines = []
> > > + md5value =
> > > bb.utils.md5_file(os.path.join(srctree, licfile))
> > > + lines.append('LICENSE = "%s"' % license)
> > > + lines.append(
> > > + 'LIC_FILES_CHKSUM = "file://%s;md5=%s"'
> > > + % (licfile, md5value)
> > > + )
> > > + lines.append("")
> > > +
> > > + # Replace the placeholder so we get the values
> > > in the right place in the recipe file
> > > + try:
> > > + pos =
> > > lines_before.index("##LICENSE_PLACEHOLDER##")
> > > + except ValueError:
> > > + pos = -1
> > > + if pos == -1:
> > > + lines_before.extend(lines)
> > > + else:
> > > + lines_before[pos : pos + 1] = lines
> > > +
> > > + handled.append(("license", [license, licfile,
> > > md5value]))
> > > + else:
> > > + info["license"] = license
> > > +
> > > + provided_packages = self.parse_pkgdata_for_python_packages()
> > > + provided_packages.update(self.known_deps_map)
> > > + native_mapped_deps, native_unmapped_deps = set(), set()
> > > + mapped_deps, unmapped_deps = set(), set()
> > > +
> > > + if "requires" in info:
> > > + for require in info["requires"]:
> > > + mapped = provided_packages.get(require)
> > > +
> > > + if mapped:
> > > + logger.error("Mapped %s to %s" % (require,
> > > mapped))
> > > + native_mapped_deps.add(mapped)
> > > + else:
> > > + logger.error("Could not map %s" % require)
> > > + native_unmapped_deps.add(require)
> > > +
> > > + info.pop("requires")
> > > +
> > > + if native_mapped_deps != set():
> > > + native_mapped_deps = {
> > > + item + "-native" for item in native_mapped_deps
> > > + }
> > > + native_mapped_deps -=
> > > set(self.excluded_native_pkgdeps)
> > > + if native_mapped_deps != set():
> > > + info["requires"] = "
> > > ".join(sorted(native_mapped_deps))
> > > +
> > > + if native_unmapped_deps:
> > > + lines_after.append("")
> > > + lines_after.append(
> > > + "# WARNING: We were unable to map the following
> > > python package/module"
> > > + )
> > > + lines_after.append(
> > > + "# dependencies to the bitbake packages which
> > > include them:"
> > > + )
> > > + lines_after.extend(
> > > + "# {}".format(d) for d in
> > > sorted(native_unmapped_deps)
> > > + )
> > > +
> > > + if "dependencies" in info:
> > > + for dependency in info["dependencies"]:
> > > + mapped = provided_packages.get(dependency)
> > > + if mapped:
> > > + logger.error("Mapped %s to %s" % (dependency,
> > > mapped))
> > > + mapped_deps.add(mapped)
> > > + else:
> > > + logger.error("Could not map %s" % dependency)
> > > + unmapped_deps.add(dependency)
> > > +
> > > + info.pop("dependencies")
> > > +
> > > + if mapped_deps != set():
> > > + if mapped_deps != set():
> > > + info["dependencies"] = "
> > > ".join(sorted(mapped_deps))
> > > +
> > > + if unmapped_deps:
> > > + lines_after.append("")
> > > + lines_after.append(
> > > + "# WARNING: We were unable to map the following
> > > python package/module"
> > > + )
> > > + lines_after.append(
> > > + "# runtime dependencies to the bitbake packages
> > > which include them:"
> > > + )
> > > + lines_after.extend(
> > > + "# {}".format(d) for d in
> > > sorted(unmapped_deps)
> > > + )
> > > +
> > > + self.map_info_to_bbvar(info, extravalues)
> > > +
> > > + handled.append("buildsystem")
> > > + except Exception:
> > > + logger.exception("Failed to parse pyproject.toml")
> > > + return False
> > > +
> > > + def parse_pyproject_toml(self, setupscript):
> > > + with open(setupscript, "r") as f:
> > > + config = toml.load(f)
> > > + return config
> > > +
> > > +
> > > def gather_setup_info(fileobj):
> > > parsed = ast.parse(fileobj.read(), fileobj.name)
> > > visitor = SetupScriptVisitor()
> > > @@ -769,5 +999,7 @@ def has_non_literals(value):
> > >
> > >
> > > def register_recipe_handlers(handlers):
> > > - # We need to make sure this is ahead of the makefile fallback handler
> > > + # We need to make sure these are ahead of the makefile fallback
> > > handler
> > > + # and the pyproject.toml handler ahead of the setup.py handler
> > > + handlers.append((PythonPyprojectTomlRecipeHandler(), 75))
> > > handlers.append((PythonSetupPyRecipeHandler(), 70))
> > > --
> > > 2.42.0
> > >
> >
> > >
> > >
> > >
> >
> >
> > --
> > Alexandre Belloni, co-owner and COO, Bootlin
> > Embedded Linux and Kernel engineering
> > https://bootlin.com
--
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#189481):
https://lists.openembedded.org/g/openembedded-core/message/189481
Mute This Topic: https://lists.openembedded.org/mt/102055999/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
