On Mon, 2017-02-13 at 17:05 +0200, Alexander Kanavin wrote: > To properly look at this patch, you probably need a side-by-side diff > viewing tool.
I don't normally use rpm, but it seems to me that the new 'remove' is not right. Looking at the description, clean_requirements_on_remove is meant to tell DNF whether to remove packages that are no longer needed after the package given is uninstalled, not to tell it to force removal of a package even though other packages still depend on it http://dnf.readthedocs.io/en/latest /command_ref.html#remove-command- The with_dependencies flag is meant to say whether or not packages that depend on the package about to be removed should be removed as well or not. Matching clean_requirements_on_remove with with_dependencies isn't right in other words. Looking at the debian package manager, and at the previous version: - The high level tool (apt/smart) is used to remove packages recursively if with_dependencies is True, i.e. the given package and all packages that depend on it - The low level tool (dpkg/rpm) is forcefully used to remove packages if with_dependencies is False, i.e. the given package only With this patch here, the high level tool is used in both cases, but this won't work, you should still use rpm -e --nodeps I guess this is the reason you need patch 31 ("image.bbclass: do not uninstall update-alternatives from readonly rootfs") Was there a specific reason to switch from rpm -e --nodeps to (the differently-working) dnf remove? Cheers, Andre' > > Signed-off-by: Alexander Kanavin <alexander.kana...@linux.intel.com> > --- > meta/classes/rootfs_rpm.bbclass | 21 +- > meta/lib/oe/package_manager.py | 1186 +++++++--------------------------- > ----- > meta/lib/oe/rootfs.py | 18 +- > meta/lib/oe/sdk.py | 7 +- > 4 files changed, 229 insertions(+), 1003 deletions(-) > > diff --git a/meta/classes/rootfs_rpm.bbclass > b/meta/classes/rootfs_rpm.bbclass > index b8ff4cb7b6a..65881a60a7b 100644 > --- a/meta/classes/rootfs_rpm.bbclass > +++ b/meta/classes/rootfs_rpm.bbclass > @@ -2,20 +2,23 @@ > # Creates a root filesystem out of rpm packages > # > > -ROOTFS_PKGMANAGE = "rpm smartpm" > +ROOTFS_PKGMANAGE = "rpm dnf" > ROOTFS_PKGMANAGE_BOOTSTRAP = "run-postinsts" > > -# Add 100Meg of extra space for Smart > -IMAGE_ROOTFS_EXTRA_SPACE_append = "${@bb.utils.contains("PACKAGE_INSTALL" > , "smartpm", " + 102400", "" ,d)}" > +# dnf is using our custom distutils, and so will fail without these > +export STAGING_INCDIR > +export STAGING_LIBDIR > > -# Smart is python based, so be sure python-native is available to us. > +# Add 100Meg of extra space for dnf > +IMAGE_ROOTFS_EXTRA_SPACE_append = "${@bb.utils.contains("PACKAGE_INSTALL" > , "dnf", " + 102400", "" ,d)}" > + > +# Dnf is python based, so be sure python-native is available to us. > EXTRANATIVEPATH += "python-native" > > # opkg is needed for update-alternatives > RPMROOTFSDEPENDS = "rpm-native:do_populate_sysroot \ > - rpmresolve-native:do_populate_sysroot \ > - python-smartpm-native:do_populate_sysroot \ > - createrepo-native:do_populate_sysroot \ > + dnf-native:do_populate_sysroot \ > + createrepo-c-native:do_populate_sysroot \ > opkg-native:do_populate_sysroot" > > do_rootfs[depends] += "${RPMROOTFSDEPENDS}" > @@ -35,7 +38,3 @@ python () { > d.setVar('RPM_POSTPROCESS_COMMANDS', '') > > } > -# Smart is python based, so be sure python-native is available to us. > -EXTRANATIVEPATH += "python-native" > - > -rpmlibdir = "/var/lib/rpm" > diff --git a/meta/lib/oe/package_manager.py > b/meta/lib/oe/package_manager.py > index c5a9b3955a4..aa431b37b59 100644 > --- a/meta/lib/oe/package_manager.py > +++ b/meta/lib/oe/package_manager.py > @@ -102,110 +102,16 @@ class Indexer(object, metaclass=ABCMeta): > > > class RpmIndexer(Indexer): > - def get_ml_prefix_and_os_list(self, arch_var=None, os_var=None): > - package_archs = collections.OrderedDict() > - target_os = collections.OrderedDict() > - > - if arch_var is not None and os_var is not None: > - package_archs['default'] = self.d.getVar(arch_var).split() > - package_archs['default'].reverse() > - target_os['default'] = self.d.getVar(os_var).strip() > - else: > - package_archs['default'] = > self.d.getVar("PACKAGE_ARCHS").split() > - # arch order is reversed. This ensures the -best- match is > - # listed first! > - package_archs['default'].reverse() > - target_os['default'] = self.d.getVar("TARGET_OS").strip() > - multilibs = self.d.getVar('MULTILIBS') or "" > - for ext in multilibs.split(): > - eext = ext.split(':') > - if len(eext) > 1 and eext[0] == 'multilib': > - localdata = bb.data.createCopy(self.d) > - default_tune_key = "DEFAULTTUNE_virtclass-multilib-" > + eext[1] > - default_tune = localdata.getVar(default_tune_key, > False) > - if default_tune is None: > - default_tune_key = "DEFAULTTUNE_ML_" + eext[1] > - default_tune = localdata.getVar(default_tune_key, > False) > - if default_tune: > - localdata.setVar("DEFAULTTUNE", default_tune) > - bb.data.update_data(localdata) > - package_archs[eext[1]] = > localdata.getVar('PACKAGE_ARCHS').split() > - package_archs[eext[1]].reverse() > - target_os[eext[1]] = > localdata.getVar("TARGET_OS").strip() > - > - ml_prefix_list = collections.OrderedDict() > - for mlib in package_archs: > - if mlib == 'default': > - ml_prefix_list[mlib] = package_archs[mlib] > - else: > - ml_prefix_list[mlib] = list() > - for arch in package_archs[mlib]: > - if arch in ['all', 'noarch', 'any']: > - ml_prefix_list[mlib].append(arch) > - else: > - ml_prefix_list[mlib].append(mlib + "_" + arch) > - > - return (ml_prefix_list, target_os) > - > def write_index(self): > - sdk_pkg_archs = (self.d.getVar('SDK_PACKAGE_ARCHS') or > "").replace('-', '_').split() > - all_mlb_pkg_archs = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS') > or "").replace('-', '_').split() > - > - mlb_prefix_list = self.get_ml_prefix_and_os_list()[0] > - > - archs = set() > - for item in mlb_prefix_list: > - archs = archs.union(set(i.replace('-', '_') for i in > mlb_prefix_list[item])) > - > - if len(archs) == 0: > - archs = archs.union(set(all_mlb_pkg_archs)) > - > - archs = archs.union(set(sdk_pkg_archs)) > - > - rpm_createrepo = bb.utils.which(os.environ['PATH'], "createrepo") > - if not rpm_createrepo: > - bb.error("Cannot rebuild index as createrepo was not found in > %s" % os.environ['PATH']) > - return > - > if self.d.getVar('PACKAGE_FEED_SIGN') == '1': > - signer = get_signer(self.d, > self.d.getVar('PACKAGE_FEED_GPG_BACKEND')) > - else: > - signer = None > - index_cmds = [] > - repomd_files = [] > - rpm_dirs_found = False > - for arch in archs: > - dbpath = os.path.join(self.d.getVar('WORKDIR'), 'rpmdb', > arch) > - if os.path.exists(dbpath): > - bb.utils.remove(dbpath, True) > - arch_dir = os.path.join(self.deploy_dir, arch) > - if not os.path.isdir(arch_dir): > - continue > + raise NotImplementedError('Package feed signing not yet > implementd for rpm') > > - index_cmds.append("%s --dbpath %s --update -q %s" % \ > - (rpm_createrepo, dbpath, arch_dir)) > - repomd_files.append(os.path.join(arch_dir, 'repodata', > 'repomd.xml')) > + os.environ['RPM_CONFIGDIR'] = > oe.path.join(self.d.getVar('STAGING_LIBDIR_NATIVE'), "rpm") > > - rpm_dirs_found = True > - > - if not rpm_dirs_found: > - bb.note("There are no packages in %s" % self.deploy_dir) > - return > - > - # Create repodata > - result = oe.utils.multiprocess_exec(index_cmds, create_index) > + createrepo_c = bb.utils.which(os.environ['PATH'], "createrepo_c") > + result = create_index("%s --update -q %s" % (createrepo_c, > self.deploy_dir)) > if result: > - bb.fatal('%s' % ('\n'.join(result))) > - # Sign repomd > - if signer: > - for repomd in repomd_files: > - feed_sig_type = > self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE') > - is_ascii_sig = (feed_sig_type.upper() != "BIN") > - signer.detach_sign(repomd, > - self.d.getVar('PACKAGE_FEED_GPG_NAME') > , > - self.d.getVar('PACKAGE_FEED_GPG_PASSPH > RASE_FILE'), > - armor=is_ascii_sig) > - > + bb.fatal(result) > > class OpkgIndexer(Indexer): > def write_index(self): > @@ -348,117 +254,9 @@ class PkgsList(object, metaclass=ABCMeta): > def list_pkgs(self): > pass > > - > class RpmPkgsList(PkgsList): > - def __init__(self, d, rootfs_dir, arch_var=None, os_var=None): > - super(RpmPkgsList, self).__init__(d, rootfs_dir) > - > - self.rpm_cmd = bb.utils.which(os.getenv('PATH'), "rpm") > - self.image_rpmlib = os.path.join(self.rootfs_dir, 'var/lib/rpm') > - > - self.ml_prefix_list, self.ml_os_list = \ > - RpmIndexer(d, rootfs_dir).get_ml_prefix_and_os_list(arch_var, > os_var) > - > - # Determine rpm version > - try: > - output = subprocess.check_output([self.rpm_cmd, "--version"], > stderr=subprocess.STDOUT).decode("utf-8") > - except subprocess.CalledProcessError as e: > - bb.fatal("Getting rpm version failed. Command '%s' " > - "returned %d:\n%s" % (self.rpm_cmd, e.returncode, > e.output.decode("utf-8"))) > - > - ''' > - Translate the RPM/Smart format names to the OE multilib format names > - ''' > - def _pkg_translate_smart_to_oe(self, pkg, arch): > - new_pkg = pkg > - new_arch = arch > - fixed_arch = arch.replace('_', '-') > - found = 0 > - for mlib in self.ml_prefix_list: > - for cmp_arch in self.ml_prefix_list[mlib]: > - fixed_cmp_arch = cmp_arch.replace('_', '-') > - if fixed_arch == fixed_cmp_arch: > - if mlib == 'default': > - new_pkg = pkg > - new_arch = cmp_arch > - else: > - new_pkg = mlib + '-' + pkg > - # We need to strip off the ${mlib}_ prefix on the > arch > - new_arch = cmp_arch.replace(mlib + '_', '') > - > - # Workaround for bug 3565. Simply look to see if we > - # know of a package with that name, if not try again! > - filename = os.path.join(self.d.getVar('PKGDATA_DIR'), > - 'runtime-reverse', > - new_pkg) > - if os.path.exists(filename): > - found = 1 > - break > - > - if found == 1 and fixed_arch == fixed_cmp_arch: > - break > - #bb.note('%s, %s -> %s, %s' % (pkg, arch, new_pkg, new_arch)) > - return new_pkg, new_arch > - > - def _list_pkg_deps(self): > - cmd = [bb.utils.which(os.getenv('PATH'), "rpmresolve"), > - "-t", self.image_rpmlib] > - > - try: > - output = subprocess.check_output(cmd, > stderr=subprocess.STDOUT).strip().decode("utf-8") > - except subprocess.CalledProcessError as e: > - bb.fatal("Cannot get the package dependencies. Command '%s' " > - "returned %d:\n%s" % (' '.join(cmd), e.returncode, > e.output.decode("utf-8"))) > - > - return output > - > def list_pkgs(self): > - cmd = [self.rpm_cmd, '--root', self.rootfs_dir] > - cmd.extend(['-D', '_dbpath /var/lib/rpm']) > - cmd.extend(['-qa', '--qf', '[%{NAME} %{ARCH} %{VERSION} > %{PACKAGEORIGIN}\n]']) > - > - try: > - tmp_output = subprocess.check_output(cmd, > stderr=subprocess.STDOUT).strip().decode("utf-8") > - except subprocess.CalledProcessError as e: > - bb.fatal("Cannot get the installed packages list. Command > '%s' " > - "returned %d:\n%s" % (' '.join(cmd), e.returncode, > e.output.decode("utf-8"))) > - > - output = dict() > - deps = dict() > - dependencies = self._list_pkg_deps() > - > - # Populate deps dictionary for better manipulation > - for line in dependencies.splitlines(): > - try: > - pkg, dep = line.split("|") > - if not pkg in deps: > - deps[pkg] = list() > - if not dep in deps[pkg]: > - deps[pkg].append(dep) > - except: > - # Ignore any other lines they're debug or errors > - pass > - > - for line in tmp_output.split('\n'): > - if len(line.strip()) == 0: > - continue > - pkg = line.split()[0] > - arch = line.split()[1] > - ver = line.split()[2] > - dep = deps.get(pkg, []) > - > - # Skip GPG keys > - if pkg == 'gpg-pubkey': > - continue > - > - pkgorigin = line.split()[3] > - new_pkg, new_arch = self._pkg_translate_smart_to_oe(pkg, > arch) > - > - output[new_pkg] = {"arch":new_arch, "ver":ver, > - "filename":pkgorigin, "deps":dep} > - > - return output > - > + return RpmPM(self.d, self.rootfs_dir, > self.d.getVar('TARGET_VENDOR')).list_installed() > > class OpkgPkgsList(PkgsList): > def __init__(self, d, rootfs_dir, config_file): > @@ -554,6 +352,16 @@ class PackageManager(object, metaclass=ABCMeta): > pass > > """ > + Returns the path to a tmpdir where resides the contents of a package. > + > + Deleting the tmpdir is responsability of the caller. > + > + """ > + @abstractmethod > + def extract(self, pkg): > + pass > + > + """ > Add remote package feeds into repository manager configuration. The > parameters > for the feeds are set by feed_uris, feed_base_paths and feed_archs. > See http://www.yoctoproject.org/docs/current/ref-manual/ref-manual.ht > ml#var-PACKAGE_FEED_URIS > @@ -662,821 +470,249 @@ class RpmPM(PackageManager): > self.target_rootfs = target_rootfs > self.target_vendor = target_vendor > self.task_name = task_name > - self.providename = providename > - self.fullpkglist = list() > - self.deploy_dir = self.d.getVar('DEPLOY_DIR_RPM') > - self.etcrpm_dir = os.path.join(self.target_rootfs, "etc/rpm") > - self.install_dir_name = "oe_install" > - self.install_dir_path = os.path.join(self.target_rootfs, > self.install_dir_name) > - self.rpm_cmd = bb.utils.which(os.getenv('PATH'), "rpm") > - self.smart_cmd = bb.utils.which(os.getenv('PATH'), "smart") > - # 0 = --log-level=warning, only warnings > - # 1 = --log-level=info (includes information about executing > scriptlets and their output), default > - # 2 = --log-level=debug > - # 3 = --log-level=debug plus dumps of scriplet content and > command invocation > - self.debug_level = int(d.getVar('ROOTFS_RPM_DEBUG') or "1") > - self.smart_opt = ["--log-level=%s" % > - ("warning" if self.debug_level == 0 else > - "info" if self.debug_level == 1 else > - "debug"), "--data-dir=%s" % > - os.path.join(target_rootfs, 'var/lib/smart')] > - self.scriptlet_wrapper = > self.d.expand('${WORKDIR}/scriptlet_wrapper') > + if arch_var == None: > + self.archs = self.d.getVar('PACKAGE_ARCHS').replace("-","_") > + else: > + self.archs = self.d.getVar(arch_var).replace("-","_") > + if task_name == "host": > + self.primary_arch = self.d.getVar('SDK_ARCH') > + else: > + self.primary_arch = self.d.getVar('MACHINE_ARCH') > + > + self.rpm_repo_dir = oe.path.join(self.d.getVar('WORKDIR'), "rpm- > repo") > + bb.utils.mkdirhier(self.rpm_repo_dir) > + oe.path.symlink(self.d.getVar('DEPLOY_DIR_RPM'), > oe.path.join(self.rpm_repo_dir, "rpm"), True) > + > + self.saved_packaging_data = > self.d.expand('${T}/saved_packaging_data/%s' % self.task_name) > + if not > os.path.exists(self.d.expand('${T}/saved_packaging_data')): > + bb.utils.mkdirhier(self.d.expand('${T}/saved_packaging_data') > ) > + self.packaging_data_dirs = ['var/lib/rpm', 'var/lib/dnf', > 'var/cache/dnf'] > self.solution_manifest = self.d.expand('${T}/saved/%s_solution' % > self.task_name) > - self.saved_rpmlib = self.d.expand('${T}/saved/%s' % > self.task_name) > - self.image_rpmlib = os.path.join(self.target_rootfs, > 'var/lib/rpm') > - > if not os.path.exists(self.d.expand('${T}/saved')): > bb.utils.mkdirhier(self.d.expand('${T}/saved')) > > - packageindex_dir = os.path.join(self.d.getVar('WORKDIR'), 'rpms') > - self.indexer = RpmIndexer(self.d, packageindex_dir) > - self.pkgs_list = RpmPkgsList(self.d, self.target_rootfs, > arch_var, os_var) > - > - self.ml_prefix_list, self.ml_os_list = > self.indexer.get_ml_prefix_and_os_list(arch_var, os_var) > - > - def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): > - if feed_uris == "": > - return > - > - arch_list = [] > - if feed_archs is not None: > - # User define feed architectures > - arch_list = feed_archs.split() > - else: > - # List must be prefered to least preferred order > - default_platform_extra = list() > - platform_extra = list() > - bbextendvariant = self.d.getVar('BBEXTENDVARIANT') or "" > - for mlib in self.ml_os_list: > - for arch in self.ml_prefix_list[mlib]: > - plt = arch.replace('-', '_') + '-.*-' + > self.ml_os_list[mlib] > - if mlib == bbextendvariant: > - if plt not in default_platform_extra: > - default_platform_extra.append(plt) > - else: > - if plt not in platform_extra: > - platform_extra.append(plt) > - platform_extra = default_platform_extra + platform_extra > + def _configure_dnf(self): > + # libsolv handles 'noarch' internally, we don't need to specify > it explicitly > + archs = [i for i in self.archs.split() if i not in ["any", "all", > "noarch"]] > + # This prevents accidental matching against libsolv's built-in > policies > + if len(archs) <= 1: > + archs = archs + ["bogusarch"] > + archconfdir = "%s/%s" %(self.target_rootfs, "etc/dnf/vars/") > + bb.utils.mkdirhier(archconfdir) > + open(archconfdir + "arch", 'w').write(":".join(archs)) > + > + open(oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"), > 'w').write("") > + > + > + def _configure_rpm(self): > + # We need to configure rpm to use our primary package > architecture as the installation architecture, > + # and to make it compatible with other package architectures that > we use. > + # Otherwise it will refuse to proceed with packages installation. > + platformconfdir = "%s/%s" %(self.target_rootfs, "etc/rpm/") > + rpmrcconfdir = "%s/%s" %(self.target_rootfs, "etc/") > + bb.utils.mkdirhier(platformconfdir) > + open(platformconfdir + "platform", 'w').write("%s-pc-linux" % > self.primary_arch) > + open(rpmrcconfdir + "rpmrc", 'w').write("arch_compat: %s: %s" % > (self.primary_arch, self.archs if len(self.archs) > 0 else > self.primary_arch)) > > - for canonical_arch in platform_extra: > - arch = canonical_arch.split('-')[0] > - if not os.path.exists(os.path.join(self.deploy_dir, > arch)): > - continue > - arch_list.append(arch) > - > - feed_uris = self.construct_uris(feed_uris.split(), > feed_base_paths.split()) > + if self.d.getVar('RPM_SIGN_PACKAGES') == '1': > + raise NotImplementedError("Signature verification with rpm > not yet supported.") > > - uri_iterator = 0 > - channel_priority = 10 + 5 * len(feed_uris) * (len(arch_list) if > arch_list else 1) > - > - for uri in feed_uris: > - if arch_list: > - for arch in arch_list: > - bb.note('Adding Smart channel url%d%s (%s)' % > - (uri_iterator, arch, channel_priority)) > - self._invoke_smart(['channel', '--add', 'url%d-%s' % > (uri_iterator, arch), > - 'type=rpm-md', 'baseurl=%s/%s' % (uri, arch), '- > y']) > - self._invoke_smart(['channel', '--set', 'url%d-%s' % > (uri_iterator, arch), > - 'priority=%d' % channel_priority]) > - channel_priority -= 5 > - else: > - bb.note('Adding Smart channel url%d (%s)' % > - (uri_iterator, channel_priority)) > - self._invoke_smart(['channel', '--add', 'url%d' % > uri_iterator, > - 'type=rpm-md', 'baseurl=%s' % uri, '-y']) > - self._invoke_smart(['channel', '--set', 'url%d' % > uri_iterator, > - 'priority=%d' % channel_priority]) > - channel_priority -= 5 > - > - uri_iterator += 1 > + if self.d.getVar('RPM_PREFER_ELF_ARCH'): > + raise NotImplementedError("RPM_PREFER_ELF_ARCH not yet > checked/tested/implemented with rpm4/dnf.") > > - ''' > - Create configs for rpm and smart, and multilib is supported > - ''' > def create_configs(self): > - target_arch = self.d.getVar('TARGET_ARCH') > - platform = '%s%s-%s' % (target_arch.replace('-', '_'), > - self.target_vendor, > - self.ml_os_list['default']) > - > - # List must be prefered to least preferred order > - default_platform_extra = list() > - platform_extra = list() > - bbextendvariant = self.d.getVar('BBEXTENDVARIANT') or "" > - for mlib in self.ml_os_list: > - for arch in self.ml_prefix_list[mlib]: > - plt = arch.replace('-', '_') + '-.*-' + > self.ml_os_list[mlib] > - if mlib == bbextendvariant: > - if plt not in default_platform_extra: > - default_platform_extra.append(plt) > - else: > - if plt not in platform_extra: > - platform_extra.append(plt) > - platform_extra = default_platform_extra + platform_extra > - > - self._create_configs(platform, platform_extra) > + self._configure_dnf() > + self._configure_rpm() > > - #takes array args > - def _invoke_smart(self, args): > - cmd = [self.smart_cmd] + self.smart_opt + args > - # bb.note(cmd) > - try: > - complementary_pkgs = > subprocess.check_output(cmd,stderr=subprocess.STDOUT).decode("utf-8") > - # bb.note(complementary_pkgs) > - return complementary_pkgs > - except subprocess.CalledProcessError as e: > - bb.fatal("Could not invoke smart. Command " > - "'%s' returned %d:\n%s" % (' '.join(cmd), > e.returncode, e.output.decode("utf-8"))) > - > - def _search_pkg_name_in_feeds(self, pkg, feed_archs): > - for arch in feed_archs: > - arch = arch.replace('-', '_') > - regex_match = re.compile(r"^%s-[^-]*-[^-]*@%s$" % \ > - (re.escape(pkg), re.escape(arch))) > - for p in self.fullpkglist: > - if regex_match.match(p) is not None: > - # First found is best match > - # bb.note('%s -> %s' % (pkg, pkg + '@' + arch)) > - return pkg + '@' + arch > - > - # Search provides if not found by pkgname. > - bb.note('Not found %s by name, searching provides ...' % pkg) > - cmd = [self.smart_cmd] + self.smart_opt + ["query", "--provides", > pkg, > - "--show-format=$name-$version"] > - bb.note('cmd: %s' % ' '.join(cmd)) > - ps = subprocess.Popen(cmd, stdout=subprocess.PIPE) > - try: > - output = subprocess.check_output(["sed", "-ne", "s/ > *Provides://p"], > - stdin=ps.stdout, stderr=subprocess.STDOUT).decode("utf- > 8") > - # Found a provider > - if output: > - bb.note('Found providers for %s: %s' % (pkg, output)) > - for p in output.split(): > - for arch in feed_archs: > - arch = arch.replace('-', '_') > - if p.rstrip().endswith('@' + arch): > - return p > - except subprocess.CalledProcessError as e: > - bb.error("Failed running smart query on package %s." % pkg) > - > - return "" > + def write_index(self): > + RpmIndexer(self.d, self.rpm_repo_dir).write_index() > > - ''' > - Translate the OE multilib format names to the RPM/Smart format names > - It searched the RPM/Smart format names in probable multilib feeds > first, > - and then searched the default base feed. > - ''' > - def _pkg_translate_oe_to_smart(self, pkgs, attempt_only=False): > - new_pkgs = list() > - > - for pkg in pkgs: > - new_pkg = pkg > - # Search new_pkg in probable multilibs first > - for mlib in self.ml_prefix_list: > - # Jump the default archs > - if mlib == 'default': > - continue > + def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): > + if feed_uris == "": > + return > > - subst = pkg.replace(mlib + '-', '') > - # if the pkg in this multilib feed > - if subst != pkg: > - feed_archs = self.ml_prefix_list[mlib] > - new_pkg = self._search_pkg_name_in_feeds(subst, > feed_archs) > - if not new_pkg: > - # Failed to translate, package not found! > - err_msg = '%s not found in the %s feeds (%s) in > %s.' % \ > - (pkg, mlib, " ".join(feed_archs), > self.d.getVar('DEPLOY_DIR_RPM')) > - if not attempt_only: > - bb.error(err_msg) > - bb.fatal("This is often caused by an empty > package declared " \ > - "in a recipe's PACKAGES variable. > (Empty packages are " \ > - "not constructed unless > ALLOW_EMPTY_<pkg> = '1' is used.)") > - bb.warn(err_msg) > - else: > - new_pkgs.append(new_pkg) > - > - break > - > - # Apparently not a multilib package... > - if pkg == new_pkg: > - # Search new_pkg in default archs > - default_archs = self.ml_prefix_list['default'] > - new_pkg = self._search_pkg_name_in_feeds(pkg, > default_archs) > - if not new_pkg: > - err_msg = '%s not found in the feeds (%s) in %s.' % \ > - (pkg, " ".join(default_archs), > self.d.getVar('DEPLOY_DIR_RPM')) > - if not attempt_only: > - bb.error(err_msg) > - bb.fatal("This is often caused by an empty > package declared " \ > - "in a recipe's PACKAGES variable. (Empty > packages are " \ > - "not constructed unless > ALLOW_EMPTY_<pkg> = '1' is used.)") > - bb.warn(err_msg) > - else: > - new_pkgs.append(new_pkg) > - > - return new_pkgs > - > - def _create_configs(self, platform, platform_extra): > - # Setup base system configuration > - bb.note("configuring RPM platform settings") > - > - # Configure internal RPM environment when using Smart > - os.environ['RPM_ETCRPM'] = self.etcrpm_dir > - bb.utils.mkdirhier(self.etcrpm_dir) > - > - # Setup temporary directory -- install... > - if os.path.exists(self.install_dir_path): > - bb.utils.remove(self.install_dir_path, True) > - bb.utils.mkdirhier(os.path.join(self.install_dir_path, 'tmp')) > - > - channel_priority = 5 > - platform_dir = os.path.join(self.etcrpm_dir, "platform") > - sdkos = self.d.getVar("SDK_OS") > - with open(platform_dir, "w+") as platform_fd: > - platform_fd.write(platform + '\n') > - for pt in platform_extra: > - channel_priority += 5 > - if sdkos: > - tmp = re.sub("-%s$" % sdkos, "-%s\n" % sdkos, pt) > - tmp = re.sub("-linux.*$", "-linux.*\n", tmp) > - platform_fd.write(tmp) > - > - # Tell RPM that the "/" directory exist and is available > - bb.note("configuring RPM system provides") > - sysinfo_dir = os.path.join(self.etcrpm_dir, "sysinfo") > - bb.utils.mkdirhier(sysinfo_dir) > - with open(os.path.join(sysinfo_dir, "Dirnames"), "w+") as > dirnames: > - dirnames.write("/\n") > - > - if self.providename: > - providename_dir = os.path.join(sysinfo_dir, "Providename") > - if not os.path.exists(providename_dir): > - providename_content = '\n'.join(self.providename) > - providename_content += '\n' > - open(providename_dir, "w+").write(providename_content) > - > - # Configure RPM... we enforce these settings! > - bb.note("configuring RPM DB settings") > - # After change the __db.* cache size, log file will not be > - # generated automatically, that will raise some warnings, > - # so touch a bare log for rpm write into it. > - rpmlib_log = os.path.join(self.image_rpmlib, 'log', > 'log.0000000001') > - if not os.path.exists(rpmlib_log): > - bb.utils.mkdirhier(os.path.join(self.image_rpmlib, 'log')) > - open(rpmlib_log, 'w+').close() > - > - DB_CONFIG_CONTENT = "# ================ Environment\n" \ > - "set_data_dir .\n" \ > - "set_create_dir .\n" \ > - "set_lg_dir ./log\n" \ > - "set_tmp_dir ./tmp\n" \ > - "set_flags db_log_autoremove on\n" \ > - "\n" \ > - "# -- thread_count must be >= 8\n" \ > - "set_thread_count 64\n" \ > - "\n" \ > - "# ================ Logging\n" \ > - "\n" \ > - "# ================ Memory Pool\n" \ > - "set_cachesize 0 1048576 0\n" \ > - "set_mp_mmapsize 268435456\n" \ > - "\n" \ > - "# ================ Locking\n" \ > - "set_lk_max_locks 16384\n" \ > - "set_lk_max_lockers 16384\n" \ > - "set_lk_max_objects 16384\n" \ > - "mutex_set_max 163840\n" \ > - "\n" \ > - "# ================ Replication\n" > - > - db_config_dir = os.path.join(self.image_rpmlib, 'DB_CONFIG') > - if not os.path.exists(db_config_dir): > - open(db_config_dir, 'w+').write(DB_CONFIG_CONTENT) > - > - # Create database so that smart doesn't complain (lazy init) > - cmd = [self.rpm_cmd, '--root', self.target_rootfs, '--dbpath', > '/var/lib/rpm', '-qa'] > - try: > - subprocess.check_output(cmd, stderr=subprocess.STDOUT) > - except subprocess.CalledProcessError as e: > - bb.fatal("Create rpm database failed. Command '%s' " > - "returned %d:\n%s" % (' '.join(cmd), e.returncode, > e.output.decode("utf-8"))) > - # Import GPG key to RPM database of the target system > - if self.d.getVar('RPM_SIGN_PACKAGES') == '1': > - pubkey_path = self.d.getVar('RPM_GPG_PUBKEY') > - cmd = [self.rpm_cmd, '--root', self.target_rootfs, ' > --dbpath', '/var/lib/rpm', '--import', pubkey_path] > - try: > - subprocess.check_output(cmd, stderr=subprocess.STDOUT) > - except subprocess.CalledProcessError as e: > - bb.fatal("Import GPG key failed. Command '%s' " > - "returned %d:\n%s" % (' '.join(cmd), > e.returncode, e.output.decode("utf-8"))) > - > - > - # Configure smart > - bb.note("configuring Smart settings") > - bb.utils.remove(os.path.join(self.target_rootfs, > 'var/lib/smart'), > - True) > - self._invoke_smart(['config', '--set', 'rpm-root=%s' % > self.target_rootfs]) > - self._invoke_smart(['config', '--set', 'rpm- > dbpath=/var/lib/rpm']) > - self._invoke_smart(['config', '--set', 'rpm-extra-macros._var=%s' > % > - self.d.getVar('localstatedir')]) > - cmd = ["config", "--set", "rpm-extra-macros._tmppath=/%s/tmp" % > self.install_dir_name] > - > - prefer_color = self.d.getVar('RPM_PREFER_ELF_ARCH') > - if prefer_color: > - if prefer_color not in ['0', '1', '2', '4']: > - bb.fatal("Invalid RPM_PREFER_ELF_ARCH: %s, it should be > one of:\n" > - "\t1: ELF32 wins\n" > - "\t2: ELF64 wins\n" > - "\t4: ELF64 N32 wins (mips64 or mips64el only)" % > - prefer_color) > - if prefer_color == "4" and self.d.getVar("TUNE_ARCH") not in > \ > - ['mips64', 'mips64el']: > - bb.fatal("RPM_PREFER_ELF_ARCH = \"4\" is for mips64 or > mips64el " > - "only.") > - self._invoke_smart(['config', '--set', 'rpm-extra- > macros._prefer_color=%s' > - % prefer_color]) > - > - self._invoke_smart(cmd) > - self._invoke_smart(['config', '--set', 'rpm-ignoresize=1']) > - > - # Write common configuration for host and target usage > - self._invoke_smart(['config', '--set', 'rpm-nolinktos=1']) > - self._invoke_smart(['config', '--set', 'rpm-noparentdirs=1']) > - check_signature = self.d.getVar('RPM_CHECK_SIGNATURES') > - if check_signature and check_signature.strip() == "0": > - self._invoke_smart(['config', '--set rpm-check- > signatures=false']) > - for i in self.d.getVar('BAD_RECOMMENDATIONS').split(): > - self._invoke_smart(['flag', '--set', 'ignore-recommends', i]) > - > - # Do the following configurations here, to avoid them being > - # saved for field upgrade > - if self.d.getVar('NO_RECOMMENDATIONS').strip() == "1": > - self._invoke_smart(['config', '--set', 'ignore-all- > recommends=1']) > - pkg_exclude = self.d.getVar('PACKAGE_EXCLUDE') or "" > - for i in pkg_exclude.split(): > - self._invoke_smart(['flag', '--set', 'exclude-packages', i]) > - > - # Optional debugging > - # self._invoke_smart(['config', '--set', 'rpm-log-level=debug']) > - # cmd = ['config', '--set', 'rpm-log-file=/tmp/smart-debug- > logfile'] > - # self._invoke_smart(cmd) > - ch_already_added = [] > - for canonical_arch in platform_extra: > - arch = canonical_arch.split('-')[0] > - arch_channel = os.path.join(self.d.getVar('WORKDIR'), 'rpms', > arch) > - oe.path.remove(arch_channel) > - deploy_arch_dir = os.path.join(self.deploy_dir, arch) > - if not os.path.exists(deploy_arch_dir): > - continue > + raise NotImplementedError("Adding remote dnf feeds not yet > supported.") > > - lockfilename = self.d.getVar('DEPLOY_DIR_RPM') + "/rpm.lock" > - lf = bb.utils.lockfile(lockfilename, False) > - oe.path.copyhardlinktree(deploy_arch_dir, arch_channel) > - bb.utils.unlockfile(lf) > - > - if not arch in ch_already_added: > - bb.note('Adding Smart channel %s (%s)' % > - (arch, channel_priority)) > - self._invoke_smart(['channel', '--add', arch, 'type=rpm- > md', > - 'baseurl=%s' % arch_channel, '-y']) > - self._invoke_smart(['channel', '--set', arch, > 'priority=%d' % > - channel_priority]) > - channel_priority -= 5 > - > - ch_already_added.append(arch) > - > - bb.note('adding Smart RPM DB channel') > - self._invoke_smart(['channel', '--add', 'rpmsys', 'type=rpm-sys', > '-y']) > - > - # Construct install scriptlet wrapper. > - # Scripts need to be ordered when executed, this ensures numeric > order. > - # If we ever run into needing more the 899 scripts, we'll have > to. > - # change num to start with 1000. > - # > - scriptletcmd = "$2 $1/$3 $4\n" > - scriptpath = "$1/$3" > - > - # When self.debug_level >= 3, also dump the content of the > - # executed scriptlets and how they get invoked. We have to > - # replace "exit 1" and "ERR" because printing those as-is > - # would trigger a log analysis failure. > - if self.debug_level >= 3: > - dump_invocation = 'echo "Executing ${name} ${kind} with: ' + > scriptletcmd + '"\n' > - dump_script = 'cat ' + scriptpath + '| sed -e "s/exit > 1/exxxit 1/g" -e "s/ERR/IRR/g"; echo\n' > - else: > - dump_invocation = 'echo "Executing ${name} ${kind}"\n' > - dump_script = '' > - > - SCRIPTLET_FORMAT = "#!/bin/bash\n" \ > - "\n" \ > - "export PATH=%s\n" \ > - "export D=%s\n" \ > - 'export OFFLINE_ROOT="$D"\n' \ > - 'export IPKG_OFFLINE_ROOT="$D"\n' \ > - 'export OPKG_OFFLINE_ROOT="$D"\n' \ > - "export INTERCEPT_DIR=%s\n" \ > - "export NATIVE_ROOT=%s\n" \ > - "\n" \ > - "name=`head -1 " + scriptpath + " | cut -d\' \' -f 2`\n" \ > - "kind=`head -1 " + scriptpath + " | cut -d\' \' -f 4`\n" \ > - + dump_invocation \ > - + dump_script \ > - + scriptletcmd + \ > - "ret=$?\n" \ > - "echo Result of ${name} ${kind}: ${ret}\n" \ > - "if [ ${ret} -ne 0 ]; then\n" \ > - " if [ $4 -eq 1 ]; then\n" \ > - " mkdir -p $1/etc/rpm-postinsts\n" \ > - " num=100\n" \ > - " while [ -e $1/etc/rpm-postinsts/${num}-* ]; do > num=$((num + 1)); done\n" \ > - ' echo "#!$2" > $1/etc/rpm-postinsts/${num}-${name}\n' \ > - ' echo "# Arg: $4" >> $1/etc/rpm-postinsts/${num}- > ${name}\n' \ > - " cat " + scriptpath + " >> $1/etc/rpm-postinsts/${num}- > ${name}\n" \ > - " chmod +x $1/etc/rpm-postinsts/${num}-${name}\n" \ > - ' echo "Info: deferring ${name} ${kind} install scriptlet > to first boot"\n' \ > - " else\n" \ > - ' echo "Error: ${name} ${kind} remove scriptlet failed"\n' > \ > - " fi\n" \ > - "fi\n" > - > - intercept_dir = self.d.expand('${WORKDIR}/intercept_scripts') > - native_root = self.d.getVar('STAGING_DIR_NATIVE') > - scriptlet_content = SCRIPTLET_FORMAT % (os.environ['PATH'], > - self.target_rootfs, > - intercept_dir, > - native_root) > - open(self.scriptlet_wrapper, 'w+').write(scriptlet_content) > - > - bb.note("configuring RPM cross-install scriptlet_wrapper") > - os.chmod(self.scriptlet_wrapper, 0o755) > - cmd = ['config', '--set', 'rpm-extra- > macros._cross_scriptlet_wrapper=%s' % > - self.scriptlet_wrapper] > - self._invoke_smart(cmd) > - > - # Debug to show smart config info > - # bb.note(self._invoke_smart(['config', '--show'])) > + def _prepare_pkg_transaction(self): > + os.environ['D'] = self.target_rootfs > + os.environ['OFFLINE_ROOT'] = self.target_rootfs > + os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs > + os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs > + os.environ['INTERCEPT_DIR'] = > oe.path.join(self.d.getVar('WORKDIR'), > + "intercept_scripts") > + os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE') > > - def update(self): > - self._invoke_smart(['update', 'rpmsys']) > - > - def get_rdepends_recursively(self, pkgs): > - # pkgs will be changed during the loop, so use [:] to make a > copy. > - for pkg in pkgs[:]: > - sub_data = oe.packagedata.read_subpkgdata(pkg, self.d) > - sub_rdep = sub_data.get("RDEPENDS_" + pkg) > - if not sub_rdep: > - continue > - done = list(bb.utils.explode_dep_versions2(sub_rdep).keys()) > - next = done > - # Find all the rdepends on dependency chain > - while next: > - new = [] > - for sub_pkg in next: > - sub_data = oe.packagedata.read_subpkgdata(sub_pkg, > self.d) > - sub_pkg_rdep = sub_data.get("RDEPENDS_" + sub_pkg) > - if not sub_pkg_rdep: > - continue > - for p in > bb.utils.explode_dep_versions2(sub_pkg_rdep): > - # Already handled, skip it. > - if p in done or p in pkgs: > - continue > - # It's a new dep > - if oe.packagedata.has_subpkgdata(p, self.d): > - done.append(p) > - new.append(p) > - next = new > - pkgs.extend(done) > - return pkgs > + os.environ['RPM_NO_CHROOT_FOR_SCRIPTS'] = "1" > > - ''' > - Install pkgs with smart, the pkg name is oe format > - ''' > - def install(self, pkgs, attempt_only=False): > > - if not pkgs: > - bb.note("There are no packages to install") > + def install(self, pkgs, attempt_only = False): > + if len(pkgs) == 0: > return > - bb.note("Installing the following packages: %s" % ' '.join(pkgs)) > - if not attempt_only: > - # Pull in multilib requires since rpm may not pull in them > - # correctly, for example, > - # lib32-packagegroup-core-standalone-sdk-target requires > - # lib32-libc6, but rpm may pull in libc6 rather than lib32- > libc6 > - # since it doesn't know mlprefix (lib32-), bitbake knows it > and > - # can handle it well, find out the RDEPENDS on the chain will > - # fix the problem. Both do_rootfs and do_populate_sdk have > this > - # issue. > - # The attempt_only packages don't need this since they are > - # based on the installed ones. > - # > - # Separate pkgs into two lists, one is multilib, the other > one > - # is non-multilib. > - ml_pkgs = [] > - non_ml_pkgs = pkgs[:] > - for pkg in pkgs: > - for mlib in (self.d.getVar("MULTILIB_VARIANTS") or > "").split(): > - if pkg.startswith(mlib + '-'): > - ml_pkgs.append(pkg) > - non_ml_pkgs.remove(pkg) > - > - if len(ml_pkgs) > 0 and len(non_ml_pkgs) > 0: > - # Found both foo and lib-foo > - ml_pkgs = self.get_rdepends_recursively(ml_pkgs) > - non_ml_pkgs = self.get_rdepends_recursively(non_ml_pkgs) > - # Longer list makes smart slower, so only keep the pkgs > - # which have the same BPN, and smart can handle others > - # correctly. > - pkgs_new = [] > - for pkg in non_ml_pkgs: > - for mlib in (self.d.getVar("MULTILIB_VARIANTS") or > "").split(): > - mlib_pkg = mlib + "-" + pkg > - if mlib_pkg in ml_pkgs: > - pkgs_new.append(pkg) > - pkgs_new.append(mlib_pkg) > - for pkg in pkgs: > - if pkg not in pkgs_new: > - pkgs_new.append(pkg) > - pkgs = pkgs_new > - new_depends = {} > - deps = bb.utils.explode_dep_versions2(" ".join(pkgs)) > - for depend in deps: > - data = oe.packagedata.read_subpkgdata(depend, self.d) > - key = "PKG_%s" % depend > - if key in data: > - new_depend = data[key] > - else: > - new_depend = depend > - new_depends[new_depend] = deps[depend] > - pkgs = bb.utils.join_deps(new_depends, > commasep=True).split(', ') > - pkgs = self._pkg_translate_oe_to_smart(pkgs, attempt_only) > - if not pkgs: > - bb.note("There are no packages to install") > + self._prepare_pkg_transaction() > + > + bad_recommendations = self.d.getVar('BAD_RECOMMENDATIONS') > + package_exclude = self.d.getVar('PACKAGE_EXCLUDE') > + exclude_pkgs = (bad_recommendations.split() if > bad_recommendations else []) + (package_exlcude.split() if package_exclude > else []) > + > + output = self._invoke_dnf((["--skip-broken"] if attempt_only else > []) + > + (["-x", ",".join(exclude_pkgs)] if > len(exclude_pkgs) > 0 else []) + > + (["--setopt=install_weak_deps=False"] if > self.d.getVar('NO_RECOMMENDATIONS') == 1 else []) + > + ["--nogpgcheck", "install"] + > + pkgs) > + failed_scriptlets_pkgnames = collections.OrderedDict() > + for line in output.splitlines(): > + if line.startswith("Non-fatal POSTIN scriptlet failure in rpm > package"): > + failed_scriptlets_pkgnames[line.split()[-1]] = True > + > + if len(failed_scriptlets_pkgnames) > 0: > + bb.warn("Intentionally using 'exit 1' to defer postinstall > scriptlets of %s to first boot is deprecated. Please place them into > pkg_postinst_ontarget_${PN} ()." > %(list(failed_scriptlets_pkgnames.keys()))) > + bb.warn("If deferring to first boot wasn't the intent, then > scriptlet failure may mean an issue in the recipe, or a regression > elsewhere.") > + for pkg in failed_scriptlets_pkgnames.keys(): > + # Path does not exist when building SDKs > + if os.path.exists(oe.path.join(os.environ['INTERCEPT_DIR'], > "postinst_intercept")): > + subprocess.check_output([oe.path.join(os.environ['INTERCE > PT_DIR'], "postinst_intercept"), "delay_to_first_boot", pkg, "mlprefix=%s" > %(self.d.getVar('MLPREFIX'))]) > + > + def remove(self, pkgs, with_dependencies = True): > + if len(pkgs) == 0: > return > - if not attempt_only: > - bb.note('to be installed: %s' % ' '.join(pkgs)) > - cmd = [self.smart_cmd] + self.smart_opt + ["install", "-y"] + > pkgs > - bb.note(' '.join(cmd)) > - else: > - bb.note('installing attempt only packages...') > - bb.note('Attempting %s' % ' '.join(pkgs)) > - cmd = [self.smart_cmd] + self.smart_opt + ["install", " > --attempt", > - "-y"] + pkgs > - try: > - output = subprocess.check_output(cmd, > stderr=subprocess.STDOUT).decode("utf-8") > - bb.note(output) > - except subprocess.CalledProcessError as e: > - bb.fatal("Unable to install packages. Command '%s' " > - "returned %d:\n%s" % (' '.join(cmd), e.returncode, > e.output.decode("utf-8"))) > + self._prepare_pkg_transaction() > > - ''' > - Remove pkgs with smart, the pkg name is smart/rpm format > - ''' > - def remove(self, pkgs, with_dependencies=True): > - bb.note('to be removed: ' + ' '.join(pkgs)) > - > - if not with_dependencies: > - cmd = [self.rpm_cmd] + ["-e", "--nodeps", "--root=%s" % > - self.target_rootfs, "--dbpath=/var/lib/rpm", > - "--define='_cross_scriptlet_wrapper %s'" % > - self.scriptlet_wrapper, > - "--define='_tmppath /%s/tmp'" % > self.install_dir_name] + pkgs > - else: > - # for pkg in pkgs: > - # bb.note('Debug: What required: %s' % pkg) > - # bb.note(self._invoke_smart(['query', pkg, '--show- > requiredby'])) > - cmd = [self.smart_cmd] + self.smart_opt + ["remove", "-y"] + > pkgs > - try: > - bb.note(' '.join(cmd)) > - output = subprocess.check_output(cmd, > stderr=subprocess.STDOUT).decode("utf-8") > - bb.note(output) > - except subprocess.CalledProcessError as e: > - bb.note("Unable to remove packages. Command '%s' " > - "returned %d:\n%s" % (cmd, e.returncode, > e.output.decode("utf-8"))) > + self._invoke_dnf((["--setopt=clean_requirements_on_remove=False"] > if with_dependencies == False else []) + > + ["remove"] + pkgs) > > def upgrade(self): > - bb.note('smart upgrade') > - self._invoke_smart(['upgrade']) > + self._prepare_pkg_transaction() > + self._invoke_dnf(["upgrade"]) > > - def write_index(self): > - result = self.indexer.write_index() > - > - if result is not None: > - bb.fatal(result) > + def autoremove(self): > + self._prepare_pkg_transaction() > + self._invoke_dnf(["autoremove"]) > > def remove_packaging_data(self): > - bb.utils.remove(self.image_rpmlib, True) > - bb.utils.remove(os.path.join(self.target_rootfs, > 'var/lib/smart'), > - True) > - bb.utils.remove(os.path.join(self.target_rootfs, 'var/lib/opkg'), > True) > - > - # remove temp directory > - bb.utils.remove(self.install_dir_path, True) > + self._invoke_dnf(["clean", "all"]) > + for dir in self.packaging_data_dirs: > + bb.utils.remove(oe.path.join(self.target_rootfs, dir), True) > > def backup_packaging_data(self): > - # Save the rpmlib for increment rpm image generation > - if os.path.exists(self.saved_rpmlib): > - bb.utils.remove(self.saved_rpmlib, True) > - shutil.copytree(self.image_rpmlib, > - self.saved_rpmlib, > - symlinks=True) > + # Save the packaging dirs for increment rpm image generation > + if os.path.exists(self.saved_packaging_data): > + bb.utils.remove(self.saved_packaging_data, True) > + for i in self.packaging_data_dirs: > + source_dir = oe.path.join(self.target_rootfs, i) > + target_dir = oe.path.join(self.saved_packaging_data, i) > + shutil.copytree(source_dir, target_dir, symlinks=True) > > def recovery_packaging_data(self): > # Move the rpmlib back > - if os.path.exists(self.saved_rpmlib): > - if os.path.exists(self.image_rpmlib): > - bb.utils.remove(self.image_rpmlib, True) > - > - bb.note('Recovery packaging data') > - shutil.copytree(self.saved_rpmlib, > - self.image_rpmlib, > + if os.path.exists(self.saved_packaging_data): > + for i in self.packaging_data_dirs: > + target_dir = oe.path.join(self.target_rootfs, i) > + if os.path.exists(target_dir): > + bb.utils.remove(target_dir, True) > + source_dir = oe.path.join(self.saved_packaging_data, i) > + shutil.copytree(source_dir, > + target_dir, > symlinks=True) > > def list_installed(self): > - return self.pkgs_list.list_pkgs() > - > - ''' > - If incremental install, we need to determine what we've got, > - what we need to add, and what to remove... > - The dump_install_solution will dump and save the new install > - solution. > - ''' > - def dump_install_solution(self, pkgs): > - bb.note('creating new install solution for incremental install') > - if len(pkgs) == 0: > - return > - > - pkgs = self._pkg_translate_oe_to_smart(pkgs, False) > - install_pkgs = list() > + output = self._invoke_dnf(["repoquery", "--installed", " > --queryformat", "Package: %{name} %{arch} %{version} > %{sourcerpm}\nDependencies:\n%{requires}\nRecommendations:\n%{recommends}\ > nDependenciesEndHere:\n"], > + bb_note = False) > + packages = {} > + current_package = None > + current_deps = None > + current_state = "initial" > + for line in output.splitlines(): > + if line.startswith("Package:"): > + package_info = line.split(" ")[1:] > + current_package = package_info[0] > + package_arch = package_info[1] > + package_version = package_info[2] > + package_srpm = package_info[3] > + packages[current_package] = {"arch":package_arch, > "ver":package_version, "filename":package_srpm} > + current_deps = [] > + elif line.startswith("Dependencies:"): > + current_state = "dependencies" > + elif line.startswith("Recommendations"): > + current_state = "recommendations" > + elif line.startswith("DependenciesEndHere:"): > + current_state = "initial" > + packages[current_package]["deps"] = current_deps > + elif len(line) > 0: > + if current_state == "dependencies": > + current_deps.append(line) > + elif current_state == "recommendations": > + current_deps.append("%s [REC]" % line) > + > + return packages > > - cmd = [self.smart_cmd] + self.smart_opt + ['install', '-y', ' > --dump'] + pkgs > + def update(self): > + self._invoke_dnf(["makecache"]) > + > + def _invoke_dnf(self, dnf_args, bb_fatal = True, bb_note = True ): > + os.environ['RPM_CONFIGDIR'] = > oe.path.join(self.d.getVar('STAGING_LIBDIR_NATIVE'), "rpm") > + os.environ['RPM_ETCCONFIGDIR'] = self.target_rootfs > + > + dnf_cmd = bb.utils.which(os.getenv('PATH'), "dnf-2") > + standard_dnf_args = (["-v", "--rpmverbosity=debug"] if > self.d.getVar('ROOTFS_RPM_DEBUG') else []) + ["-y", > + "-c", oe.path.join(self.target_rootfs, > "etc/dnf/dnf.conf"), > + "--repofrompath=oe-repo,%s" % > (self.rpm_repo_dir), > + "--installroot=%s" % (self.target_rootfs), > + "--setopt=logdir=%s" % (self.d.getVar('T')) > + ] > + cmd = [dnf_cmd] + standard_dnf_args + dnf_args > try: > - # Disable rpmsys channel for the fake install > - self._invoke_smart(['channel', '--disable', 'rpmsys']) > - > - output = > subprocess.check_output(cmd,stderr=subprocess.STDOUT).decode('utf-8') > - f = open(self.solution_manifest, 'w') > - f.write(output) > - f.close() > - with open(self.solution_manifest, 'r') as manifest: > - for pkg in manifest.read().split('\n'): > - if '@' in pkg: > - install_pkgs.append(pkg.strip()) > + output = > subprocess.check_output(cmd,stderr=subprocess.STDOUT).decode("utf-8") > + if bb_note: > + bb.note(output) > + return output > except subprocess.CalledProcessError as e: > - bb.note("Unable to dump install packages. Command '%s' " > - "returned %d:\n%s" % (cmd, e.returncode, > e.output.decode("utf-8"))) > - # Recovery rpmsys channel > - self._invoke_smart(['channel', '--enable', 'rpmsys']) > - return install_pkgs > + (bb.note, bb.fatal)[bb_fatal]("Could not invoke dnf. Command > " > + "'%s' returned %d:\n%s" % (' '.join(cmd), > e.returncode, e.output.decode("utf-8"))) > + > + def dump_install_solution(self, pkgs): > + open(self.solution_manifest, 'w').write(" ".join(pkgs)) > + return pkgs > > - ''' > - If incremental install, we need to determine what we've got, > - what we need to add, and what to remove... > - The load_old_install_solution will load the previous install > - solution > - ''' > def load_old_install_solution(self): > - bb.note('load old install solution for incremental install') > - installed_pkgs = list() > if not os.path.exists(self.solution_manifest): > - bb.note('old install solution not exist') > - return installed_pkgs > + return [] > > - with open(self.solution_manifest, 'r') as manifest: > - for pkg in manifest.read().split('\n'): > - if '@' in pkg: > - installed_pkgs.append(pkg.strip()) > + return open(self.solution_manifest, 'r').read().split() > > - return installed_pkgs > - > - ''' > - Dump all available packages in feeds, it should be invoked after the > - newest rpm index was created > - ''' > - def dump_all_available_pkgs(self): > - available_manifest = > self.d.expand('${T}/saved/available_pkgs.txt') > - available_pkgs = list() > - cmd = [self.smart_cmd] + self.smart_opt + ['query', '--output', > available_manifest] > - try: > - subprocess.check_output(cmd, stderr=subprocess.STDOUT) > - with open(available_manifest, 'r') as manifest: > - for pkg in manifest.read().split('\n'): > - if '@' in pkg: > - available_pkgs.append(pkg.strip()) > - except subprocess.CalledProcessError as e: > - bb.note("Unable to list all available packages. Command '%s' > " > - "returned %d:\n%s" % (' '.join(cmd), e.returncode, > e.output.decode("utf-8"))) > - > - self.fullpkglist = available_pkgs > - > - return > + def _script_num_prefix(self, path): > + files = os.listdir(path) > + numbers = set() > + numbers.add(99) > + for f in files: > + numbers.add(int(f.split("-")[0])) > + return max(numbers) + 1 > > def save_rpmpostinst(self, pkg): > - mlibs = (self.d.getVar('MULTILIB_GLOBAL_VARIANTS', False) or > "").split() > - > - new_pkg = pkg > - # Remove any multilib prefix from the package name > - for mlib in mlibs: > - if mlib in pkg: > - new_pkg = pkg.replace(mlib + '-', '') > - break > - > - bb.note(' * postponing %s' % new_pkg) > - saved_dir = self.target_rootfs + > self.d.expand('${sysconfdir}/rpm-postinsts/') + new_pkg > - > - cmd = self.rpm_cmd + ' -q --scripts --root ' + self.target_rootfs > - cmd += ' --dbpath=/var/lib/rpm ' + new_pkg > - cmd += ' | sed -n -e "/^postinstall scriptlet (using .*):$/,/^.* > scriptlet (using .*):$/ {/.*/p}"' > - cmd += ' | sed -e "/postinstall scriptlet (using \(.*\)):$/d"' > - cmd += ' -e "/^.* scriptlet (using .*):$/d" > %s' % saved_dir > - > - try: > - bb.note(cmd) > - output = subprocess.check_output(cmd, > stderr=subprocess.STDOUT, shell=True).strip().decode("utf-8") > - bb.note(output) > - os.chmod(saved_dir, 0o755) > - except subprocess.CalledProcessError as e: > - bb.fatal("Invoke save_rpmpostinst failed. Command '%s' " > - "returned %d:\n%s" % (cmd, e.returncode, > e.output.decode("utf-8"))) > - > - '''Write common configuration for target usage''' > - def rpm_setup_smart_target_config(self): > - bb.utils.remove(os.path.join(self.target_rootfs, > 'var/lib/smart'), > - True) > - > - self._invoke_smart(['config', '--set', 'rpm-nolinktos=1']) > - self._invoke_smart(['config', '--set', 'rpm-noparentdirs=1']) > - for i in self.d.getVar('BAD_RECOMMENDATIONS').split(): > - self._invoke_smart(['flag', '--set', 'ignore-recommends', i]) > - self._invoke_smart(['channel', '--add', 'rpmsys', 'type=rpm-sys', > '-y']) > - > - ''' > - The rpm db lock files were produced after invoking rpm to query on > - build system, and they caused the rpm on target didn't work, so we > - need to unlock the rpm db by removing the lock files. > - ''' > - def unlock_rpm_db(self): > - # Remove rpm db lock files > - rpm_db_locks = glob.glob('%s/var/lib/rpm/__db.*' % > self.target_rootfs) > - for f in rpm_db_locks: > - bb.utils.remove(f, True) > + bb.note("Saving postinstall script of %s" % (pkg)) > + cmd = bb.utils.which(os.getenv('PATH'), "rpm") > + args = ["-q", "--root=%s" % self.target_rootfs, "--queryformat", > "%{postin}", pkg] > > - """ > - Returns a dictionary with the package info. > - """ > - def package_info(self, pkg): > - cmd = [self.smart_cmd] + self.smart_opt + ['info', '--urls', pkg] > try: > - output = subprocess.check_output(cmd, > stderr=subprocess.STDOUT).decode("utf-8") > + output = subprocess.check_output([cmd] + > args,stderr=subprocess.STDOUT).decode("utf-8") > except subprocess.CalledProcessError as e: > - bb.fatal("Unable to list available packages. Command '%s' " > - "returned %d:\n%s" % (' '.join(cmd), e.returncode, > e.output.decode("utf-8"))) > + bb_fatal("Could not invoke rpm. Command " > + "'%s' returned %d:\n%s" % (' '.join([cmd] + args), > e.returncode, e.output.decode("utf-8"))) > > - # Set default values to avoid UnboundLocalError > - arch = "" > - ver = "" > - filename = "" > + # may need to prepend #!/bin/sh to output > > - #Parse output > - for line in output.splitlines(): > - line = line.rstrip() > - if line.startswith("Name:"): > - pkg = line.split(": ")[1] > - elif line.startswith("Version:"): > - tmp_str = line.split(": ")[1] > - ver, arch = tmp_str.split("@") > - break > - > - # Get filename > - index = re.search("^URLs", output, re.MULTILINE) > - tmp_str = output[index.end():] > - for line in tmp_str.splitlines(): > - if "/" in line: > - line = line.lstrip() > - filename = line.split(" ")[0] > - break > - > - # To have the same data type than other package_info methods > - filepath = os.path.join(self.deploy_dir, arch, filename) > - pkg_dict = {} > - pkg_dict[pkg] = {"arch":arch, "ver":ver, "filename":filename, > - "filepath": filepath} > - > - return pkg_dict > - > - """ > - Returns the path to a tmpdir where resides the contents of a package. > + target_path = oe.path.join(self.target_rootfs, > self.d.expand('${sysconfdir}/rpm-postinsts/')) > + num = self._script_num_prefix(target_path) > + saved_script_name = oe.path.join(target_path, "%d-%s" % (num, > pkg)) > + open(saved_script_name, 'w').write(output) > + os.chmod(saved_script_name, 0o755) > > - Deleting the tmpdir is responsability of the caller. > - > - """ > def extract(self, pkg): > - pkg_info = self.package_info(pkg) > - if not pkg_info: > - bb.fatal("Unable to get information for package '%s' while " > - "trying to extract the package." % pkg) > - > - pkg_path = pkg_info[pkg]["filepath"] > + output = self._invoke_dnf(["repoquery", "--queryformat", > "%{location}", pkg]) > + pkg_name = output.splitlines()[-1] > + if not pkg_name.endswith(".rpm"): > + bb.fatal("dnf could not find package %s in repository: %s" > %(pkg, output)) > + pkg_path = oe.path.join(self.rpm_repo_dir, pkg_name) > > cpio_cmd = bb.utils.which(os.getenv("PATH"), "cpio") > rpm2cpio_cmd = bb.utils.which(os.getenv("PATH"), "rpm2cpio") > @@ -1726,7 +962,7 @@ class OpkgPM(OpkgDpkgPM): > for uri in feed_uris: > if archs: > for arch in archs: > - if (feed_archs is None) and (not > os.path.exists(os.path.join(self.deploy_dir, arch))): > + if (feed_archs is None) and (not > os.path.exists(oe.path.join(self.deploy_dir, arch))): > continue > bb.note('Adding opkg feed url-%s-%d (%s)' % > (arch, uri_iterator, uri)) > diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py > index abcc852ba45..643508e6741 100644 > --- a/meta/lib/oe/rootfs.py > +++ b/meta/lib/oe/rootfs.py > @@ -442,6 +442,8 @@ class RpmRootfs(Rootfs): > bb.note('incremental removed: %s' % ' > '.join(pkg_to_remove)) > self.pm.remove(pkg_to_remove) > > + self.pm.autoremove() > + > def _create(self): > pkgs_to_install = self.manifest.parse_initial_manifest() > rpm_pre_process_cmds = self.d.getVar('RPM_PREPROCESS_COMMANDS') > @@ -455,8 +457,6 @@ class RpmRootfs(Rootfs): > if self.progress_reporter: > self.progress_reporter.next_stage() > > - self.pm.dump_all_available_pkgs() > - > if self.inc_rpm_image_gen == "1": > self._create_incremental(pkgs_to_install) > > @@ -491,15 +491,13 @@ class RpmRootfs(Rootfs): > if self.progress_reporter: > self.progress_reporter.next_stage() > > - self._setup_dbg_rootfs(['/etc/rpm', '/var/lib/rpm', > '/var/lib/smart']) > + self._setup_dbg_rootfs(['/etc', '/var/lib/rpm', '/var/cache/dnf', > '/var/lib/dnf']) > > execute_pre_post_process(self.d, rpm_post_process_cmds) > > if self.inc_rpm_image_gen == "1": > self.pm.backup_packaging_data() > > - self.pm.rpm_setup_smart_target_config() > - > if self.progress_reporter: > self.progress_reporter.next_stage() > > @@ -537,15 +535,7 @@ class RpmRootfs(Rootfs): > self.pm.save_rpmpostinst(pkg) > > def _cleanup(self): > - # during the execution of postprocess commands, rpm is called > several > - # times to get the files installed, dependencies, etc. This > creates the > - # __db.00* (Berkeley DB files that hold locks, rpm specific > environment > - # settings, etc.), that should not get into the final rootfs > - self.pm.unlock_rpm_db() > - if os.path.isdir(self.pm.install_dir_path + "/tmp") and not > os.listdir(self.pm.install_dir_path + "/tmp"): > - bb.utils.remove(self.pm.install_dir_path + "/tmp", True) > - if os.path.isdir(self.pm.install_dir_path) and not > os.listdir(self.pm.install_dir_path): > - bb.utils.remove(self.pm.install_dir_path, True) > + pass > > class DpkgOpkgRootfs(Rootfs): > def __init__(self, d, progress_reporter=None, logcatcher=None): > diff --git a/meta/lib/oe/sdk.py b/meta/lib/oe/sdk.py > index fef02d0777e..deb823b6ec1 100644 > --- a/meta/lib/oe/sdk.py > +++ b/meta/lib/oe/sdk.py > @@ -130,7 +130,6 @@ class RpmSdk(Sdk): > > pm.create_configs() > pm.write_index() > - pm.dump_all_available_pkgs() > pm.update() > > pkgs = [] > @@ -188,7 +187,9 @@ class RpmSdk(Sdk): > True).strip('/'), > ) > self.mkdirhier(native_sysconf_dir) > - for f in glob.glob(os.path.join(self.sdk_output, "etc", "*")): > + for f in glob.glob(os.path.join(self.sdk_output, "etc", "rpm*")): > + self.movefile(f, native_sysconf_dir) > + for f in glob.glob(os.path.join(self.sdk_output, "etc", "dnf", > "*")): > self.movefile(f, native_sysconf_dir) > self.remove(os.path.join(self.sdk_output, "etc"), True) > > @@ -350,7 +351,7 @@ def sdk_list_installed_packages(d, target, > rootfs_dir=None): > if img_type == "rpm": > arch_var = ["SDK_PACKAGE_ARCHS", None][target is True] > os_var = ["SDK_OS", None][target is True] > - return RpmPkgsList(d, rootfs_dir, arch_var, os_var).list_pkgs() > + return RpmPkgsList(d, rootfs_dir).list_pkgs() > elif img_type == "ipk": > conf_file_var = ["IPKGCONF_SDK", "IPKGCONF_TARGET"][target is > True] > return OpkgPkgsList(d, rootfs_dir, > d.getVar(conf_file_var)).list_pkgs() > -- > 2.11.0 > -- _______________________________________________ Openembedded-core mailing list Openembedded-core@lists.openembedded.org http://lists.openembedded.org/mailman/listinfo/openembedded-core
