The -r option still uses the old make-based move_config() path, which requires cross-compiler toolchains and is much slower.
Handle -r in do_sync_defconfigs() instead: clone the reference commit, create a kconfiglib.Kconfig instance from that tree, load each defconfig against it, write a full .config, then load that into the current tree's Kconfig and write_min_config(). This gives the same result without any toolchain requirement. With move_config() now unreachable, remove it along with all the infrastructure it depended on: KconfigParser, Slot, Slots, DatabaseThread, get_make_cmd(), check_clean_directory(), the STATE_* constants and the now-unused queue and threading imports. Much of this is already dead before this patch, since -b uses do_build_db() and -s uses do_sync_defconfigs(), leaving move_config() reachable only for the -s -r combination. Defconfigs which use #include are left alone in this mode, since writing a minimal config would flatten them. Signed-off-by: Simon Glass <[email protected]> --- doc/develop/qconfig.rst | 24 +- tools/qconfig.py | 575 +++++----------------------------------- 2 files changed, 77 insertions(+), 522 deletions(-) diff --git a/doc/develop/qconfig.rst b/doc/develop/qconfig.rst index 4a85c78cbbb..7fdbad8923d 100644 --- a/doc/develop/qconfig.rst +++ b/doc/develop/qconfig.rst @@ -51,25 +51,21 @@ that use ``#include`` directives are handled by computing the delta between the full expanded config and the base provided by the included files, so the include structure is preserved. -The ``-r`` (git-ref) option still uses the old make-based path, since it -needs to build against a different source tree. +The ``-r`` (git-ref) option also uses kconfiglib: it clones the reference +commit, creates a Kconfig instance from that tree, loads each defconfig +against it, then normalises the result against the current tree's Kconfig. +No toolchains are needed. Toolchains ---------- -Toolchains are **not** needed for ``-b`` or ``-s``, since both use -kconfiglib to evaluate Kconfig files directly in Python. The only +Toolchains are **not** needed for any qconfig operation. The only difference from using a real toolchain is that ``CONFIG_GCC_VERSION`` reflects the host compiler rather than each board's cross-compiler. This does not affect database queries, imply analysis, or defconfig sync, since ``CONFIG_GCC_VERSION`` is a build-time value that never appears in defconfig files or influences Kconfig defaults. -The ``-r`` (git-ref) option still requires toolchains, as it falls back -to the make-based path. Most toolchains are available at the kernel.org -site. This tool uses the same tools as :doc:`../build/buildman`, so you -can use ``buildman --fetch-arch`` to fetch them. - Examples -------- @@ -256,12 +252,12 @@ Available options the number of threads is the same as the number of CPU cores. -r, --git-ref - Specify the git ref to clone for building the autoconf.mk. If unspecified - use the CWD. This is useful for when changes to the Kconfig affect the - default values and you want to capture the state of the defconfig from + Specify the git ref to clone for evaluating the Kconfig tree. If + unspecified, use the CWD. This is useful when changes to Kconfig affect + default values and you want to capture the state of defconfigs from before that change was in effect. If in doubt, specify a ref pre-Kconfig - changes (use HEAD if Kconfig changes are not committed). Worst case it will - take a bit longer to run, but will always do the right thing. + changes (use HEAD if Kconfig changes are not committed). Worst case it + will take a bit longer to run, but will always do the right thing. -v, --verbose Show any build errors as boards are built diff --git a/tools/qconfig.py b/tools/qconfig.py index a19a8e0118d..e0c4ae37021 100755 --- a/tools/qconfig.py +++ b/tools/qconfig.py @@ -18,32 +18,21 @@ import fnmatch import glob import multiprocessing import os -import queue import re import shutil import subprocess import sys import tempfile -import threading import time import unittest -from buildman import bsettings from buildman import kconfiglib -from buildman import toolchain from u_boot_pylib import terminal from u_boot_pylib.terminal import tprint from u_boot_pylib import tools -SHOW_GNU_MAKE = 'scripts/show-gnu-make' SLEEP_TIME=0.03 -STATE_IDLE = 0 -STATE_DEFCONFIG = 1 -STATE_AUTOCONF = 2 -STATE_SAVEDEFCONFIG = 3 - -AUTO_CONF_PATH = 'include/config/auto.conf' CONFIG_DATABASE = 'qconfig.db' FAILED_LIST = 'qconfig.failed' @@ -90,25 +79,6 @@ def check_top_directory(): if not os.path.exists(fname): sys.exit('Please run at the top of source directory.') -def check_clean_directory(): - """Exit if the source tree is not clean.""" - for fname in '.config', 'include/config': - if os.path.exists(fname): - sys.exit("source tree is not clean, please run 'make mrproper'") - -def get_make_cmd(): - """Get the command name of GNU Make. - - U-Boot needs GNU Make for building, but the command name is not - necessarily "make". (for example, "gmake" on FreeBSD). - Returns the most appropriate command name on your system. - """ - with subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) as proc: - ret = proc.communicate() - if proc.returncode: - sys.exit('GNU Make not found') - return ret[0].rstrip() - def get_matched_defconfig(line): """Get the defconfig files that match a pattern @@ -738,12 +708,16 @@ def _sync_include_defconfig(kconf, srcdir, orig, dry_run): def _sync_defconfigs_worker(srcdir, defconfigs, result_queue, error_queue, - dry_run): + dry_run, ref_srcdir=None): """Worker process that syncs defconfigs using kconfiglib For each defconfig, loads it via kconfiglib and writes a minimal config (equivalent to 'make savedefconfig'), then compares with the original. + When ref_srcdir is set (the -r option), loads each defconfig against the + reference Kconfig tree first, writes a full .config, then loads that + .config into the current tree's Kconfig and writes a minimal config. + Args: srcdir (str): Source-tree directory defconfigs (list of str): Defconfig filenames to process @@ -751,24 +725,63 @@ def _sync_defconfigs_worker(srcdir, defconfigs, result_queue, error_queue, (defconfig, updated) tuples error_queue (multiprocessing.Queue): Output queue for failed defconfigs dry_run (bool): If True, do not update defconfig files + ref_srcdir (str or None): Reference source tree for -r option """ - os.environ['srctree'] = srcdir os.environ['UBOOTVERSION'] = 'dummy' os.environ['KCONFIG_OBJDIR'] = '' os.environ['CC'] = 'gcc' + + os.environ['srctree'] = srcdir kconf = kconfiglib.Kconfig(warn=False) + if ref_srcdir: + os.environ['srctree'] = ref_srcdir + ref_kconf = kconfiglib.Kconfig(warn=False) + os.environ['srctree'] = srcdir + for defconfig in defconfigs: orig = os.path.join(srcdir, 'configs', defconfig) try: - raw = tools.read_file(orig) - has_include = b'#include' in raw + if ref_srcdir: + # Leave #include defconfigs alone: write_min_config() would + # flatten them, destroying the include structure + if b'#include' in tools.read_file(orig): + result_queue.put((defconfig, False, None)) + continue - if has_include: - updated = _sync_include_defconfig(kconf, srcdir, orig, - dry_run) + # Load defconfig against the reference Kconfig tree, write + # a full .config, then load it into the current tree + ref_orig = os.path.join(ref_srcdir, 'configs', defconfig) + if not os.path.exists(ref_orig): + ref_orig = orig + _load_defconfig(ref_kconf, ref_srcdir, ref_orig) + tmp_config = tempfile.NamedTemporaryFile( + prefix='qconfig-cfg-', delete=False) + tmp_config.close() + ref_kconf.write_config(tmp_config.name) + kconf.load_config(tmp_config.name, replace=True) + os.unlink(tmp_config.name) + else: + raw = tools.read_file(orig) + if b'#include' in raw: + updated = _sync_include_defconfig(kconf, srcdir, orig, + dry_run) + result_queue.put((defconfig, updated, None)) + continue + _load_defconfig(kconf, srcdir, orig) + + confdir = os.path.dirname(orig) + tmp = tempfile.NamedTemporaryFile( + mode='w', prefix='qconfig-', suffix='_defconfig', + dir=confdir, delete=False) + tmp.close() + kconf.write_min_config(tmp.name) + + updated = not filecmp.cmp(orig, tmp.name) + if updated and not dry_run: + shutil.move(tmp.name, orig) else: - updated = _sync_plain_defconfig(kconf, orig, dry_run) + os.unlink(tmp.name) result_queue.put((defconfig, updated, None)) except Exception as exc: error_queue.put((defconfig, str(exc))) @@ -780,15 +793,25 @@ def do_sync_defconfigs(args): Evaluates each defconfig through kconfiglib and writes a minimal config (equivalent to 'make savedefconfig'), updating the original if it differs. + When -r (git-ref) is given, loads each defconfig against the Kconfig tree + from the reference commit first, then normalises it against the current + tree, capturing any Kconfig default changes. + Args: args (Namespace): Program arguments (uses jobs, defconfigs, - defconfiglist, nocolour, dry_run, force_sync) + defconfiglist, nocolour, dry_run, force_sync, git_ref) Returns: Progress: progress indicator """ srcdir = os.getcwd() + if args.git_ref: + reference_src = ReferenceSource(args.git_ref) + ref_srcdir = reference_src.get_dir() + else: + ref_srcdir = None + if args.defconfigs: defconfigs = [os.path.basename(d) for d in get_matched_defconfigs(args.defconfigs)] @@ -813,7 +836,8 @@ def do_sync_defconfigs(args): continue proc = multiprocessing.Process( target=_sync_defconfigs_worker, - args=(srcdir, chunk, result_queue, error_queue, args.dry_run)) + args=(srcdir, chunk, result_queue, error_queue, args.dry_run, + ref_srcdir)) proc.start() processes.append(proc) @@ -859,403 +883,6 @@ def do_sync_defconfigs(args): return progress -# pylint: disable=R0903 -class KconfigParser: - """A parser of .config and include/autoconf.mk.""" - - re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') - re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') - - def __init__(self, build_dir): - """Create a new parser. - - Args: - build_dir: Build directory. - """ - self.dotconfig = os.path.join(build_dir, '.config') - self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') - self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', - 'autoconf.mk') - self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH) - self.defconfig = os.path.join(build_dir, 'defconfig') - - def get_arch(self): - """Parse .config file and return the architecture. - - Returns: - Architecture name (e.g. 'arm'). - """ - arch = '' - cpu = '' - for line in read_file(self.dotconfig): - m_arch = self.re_arch.match(line) - if m_arch: - arch = m_arch.group(1) - continue - m_cpu = self.re_cpu.match(line) - if m_cpu: - cpu = m_cpu.group(1) - - if not arch: - return None - - # fix-up for aarch64 - if arch == 'arm' and cpu == 'armv8': - arch = 'aarch64' - - return arch - - -class DatabaseThread(threading.Thread): - """This thread processes results from Slot threads. - - It collects the data in the master config directary. There is only one - result thread, and this helps to serialise the build output. - """ - def __init__(self, config_db, db_queue): - """Set up a new result thread - - Args: - builder: Builder which will be sent each result - """ - threading.Thread.__init__(self) - self.config_db = config_db - self.db_queue= db_queue - - def run(self): - """Called to start up the result thread. - - We collect the next result job and pass it on to the build. - """ - while True: - defconfig, configs = self.db_queue.get() - self.config_db[defconfig] = configs - self.db_queue.task_done() - - -class Slot: - - """A slot to store a subprocess. - - Each instance of this class handles one subprocess. - This class is useful to control multiple threads - for faster processing. - """ - - def __init__(self, toolchains, args, progress, devnull, make_cmd, - reference_src_dir, db_queue): - """Create a new process slot. - - Args: - toolchains: Toolchains object containing toolchains. - args: Program arguments - progress: A progress indicator. - devnull: A file object of '/dev/null'. - make_cmd: command name of GNU Make. - reference_src_dir: Determine the true starting config state from this - source tree. - db_queue: output queue to write config info for the database - col (terminal.Color): Colour object - """ - self.toolchains = toolchains - self.args = args - self.progress = progress - self.build_dir = tempfile.mkdtemp() - self.devnull = devnull - self.make_cmd = (make_cmd, 'O=' + self.build_dir) - self.reference_src_dir = reference_src_dir - self.db_queue = db_queue - self.col = progress.col - self.parser = KconfigParser(self.build_dir) - self.state = STATE_IDLE - self.failed_boards = set() - self.defconfig = None - self.log = [] - self.current_src_dir = None - self.proc = None - - def __del__(self): - """Delete the working directory - - This function makes sure the temporary directory is cleaned away - even if Python suddenly dies due to error. It should be done in here - because it is guaranteed the destructor is always invoked when the - instance of the class gets unreferenced. - - If the subprocess is still running, wait until it finishes. - """ - if self.state != STATE_IDLE: - while self.proc.poll() is None: - pass - shutil.rmtree(self.build_dir) - - def add(self, defconfig): - """Assign a new subprocess for defconfig and add it to the slot. - - If the slot is vacant, create a new subprocess for processing the - given defconfig and add it to the slot. Just returns False if - the slot is occupied (i.e. the current subprocess is still running). - - Args: - defconfig (str): defconfig name. - - Returns: - Return True on success or False on failure - """ - if self.state != STATE_IDLE: - return False - - self.defconfig = defconfig - self.log = [] - self.current_src_dir = self.reference_src_dir - self.do_defconfig() - return True - - def poll(self): - """Check the status of the subprocess and handle it as needed. - - Returns True if the slot is vacant (i.e. in idle state). - If the configuration is successfully finished, assign a new - subprocess to build include/autoconf.mk. - If include/autoconf.mk is generated, invoke the parser to - parse the .config and the include/autoconf.mk, moving - config options to the .config as needed. - If the .config was updated, run "make savedefconfig" to sync - it, update the original defconfig, and then set the slot back - to the idle state. - - Returns: - Return True if the subprocess is terminated, False otherwise - """ - if self.state == STATE_IDLE: - return True - - if self.proc.poll() is None: - return False - - if self.proc.poll() != 0: - self.handle_error() - elif self.state == STATE_DEFCONFIG: - if self.reference_src_dir and not self.current_src_dir: - self.do_savedefconfig() - else: - self.do_autoconf() - elif self.state == STATE_AUTOCONF: - if self.current_src_dir: - self.current_src_dir = None - self.do_defconfig() - elif self.args.build_db: - self.do_add_to_db() - else: - self.do_savedefconfig() - elif self.state == STATE_SAVEDEFCONFIG: - self.update_defconfig() - else: - sys.exit('Internal Error. This should not happen.') - - return self.state == STATE_IDLE - - def handle_error(self): - """Handle error cases.""" - - self.log.append(self.col.build(self.col.RED, 'Failed to process', - bright=True)) - if self.args.verbose: - for line in self.proc.stderr.read().decode().splitlines(): - self.log.append(self.col.build(self.col.CYAN, line, True)) - self.finish(False) - - def do_defconfig(self): - """Run 'make <board>_defconfig' to create the .config file.""" - - cmd = list(self.make_cmd) - cmd.append(self.defconfig) - # pylint: disable=R1732 - self.proc = subprocess.Popen(cmd, stdout=self.devnull, - stderr=subprocess.PIPE, - cwd=self.current_src_dir) - self.state = STATE_DEFCONFIG - - def do_autoconf(self): - """Run 'make AUTO_CONF_PATH'.""" - - arch = self.parser.get_arch() - try: - tchain = self.toolchains.Select(arch) - except ValueError: - self.log.append(self.col.build( - self.col.YELLOW, - f"Tool chain for '{arch}' is missing: do nothing")) - self.finish(False) - return - env = tchain.MakeEnvironment(False) - - cmd = list(self.make_cmd) - cmd.append('KCONFIG_IGNORE_DUPLICATES=1') - cmd.append(AUTO_CONF_PATH) - # pylint: disable=R1732 - self.proc = subprocess.Popen(cmd, stdout=self.devnull, env=env, - stderr=subprocess.PIPE, - cwd=self.current_src_dir) - self.state = STATE_AUTOCONF - - def do_add_to_db(self): - """Add the board to the database""" - configs = {} - for line in read_file(os.path.join(self.build_dir, AUTO_CONF_PATH)): - if line.startswith('CONFIG'): - config, value = line.split('=', 1) - configs[config] = value.rstrip() - self.db_queue.put([self.defconfig, configs]) - self.finish(True) - - def do_savedefconfig(self): - """Update the .config and run 'make savedefconfig'.""" - if not self.args.force_sync: - self.finish(True) - return - - cmd = list(self.make_cmd) - cmd.append('savedefconfig') - # pylint: disable=R1732 - self.proc = subprocess.Popen(cmd, stdout=self.devnull, - stderr=subprocess.PIPE) - self.state = STATE_SAVEDEFCONFIG - - def update_defconfig(self): - """Update the input defconfig and go back to the idle state.""" - orig_defconfig = os.path.join('configs', self.defconfig) - new_defconfig = os.path.join(self.build_dir, 'defconfig') - updated = not filecmp.cmp(orig_defconfig, new_defconfig) - success = True - - if updated: - # Files with #include get mangled as savedefconfig doesn't - # know how to deal with them. Ignore them - success = b'#include' not in tools.read_file(orig_defconfig) - if success: - self.log.append( - self.col.build(self.col.BLUE, 'defconfig updated', - bright=True)) - else: - self.log.append( - self.col.build(self.col.RED, 'ignored due to #include', - bright=True)) - updated = False - - if not self.args.dry_run and updated: - shutil.move(new_defconfig, orig_defconfig) - self.finish(success) - - def finish(self, success): - """Display log along with progress and go to the idle state. - - Args: - success (bool): Should be True when the defconfig was processed - successfully, or False when it fails. - """ - # output at least 30 characters to hide the "* defconfigs out of *". - name = self.defconfig[:-len('_defconfig')] - if self.log: - - # Put the first log line on the first line - log = name.ljust(20) + ' ' + self.log[0] - - if len(self.log) > 1: - log += '\n' + '\n'.join([' ' + s for s in self.log[1:]]) - # Some threads are running in parallel. - # Print log atomically to not mix up logs from different threads. - print(log, file=(sys.stdout if success else sys.stderr)) - - if not success: - if self.args.exit_on_error: - sys.exit('Exit on error.') - # If --exit-on-error flag is not set, skip this board and continue. - # Record the failed board. - self.failed_boards.add(name) - - self.progress.inc(success) - self.progress.show() - self.state = STATE_IDLE - - def get_failed_boards(self): - """Returns a set of failed boards (defconfigs) in this slot. - """ - return self.failed_boards - -class Slots: - """Controller of the array of subprocess slots.""" - - def __init__(self, toolchains, args, progress, reference_src_dir, db_queue): - """Create a new slots controller. - - Args: - toolchains (Toolchains): Toolchains object containing toolchains - args (Namespace): Program arguments - progress (Progress): A progress indicator. - reference_src_dir (str): Determine the true starting config state - from this source tree (None for none) - db_queue (Queue): output queue to write config info for the database - """ - self.args = args - self.slots = [] - self.progress = progress - self.col = progress.col - devnull = subprocess.DEVNULL - make_cmd = get_make_cmd() - for _ in range(args.jobs): - self.slots.append(Slot(toolchains, args, progress, devnull, - make_cmd, reference_src_dir, db_queue)) - - def add(self, defconfig): - """Add a new subprocess if a vacant slot is found. - - Args: - defconfig (str): defconfig name to be put into. - - Returns: - Return True on success or False on failure - """ - for slot in self.slots: - if slot.add(defconfig): - return True - return False - - def available(self): - """Check if there is a vacant slot. - - Returns: - Return True if at lease one vacant slot is found, False otherwise. - """ - for slot in self.slots: - if slot.poll(): - return True - return False - - def empty(self): - """Check if all slots are vacant. - - Returns: - Return True if all the slots are vacant, False otherwise. - """ - ret = True - for slot in self.slots: - if not slot.poll(): - ret = False - return ret - - def write_failed_boards(self): - """Show the results of processing""" - boards = set() - - for slot in self.slots: - boards |= slot.get_failed_boards() - - if boards: - boards = '\n'.join(sorted(boards)) + '\n' - write_file(FAILED_LIST, boards) - - class ReferenceSource: """Reference source against which original configs should be parsed.""" @@ -1291,71 +918,6 @@ class ReferenceSource: return self.src_dir -def move_config(args): - """Build database or sync config options to defconfig files. - - Args: - args (Namespace): Program arguments - - Returns: - tuple: - config_db (dict of configs for each defconfig): - key: defconfig name, e.g. "MPC8548CDS_legacy_defconfig" - value: dict: - key: CONFIG option - value: Value of option - Progress: Progress indicator - """ - config_db = {} - db_queue = queue.Queue() - dbt = DatabaseThread(config_db, db_queue) - dbt.daemon = True - dbt.start() - - check_clean_directory() - bsettings.setup('') - - # Get toolchains to use - toolchains = toolchain.Toolchains() - toolchains.GetSettings() - toolchains.Scan(verbose=False) - - if args.git_ref: - reference_src = ReferenceSource(args.git_ref) - reference_src_dir = reference_src.get_dir() - else: - reference_src_dir = None - - if args.defconfigs: - defconfigs = get_matched_defconfigs(args.defconfigs) - elif args.defconfiglist: - defconfigs = get_matched_defconfigs(args.defconfiglist) - else: - defconfigs = get_all_defconfigs() - - col = terminal.Color(terminal.COLOR_NEVER if args.nocolour - else terminal.COLOR_IF_TERMINAL) - progress = Progress(col, len(defconfigs)) - slots = Slots(toolchains, args, progress, reference_src_dir, db_queue) - - # Main loop to process defconfig files: - # Add a new subprocess into a vacant slot. - # Sleep if there is no available slot. - for defconfig in defconfigs: - while not slots.add(defconfig): - while not slots.available(): - # No available slot: sleep for a while - time.sleep(SLEEP_TIME) - - # wait until all the subprocesses finish - while not slots.empty(): - time.sleep(SLEEP_TIME) - - slots.write_failed_boards() - db_queue.join() - progress.completed() - return config_db, progress - def find_kconfig_rules(kconf, config, imply_config): """Check whether a config has a 'select' or 'imply' keyword @@ -2522,17 +2084,14 @@ def main(): config_db, progress = do_build_db(args) return write_db(config_db, progress) - if args.force_sync and not args.git_ref: + if args.force_sync: progress = do_sync_defconfigs(args) if args.commit: add_commit(args.configs) return move_done(progress) - config_db, progress = move_config(args) - - if args.commit: - add_commit(args.configs) - return move_done(progress) + parser.print_usage() + return 1 if __name__ == '__main__': -- 2.43.0
