Hello Zinovy, This commit broke test on the next builder: http://lab.llvm.org:8011/builders/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/builds/45557 . . . Failing Tests (1): Clang Tools :: clang-tidy/clang-tidy-diff.cpp
Please have a look? Thanks Galina On Wed, Mar 20, 2019 at 8:49 AM Zinovy Nis via cfe-commits < cfe-commits@lists.llvm.org> wrote: > Author: zinovy.nis > Date: Wed Mar 20 08:50:26 2019 > New Revision: 356565 > > URL: http://llvm.org/viewvc/llvm-project?rev=356565&view=rev > Log: > Reland r356547 after fixing the tests for Linux. > > [clang-tidy] Parallelize clang-tidy-diff.py > > This patch has 2 rationales: > > - large patches lead to long command lines and often cause max command > line length restrictions imposed by OS; > - clang-tidy runs on modified files are independent and can be done in > parallel, the same as done for run-clang-tidy. > > Differential Revision: https://reviews.llvm.org/D57662 > > > Modified: > clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py > > Modified: clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py > URL: > http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py?rev=356565&r1=356564&r2=356565&view=diff > > ============================================================================== > --- clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py (original) > +++ clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py Wed Mar 20 > 08:50:26 2019 > @@ -24,10 +24,90 @@ Example usage for git/svn users: > """ > > import argparse > +import glob > import json > +import multiprocessing > +import os > import re > +import shutil > import subprocess > import sys > +import tempfile > +import threading > +import traceback > +import yaml > + > +is_py2 = sys.version[0] == '2' > + > +if is_py2: > + import Queue as queue > +else: > + import queue as queue > + > + > +def run_tidy(task_queue, lock, timeout): > + watchdog = None > + while True: > + command = task_queue.get() > + try: > + proc = subprocess.Popen(command, > + stdout=subprocess.PIPE, > + stderr=subprocess.PIPE) > + > + if timeout is not None: > + watchdog = threading.Timer(timeout, proc.kill) > + watchdog.start() > + > + stdout, stderr = proc.communicate() > + > + with lock: > + sys.stdout.write(stdout.decode('utf-8') + '\n') > + sys.stdout.flush() > + if stderr: > + sys.stderr.write(stderr.decode('utf-8') + '\n') > + sys.stderr.flush() > + except Exception as e: > + with lock: > + sys.stderr.write('Failed: ' + str(e) + ': '.join(command) + '\n') > + finally: > + with lock: > + if (not timeout is None) and (not watchdog is None): > + if not watchdog.is_alive(): > + sys.stderr.write('Terminated by timeout: ' + > + ' '.join(command) + '\n') > + watchdog.cancel() > + task_queue.task_done() > + > + > +def start_workers(max_tasks, tidy_caller, task_queue, lock, timeout): > + for _ in range(max_tasks): > + t = threading.Thread(target=tidy_caller, args=(task_queue, lock, > timeout)) > + t.daemon = True > + t.start() > + > +def merge_replacement_files(tmpdir, mergefile): > + """Merge all replacement files in a directory into a single file""" > + # The fixes suggested by clang-tidy >= 4.0.0 are given under > + # the top level key 'Diagnostics' in the output yaml files > + mergekey = "Diagnostics" > + merged = [] > + for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')): > + content = yaml.safe_load(open(replacefile, 'r')) > + if not content: > + continue # Skip empty files. > + merged.extend(content.get(mergekey, [])) > + > + if merged: > + # MainSourceFile: The key is required by the definition inside > + # include/clang/Tooling/ReplacementsYaml.h, but the value > + # is actually never used inside clang-apply-replacements, > + # so we set it to '' here. > + output = { 'MainSourceFile': '', mergekey: merged } > + with open(mergefile, 'w') as out: > + yaml.safe_dump(output, out) > + else: > + # Empty the file: > + open(mergefile, 'w').close() > > > def main(): > @@ -47,7 +127,10 @@ def main(): > r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)', > help='custom pattern selecting file paths to check ' > '(case insensitive, overridden by -regex)') > - > + parser.add_argument('-j', type=int, default=1, > + help='number of tidy instances to be run in > parallel.') > + parser.add_argument('-timeout', type=int, default=None, > + help='timeout per each file in seconds.') > parser.add_argument('-fix', action='store_true', default=False, > help='apply suggested fixes') > parser.add_argument('-checks', > @@ -84,7 +167,7 @@ def main(): > match = re.search('^\+\+\+\ \"?(.*?/){%s}([^ \t\n\"]*)' % args.p, > line) > if match: > filename = match.group(2) > - if filename == None: > + if filename is None: > continue > > if args.regex is not None: > @@ -102,44 +185,79 @@ def main(): > line_count = int(match.group(3)) > if line_count == 0: > continue > - end_line = start_line + line_count - 1; > + end_line = start_line + line_count - 1 > lines_by_file.setdefault(filename, []).append([start_line, > end_line]) > > - if len(lines_by_file) == 0: > + if not any(lines_by_file): > print("No relevant changes found.") > sys.exit(0) > > - line_filter_json = json.dumps( > - [{"name" : name, "lines" : lines_by_file[name]} for name in > lines_by_file], > - separators = (',', ':')) > - > - quote = ""; > - if sys.platform == 'win32': > - line_filter_json=re.sub(r'"', r'"""', line_filter_json) > - else: > - quote = "'"; > + max_task_count = args.j > + if max_task_count == 0: > + max_task_count = multiprocessing.cpu_count() > + max_task_count = min(len(lines_by_file), max_task_count) > > - # Run clang-tidy on files containing changes. > - command = [args.clang_tidy_binary] > - command.append('-line-filter=' + quote + line_filter_json + quote) > - if args.fix: > - command.append('-fix') > + tmpdir = None > if args.export_fixes: > - command.append('-export-fixes=' + args.export_fixes) > + tmpdir = tempfile.mkdtemp() > + > + # Tasks for clang-tidy. > + task_queue = queue.Queue(max_task_count) > + # A lock for console output. > + lock = threading.Lock() > + > + # Run a pool of clang-tidy workers. > + start_workers(max_task_count, run_tidy, task_queue, lock, args.timeout) > + > + # Form the common args list. > + common_clang_tidy_args = [] > + if args.fix: > + common_clang_tidy_args.append('-fix') > if args.checks != '': > - command.append('-checks=' + quote + args.checks + quote) > + common_clang_tidy_args.append('-checks=' + args.checks) > if args.quiet: > - command.append('-quiet') > + common_clang_tidy_args.append('-quiet') > if args.build_path is not None: > - command.append('-p=%s' % args.build_path) > - command.extend(lines_by_file.keys()) > + common_clang_tidy_args.append('-p=%s' % args.build_path) > for arg in args.extra_arg: > - command.append('-extra-arg=%s' % arg) > + common_clang_tidy_args.append('-extra-arg=%s' % arg) > for arg in args.extra_arg_before: > - command.append('-extra-arg-before=%s' % arg) > - command.extend(clang_tidy_args) > + common_clang_tidy_args.append('-extra-arg-before=%s' % arg) > + > + for name in lines_by_file: > + line_filter_json = json.dumps( > + [{"name": name, "lines": lines_by_file[name]}], > + separators=(',', ':')) > + > + # Run clang-tidy on files containing changes. > + command = [args.clang_tidy_binary] > + command.append('-line-filter=' + line_filter_json) > + if args.export_fixes: > + # Get a temporary file. We immediately close the handle so > clang-tidy can > + # overwrite it. > + (handle, tmp_name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir) > + os.close(handle) > + command.append('-export-fixes=' + tmp_name) > + command.extend(common_clang_tidy_args) > + command.append(name) > + command.extend(clang_tidy_args) > + > + task_queue.put(command) > + > + # Wait for all threads to be done. > + task_queue.join() > + > + if args.export_fixes: > + print('Writing fixes to ' + args.export_fixes + ' ...') > + try: > + merge_replacement_files(tmpdir, args.export_fixes) > + except: > + sys.stderr.write('Error exporting fixes.\n') > + traceback.print_exc() > + > + if tmpdir: > + shutil.rmtree(tmpdir) > > - sys.exit(subprocess.call(' '.join(command), shell=True)) > > if __name__ == '__main__': > main() > > > _______________________________________________ > cfe-commits mailing list > cfe-commits@lists.llvm.org > https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits >
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
