Hi Aaron, I tried using this script to update a json dump test, and it seems to not really work very well:
1) it's really inconvenient to invoke; you need to pass in a --clang, copy some options out from two different places in the test file (from the RUN line and from a "--filters" line), and guess how to form the proper command line for the tool 2) it generates a file with the wrong name (adding a -json before the extension) 3) it doesn't strip out the CHECK lines from the previous run of the tool 4) it adds in a bunch of trailing whitespace, causing the file to not match the one in the repository Have you had a chance to look at making this more usable? I think the approach taken by utils/make-ast-dump-check.sh helps a lot: run the test in its normal configuration to generate the output, and then replace FileCheck with a tool that updates the CHECK lines instead of checking them. (That saves you needing to pass in --clang and --opts, at least, and makes it really easy to update a whole bunch of tests at once by running them all with lit.) On Wed, 19 Jun 2019 at 08:21, Aaron Ballman via cfe-commits < cfe-commits@lists.llvm.org> wrote: > Author: aaronballman > Date: Wed Jun 19 08:25:24 2019 > New Revision: 363820 > > URL: http://llvm.org/viewvc/llvm-project?rev=363820&view=rev > Log: > Add a script to help generate expected test output for dumping the AST to > JSON. > > Patch by Abhishek Bhaskar. > > Added: > cfe/trunk/test/AST/gen_ast_dump_json_test.py > > Added: cfe/trunk/test/AST/gen_ast_dump_json_test.py > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/test/AST/gen_ast_dump_json_test.py?rev=363820&view=auto > > ============================================================================== > --- cfe/trunk/test/AST/gen_ast_dump_json_test.py (added) > +++ cfe/trunk/test/AST/gen_ast_dump_json_test.py Wed Jun 19 08:25:24 2019 > @@ -0,0 +1,137 @@ > +#!/usr/bin/env python > + > +from collections import OrderedDict > +from sets import Set > +from shutil import copyfile > +import argparse > +import json > +import os > +import pprint > +import re > +import subprocess > + > +def normalize(dict_var): > + for k, v in dict_var.items(): > + if isinstance(v, OrderedDict): > + normalize(v) > + elif isinstance(v, list): > + for e in v: > + if isinstance(e, OrderedDict): > + normalize(e) > + elif type(v) is unicode: > + st = v.encode('utf-8') > + if re.match(r"0x[0-9A-Fa-f]+", v): > + dict_var[k] = u'0x{{.*}}' > + elif os.path.isfile(v): > + dict_var[k] = u'{{.*}}' > + else: > + splits = (v.split(u' ')) > + out_splits = [] > + for split in splits: > + inner_splits = split.rsplit(u':',2) > + if os.path.isfile(inner_splits[0]): > + out_splits.append( > + u'{{.*}}:%s:%s' > + %(inner_splits[1], > + inner_splits[2])) > + continue > + out_splits.append(split) > + > + dict_var[k] = ' '.join(out_splits) > + > +def filter_json(dict_var, filters, out): > + for k, v in dict_var.items(): > + if type(v) is unicode: > + st = v.encode('utf-8') > + if st in filters: > + out.append(dict_var) > + break > + elif isinstance(v, OrderedDict): > + filter_json(v, filters, out) > + elif isinstance(v, list): > + for e in v: > + if isinstance(e, OrderedDict): > + filter_json(e, filters, out) > + > +def main(): > + parser = argparse.ArgumentParser() > + parser.add_argument("--clang", help="The clang binary (could be a > relative or absolute path)", > + action="store", required=True) > + parser.add_argument("--opts", help="other options", > + action="store", default='', type=str) > + parser.add_argument("--source", help="the source file. Command used > to generate the json will be of the format <clang> -cc1 -ast-dump=json > <opts> <source>", > + action="store", required=True) > + parser.add_argument("--filters", help="comma separated list of AST > filters. Ex: --filters=TypedefDecl,BuiltinType", > + action="store", default='') > + > + args = parser.parse_args() > + > + if not args.source: > + print("Specify the source file to give to clang.") > + return -1 > + > + clang_binary = os.path.abspath(args.clang) > + if not os.path.isfile(clang_binary): > + print("clang binary specified not present.") > + return -1 > + > + options = args.opts.split(' ') > + filters = Set(args.filters.split(',')) if args.filters else Set([]) > + > + cmd = [clang_binary, "-cc1"] > + cmd.extend(options) > + > + using_ast_dump_filter = 'ast-dump-filter' in args.opts > + > + cmd.extend(["-ast-dump=json", args.source]) > + > + try: > + json_str = subprocess.check_output(cmd) > + except Exception as ex: > + print("The clang command failed with %s" % ex) > + return -1 > + > + out_asts = [] > + if using_ast_dump_filter: > + splits = re.split('Dumping .*:\n', json_str) > + if len(splits) > 1: > + for split in splits[1:]: > + j = json.loads(split.decode('utf-8'), > object_pairs_hook=OrderedDict) > + normalize(j) > + out_asts.append(j) > + else: > + j = json.loads(json_str.decode('utf-8'), > object_pairs_hook=OrderedDict) > + normalize(j) > + > + if len(filters) == 0: > + out_asts.append(j) > + else: > + #assert using_ast_dump_filter is False,\ > + # "Does not support using compiler's ast-dump-filter "\ > + # "and the tool's filter option at the same time yet." > + > + filter_json(j, filters, out_asts) > + > + partition = args.source.rpartition('.') > + dest_path = '%s-json%s%s' % (partition[0], partition[1], partition[2]) > + > + print("Writing json appended source file to %s." %(dest_path)) > + copyfile(args.source, dest_path) > + with open(dest_path, "a") as f: > + for out_ast in out_asts: > + append_str = json.dumps(out_ast, indent=1, ensure_ascii=False) > + out_str = '\n\n' > + index = 0 > + for append_line in append_str.splitlines()[2:]: > + if index == 0: > + out_str += '// CHECK: %s\n' %(append_line) > + index += 1 > + else: > + out_str += '// CHECK-NEXT: %s\n' %(append_line) > + > + f.write(out_str) > + > + return 0 > + > +if __name__ == '__main__': > + main() > > > _______________________________________________ > cfe-commits mailing list > cfe-commits@lists.llvm.org > https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits >
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
