zturner created this revision.
Herald added a reviewer: modocache.
Herald added a subscriber: fedor.sergeev.
This is probably the last work I'm going to do here for a while. I still see
some room for improvement, but I think after this patch:
a) refactor begins to have diminishing returns
b) There's a sufficient amount of sharing and re-use to validate the design
c) There's enough specific examples of how to use this that other projects
(e.g. compiler-rt, etc) can use those as a model of how to simplify their own
configs.
In other words, I'm not brave enough to touch compiler-rt's config files :)
There's some minor changes in the config api that increase flexibility in how
you override environment variables and spawn an external `llvm-config`, but
otherwise this is mostly just deleting code.
https://reviews.llvm.org/D37818
Files:
clang/test/lit.cfg
clang/test/lit.site.cfg.in
lld/test/lit.cfg
lld/test/lit.site.cfg.in
llvm/test/lit.cfg
llvm/utils/lit/lit/llvm/config.py
Index: llvm/utils/lit/lit/llvm/config.py
===================================================================
--- llvm/utils/lit/lit/llvm/config.py
+++ llvm/utils/lit/lit/llvm/config.py
@@ -1,4 +1,5 @@
import os
+import platform
import re
import subprocess
import sys
@@ -23,35 +24,40 @@
# Tweak PATH for Win32 to decide to use bash.exe or not.
if sys.platform == 'win32':
- # For tests that require Windows to run.
- features.add('system-windows')
-
# Seek sane tools in directories and set to $PATH.
path = self.lit_config.getToolsPath(config.lit_tools_dir,
config.environment['PATH'],
['cmp.exe', 'grep.exe', 'sed.exe'])
self.with_environment('PATH', path, append_path=True)
- if use_lit_shell:
- # 0 is external, "" is default, and everything else is internal.
- self.use_lit_shell = (use_lit_shell != "0")
- else:
- # Otherwise we default to internal on Windows and external elsewhere, as
- # bash on Windows is usually very slow.
- self.use_lit_shell = (sys.platform in ['win32'])
+ self.with_environment('LLVM_SRC_ROOT', config.llvm_src_root)
self.use_lit_shell = litsh
- if not self.litsh:
+ if not litsh:
features.add('shell')
+
+ # Running on Darwin OS
+ if platform.system() in ['Darwin']:
+ # FIXME: lld uses the first, other projects use the second.
+ # We should standardize on the former.
+ features.add('system-linker-mach-o')
+ features.add('system-darwin')
+ elif platform.system() in ['Windows']:
+ # For tests that require Windows to run.
+ features.add('system-windows')
+
# Native compilation: host arch == default triple arch
- # FIXME: Consider cases that target can be executed
- # even if host_triple were different from target_triple.
- if config.host_triple == config.target_triple:
+ # Both of these values should probably be in every site config (e.g. as
+ # part of the standard header. But currently they aren't)
+ host_triple = getattr(config, 'host_triple', None)
+ target_triple = getattr(config, 'target_triple', None)
+ if host_triple and host_triple == target_triple:
features.add("native")
# Sanitizers.
- sanitizers = frozenset(x.lower() for x in getattr(config, 'llvm_use_sanitizer', []).split(';'))
+ sanitizers = getattr(config, 'llvm_use_sanitizer', '')
+ sanitizers = frozenset(x.lower() for x in sanitizers.split(';'))
features.add(binary_feature('address' in sanitizers, 'asan', 'not_'))
features.add(binary_feature('memory' in sanitizers, 'msan', 'not_'))
features.add(binary_feature('undefined' in sanitizers, 'ubsan', 'not_'))
@@ -64,7 +70,6 @@
if lit.util.pythonize_bool(long_tests):
features.add("long_tests")
- target_triple = getattr(config, 'target_triple', None)
if target_triple:
if re.match(r'^x86_64.*-linux', target_triple):
features.add("x86_64-linux")
@@ -80,25 +85,34 @@
if gmalloc_path_str is not None:
self.with_environment('DYLD_INSERT_LIBRARIES', gmalloc_path_str)
- breaking_checks = getattr(config, 'enable_abi_breaking_checks')
- if lit.util.pythonize_bool(lit_config, breaking_checks):
+ breaking_checks = getattr(config, 'enable_abi_breaking_checks', None)
+ if lit.util.pythonize_bool(breaking_checks):
features.add('abi-breaking-checks')
def with_environment(self, variable, value, append_path = False):
- if append_path and variable in self.config.environment:
+ if append_path:
+ # For paths, we should be able to take a list of them and process all
+ # of them.
+ paths_to_add = value
+ if isinstance(paths_to_add, basestring):
+ paths_to_add = [paths_to_add]
+
def norm(x):
return os.path.normcase(os.path.normpath(x))
- # Move it to the front if it already exists, otherwise insert it at the
- # beginning.
- value = norm(value)
- current_value = self.config.environment[variable]
- items = [norm(x) for x in current_value.split(os.path.pathsep)]
- try:
- items.remove(value)
- except ValueError:
- pass
- value = os.path.pathsep.join([value] + items)
+ current_paths = self.config.environment.get(variable, "")
+ current_paths = current_paths.split(os.path.pathsep)
+ paths = [norm(p) for p in current_paths]
+ for p in paths_to_add:
+ # Move it to the front if it already exists, otherwise insert it at the
+ # beginning.
+ p = norm(p)
+ try:
+ paths.remove(p)
+ except ValueError:
+ pass
+ paths = [p] + paths
+ value = os.pathsep.join(paths)
self.config.environment[variable] = value
@@ -110,16 +124,35 @@
if value:
self.with_environment(v, value, append_path)
- def feature_config(self, flag, feature):
- # Ask llvm-config about assertion mode.
+ def clear_environment(self, variables):
+ for name in variables:
+ if name in self.config.environment:
+ del self.config.environment[name]
+
+ def feature_config(self, features, encoding = 'ascii'):
+ # Ask llvm-config about the specified feature.
+ arguments = [x for (x, _) in features]
try:
+ config_path = os.path.join(self.config.llvm_tools_dir, 'llvm-config')
+
llvm_config_cmd = subprocess.Popen(
- [os.path.join(self.config.llvm_tools_dir, 'llvm-config'), flag],
+ [config_path] + arguments,
stdout = subprocess.PIPE,
env=self.config.environment)
except OSError:
self.lit_config.fatal("Could not find llvm-config in " + self.config.llvm_tools_dir)
output, _ = llvm_config_cmd.communicate()
- if re.search(r'ON', llvm_config_cmd.stdout.read().decode('ascii')):
- self.config.available_features.add(feature)
+ output = output.decode(encoding)
+ lines = output.split('\n')
+ for (line, (_, patterns)) in zip(lines, features):
+ # We should have either a callable or a dictionary. If it's a
+ # dictionary, grep each key against the output and use the value if
+ # it matches. If it's a callable, it does the entire translation.
+ if callable(patterns):
+ features_to_add = patterns(line)
+ self.config.available_features.update(features_to_add)
+ else:
+ for (match, feature) in patterns.items():
+ if re.search(line, match):
+ self.config.available_features.add(feature)
Index: llvm/test/lit.cfg
===================================================================
--- llvm/test/lit.cfg
+++ llvm/test/lit.cfg
@@ -39,8 +39,6 @@
# Propagate some variables from the host environment.
llvm_config.with_system_environment(['HOME', 'INCLUDE', 'LIB', 'TMP', 'TEMP', 'ASAN_SYMBOLIZER_PATH', 'MSAN_SYMBOLIZER_PATH'])
-# Propagate LLVM_SRC_ROOT into the environment.
-config.environment['LLVM_SRC_ROOT'] = config.llvm_src_root
# Set up OCAMLPATH to include newly built OCaml libraries.
top_ocaml_lib = os.path.join(config.llvm_lib_dir, 'ocaml')
@@ -330,8 +328,9 @@
config.available_features.add('ld64_plugin')
# Ask llvm-config about asserts and global-isel.
-llvm_config.feature_config('--assertion-mode', 'asserts')
-llvm_config.feature_config('--has-global-isel', 'global-isel')
+llvm_config.feature_config(
+ [('--assertion-mode', {'ON' : 'asserts'}),
+ ('--has-global-isel', {'ON' : 'global-isel'})])
if 'darwin' == sys.platform:
try:
Index: lld/test/lit.site.cfg.in
===================================================================
--- lld/test/lit.site.cfg.in
+++ lld/test/lit.site.cfg.in
@@ -22,5 +22,7 @@
key, = e.args
lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
+@LIT_SITE_CFG_IN_FOOTER@
+
# Let the main config do the real work.
lit_config.load_config(config, "@LLD_SOURCE_DIR@/test/lit.cfg")
Index: lld/test/lit.cfg
===================================================================
--- lld/test/lit.cfg
+++ lld/test/lit.cfg
@@ -9,40 +9,18 @@
import lit.formats
import lit.util
+from lit.llvm import llvm_config
+
# Configuration file for the 'lit' test runner.
# name: The name of this test suite.
config.name = 'lld'
-# Tweak PATH for Win32
-if sys.platform in ['win32']:
- # Seek sane tools in directories and set to $PATH.
- path = getattr(config, 'lit_tools_dir', None)
- path = lit_config.getToolsPath(path,
- config.environment['PATH'],
- ['cmp.exe', 'grep.exe', 'sed.exe'])
- if path is not None:
- path = os.path.pathsep.join((path,
- config.environment['PATH']))
- config.environment['PATH'] = path
-
-# Choose between lit's internal shell pipeline runner and a real shell. If
-# LIT_USE_INTERNAL_SHELL is in the environment, we use that as an override.
-use_lit_shell = os.environ.get("LIT_USE_INTERNAL_SHELL")
-if use_lit_shell:
- # 0 is external, "" is default, and everything else is internal.
- execute_external = (use_lit_shell == "0")
-else:
- # Otherwise we default to internal on Windows and external elsewhere, as
- # bash on Windows is usually very slow.
- execute_external = (not sys.platform in ['win32'])
-
-
# testFormat: The test format to use to interpret tests.
#
# For now we require '&&' between commands, until they get globally killed and
# the test runner updated.
-config.test_format = lit.formats.ShTest(execute_external)
+config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
# suffixes: A list of file extensions to treat as test files.
config.suffixes = ['.ll', '.s', '.test', '.yaml', '.objtxt']
@@ -57,21 +35,10 @@
config.test_exec_root = os.path.join(config.lld_obj_root, 'test')
-
# Tweak the PATH to include the tools dir and the scripts dir.
-path = os.path.pathsep.join((config.lld_tools_dir, config.llvm_tools_dir, config.environment['PATH']))
-
-config.environment['PATH'] = path
-
-path = os.path.pathsep.join((config.lld_libs_dir, config.llvm_libs_dir,
- config.environment.get('LD_LIBRARY_PATH','')))
-config.environment['LD_LIBRARY_PATH'] = path
+llvm_config.with_environment('PATH', [config.llvm_tools_dir, config.lld_tools_dir], append_path=True)
-# Propagate LLVM_SRC_ROOT into the environment.
-config.environment['LLVM_SRC_ROOT'] = config.llvm_src_root
-
-# Propagate PYTHON_EXECUTABLE into the environment
-config.environment['PYTHON_EXECUTABLE'] = sys.executable
+llvm_config.with_environment('LD_LIBRARY_PATH', [config.lld_libs_dir, config.llvm_libs_dir], append_path=True)
# For each occurrence of a lld tool name as its own word, replace it
# with the full path to the build directory holding that tool. This
@@ -126,68 +93,26 @@
if lit_config.useValgrind:
config.target_triple += '-vg'
-# Shell execution
-if execute_external:
- config.available_features.add('shell')
-
-# zlib compression library
-if config.have_zlib:
- config.available_features.add("zlib")
-
-# Running on Darwin OS
-if platform.system() in ['Darwin']:
- config.available_features.add('system-linker-mach-o')
-
# Running on ELF based *nix
if platform.system() in ['FreeBSD', 'Linux']:
config.available_features.add('system-linker-elf')
-# Running on Windows
-if platform.system() in ['Windows']:
- config.available_features.add('system-windows')
-
# Set if host-cxxabi's demangler can handle target's symbols.
if platform.system() not in ['Windows']:
config.available_features.add('demangler')
-# llvm-config knows whether it is compiled with asserts (and)
-# whether we are operating in release/debug mode.
-import subprocess
-try:
- llvm_config_cmd = \
- subprocess.Popen([os.path.join(config.llvm_tools_dir, 'llvm-config'),
- '--build-mode', '--assertion-mode', '--targets-built'],
- stdout = subprocess.PIPE)
-except OSError as why:
- print("Could not find llvm-config in " + config.llvm_tools_dir)
- exit(42)
-
-llvm_config_output = llvm_config_cmd.stdout.read().decode('utf_8')
-llvm_config_output_list = llvm_config_output.split("\n")
-
-if re.search(r'DEBUG', llvm_config_output_list[0]):
- config.available_features.add('debug')
-if re.search(r'ON', llvm_config_output_list[1]):
- config.available_features.add('asserts')
-
-archs = llvm_config_output_list[2]
-if re.search(r'AArch64', archs):
- config.available_features.add('aarch64')
-if re.search(r'AMDGPU', archs):
- config.available_features.add('amdgpu')
-if re.search(r'ARM', archs):
- config.available_features.add('arm')
-if re.search(r'AVR', archs):
- config.available_features.add('avr')
-if re.search(r'Mips', archs):
- config.available_features.add('mips')
-if re.search(r'PowerPC', archs):
- config.available_features.add('ppc')
-if re.search(r'Sparc', archs):
- config.available_features.add('sparc')
-if re.search(r'X86', archs):
- config.available_features.add('x86')
-llvm_config_cmd.wait()
+llvm_config.feature_config(
+ [('--build-mode', {'DEBUG' : 'debug'}),
+ ('--assertion-mode', {'ON' : 'asserts'}),
+ ('--targets-built', {'AArch64' : 'aarch64',
+ 'AMDGPU' : 'amdgpu',
+ 'ARM' : 'arm',
+ 'AVR' : 'avr',
+ 'Mips' : 'mips',
+ 'PowerPC' : 'ppc',
+ 'Sparc' : 'sparc',
+ 'X86' : 'x86'})
+ ])
# Set a fake constant version so that we get consitent output.
config.environment['LLD_VERSION'] = 'LLD 1.0'
Index: clang/test/lit.site.cfg.in
===================================================================
--- clang/test/lit.site.cfg.in
+++ clang/test/lit.site.cfg.in
@@ -39,5 +39,7 @@
key, = e.args
lit_config.fatal("unable to find %r parameter, use '--param=%s=VALUE'" % (key,key))
+@LIT_SITE_CFG_IN_FOOTER@
+
# Let the main config do the real work.
lit_config.load_config(config, "@CLANG_SOURCE_DIR@/test/lit.cfg")
Index: clang/test/lit.cfg
===================================================================
--- clang/test/lit.cfg
+++ clang/test/lit.cfg
@@ -9,39 +9,18 @@
import lit.formats
import lit.util
+from lit.llvm import llvm_config
+
# Configuration file for the 'lit' test runner.
# name: The name of this test suite.
config.name = 'Clang'
-# Tweak PATH for Win32
-if platform.system() == 'Windows':
- # Seek sane tools in directories and set to $PATH.
- path = getattr(config, 'lit_tools_dir', None)
- path = lit_config.getToolsPath(path,
- config.environment['PATH'],
- ['cmp.exe', 'grep.exe', 'sed.exe'])
- if path is not None:
- path = os.path.pathsep.join((path,
- config.environment['PATH']))
- config.environment['PATH'] = path
-
-# Choose between lit's internal shell pipeline runner and a real shell. If
-# LIT_USE_INTERNAL_SHELL is in the environment, we use that as an override.
-use_lit_shell = os.environ.get("LIT_USE_INTERNAL_SHELL")
-if use_lit_shell:
- # 0 is external, "" is default, and everything else is internal.
- execute_external = (use_lit_shell == "0")
-else:
- # Otherwise we default to internal on Windows and external elsewhere, as
- # bash on Windows is usually very slow.
- execute_external = (not sys.platform in ['win32'])
-
# testFormat: The test format to use to interpret tests.
#
# For now we require '&&' between commands, until they get globally killed and
# the test runner updated.
-config.test_format = lit.formats.ShTest(execute_external)
+config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
# suffixes: A list of file extensions to treat as test files.
config.suffixes = ['.c', '.cpp', '.cppm', '.m', '.mm', '.cu', '.ll', '.cl', '.s', '.S', '.modulemap', '.test', '.rs']
@@ -81,22 +60,16 @@
# Clang/Win32 may refer to %INCLUDE%. vsvarsall.bat sets it.
if platform.system() != 'Windows':
possibly_dangerous_env_vars.append('INCLUDE')
-for name in possibly_dangerous_env_vars:
- if name in config.environment:
- del config.environment[name]
+
+llvm_config.clear_environment(possibly_dangerous_env_vars)
# Tweak the PATH to include the tools dir and the scripts dir.
-path = os.path.pathsep.join((
- config.clang_tools_dir, config.llvm_tools_dir, config.environment['PATH']))
-config.environment['PATH'] = path
-path = os.path.pathsep.join((config.llvm_shlib_dir, config.llvm_libs_dir,
- config.environment.get('LD_LIBRARY_PATH','')))
-config.environment['LD_LIBRARY_PATH'] = path
+llvm_config.with_environment('PATH', [config.llvm_tools_dir, config.clang_tools_dir], append_path=True)
+
+llvm_config.with_environment('LD_LIBRARY_PATH', [config.llvm_shlib_dir, config.llvm_libs_dir], append_path=True)
# Propagate path to symbolizer for ASan/MSan.
-for symbolizer in ['ASAN_SYMBOLIZER_PATH', 'MSAN_SYMBOLIZER_PATH']:
- if symbolizer in os.environ:
- config.environment[symbolizer] = os.environ[symbolizer]
+llvm_config.with_system_environment(['ASAN_SYMBOLIZER_PATH', 'MSAN_SYMBOLIZER_PATH'])
# Discover the 'clang' and 'clangcc' to use.
@@ -151,7 +124,7 @@
if not cmd.stdout:
lit_config.fatal("Couldn't find the include dir for Clang ('%s')" % clang)
dir = cmd.stdout.read().strip()
- if sys.platform in ['win32'] and execute_external:
+ if sys.platform in ['win32'] and not llvm_config.use_lit_shell:
# Don't pass dosish path separator to msys bash.exe.
dir = dir.replace('\\', '/')
# Ensure the result is an ascii string, across Python2.5+ - Python3.
@@ -295,18 +268,6 @@
if platform.system() not in ['FreeBSD']:
config.available_features.add('crash-recovery')
-# Shell execution
-if execute_external:
- config.available_features.add('shell')
-
-# For tests that require Darwin to run.
-# This is used by debuginfo-tests/*block*.m and debuginfo-tests/foreach.m.
-if platform.system() in ['Darwin']:
- config.available_features.add('system-darwin')
-elif platform.system() in ['Windows']:
- # For tests that require Windows to run.
- config.available_features.add('system-windows')
-
# ANSI escape sequences in non-dumb terminal
if platform.system() not in ['Windows']:
config.available_features.add('ansi-escape-sequences')
@@ -321,12 +282,6 @@
if platform.system() not in ['Darwin', 'Fuchsia']:
config.available_features.add('libgcc')
-# Native compilation: Check if triples match.
-# FIXME: Consider cases that target can be executed
-# even if host_triple were different from target_triple.
-if config.host_triple == config.target_triple:
- config.available_features.add("native")
-
# Case-insensitive file system
def is_filesystem_case_insensitive():
handle, path = tempfile.mkstemp(prefix='case-test', dir=config.test_exec_root)
@@ -367,87 +322,24 @@
if platform.system() not in ['Windows']:
config.available_features.add('can-remove-opened-file')
-# Returns set of available features, registered-target(s), asserts and
-# compile definitions.
-def get_llvm_config_props():
- set_of_features = set()
-
- cmd = subprocess.Popen(
- [
- os.path.join(config.llvm_tools_dir, 'llvm-config'),
- '--assertion-mode',
- '--targets-built',
- '--cxxflags'
- ],
- stdout=subprocess.PIPE,
- env=config.environment
- )
- # 1st line corresponds to --assertion-mode, "ON" or "OFF".
- line = cmd.stdout.readline().strip().decode('ascii')
- if line == "ON":
- set_of_features.add('asserts')
-
- # 2nd line corresponds to --targets-built, like;
- # AArch64 ARM CppBackend X86
- for arch in cmd.stdout.readline().decode('ascii').split():
- set_of_features.add(arch.lower() + '-registered-target')
-
- # 3rd line contains compile definitions, search it to define if
- # libstdc++ safe mode is set.
- if re.search(r'-D_GLIBCXX_DEBUG\b', cmd.stdout.readline().decode('ascii')):
- set_of_features.add('libstdcxx-safe-mode')
-
- return set_of_features
-
-config.available_features.update(get_llvm_config_props())
+def calculate_arch_features(arch_string):
+ features = []
+ for arch in arch_string.split():
+ features.append(arch.lower() + '-registered-target')
+ return features
+
+llvm_config.feature_config(
+ [('--assertion-mode', {'ON' : 'asserts'}),
+ ('--cxxflags', {r'-D_GLIBCXX_DEBUG\b' : 'libstdcxx-safe-mode'}),
+ ('--targets-built', calculate_arch_features)
+ ])
if lit.util.which('xmllint'):
config.available_features.add('xmllint')
-# Sanitizers.
-if 'Address' in config.llvm_use_sanitizer:
- config.available_features.add("asan")
-else:
- config.available_features.add("not_asan")
-if 'Memory' in config.llvm_use_sanitizer:
- config.available_features.add("msan")
-if 'Undefined' in config.llvm_use_sanitizer:
- config.available_features.add("ubsan")
-else:
- config.available_features.add("not_ubsan")
-
if config.enable_backtrace:
config.available_features.add("backtrace")
-if config.have_zlib:
- config.available_features.add("zlib")
-else:
- config.available_features.add("nozlib")
-
-# Check if we should run long running tests.
-if lit_config.params.get("run_long_tests", None) == "true":
- config.available_features.add("long_tests")
-
-# Check if we should use gmalloc.
-use_gmalloc_str = lit_config.params.get('use_gmalloc', None)
-if use_gmalloc_str is not None:
- if use_gmalloc_str.lower() in ('1', 'true'):
- use_gmalloc = True
- elif use_gmalloc_str.lower() in ('', '0', 'false'):
- use_gmalloc = False
- else:
- lit_config.fatal('user parameter use_gmalloc should be 0 or 1')
-else:
- # Default to not using gmalloc
- use_gmalloc = False
-
-# Allow use of an explicit path for gmalloc library.
-# Will default to '/usr/lib/libgmalloc.dylib' if not set.
-gmalloc_path_str = lit_config.params.get('gmalloc_path',
- '/usr/lib/libgmalloc.dylib')
-if use_gmalloc:
- config.environment.update({'DYLD_INSERT_LIBRARIES' : gmalloc_path_str})
-
# Check if we should allow outputs to console.
run_console_tests = int(lit_config.params.get('enable_console', '0'))
if run_console_tests != 0:
@@ -457,6 +349,3 @@
macOSSDKVersion = lit.util.findPlatformSdkVersionOnMacOS(config, lit_config)
if macOSSDKVersion is not None:
config.available_features.add('macos-sdk-' + macOSSDKVersion)
-
-if config.enable_abi_breaking_checks == "1":
- config.available_features.add('abi-breaking-checks')
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
