Modified: trunk/Tools/Scripts/webkitpy/thirdparty/__init__.py (261606 => 261607)
--- trunk/Tools/Scripts/webkitpy/thirdparty/__init__.py 2020-05-13 08:34:28 UTC (rev 261606)
+++ trunk/Tools/Scripts/webkitpy/thirdparty/__init__.py 2020-05-13 08:57:55 UTC (rev 261607)
@@ -128,6 +128,8 @@
self._install_beautifulsoup()
elif '.html5lib' in fullname:
self._install_html5lib()
+ elif '.toml' in fullname:
+ self._install_toml()
def _install_six(self):
self._install("https://files.pythonhosted.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz",
@@ -323,6 +325,11 @@
full_firefox_url = re.findall(r'.*%s.*' % filename_postfix, all_firefox_release_urls)[0].split('"')[3]
self.install_binary(full_firefox_url, 'geckodriver')
+ def _install_toml(self):
+ toml_dir = self._fs.join(_AUTOINSTALLED_DIR, "toml")
+ installer = AutoInstaller(prepend_to_search_path=True, target_dir=toml_dir)
+ installer.install(url="" url_subpath="toml-0.10.0/toml")
+
def _install(self, url, url_subpath=None, target_name=None):
installer = AutoInstaller(target_dir=_AUTOINSTALLED_DIR)
return installer.install(url="" url_subpath=url_subpath, target_name=target_name)
Modified: trunk/Tools/flatpak/flatpakutils.py (261606 => 261607)
--- trunk/Tools/flatpak/flatpakutils.py 2020-05-13 08:34:28 UTC (rev 261606)
+++ trunk/Tools/flatpak/flatpakutils.py 2020-05-13 08:57:55 UTC (rev 261607)
@@ -37,6 +37,8 @@
from webkitpy.common.system.systemhost import SystemHost
from webkitpy.port.factory import PortFactory
from webkitpy.common.system.logutils import configure_logging
+import webkitpy.thirdparty.autoinstalled.toml
+import toml
try:
from urllib.parse import urlparse # pylint: disable=E0611
@@ -417,11 +419,17 @@
help="The command to run in the sandbox",
dest="user_command")
general.add_argument('--available', action='', dest="check_available", help='Check if required dependencies are available.'),
- general.add_argument("--use-icecream", dest="use_icecream", help="Use the distributed icecream (icecc) compiler.", action=""
- general.add_argument("-r", "--regenerate-toolchains", dest="regenerate_toolchains", action=""
- help="Regenerate IceCC distribuable toolchain archives")
general.add_argument("--repo", help="Filesystem absolute path to the Flatpak repository to use", dest="user_repo")
+ distributed_build_options = parser.add_argument_group("Distributed building")
+ distributed_build_options.add_argument("--use-icecream", dest="use_icecream", help="Use the distributed icecream (icecc) compiler.", action=""
+ distributed_build_options.add_argument("-r", "--regenerate-toolchains", dest="regenerate_toolchains", action=""
+ help="Regenerate IceCC distribuable toolchain archives")
+ distributed_build_options.add_argument("-t", "--sccache-token", dest="sccache_token",
+ help="sccache authentication token")
+ distributed_build_options.add_argument("-s", "--sccache-scheduler", dest="sccache_scheduler", default='https://sccache.igalia.com',
+ help="sccache scheduler URL")
+
debugoptions = parser.add_argument_group("Debugging")
debugoptions.add_argument("--gdb", nargs="?", help="Activate gdb, passing extra args to it if wanted.")
debugoptions.add_argument("--gdb-stack-trace", dest="gdb_stack_trace", nargs="?",
@@ -475,6 +483,8 @@
self.use_icecream = False
self.icc_version = {}
self.regenerate_toolchains = False
+ self.sccache_token = ""
+ self.sccache_scheduler = ""
def execute_command(self, args, stdout=None, stderr=None):
_log.debug('Running in sandbox: %s\n' % ' '.join(args))
@@ -520,6 +530,7 @@
self.build_root = os.path.join(self.source_root, 'WebKitBuild')
self.build_path = os.path.join(self.build_root, self.platform, self.build_type)
self.config_file = os.path.join(self.flatpak_build_path, 'webkit_flatpak_config.json')
+ self.sccache_config_file = os.path.join(self.flatpak_build_path, 'sccache.toml')
Console.quiet = self.quiet
if not check_flatpak():
@@ -607,6 +618,10 @@
return True
return False
+ def host_path_to_sandbox_path(self, host_path):
+ # For now this supports only files in the WebKit path
+ return host_path.replace(self.source_root, self.sandbox_source_root)
+
def run_in_sandbox(self, *args, **kwargs):
self.setup_builddir(stdout=kwargs.get("stdout", sys.stdout))
cwd = kwargs.pop("cwd", None)
@@ -768,6 +783,9 @@
_log.debug("Enabling network access for the remote sccache")
flatpak_command.append(share_network_option)
+ if os.path.isfile(self.sccache_config_file) and not self.regenerate_toolchains and "SCCACHE_CONF" not in os.environ.keys():
+ sandbox_environment["SCCACHE_CONF"] = self.host_path_to_sandbox_path(self.sccache_config_file)
+
override_sccache_server_port = os.environ.get("WEBKIT_SCCACHE_SERVER_PORT")
if override_sccache_server_port:
_log.debug("Overriding sccache server port to %s" % override_sccache_server_port)
@@ -836,9 +854,9 @@
if regenerate_toolchains:
self.icc_version = {}
- self.setup_icecc("gcc", "g++")
- self.setup_icecc("clang", "clang++")
- self.save_config()
+ toolchains = self.pack_toolchain(("gcc", "g++"), {"/usr/bin/c++": "/usr/bin/g++"})
+ toolchains.extend(self.pack_toolchain(("clang", "clang++"), {"/usr/bin/clang++": "/usr/bin/clang++"}))
+ self.save_config(toolchains)
return self.setup_dev_env()
@@ -854,12 +872,24 @@
def has_environment(self):
return os.path.exists(self.flatpak_build_path)
- def save_config(self):
+ def save_config(self, toolchains):
with open(self.config_file, 'w') as config:
json_config = {'icecc_version': self.icc_version}
json.dump(json_config, config)
- def setup_icecc(self, *compilers):
+ if not self.sccache_token:
+ Console.message("No authentication token provided. Re-run this with the -t option if an sccache token was provided to you. Skipping sccache configuration for now.")
+ return
+
+ with open(self.sccache_config_file, 'w') as config:
+ sccache_config = {'dist': {'scheduler_url': self.sccache_scheduler,
+ 'auth': {'type': 'token',
+ 'token': self.sccache_token},
+ 'toolchains': toolchains}}
+ toml.dump(sccache_config, config)
+ Console.message("Created %s sccache config file. It will automatically be used when building WebKit", self.sccache_config_file)
+
+ def pack_toolchain(self, compilers, path_mapping):
with tempfile.NamedTemporaryFile() as tmpfile:
command = ['icecc', '--build-native']
command.extend(["/usr/bin/%s" % compiler for compiler in compilers])
@@ -875,6 +905,15 @@
self.icc_version[compilers[0]] = archive_filename
Console.message("Created %s self-contained toolchain archive", archive_filename)
+ sccache_toolchains = []
+ for (compiler_executable, archive_compiler_executable) in path_mapping.iteritems():
+ item = {'type': 'path_override',
+ 'compiler_executable': compiler_executable,
+ 'archive': archive_filename,
+ 'archive_compiler_executable': archive_compiler_executable}
+ sccache_toolchains.append(item)
+ return sccache_toolchains
+
def setup_dev_env(self):
if not os.path.exists(os.path.join(self.flatpak_build_path, "runtime", "org.webkit.Sdk")) or self.update:
self.install_all()