Title: [261607] trunk/Tools
Revision
261607
Author
ph...@webkit.org
Date
2020-05-13 01:57:55 -0700 (Wed, 13 May 2020)

Log Message

[Flatpak SDK] Craft a custom sccache config file from SDK toolchains
https://bugs.webkit.org/show_bug.cgi?id=211781

Reviewed by Carlos Alberto Lopez Perez.

Optionally generate a sccache config file and use it when building WebKit.

* Scripts/webkitpy/thirdparty/__init__.py:
(AutoinstallImportHook.find_module):
(AutoinstallImportHook._install_toml):
* flatpak/flatpakutils.py:
(WebkitFlatpak.load_from_args):
(WebkitFlatpak.__init__):
(WebkitFlatpak.clean_args):
(WebkitFlatpak.host_path_to_sandbox_path):
(WebkitFlatpak.run_in_sandbox):
(WebkitFlatpak.main):
(WebkitFlatpak.save_config):
(WebkitFlatpak):
(WebkitFlatpak.pack_toolchain):
(WebkitFlatpak.setup_icecc): Deleted.

Modified Paths

Diff

Modified: trunk/Tools/ChangeLog (261606 => 261607)


--- trunk/Tools/ChangeLog	2020-05-13 08:34:28 UTC (rev 261606)
+++ trunk/Tools/ChangeLog	2020-05-13 08:57:55 UTC (rev 261607)
@@ -1,3 +1,27 @@
+2020-05-13  Philippe Normand  <pnorm...@igalia.com>
+
+        [Flatpak SDK] Craft a custom sccache config file from SDK toolchains
+        https://bugs.webkit.org/show_bug.cgi?id=211781
+
+        Reviewed by Carlos Alberto Lopez Perez.
+
+        Optionally generate a sccache config file and use it when building WebKit.
+
+        * Scripts/webkitpy/thirdparty/__init__.py:
+        (AutoinstallImportHook.find_module):
+        (AutoinstallImportHook._install_toml):
+        * flatpak/flatpakutils.py:
+        (WebkitFlatpak.load_from_args):
+        (WebkitFlatpak.__init__):
+        (WebkitFlatpak.clean_args):
+        (WebkitFlatpak.host_path_to_sandbox_path):
+        (WebkitFlatpak.run_in_sandbox):
+        (WebkitFlatpak.main):
+        (WebkitFlatpak.save_config):
+        (WebkitFlatpak):
+        (WebkitFlatpak.pack_toolchain):
+        (WebkitFlatpak.setup_icecc): Deleted.
+
 2020-05-13  Víctor Manuel Jáquez Leal  <vjaq...@igalia.com>
 
         [Flatpak] *PROCESS_CMD_PREFIX are ignored

Modified: trunk/Tools/Scripts/webkitpy/thirdparty/__init__.py (261606 => 261607)


--- trunk/Tools/Scripts/webkitpy/thirdparty/__init__.py	2020-05-13 08:34:28 UTC (rev 261606)
+++ trunk/Tools/Scripts/webkitpy/thirdparty/__init__.py	2020-05-13 08:57:55 UTC (rev 261607)
@@ -128,6 +128,8 @@
             self._install_beautifulsoup()
         elif '.html5lib' in fullname:
             self._install_html5lib()
+        elif '.toml' in fullname:
+            self._install_toml()
 
     def _install_six(self):
         self._install("https://files.pythonhosted.org/packages/16/d8/bc6316cf98419719bd59c91742194c111b6f2e85abac88e496adefaf7afe/six-1.11.0.tar.gz",
@@ -323,6 +325,11 @@
             full_firefox_url = re.findall(r'.*%s.*' % filename_postfix, all_firefox_release_urls)[0].split('"')[3]
             self.install_binary(full_firefox_url, 'geckodriver')
 
+    def _install_toml(self):
+        toml_dir = self._fs.join(_AUTOINSTALLED_DIR, "toml")
+        installer = AutoInstaller(prepend_to_search_path=True, target_dir=toml_dir)
+        installer.install(url="" url_subpath="toml-0.10.0/toml")
+
     def _install(self, url, url_subpath=None, target_name=None):
         installer = AutoInstaller(target_dir=_AUTOINSTALLED_DIR)
         return installer.install(url="" url_subpath=url_subpath, target_name=target_name)

Modified: trunk/Tools/flatpak/flatpakutils.py (261606 => 261607)


--- trunk/Tools/flatpak/flatpakutils.py	2020-05-13 08:34:28 UTC (rev 261606)
+++ trunk/Tools/flatpak/flatpakutils.py	2020-05-13 08:57:55 UTC (rev 261607)
@@ -37,6 +37,8 @@
 from webkitpy.common.system.systemhost import SystemHost
 from webkitpy.port.factory import PortFactory
 from webkitpy.common.system.logutils import configure_logging
+import webkitpy.thirdparty.autoinstalled.toml
+import toml
 
 try:
     from urllib.parse import urlparse  # pylint: disable=E0611
@@ -417,11 +419,17 @@
                              help="The command to run in the sandbox",
                              dest="user_command")
         general.add_argument('--available', action='', dest="check_available", help='Check if required dependencies are available.'),
-        general.add_argument("--use-icecream", dest="use_icecream", help="Use the distributed icecream (icecc) compiler.", action=""
-        general.add_argument("-r", "--regenerate-toolchains", dest="regenerate_toolchains", action=""
-                             help="Regenerate IceCC distribuable toolchain archives")
         general.add_argument("--repo", help="Filesystem absolute path to the Flatpak repository to use", dest="user_repo")
 
+        distributed_build_options = parser.add_argument_group("Distributed building")
+        distributed_build_options.add_argument("--use-icecream", dest="use_icecream", help="Use the distributed icecream (icecc) compiler.", action=""
+        distributed_build_options.add_argument("-r", "--regenerate-toolchains", dest="regenerate_toolchains", action=""
+                             help="Regenerate IceCC distribuable toolchain archives")
+        distributed_build_options.add_argument("-t", "--sccache-token", dest="sccache_token",
+                                               help="sccache authentication token")
+        distributed_build_options.add_argument("-s", "--sccache-scheduler", dest="sccache_scheduler", default='https://sccache.igalia.com',
+                                               help="sccache scheduler URL")
+
         debugoptions = parser.add_argument_group("Debugging")
         debugoptions.add_argument("--gdb", nargs="?", help="Activate gdb, passing extra args to it if wanted.")
         debugoptions.add_argument("--gdb-stack-trace", dest="gdb_stack_trace", nargs="?",
@@ -475,6 +483,8 @@
         self.use_icecream = False
         self.icc_version = {}
         self.regenerate_toolchains = False
+        self.sccache_token = ""
+        self.sccache_scheduler = ""
 
     def execute_command(self, args, stdout=None, stderr=None):
         _log.debug('Running in sandbox: %s\n' % ' '.join(args))
@@ -520,6 +530,7 @@
         self.build_root = os.path.join(self.source_root, 'WebKitBuild')
         self.build_path = os.path.join(self.build_root, self.platform, self.build_type)
         self.config_file = os.path.join(self.flatpak_build_path, 'webkit_flatpak_config.json')
+        self.sccache_config_file = os.path.join(self.flatpak_build_path, 'sccache.toml')
 
         Console.quiet = self.quiet
         if not check_flatpak():
@@ -607,6 +618,10 @@
                 return True
         return False
 
+    def host_path_to_sandbox_path(self, host_path):
+        # For now this supports only files in the WebKit path
+        return host_path.replace(self.source_root, self.sandbox_source_root)
+
     def run_in_sandbox(self, *args, **kwargs):
         self.setup_builddir(stdout=kwargs.get("stdout", sys.stdout))
         cwd = kwargs.pop("cwd", None)
@@ -768,6 +783,9 @@
             _log.debug("Enabling network access for the remote sccache")
             flatpak_command.append(share_network_option)
 
+            if os.path.isfile(self.sccache_config_file) and not self.regenerate_toolchains and "SCCACHE_CONF" not in os.environ.keys():
+                sandbox_environment["SCCACHE_CONF"] = self.host_path_to_sandbox_path(self.sccache_config_file)
+
         override_sccache_server_port = os.environ.get("WEBKIT_SCCACHE_SERVER_PORT")
         if override_sccache_server_port:
             _log.debug("Overriding sccache server port to %s" % override_sccache_server_port)
@@ -836,9 +854,9 @@
 
         if regenerate_toolchains:
             self.icc_version = {}
-            self.setup_icecc("gcc", "g++")
-            self.setup_icecc("clang", "clang++")
-            self.save_config()
+            toolchains = self.pack_toolchain(("gcc", "g++"), {"/usr/bin/c++": "/usr/bin/g++"})
+            toolchains.extend(self.pack_toolchain(("clang", "clang++"), {"/usr/bin/clang++": "/usr/bin/clang++"}))
+            self.save_config(toolchains)
 
         return self.setup_dev_env()
 
@@ -854,12 +872,24 @@
     def has_environment(self):
         return os.path.exists(self.flatpak_build_path)
 
-    def save_config(self):
+    def save_config(self, toolchains):
         with open(self.config_file, 'w') as config:
             json_config = {'icecc_version': self.icc_version}
             json.dump(json_config, config)
 
-    def setup_icecc(self, *compilers):
+        if not self.sccache_token:
+            Console.message("No authentication token provided. Re-run this with the -t option if an sccache token was provided to you. Skipping sccache configuration for now.")
+            return
+
+        with open(self.sccache_config_file, 'w') as config:
+            sccache_config = {'dist': {'scheduler_url': self.sccache_scheduler,
+                                       'auth': {'type': 'token',
+                                                'token': self.sccache_token},
+                                       'toolchains': toolchains}}
+            toml.dump(sccache_config, config)
+            Console.message("Created %s sccache config file. It will automatically be used when building WebKit", self.sccache_config_file)
+
+    def pack_toolchain(self, compilers, path_mapping):
         with tempfile.NamedTemporaryFile() as tmpfile:
             command = ['icecc', '--build-native']
             command.extend(["/usr/bin/%s" % compiler for compiler in compilers])
@@ -875,6 +905,15 @@
             self.icc_version[compilers[0]] = archive_filename
             Console.message("Created %s self-contained toolchain archive", archive_filename)
 
+            sccache_toolchains = []
+            for (compiler_executable, archive_compiler_executable) in path_mapping.iteritems():
+                item = {'type': 'path_override',
+                        'compiler_executable': compiler_executable,
+                        'archive': archive_filename,
+                        'archive_compiler_executable': archive_compiler_executable}
+                sccache_toolchains.append(item)
+            return sccache_toolchains
+
     def setup_dev_env(self):
         if not os.path.exists(os.path.join(self.flatpak_build_path, "runtime", "org.webkit.Sdk")) or self.update:
             self.install_all()
_______________________________________________
webkit-changes mailing list
webkit-changes@lists.webkit.org
https://lists.webkit.org/mailman/listinfo/webkit-changes

Reply via email to