Diff
Modified: trunk/Tools/ChangeLog (287044 => 287045)
--- trunk/Tools/ChangeLog 2021-12-14 21:16:48 UTC (rev 287044)
+++ trunk/Tools/ChangeLog 2021-12-14 21:31:59 UTC (rev 287045)
@@ -1,3 +1,33 @@
+2021-12-13 Jonathan Bedard <[email protected]>
+
+ [reporelaypy] Update checkout with hook instead of polling
+ https://bugs.webkit.org/show_bug.cgi?id=234243
+ <rdar://problem/86413065>
+
+ Reviewed by Dewei Zhu.
+
+ * Scripts/libraries/reporelaypy/reporelaypy/__init__.py: Bump version,
+ export HookProcessor and HookReceiver.
+ * Scripts/libraries/reporelaypy/reporelaypy/checkout.py:
+ (Checkout.update_for): If a branch is new, we need to track it.
+ * Scripts/libraries/reporelaypy/reporelaypy/database.py:
+ (Database.__init__): Use self.host and self.password.
+ * Scripts/libraries/reporelaypy/reporelaypy/hooks.py: Added.
+ (HookProcessor): Class to process received hooks.
+ (HookReceiver): Class to receive hooks and queue them for processing.
+ * Scripts/libraries/reporelaypy/reporelaypy/tests/hooks_unittest.py: Added.
+ (HooksUnittest):
+ (HooksUnittest.setUp):
+ (HooksUnittest.test_receive):
+ (HooksUnittest.test_invalid):
+ (HooksUnittest.test_process):
+ (HooksUnittest.test_hmac):
+ (HooksUnittest.test_invalid_hmac):
+ * Scripts/libraries/reporelaypy/reporelaypy/webserver.py: Add hook routes,
+ if hooks are enabled.
+ * Scripts/libraries/reporelaypy/run: Allow caller to enable hooks.
+ * Scripts/libraries/reporelaypy/setup.py: Bump version.
+
2021-12-14 Alex Christensen <[email protected]>
Add _WKContentRuleListAction.redirected and .modifiedHeaders
Modified: trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/__init__.py (287044 => 287045)
--- trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/__init__.py 2021-12-14 21:16:48 UTC (rev 287044)
+++ trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/__init__.py 2021-12-14 21:31:59 UTC (rev 287045)
@@ -44,7 +44,7 @@
"Please install webkitcorepy with `pip install webkitcorepy --extra-index-url <package index URL>`"
)
-version = Version(0, 2, 0)
+version = Version(0, 3, 0)
import webkitflaskpy
@@ -51,6 +51,7 @@
from reporelaypy.checkout import Checkout
from reporelaypy.database import Database
from reporelaypy.checkoutroute import CheckoutRoute, Redirector
+from reporelaypy.hooks import HookProcessor, HookReceiver
AutoInstall.register(Package('fakeredis', Version(1, 5, 2)))
AutoInstall.register(Package('hiredis', Version(1, 1, 0)))
Modified: trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/checkout.py (287044 => 287045)
--- trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/checkout.py 2021-12-14 21:16:48 UTC (rev 287044)
+++ trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/checkout.py 2021-12-14 21:31:59 UTC (rev 287045)
@@ -164,19 +164,25 @@
return ref == line.split()[0]
return False
- def update_for(self, branch=None, remote='origin'):
+ def update_for(self, branch=None, remote='origin', track=False):
if not self.repository:
sys.stderr.write("Cannot update '{}', clone still pending...\n".format(branch))
return None
branch = branch or self.repository.default_branch
- if branch == self.repository.default_branch:
+ if not self.repository.prod_branches.match(branch):
+ return False
+ elif track and branch not in self.repository.branches_for(remote=remote):
+ run(
+ [self.repository.executable(), 'branch', '--track', branch, 'remotes/{}/{}'.format(remote, branch)],
+ cwd=self.repository.root_path,
+ )
+ self.repository.cache.populate(branch=branch)
+ elif branch == self.repository.default_branch:
self.repository.pull(remote=remote)
self.repository.cache.populate(branch=branch)
return True
- if not self.repository.prod_branches.match(branch):
- return False
- if self.is_updated(branch, remote=remote):
+ elif not track and self.is_updated(branch, remote=remote):
return True
run(
Modified: trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/database.py (287044 => 287045)
--- trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/database.py 2021-12-14 21:16:48 UTC (rev 287044)
+++ trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/database.py 2021-12-14 21:31:59 UTC (rev 287045)
@@ -46,9 +46,9 @@
self.host = host or Environment.instance().get(self.HOST_ENV)
self.password = password or Environment.instance().get(self.PASSWORD_ENV)
- if host:
+ if self.host:
import redis
- self._redis = redis.Redis(host=host, password=password)
+ self._redis = redis.StrictRedis(host=self.host, password=self.password)
else:
import fakeredis
self._redis = fakeredis.FakeStrictRedis()
Added: trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/hooks.py (0 => 287045)
--- trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/hooks.py (rev 0)
+++ trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/hooks.py 2021-12-14 21:31:59 UTC (rev 287045)
@@ -0,0 +1,154 @@
+# Copyright (C) 2021 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import hashlib
+import hmac
+import json
+
+from flask import current_app, json as fjson, request
+from reporelaypy import Database
+from webkitflaskpy import AuthedBlueprint
+from webkitcorepy import Environment, string_utils
+
+TIMEOUT = 30 * 60
+
+
+class HookProcessor(object):
+ INBOUND_KEY = 'inbound-hooks'
+ WORKER_HOOKS = 'worker-hooks'
+ TYPES = ('push',)
+
+ @classmethod
+ def is_valid(cls, type, data):
+ if type not in cls.TYPES or not isinstance(data, dict):
+ return False
+ return type == 'push' and data.get('ref')
+
+ def __init__(self, checkout, database=None, num_workers=1, worker_index=0, callbacks=None):
+ self.checkout = checkout
+ self.database = database or Database()
+ self.num_workers = num_workers
+ self.worker_index = worker_index
+ self.callbacks = callbacks or dict()
+
+ def process_hook(self, type, data):
+ if not self.is_valid(type, data):
+ return None
+
+ self.callbacks.get(type, lambda _: _)(data)
+
+ def process_worker_hook(self, type, data):
+ if not self.is_valid(type, data):
+ return None
+
+ branch = data.get('ref')
+ if type == 'push' and branch:
+ try:
+ if branch.startswith('refs/heads/'):
+ branch = branch[len('refs/heads/'):]
+ self.checkout.update_for(branch, track=True)
+ except BaseException as e:
+ sys.stderr.write('{}\n'.format(e))
+
+ def process_hooks(self):
+ with self.database.lock(name='lock_{}'.format(self.INBOUND_KEY), timeout=60):
+ for key in self.database.scan_iter('{}:*'.format(self.INBOUND_KEY)):
+ data = ""
+ self.process_hook(type=data.get('type'), data=""
+
+ encoded = json.dumps(data)
+ digest = hashlib.md5()
+ digest.update(string_utils.encode(encoded))
+ digest = digest.hexdigest()
+ for i in range(self.num_workers):
+ self.database.set('{}-{}:{}'.format(HookProcessor.WORKER_HOOKS, i, digest), encoded, ex=TIMEOUT)
+ self.database.delete(string_utils.decode(key))
+
+ for key in self.database.scan_iter('{}-{}:*'.format(self.WORKER_HOOKS, self.worker_index)):
+ data = ""
+ self.process_worker_hook(type=data.get('type'), data=""
+ self.database.delete(string_utils.decode(key))
+
+
+class HookReceiver(AuthedBlueprint):
+ SECRET_ENV = 'HOOK_SECRET'
+
+ def __init__(self, import_name=__name__, auth_decorator=None, database=None, debug=False, secret=None):
+ super(HookReceiver, self).__init__('hooks', import_name, url_prefix='/hooks', auth_decorator=auth_decorator)
+
+ self.database = database or Database()
+ self.secret = secret
+
+ self.add_url_rule('', 'inbound', self.inbound, methods=('POST',))
+ if debug:
+ self.add_url_rule('', 'received', self.received, methods=('GET',))
+
+ def inbound(self):
+ if self.secret and request.headers.get('X-Hub-Signature-256', '') != 'sha256={}'.format(hmac.new(
+ string_utils.encode(self.secret), request.data, hashlib.sha256,
+ ).hexdigest()):
+ return current_app.response_class(
+ fjson.dumps(dict(
+ status='Unauthorized Hook',
+ message='HMAC verification failed failed',
+ ), indent=4),
+ mimetype='application/json',
+ status=403,
+ )
+
+ type = request.headers.get('X-GitHub-Event', '')
+ data = "" force=False)
+ if HookProcessor.is_valid(type, data):
+ encoded = json.dumps(dict(type=type, data=""
+ digest = hashlib.md5()
+ digest.update(string_utils.encode(encoded))
+
+ self.database.set('{}:{}'.format(HookProcessor.INBOUND_KEY, digest.hexdigest()), encoded, ex=TIMEOUT)
+
+ return current_app.response_class(
+ fjson.dumps(dict(
+ status='Success',
+ message='Hook queued for processing',
+ ), indent=4),
+ mimetype='application/json',
+ status=202,
+ )
+
+ return current_app.response_class(
+ fjson.dumps(dict(
+ status='Unrecognized Hook',
+ message='Provided hook is in unrecognized format',
+ ), indent=4),
+ mimetype='application/json',
+ status=400,
+ )
+
+ def received(self):
+ result = [
+ json.loads(self.database.get(string_utils.decode(key)))
+ for key in self.database.scan_iter('{}:*'.format(HookProcessor.INBOUND_KEY))
+ ]
+ return current_app.response_class(
+ fjson.dumps(result, indent=4),
+ mimetype='application/json',
+ status=200,
+ )
Added: trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/tests/hooks_unittest.py (0 => 287045)
--- trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/tests/hooks_unittest.py (rev 0)
+++ trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/tests/hooks_unittest.py 2021-12-14 21:31:59 UTC (rev 287045)
@@ -0,0 +1,136 @@
+# Copyright (C) 2021 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import json
+import unittest
+
+from reporelaypy import Checkout, Database, HookProcessor, HookReceiver
+from webkitcorepy import testing, OutputCapture
+from webkitflaskpy import mock_app
+from webkitscmpy import mocks, Commit
+
+
+class HooksUnittest(testing.PathTestCase):
+ basepath = 'mock/repository'
+
+ def setUp(self):
+ super(HooksUnittest, self).setUp()
+ os.mkdir(os.path.join(self.path, '.git'))
+
+ @mock_app
+ def test_receive(self, app=None, client=None):
+ app.register_blueprint(HookReceiver(debug=True))
+ response = client.get('/hooks')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(response.json(), [])
+
+ response = client.post('/hooks', json=dict(ref='refs/heads/main'), headers={'X-GitHub-Event': 'push'})
+ self.assertEqual(response.status_code, 202)
+ self.assertEqual(response.json(), dict(
+ status='Success',
+ message='Hook queued for processing',
+ ))
+
+ response = client.get('/hooks')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(response.json(), [dict(data="" type='push')])
+
+ @mock_app
+ def test_invalid(self, app=None, client=None):
+ app.register_blueprint(HookReceiver(debug=True))
+ response = client.get('/hooks')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(response.json(), [])
+
+ response = client.post('/hooks', json=['invalid'], headers={'X-GitHub-Event': 'push'})
+ self.assertEqual(response.status_code, 400)
+ self.assertEqual(response.json(), dict(
+ status='Unrecognized Hook',
+ message='Provided hook is in unrecognized format',
+ ))
+
+ @mock_app
+ def test_process(self, app=None, client=None):
+ with OutputCapture(), mocks.local.Git(self.path) as repo:
+ database = Database()
+ app.register_blueprint(HookReceiver(database=database, debug=True))
+
+ response = client.post('/hooks', json=dict(ref='refs/heads/main'), headers={'X-GitHub-Event': 'push'})
+ self.assertEqual(response.status_code, 202)
+
+ processor = HookProcessor(
+ Checkout(path=self.path, url="" sentinal=False),
+ database=database,
+ )
+ processor.process_hooks()
+
+ response = client.get('/hooks')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(response.json(), [])
+
+ @mock_app
+ def test_process_branch(self, app=None, client=None):
+ with OutputCapture(), mocks.local.Git(self.path) as repo:
+ database = Database()
+ app.register_blueprint(HookReceiver(database=database, debug=True))
+
+ response = client.post('/hooks', json=dict(ref='refs/heads/branch-a'), headers={'X-GitHub-Event': 'push'})
+ self.assertEqual(response.status_code, 202)
+
+ processor = HookProcessor(
+ Checkout(path=self.path, url="" sentinal=False),
+ database=database,
+ )
+ processor.process_hooks()
+
+ response = client.get('/hooks')
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(response.json(), [])
+
+ @mock_app
+ def test_hmac(self, app=None, client=None):
+ app.register_blueprint(HookReceiver(debug=True, secret='secret'))
+
+ response = client.post(
+ '/hooks', json=dict(ref='refs/heads/main'),
+ headers={
+ 'X-Hub-Signature-256': 'sha256=36a089f93b92e972b714e8c0f008873a206c690a8aee946a787eeb0f23e131b2',
+ 'X-GitHub-Event': 'push',
+ },
+ )
+ self.assertEqual(response.status_code, 202)
+ self.assertEqual(response.json(), dict(
+ status='Success',
+ message='Hook queued for processing',
+ ))
+
+ @mock_app
+ def test_invalid_hmac(self, app=None, client=None):
+ app.register_blueprint(HookReceiver(debug=True, secret='secret'))
+
+ response = client.post('/hooks', json=dict(ref='refs/heads/main'), headers={'X-GitHub-Event': 'push'})
+ self.assertEqual(response.status_code, 403)
+ self.assertEqual(response.json(), dict(
+ status='Unauthorized Hook',
+ message='HMAC verification failed failed',
+ ))
Modified: trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/webserver.py (287044 => 287045)
--- trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/webserver.py 2021-12-14 21:16:48 UTC (rev 287044)
+++ trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/webserver.py 2021-12-14 21:31:59 UTC (rev 287045)
@@ -20,6 +20,7 @@
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import json
import os
autoinstall_path = os.environ.get('AUTOINSTALL_PATH')
@@ -28,7 +29,7 @@
AutoInstall.set_directory(autoinstall_path)
from flask import Flask, current_app, json as fjson
-from reporelaypy import Checkout, CheckoutRoute, Database, Redirector
+from reporelaypy import Checkout, CheckoutRoute, Database, Redirector, HookReceiver
app = Flask(__name__)
@@ -39,7 +40,16 @@
import_name=__name__, database=database,
)
+hook_args = json.loads(os.environ.get('HOOKS', '{}'))
+if hook_args.get('enabled', False):
+ hook_routes = HookReceiver(
+ import_name=__name__, database=database,
+ debug=hook_args.get('debug', False), secret=os.environ.get(HookReceiver.SECRET_ENV),
+ )
+else:
+ hook_routes = None
+
@app.route('/__health')
def health():
return 'ready' if checkout.repository else 'cloning'
@@ -46,6 +56,8 @@
app.register_blueprint(checkout_routes)
+if hook_routes:
+ app.register_blueprint(hook_routes)
if __name__ == '__main__':
Modified: trunk/Tools/Scripts/libraries/reporelaypy/run (287044 => 287045)
--- trunk/Tools/Scripts/libraries/reporelaypy/run 2021-12-14 21:16:48 UTC (rev 287044)
+++ trunk/Tools/Scripts/libraries/reporelaypy/run 2021-12-14 21:31:59 UTC (rev 287045)
@@ -38,7 +38,7 @@
sys.path.insert(0, scripts)
import webkitpy
-from reporelaypy import Checkout, Database, Redirector
+from reporelaypy import Checkout, Database, HookProcessor, HookReceiver, Redirector
from webkitcorepy import arguments, AutoInstall
from whichcraft import which
@@ -101,6 +101,16 @@
help='Base URL to redirect user to for commit information.',
)
+ group = parser.add_argument_group('Hooks')
+ group.add_argument(
+ '--hooks', '--no-hooks', action="" dest='hooks', default=False,
+ help='Enable or disable hook end-points (disabled by default)',
+ )
+ group.add_argument(
+ '--debug', '--no-debug', action="" dest='hooks_debug', default=True,
+ help='Enable endpoint to report hooks being processed (enabled by default)',
+ )
+
args = parser.parse_args(args=args)
database = Database(host=args.redis_host, password=args.redis_password)
@@ -140,25 +150,30 @@
redirector = Redirector(url)
print(' {}: {}'.format(redirector.name, redirector.url))
- env = dict(
+ passenv = dict(
PYTHONPATH=':'.join(sys.path),
CHECKOUT=json.dumps(checkout, cls=Checkout.Encoder),
- REDIRECTORS=json.dumps([Redirector(url) for url in args.redirector or []], cls=Redirector.Encoder)
+ REDIRECTORS=json.dumps([Redirector(url) for url in args.redirector or []], cls=Redirector.Encoder),
+ HOOKS=json.dumps({'enabled': args.hooks, 'debug': args.hooks_debug}),
)
if AutoInstall.directory:
- env['AUTOINSTALL_PATH'] = AutoInstall.directory
+ passenv['AUTOINSTALL_PATH'] = AutoInstall.directory
if database.host:
- env[database.HOST_ENV] = database.host
+ passenv[database.HOST_ENV] = database.host
if database.password:
- env[database.PASSWORD_ENV] = database.password
+ passenv[database.PASSWORD_ENV] = database.password
if database.default_expiration:
- env[database.EXPIRATION_ENV] = str(database.default_expiration)
+ passenv[database.EXPIRATION_ENV] = str(database.default_expiration)
+ if os.environ.get(HookReceiver.SECRET_ENV):
+ passenv[HookReceiver.SECRET_ENV] = os.environ.get(HookReceiver.SECRET_ENV)
+ processor = HookProcessor(checkout=checkout, database=database) if args.hooks else None
+
with subprocess.Popen(
[which('gunicorn'), 'reporelaypy.webserver:app'],
cwd=os.path.dirname(os.path.dirname(reporelaypy.__file__)),
- env=env,
+ env=passenv,
) as webserver:
last_poll = time.time()
last_pull = time.time()
@@ -169,7 +184,7 @@
break
last_poll = time.time()
if last_pull + args.update_interval < time.time():
- checkout.update_all()
+ processor.process_hooks() if processor else checkout.update_all()
last_pull = time.time()
time.sleep(math.gcd(args.poll, args.update_interval))
Modified: trunk/Tools/Scripts/libraries/reporelaypy/setup.py (287044 => 287045)
--- trunk/Tools/Scripts/libraries/reporelaypy/setup.py 2021-12-14 21:16:48 UTC (rev 287044)
+++ trunk/Tools/Scripts/libraries/reporelaypy/setup.py 2021-12-14 21:31:59 UTC (rev 287045)
@@ -30,7 +30,7 @@
setup(
name='reporelaypy',
- version='0.2.0',
+ version='0.3.0',
description='Library for visualizing, processing and storing test results.',
long_description=readme(),
classifiers=[
