Title: [122394] trunk/Tools
Revision
122394
Author
[email protected]
Date
2012-07-11 17:45:58 -0700 (Wed, 11 Jul 2012)

Log Message

nrwt: add a MessagePool abstraction that the manager will call to replace the broker
https://bugs.webkit.org/show_bug.cgi?id=90511

Reviewed by Ojan Vafai.

This change introduces the new MessagePool abstraction that will
replace the classes in manager_worker_broker. It is a minimal
interface that tries to follow the conventions in
multiprocessing.Pool and concurrency.futures ... it provides a
context manager and a run() method that sends N messages to M
workers processes (starting workers as necessary) and waits for
them all to complete, handling cleanup as necessary. The caller
is responsible for providing a handle() method to handle
messages received from the workers.

This interface basically hides all of the multiprocessing logic from
the manager class.

The initial implementation of MessagePool is a simple shim
around the existing broker classes; a subsequent change will
replace all the other classes with a much simpler
implementation.

No additional tests are provided for now; existing tests should
provide adequate coverage, and I will add new unit tests for the
MessagePool class when I replace the existing implementation.

* Scripts/webkitpy/layout_tests/controllers/manager.py:
(TestRunInterruptedException.__reduce__):
(Manager.__init__):
(Manager._run_tests):
(Manager._run_tests.instead):
(Manager.handle):
(Manager._handle_started_test):
(Manager._handle_finished_test_list):
(Manager._handle_finished_test):
* Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py:
(get):
(_MessagePool):
(_MessagePool.__init__):
(_MessagePool.__enter__):
(_MessagePool.__exit__):
(_MessagePool.run):
(_MessagePool.wait):
(_MessagePool.is_done):
(_MessagePool._worker_is_done):
(_MessagePool._close):
(_MessagePool.handle_done):
(_MessagePool.handle_started_test):
(_MessagePool.handle_finished_test):
(_MessagePool.handle_finished_test_list):
(_MessagePool.handle_exception):
(_MessagePool._log_messages):
(_MessagePool._handle_worker_exception):
(_WorkerState):
(_WorkerState.for):
(_WorkerState.__init__):
(_WorkerState.__repr__):
(_get_broker):
* Scripts/webkitpy/layout_tests/controllers/manager_worker_broker_unittest.py:
(make_broker):

Modified Paths

Diff

Modified: trunk/Tools/ChangeLog (122393 => 122394)


--- trunk/Tools/ChangeLog	2012-07-12 00:23:40 UTC (rev 122393)
+++ trunk/Tools/ChangeLog	2012-07-12 00:45:58 UTC (rev 122394)
@@ -1,3 +1,67 @@
+2012-07-03  Dirk Pranke  <[email protected]>
+
+        nrwt: add a MessagePool abstraction that the manager will call to replace the broker
+        https://bugs.webkit.org/show_bug.cgi?id=90511
+
+        Reviewed by Ojan Vafai.
+
+        This change introduces the new MessagePool abstraction that will
+        replace the classes in manager_worker_broker. It is a minimal
+        interface that tries to follow the conventions in
+        multiprocessing.Pool and concurrency.futures ... it provides a
+        context manager and a run() method that sends N messages to M
+        workers processes (starting workers as necessary) and waits for
+        them all to complete, handling cleanup as necessary. The caller
+        is responsible for providing a handle() method to handle
+        messages received from the workers.
+        
+        This interface basically hides all of the multiprocessing logic from
+        the manager class.
+
+        The initial implementation of MessagePool is a simple shim
+        around the existing broker classes; a subsequent change will
+        replace all the other classes with a much simpler
+        implementation.
+
+        No additional tests are provided for now; existing tests should
+        provide adequate coverage, and I will add new unit tests for the
+        MessagePool class when I replace the existing implementation.
+
+        * Scripts/webkitpy/layout_tests/controllers/manager.py:
+        (TestRunInterruptedException.__reduce__):
+        (Manager.__init__):
+        (Manager._run_tests):
+        (Manager._run_tests.instead):
+        (Manager.handle):
+        (Manager._handle_started_test):
+        (Manager._handle_finished_test_list):
+        (Manager._handle_finished_test):
+        * Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py:
+        (get):
+        (_MessagePool):
+        (_MessagePool.__init__):
+        (_MessagePool.__enter__):
+        (_MessagePool.__exit__):
+        (_MessagePool.run):
+        (_MessagePool.wait):
+        (_MessagePool.is_done):
+        (_MessagePool._worker_is_done):
+        (_MessagePool._close):
+        (_MessagePool.handle_done):
+        (_MessagePool.handle_started_test):
+        (_MessagePool.handle_finished_test):
+        (_MessagePool.handle_finished_test_list):
+        (_MessagePool.handle_exception):
+        (_MessagePool._log_messages):
+        (_MessagePool._handle_worker_exception):
+        (_WorkerState):
+        (_WorkerState.for):
+        (_WorkerState.__init__):
+        (_WorkerState.__repr__):
+        (_get_broker):
+        * Scripts/webkitpy/layout_tests/controllers/manager_worker_broker_unittest.py:
+        (make_broker):
+
 2012-07-11  Simon Fraser  <[email protected]>
 
         Add an option to enter paginated mode in MiniBrowser

Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py (122393 => 122394)


--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py	2012-07-12 00:23:40 UTC (rev 122393)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py	2012-07-12 00:45:58 UTC (rev 122394)
@@ -266,6 +266,7 @@
         return self.__class__, (self.reason,)
 
 
+# Export this so callers don't need to know about manager_worker_broker.
 WorkerException = manager_worker_broker.WorkerException
 
 
@@ -300,7 +301,6 @@
         self._filesystem = port.host.filesystem
         self._options = options
         self._printer = printer
-        self._message_broker = None
         self._expectations = None
 
         self.HTTP_SUBDIR = 'http' + port.TEST_PATH_SEPARATOR
@@ -328,13 +328,9 @@
 
         self._all_results = []
         self._group_stats = {}
+        self._worker_stats = {}
         self._current_result_summary = None
 
-        # This maps worker names to the state we are tracking for each of them.
-        self._worker_states = {}
-
-        self.name = 'manager'
-
     def collect_tests(self, args):
         """Find all the files to test.
 
@@ -746,11 +742,10 @@
         self._current_result_summary = result_summary
         self._all_results = []
         self._group_stats = {}
-        self._worker_states = {}
+        self._worker_stats = {}
 
         keyboard_interrupted = False
         interrupted = False
-        thread_timings = []
 
         self._printer.print_update('Sharding tests ...')
         locked_shards, unlocked_shards = self._shard_tests(file_list, int(self._options.child_processes), self._options.fully_parallel)
@@ -777,65 +772,28 @@
         manager_connection = manager_worker_broker.get(num_workers, self, worker_factory, self._port.host)
 
         if self._options.dry_run:
-            return (keyboard_interrupted, interrupted, thread_timings, self._group_stats, self._all_results)
+            return (keyboard_interrupted, interrupted, self._worker_stats.values(), self._group_stats, self._all_results)
 
         self._printer.print_update('Starting %s ...' % grammar.pluralize('worker', num_workers))
-        for worker_number in xrange(num_workers):
-            worker_connection = manager_connection.start_worker(worker_number)
-            worker_state = _WorkerState(worker_number, worker_connection)
-            self._worker_states[worker_connection.name] = worker_state
 
-            time.sleep(self._port.worker_startup_delay_secs())
-
-        self._printer.print_update("Starting testing ...")
-        for shard in all_shards:
-            # FIXME: Change 'test_list' to 'shard', make sharding public.
-            manager_connection.post_message('test_list', shard.name, shard.test_inputs)
-
-        # We post one 'stop' message for each worker. Because the stop message
-        # are sent after all of the tests, and because each worker will stop
-        # reading messsages after receiving a stop, we can be sure each
-        # worker will get a stop message and hence they will all shut down.
-        for _ in xrange(num_workers):
-            manager_connection.post_message('stop')
-
         try:
-            while not self.is_done():
-                manager_connection.run_message_loop(delay_secs=1.0)
-
-            # Make sure all of the workers have shut down (if possible).
-            for worker_state in self._worker_states.values():
-                if worker_state.worker_connection.is_alive():
-                    _log.debug('Waiting for worker %d to exit' % worker_state.number)
-                    worker_state.worker_connection.join(5.0)
-                    if worker_state.worker_connection.is_alive():
-                        _log.error('Worker %d did not exit in time.' % worker_state.number)
-
+            with manager_worker_broker.get(self, worker_factory, num_workers, self._port.worker_startup_delay_secs(), self._port.host) as pool:
+                pool.run(('test_list', shard.name, shard.test_inputs) for shard in all_shards)
         except KeyboardInterrupt:
             self._printer.flush()
             self._printer.write('Interrupted, exiting ...')
-            self.cancel_workers()
             keyboard_interrupted = True
         except TestRunInterruptedException, e:
             _log.warning(e.reason)
-            self.cancel_workers()
             interrupted = True
-        except WorkerException:
-            self.cancel_workers()
-            raise
         except:
-            # Unexpected exception; don't try to clean up workers.
             _log.error("Exception raised, exiting")
-            self.cancel_workers()
             raise
         finally:
-            manager_connection.cleanup()
             self.stop_servers_with_lock()
 
-        thread_timings = [worker_state.stats for worker_state in self._worker_states.values()]
-
         # FIXME: should this be a class instead of a tuple?
-        return (interrupted, keyboard_interrupted, thread_timings, self._group_stats, self._all_results)
+        return (interrupted, keyboard_interrupted, self._worker_stats.values(), self._group_stats, self._all_results)
 
     def results_directory(self):
         if not self._retrying:
@@ -1444,40 +1402,17 @@
         results_filename = self._filesystem.join(self._results_directory, "results.html")
         self._port.show_results_html_file(results_filename)
 
+    def handle(self, name, source, *args):
+        method = getattr(self, '_handle_' + name)
+        if method:
+            return method(source, *args)
+        raise AssertionError('unknown message %s received from %s, args=%s' % (name, source, repr(args)))
 
-    def is_done(self):
-        worker_states = self._worker_states.values()
-        return worker_states and all(self._worker_is_done(worker_state) for worker_state in worker_states)
+    def _handle_started_test(self, worker_name, test_input, test_timeout_sec):
+        # FIXME: log that we've started another test.
+        pass
 
-    # FIXME: Inline this function.
-    def _worker_is_done(self, worker_state):
-        return worker_state.done
-
-    def cancel_workers(self):
-        for worker_state in self._worker_states.values():
-            worker_state.worker_connection.cancel()
-
-    def handle_started_test(self, source, test_info, hang_timeout):
-        worker_state = self._worker_states[source]
-        worker_state.current_test_name = test_info.test_name
-        worker_state.next_timeout = time.time() + hang_timeout
-
-    def handle_done(self, source, log_messages=None):
-        worker_state = self._worker_states[source]
-        worker_state.done = True
-        self._log_messages(log_messages)
-
-    def handle_exception(self, source, exception_type, exception_value, stack):
-        if exception_type in (KeyboardInterrupt, TestRunInterruptedException):
-            raise exception_type(exception_value)
-        _log.error("%s raised %s('%s'):" % (
-                   source,
-                   exception_value.__class__.__name__,
-                   str(exception_value)))
-        self._log_worker_stack(stack)
-        raise WorkerException(str(exception_value))
-
-    def handle_finished_test_list(self, source, list_name, num_tests, elapsed_time):
+    def _handle_finished_test_list(self, worker_name, list_name, num_tests, elapsed_time):
         self._group_stats[list_name] = (num_tests, elapsed_time)
 
         def find(name, test_lists):
@@ -1492,30 +1427,14 @@
             if not self._remaining_locked_shards:
                 self.stop_servers_with_lock()
 
-    def handle_finished_test(self, source, result, elapsed_time, log_messages=None):
-        worker_state = self._worker_states[source]
-        worker_state.next_timeout = None
-        worker_state.current_test_name = None
-        worker_state.stats['total_time'] += elapsed_time
-        worker_state.stats['num_tests'] += 1
-
-        self._log_messages(log_messages)
+    def _handle_finished_test(self, worker_name, result, elapsed_time, log_messages=[]):
+        self._worker_stats.setdefault(worker_name, {'name': worker_name, 'num_tests': 0, 'total_time': 0})
+        self._worker_stats[worker_name]['total_time'] += elapsed_time
+        self._worker_stats[worker_name]['num_tests'] += 1
         self._all_results.append(result)
         self._update_summary_with_result(self._current_result_summary, result)
 
-    def _log_messages(self, messages):
-        for message in messages:
-            logging.root.handle(message)
 
-    def _log_worker_stack(self, stack):
-        webkitpydir = self._port.path_from_webkit_base('Tools', 'Scripts', 'webkitpy') + self._filesystem.sep
-        for filename, line_number, function_name, text in stack:
-            if filename.startswith(webkitpydir):
-                filename = filename.replace(webkitpydir, '')
-            _log.error('  %s:%u (in %s)' % (filename, line_number, function_name))
-            _log.error('    %s' % text)
-
-
 def read_test_files(fs, filenames, test_path_separator):
     tests = []
     for filename in filenames:
@@ -1563,20 +1482,3 @@
             return val
 
     return [tryint(chunk) for chunk in re.split('(\d+)', string_to_split)]
-
-
-class _WorkerState(object):
-    """A class for the manager to use to track the current state of the workers."""
-    def __init__(self, number, worker_connection):
-        self.worker_connection = worker_connection
-        self.number = number
-        self.done = False
-        self.current_test_name = None
-        self.next_timeout = None
-        self.stats = {}
-        self.stats['name'] = worker_connection.name
-        self.stats['num_tests'] = 0
-        self.stats['total_time'] = 0
-
-    def __repr__(self):
-        return "_WorkerState(" + str(self.__dict__) + ")"

Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py (122393 => 122394)


--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py	2012-07-12 00:23:40 UTC (rev 122393)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py	2012-07-12 00:45:58 UTC (rev 122394)
@@ -26,45 +26,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-"""Module for handling messages and concurrency for run-webkit-tests.
+"""Module for handling messages and concurrency for run-webkit-tests."""
 
-This module implements a message broker that connects the manager to the
-workers: it provides a messaging abstraction and message loops (building on
-top of message_broker), and handles starting workers by launching processes.
-
-There are a lot of classes and objects involved in a fully connected system.
-They interact more or less like:
-
-  Manager  -->  _InlineManager ---> _InlineWorker <-> Worker
-     ^                    \               /              ^
-     |                     v             v               |
-     \-----------------------  Broker   ----------------/
-
-The broker simply distributes messages onto topics (named queues); the actual
-queues themselves are provided by the caller, as the queue's implementation
-requirements varies vary depending on the desired concurrency model
-(none/threads/processes).
-
-In order for shared-nothing messaging between processing to be possible,
-Messages must be picklable.
-
-The module defines one interface and two classes. Callers of this package
-must implement the BrokerClient interface, and most callers will create
-_BrokerConnections as well as Brokers.
-
-The classes relate to each other as:
-
-    BrokerClient   ------>    _BrokerConnection
-         ^                         |
-         |                         v
-         \----------------      _Broker
-
-(The BrokerClient never calls broker directly after it is created, only
-_BrokerConnection.  _BrokerConnection passes a reference to BrokerClient to
-_Broker, and _Broker only invokes that reference, never talking directly to
-BrokerConnection).
-"""
-
 import cPickle
 import logging
 import multiprocessing
@@ -72,6 +35,7 @@
 import os
 import Queue
 import sys
+import time
 import traceback
 
 
@@ -90,7 +54,106 @@
 ANY_WORKER_TOPIC = 'workers'
 
 
-def get(max_workers, client, worker_factory, host=None):
+def get(caller, worker_factory, num_workers, worker_startup_delay_secs=0.0, host=None):
+    """Returns an object that exposes a run() method that takes a list of test shards and runs them in parallel."""
+    return _MessagePool(caller, worker_factory, num_workers, worker_startup_delay_secs, host)
+
+
+class _MessagePool(object):
+    def __init__(self, caller, worker_factory, num_workers, worker_startup_delay_secs=0.0, host=None):
+        self._caller = caller
+        self._worker_factory = worker_factory
+        self._num_workers = num_workers
+        self._worker_startup_delay_secs = worker_startup_delay_secs
+        self._worker_states = {}
+        self._host = host
+        self.name = 'manager'
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self._close()
+        return False
+
+    def run(self, shards):
+        manager_connection = _get_broker(self._num_workers, self, self._worker_factory, self._host)
+        for worker_number in xrange(self._num_workers):
+            worker_connection = manager_connection.start_worker(worker_number)
+            worker_state = _WorkerState(worker_number, worker_connection)
+            self._worker_states[worker_connection.name] = worker_state
+            time.sleep(self._worker_startup_delay_secs)
+
+        messages = list(shards)
+        for message in messages:
+            manager_connection.post_message(*message)
+
+        for _ in xrange(self._num_workers):
+            manager_connection.post_message('stop')
+
+        self.wait(manager_connection)
+
+    def wait(self, manager_connection):
+        try:
+            while not self.is_done():
+                manager_connection.run_message_loop(delay_secs=1.0)
+        finally:
+            self._close()
+
+    def is_done(self):
+        worker_states = self._worker_states.values()
+        return worker_states and all(worker_state.done for worker_state in worker_states)
+
+    def _close(self):
+        for worker_state in self._worker_states.values():
+            if worker_state.worker_connection.is_alive():
+                worker_state.worker_connection.cancel()
+                _log.debug('Waiting for worker %d to exit' % worker_state.number)
+                worker_state.worker_connection.join(5.0)
+                if worker_state.worker_connection.is_alive():
+                    _log.error('Worker %d did not exit in time.' % worker_state.number)
+
+    def handle_done(self, source, log_messages=None):
+        worker_state = self._worker_states[source]
+        worker_state.done = True
+        self._log_messages(log_messages)
+
+    def handle_started_test(self, *args):
+        self._caller.handle('started_test', *args)
+
+    def handle_finished_test(self, *args):
+        self._caller.handle('finished_test', *args)
+
+    def handle_finished_test_list(self, *args):
+        self._caller.handle('finished_test_list', *args)
+
+    def handle_exception(self, *args):
+        self._handle_worker_exception(*args)
+
+    def _log_messages(self, messages):
+        for message in messages:
+            logging.root.handle(message)
+
+    @staticmethod
+    def _handle_worker_exception(source, exception_type, exception_value, stack):
+        if exception_type == KeyboardInterrupt:
+            raise exception_type(exception_value)
+        _log.error("%s raised %s('%s'):" % (
+                   source, exception_value.__class__.__name__, str(exception_value)))
+        raise WorkerException(str(exception_value))
+
+
+class _WorkerState(object):
+    def __init__(self, number, worker_connection):
+        self.worker_connection = worker_connection
+        self.number = number
+        self.done = False
+
+    def __repr__(self):
+        return "_WorkerState(" + str(self.__dict__) + ")"
+
+
+def _get_broker(max_workers, client, worker_factory, host=None):
     """Return a connection to a manager/worker message_broker
 
     Args:
@@ -397,6 +460,7 @@
     def join(self, timeout):
         raise NotImplementedError
 
+    # FIXME: rename to yield_to_caller().
     def yield_to_broker(self):
         pass
 

Modified: trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker_unittest.py (122393 => 122394)


--- trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker_unittest.py	2012-07-12 00:23:40 UTC (rev 122393)
+++ trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker_unittest.py	2012-07-12 00:45:58 UTC (rev 122394)
@@ -51,7 +51,7 @@
     global stopping_queue
     starting_queue = start_queue
     stopping_queue = stop_queue
-    return manager_worker_broker.get(max_workers, manager, _TestWorker)
+    return manager_worker_broker._get_broker(max_workers, manager, _TestWorker)
 
 
 class _TestWorker(object):
_______________________________________________
webkit-changes mailing list
[email protected]
http://lists.webkit.org/mailman/listinfo/webkit-changes

Reply via email to