jenkins-bot has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1336046?usp=email )
Change subject: threading: Release bounded capacity when submission fails
......................................................................
threading: Release bounded capacity when submission fails
Submitting to a shut down executor raises RuntimeError rather than
BrokenExecutor. The acquired semaphore slot was never released, so later
submissions could block indefinitely once the bound was exhausted.
Release the slot on any exception from the underlying submit call and
re-raise the original exception, including interruptions.
Change-Id: I09b286c39149f7d2915a3bdbbb2ea9a27fead3bf
---
M pywikibot/tools/threading.py
M tests/tools_threading_tests.py
2 files changed, 35 insertions(+), 1 deletion(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/pywikibot/tools/threading.py b/pywikibot/tools/threading.py
index 7cd7fe2..e112249 100644
--- a/pywikibot/tools/threading.py
+++ b/pywikibot/tools/threading.py
@@ -264,7 +264,7 @@
try:
f = super().submit(fn, *args, **kwargs)
- except futures.BrokenExecutor:
+ except BaseException:
self._bound_semaphore.release()
raise
diff --git a/tests/tools_threading_tests.py b/tests/tools_threading_tests.py
index abc836f..20f12bb 100755
--- a/tests/tools_threading_tests.py
+++ b/tests/tools_threading_tests.py
@@ -9,6 +9,7 @@
import unittest
from concurrent.futures import (
+ BrokenExecutor,
Executor,
Future,
ProcessPoolExecutor,
@@ -16,6 +17,7 @@
)
from contextlib import suppress
from threading import Condition, Event, Thread
+from unittest.mock import patch
from pywikibot.tools import PYTHON_VERSION
from pywikibot.tools.threading import BoundedPoolExecutor, ThreadedGenerator
@@ -136,6 +138,38 @@
with self.subTest(bound=bound):
self._check_bound(bound)
+ def test_submit_after_shutdown(self) -> None:
+ """Test failed submissions after shutdown release their capacity."""
+ for executor in (ThreadPoolExecutor, ProcessPoolExecutor):
+ with self.subTest(executor=executor):
+ pool = BoundedPoolExecutor(executor, max_bound=1,
+ max_workers=1)
+ pool.shutdown()
+ with self.assertRaises(RuntimeError):
+ pool.submit(pow, 2, 3)
+ self.assertTrue(
+ pool._bound_semaphore.acquire(blocking=False)
+ )
+ pool._bound_semaphore.release()
+
+ def test_submit_failure(self) -> None:
+ """Test submission errors propagate and leave capacity reusable."""
+ for error in (BrokenExecutor, RuntimeError, KeyboardInterrupt):
+ with self.subTest(error=error), BoundedPoolExecutor(
+ ThreadPoolExecutor, max_bound=1, max_workers=1
+ ) as pool:
+ failure = error('submission failed')
+ with patch.object(ThreadPoolExecutor, 'submit',
+ side_effect=failure):
+ with self.assertRaises(error) as caught:
+ pool.submit(pow, 2, 3)
+ self.assertIs(caught.exception, failure)
+ self.assertTrue(
+ pool._bound_semaphore.acquire(blocking=False)
+ )
+ pool._bound_semaphore.release()
+ self.assertEqual(pool.submit(pow, 2, 3).result(timeout=5), 8)
+
def test_exceptions(self) -> None:
"""Test exceptions when creating a bounded executor."""
with self.assertRaisesRegex(TypeError,
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1336046?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings?usp=email
Gerrit-MessageType: merged
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I09b286c39149f7d2915a3bdbbb2ea9a27fead3bf
Gerrit-Change-Number: 1336046
Gerrit-PatchSet: 3
Gerrit-Owner: Mahveotm <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
