Xqt has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1332201?usp=email )
Change subject: pagegenerators: Keep preload group size per site
......................................................................
pagegenerators: Keep preload group size per site
Keep the requested size unchanged when a site's API limit is lower.
Calculate effective batch sizes independently for each site's buffer.
Change-Id: I1c809208a25c8f88a884bc0aec0b29870a496f1e
---
M pywikibot/pagegenerators/__init__.py
M tests/pagegenerators_tests.py
2 files changed, 34 insertions(+), 4 deletions(-)
Approvals:
Xqt: Verified; Looks good to me, approved
diff --git a/pywikibot/pagegenerators/__init__.py
b/pywikibot/pagegenerators/__init__.py
index 575f242..06acdec 100644
--- a/pywikibot/pagegenerators/__init__.py
+++ b/pywikibot/pagegenerators/__init__.py
@@ -689,16 +689,18 @@
site = page.site
sites.setdefault(site, []).append(page)
- groupsize = min(groupsize, site.maxlimit)
- if len(sites[site]) >= groupsize:
+ site_groupsize = min(groupsize, site.maxlimit)
+ if len(sites[site]) >= site_groupsize:
# if this site is at the groupsize, process it
group = sites.pop(site)
- yield from site.preloadpages(group, groupsize=groupsize,
+ yield from site.preloadpages(group, groupsize=site_groupsize,
quiet=quiet)
for site, pages in sites.items():
# process any leftover sites that never reached the groupsize
- yield from site.preloadpages(pages, groupsize=groupsize, quiet=quiet)
+ site_groupsize = min(groupsize, site.maxlimit)
+ yield from site.preloadpages(pages, groupsize=site_groupsize,
+ quiet=quiet)
def DequePreloadingGenerator(
diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py
index e4d6f5a..99c424f 100755
--- a/tests/pagegenerators_tests.py
+++ b/tests/pagegenerators_tests.py
@@ -634,6 +634,34 @@
self._run_test(12, 13)
+class TestDryPreloadingGenerator(TestCase):
+
+ """Dry tests for PreloadingGenerator."""
+
+ net = False
+
+ def test_groupsize_per_site(self) -> None:
+ """Test that each site keeps its own preload group size."""
+ low_site = mock.Mock(maxlimit=2)
+ high_site = mock.Mock(maxlimit=5)
+ low_site.preloadpages.side_effect = (
+ lambda pages, **kwargs: iter(pages))
+ high_site.preloadpages.side_effect = (
+ lambda pages, **kwargs: iter(pages))
+
+ low_pages = [mock.Mock(site=low_site) for _ in range(2)]
+ high_pages = [mock.Mock(site=high_site) for _ in range(3)]
+ pages = [low_pages[0], high_pages[0], low_pages[1],
+ *high_pages[1:]]
+
+ list(PreloadingGenerator(pages, groupsize=5))
+
+ low_site.preloadpages.assert_called_once_with(
+ low_pages, groupsize=2, quiet=False)
+ high_site.preloadpages.assert_called_once_with(
+ high_pages, groupsize=5, quiet=False)
+
+
class TestPreloadingGenerator(DefaultSiteTestCase):
"""Test preloading generator on lists."""
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1332201?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings?usp=email
Gerrit-MessageType: merged
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I1c809208a25c8f88a884bc0aec0b29870a496f1e
Gerrit-Change-Number: 1332201
Gerrit-PatchSet: 1
Gerrit-Owner: Mahveotm <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
