jenkins-bot has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1333925?usp=email )
Change subject: pagegenerators: Support selective page preloading
......................................................................
pagegenerators: Support selective page preloading
Forward opt-in page properties through the shared preload generators and add
batched GeoData support to APISite.preloadpages.
Use it in coordinate_import and illustrate_wikidata to batch the properties
they consume without loading unused page content.
Change-Id: I5bc293683b322e16c70eb7ac1db47ec021a81f43
---
M pywikibot/data/api/_generators.py
M pywikibot/pagegenerators/__init__.py
M pywikibot/site/_generators.py
M scripts/coordinate_import.py
M scripts/illustrate_wikidata.py
M tests/pagegenerators_tests.py
M tests/site_generators_tests.py
7 files changed, 141 insertions(+), 16 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/data/api/_generators.py
b/pywikibot/data/api/_generators.py
index 002b35c..fcbf3d0 100644
--- a/pywikibot/data/api/_generators.py
+++ b/pywikibot/data/api/_generators.py
@@ -1122,11 +1122,13 @@
props = props or []
- # test for pagedict content only and call updater function
- for element in ('coordinates', 'revisions'):
- if element in pagedict:
- updater = globals()['_update_' + element]
- updater(page, pagedict[element])
+ if 'coordinates' in pagedict:
+ _update_coordinates(page, pagedict['coordinates'])
+ elif 'coordinates' in props:
+ page._coords = []
+
+ if 'revisions' in pagedict:
+ _update_revisions(page, pagedict['revisions'])
# test for pagedict and props contents, call updater or set attribute
for element in ('categories', 'langlinks', 'templates'):
diff --git a/pywikibot/pagegenerators/__init__.py
b/pywikibot/pagegenerators/__init__.py
index 06acdec..eacee9b 100644
--- a/pywikibot/pagegenerators/__init__.py
+++ b/pywikibot/pagegenerators/__init__.py
@@ -670,17 +670,49 @@
yield from reversed(list(filtered_generator()))
-def PreloadingGenerator(generator: Iterable[pywikibot.page.Page],
- groupsize: int = 50,
- quiet: bool = False
- ) -> Generator[pywikibot.page.Page]:
+def PreloadingGenerator(
+ generator: Iterable[pywikibot.page.Page],
+ groupsize: int = 50,
+ quiet: bool = False,
+ *,
+ templates: bool = False,
+ langlinks: bool = False,
+ pageprops: bool = False,
+ categories: bool = False,
+ content: bool = True,
+ coordinates: bool = False,
+) -> Generator[pywikibot.page.Page]:
"""Yield preloaded pages taken from another generator.
+ .. version-changed:: 11.8
+ Optional page properties can be selected for preloading.
+
:param generator: Pages to iterate over
:param groupsize: How many pages to preload at once
:param quiet: If False (default), show the "Retrieving pages"
message
+ :param templates: Preload transcluded pages
+ :param langlinks: Preload language links
+ :param pageprops: Preload page properties
+ :param categories: Preload page categories
+ :param content: Preload page content
+ :param coordinates: Preload page coordinates when the GeoData
+ extension is available
"""
+ preload_options = {}
+ if templates:
+ preload_options['templates'] = True
+ if langlinks:
+ preload_options['langlinks'] = True
+ if pageprops:
+ preload_options['pageprops'] = True
+ if categories:
+ preload_options['categories'] = True
+ if not content:
+ preload_options['content'] = False
+ if coordinates:
+ preload_options['coordinates'] = True
+
# pages may be on more than one site, for example if an interwiki
# generator is used, so use a separate preloader for each site
sites: PRELOAD_SITE_TYPE = {}
@@ -694,26 +726,43 @@
# if this site is at the groupsize, process it
group = sites.pop(site)
yield from site.preloadpages(group, groupsize=site_groupsize,
- quiet=quiet)
+ quiet=quiet, **preload_options)
for site, pages in sites.items():
# process any leftover sites that never reached the groupsize
site_groupsize = min(groupsize, site.maxlimit)
yield from site.preloadpages(pages, groupsize=site_groupsize,
- quiet=quiet)
+ quiet=quiet, **preload_options)
def DequePreloadingGenerator(
generator: DequeGenerator,
groupsize: int = 50,
quiet: bool = False,
+ *,
+ templates: bool = False,
+ langlinks: bool = False,
+ pageprops: bool = False,
+ categories: bool = False,
+ content: bool = True,
+ coordinates: bool = False,
) -> Generator[pywikibot.page.Page]:
"""Preload generator of type DequeGenerator.
+ .. version-changed:: 11.8
+ Optional page properties can be selected for preloading.
+
:param generator: Pages to iterate over
:param groupsize: How many pages to preload at once
:param quiet: If False (default), show the "Retrieving pages"
message
+ :param templates: Preload transcluded pages
+ :param langlinks: Preload language links
+ :param pageprops: Preload page properties
+ :param categories: Preload page categories
+ :param content: Preload page content
+ :param coordinates: Preload page coordinates when the GeoData
+ extension is available
"""
assert isinstance(generator, DequeGenerator), \
'generator must be a DequeGenerator object'
@@ -723,7 +772,17 @@
if not page_count:
return
- yield from PreloadingGenerator(generator, page_count, quiet)
+ yield from PreloadingGenerator(
+ generator,
+ page_count,
+ quiet,
+ templates=templates,
+ langlinks=langlinks,
+ pageprops=pageprops,
+ categories=categories,
+ content=content,
+ coordinates=coordinates,
+ )
def PreloadingEntityGenerator(
diff --git a/pywikibot/site/_generators.py b/pywikibot/site/_generators.py
index 4667fe9..ff1a48e 100644
--- a/pywikibot/site/_generators.py
+++ b/pywikibot/site/_generators.py
@@ -132,6 +132,7 @@
pageprops: bool = False,
categories: bool = False,
content: bool = True,
+ coordinates: bool = False,
quiet: bool = True,
) -> Generator[pywikibot.Page]:
"""Return a generator to a list of preloaded pages.
@@ -148,6 +149,8 @@
*groupsize* is maxlimit by default. *quiet* parameter was
added. No longer show the "Retrieving pages from site"
message by default.
+ .. version-changed:: 11.8
+ *coordinates* parameter was added.
:param pagelist: An iterable that returns Page objects
:param groupsize: How many Pages to query at a time. If None
@@ -161,6 +164,8 @@
content
:param categories: Preload page categories
:param content: Preload page content
+ :param coordinates: Preload page coordinates when the GeoData
+ extension is available
:param quiet: If True (default), do not show the "Retrieving
pages" message
"""
@@ -173,6 +178,9 @@
props += '|pageprops'
if categories:
props += '|categories'
+ coordinates = coordinates and self.has_extension('GeoData')
+ if coordinates:
+ props += '|coordinates'
groupsize_ = min(groupsize or self.maxlimit, self.maxlimit)
for batch in batched(pagelist, groupsize_):
@@ -200,6 +208,10 @@
else:
rvgen.request['titles'] = list(cache.keys())
rvgen.request['rvprop'] = self._rvprops(content=content)
+ if coordinates:
+ rvgen.request['coprop'] = [
+ 'type', 'name', 'dim', 'country', 'region', 'globe']
+ rvgen.request['coprimary'] = 'all'
if not quiet:
pywikibot.info(f'Retrieving {len(cache)} pages from {self}.')
diff --git a/scripts/coordinate_import.py b/scripts/coordinate_import.py
index e71c9ac..96a19f7 100755
--- a/scripts/coordinate_import.py
+++ b/scripts/coordinate_import.py
@@ -167,9 +167,10 @@
if arg == '-create':
create_new = True
- # FIXME: this preloading preloads neither coordinates nor Wikibase items
- # but preloads wikitext which we don't need
- generator = generator_factory.getCombinedGenerator(preload=True)
+ generator = generator_factory.getCombinedGenerator()
+ if generator is not None and not generator_factory.nopreload:
+ generator = pagegenerators.PreloadingGenerator(
+ generator, quiet=True, content=False, coordinates=True)
coordbot = CoordImportRobot(generator=generator, create=create_new)
coordbot.run()
diff --git a/scripts/illustrate_wikidata.py b/scripts/illustrate_wikidata.py
index 6a36693..df73210 100755
--- a/scripts/illustrate_wikidata.py
+++ b/scripts/illustrate_wikidata.py
@@ -101,7 +101,11 @@
else:
generator_factory.handle_arg(arg)
- options['generator'] = generator_factory.getCombinedGenerator(preload=True)
+ generator = generator_factory.getCombinedGenerator()
+ if generator is not None and not generator_factory.nopreload:
+ generator = pagegenerators.PreloadingGenerator(
+ generator, quiet=True, content=False, pageprops=True)
+ options['generator'] = generator
bot = IllustrateRobot(**options)
bot.run()
diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py
index 29b7f4f..69dba1f 100755
--- a/tests/pagegenerators_tests.py
+++ b/tests/pagegenerators_tests.py
@@ -679,6 +679,20 @@
high_site.preloadpages.assert_called_once_with(
high_pages, groupsize=5, quiet=False)
+ def test_preload_options(self) -> None:
+ """Test that preload options are passed to the site."""
+ site = mock.Mock(maxlimit=5)
+ site.preloadpages.side_effect = lambda pages, **kwargs: iter(pages)
+ page = mock.Mock(site=site)
+
+ pages = list(PreloadingGenerator(
+ [page], content=False, coordinates=True, pageprops=True))
+
+ self.assertEqual(pages, [page])
+ site.preloadpages.assert_called_once_with(
+ [page], groupsize=5, quiet=False, pageprops=True,
+ content=False, coordinates=True)
+
class TestPreloadingGenerator(DefaultSiteTestCase):
diff --git a/tests/site_generators_tests.py b/tests/site_generators_tests.py
index fb753a9..da7f393 100755
--- a/tests/site_generators_tests.py
+++ b/tests/site_generators_tests.py
@@ -138,6 +138,39 @@
set_query_increment.assert_called_once_with(5)
+ def test_preloadpages_coordinates(self) -> None:
+ """Test that coordinate preloading populates an empty cache."""
+ page = pywikibot.Page(self.site, 'Main Page')
+ pagedata = {'title': page.title(), 'ns': 0, 'pageid': 1}
+
+ with (
+ patch.object(type(self.site), 'maxlimit', 50),
+ patch.object(self.site, 'has_extension', return_value=True),
+ patch.object(self.site, '_rvprops', return_value=['ids'])
+ as rvprops,
+ patch.object(api, 'PropertyGenerator') as generator,
+ ):
+ rvgen = generator.return_value
+ rvgen.request = {}
+ rvgen.props = frozenset(
+ {'revisions', 'info', 'categoryinfo', 'coordinates'})
+ rvgen.__iter__.return_value = iter((pagedata,))
+
+ pages = list(self.site.preloadpages(
+ [page], content=False, coordinates=True))
+
+ self.assertEqual(pages, [page])
+ generator.assert_called_once_with(
+ 'revisions|info|categoryinfo|coordinates', site=self.site)
+ self.assertEqual(
+ rvgen.request['coprop'],
+ ['type', 'name', 'dim', 'country', 'region', 'globe'])
+ self.assertEqual(rvgen.request['coprimary'], 'all')
+ rvprops.assert_called_once_with(content=False)
+ with patch.object(self.site, 'loadcoordinfo') as loadcoordinfo:
+ self.assertEqual(page.coordinates(), [])
+ loadcoordinfo.assert_not_called()
+
class TestSiteGenerators(DefaultSiteTestCase):
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1333925?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings?usp=email
Gerrit-MessageType: merged
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I5bc293683b322e16c70eb7ac1db47ec021a81f43
Gerrit-Change-Number: 1333925
Gerrit-PatchSet: 2
Gerrit-Owner: Mahveotm <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
