jenkins-bot has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1178033?usp=email )
Change subject: [IMPR] Add `strict` parameter to unconnected_pages()
......................................................................
[IMPR] Add `strict` parameter to unconnected_pages()
Introduce a new `strict` flag to verify that pages from
Special:UnconnectedPages are still unconnected before yielding.
- Pass `total` directly to querypage() when strict=False to limit API calls
- Stop iteration when `total` is reached
- Return nothing if total<=0
- Updated docstring with description of strict and versionchanged note
- Add a new `strict` flag to UnconnectedPageGenerator
- Update pagegenerators_tests.TestUnconnectedPageGenerator
- Update site_generators_tests.TestUnconnectedPages
- Deprecation warning for UnconnectedPageGenerator was withdrawn
Bug: T401699
Change-Id: Ib88712dfd88d92d585dce4739c3696d1e9e39ae1
---
M HISTORY.rst
M pywikibot/pagegenerators/_generators.py
M pywikibot/site/_extensions.py
M tests/pagegenerators_tests.py
M tests/site_generators_tests.py
5 files changed, 53 insertions(+), 38 deletions(-)
Approvals:
jenkins-bot: Verified
Xqt: Looks good to me, approved
diff --git a/HISTORY.rst b/HISTORY.rst
index 0559415..9a4b6dc 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -2000,7 +2000,7 @@
* UploadBot got a filename prefix parameter (:phab:`T170123`)
* cosmetic_changes is able to remove empty sections (:phab:`T140570`)
* Pywikibot is following :pep:`396` versioning
-* pagegenerators AllpagesPageGenerator, CombinedPageGenerator,
UnconnectedPageGenerator are deprecated
+* CombinedPageGenerator is deprecated, itertools.chain can be used instead
* Some DayPageGenerator parameters has been renamed
* unicodedata2, httpbin and Flask dependency was removed
(:phab:`T102461`, :phab:`T108068`, :phab:`T178864`, :phab:`T193383`)
diff --git a/pywikibot/pagegenerators/_generators.py
b/pywikibot/pagegenerators/_generators.py
index 4e04b8c..c74a972 100644
--- a/pywikibot/pagegenerators/_generators.py
+++ b/pywikibot/pagegenerators/_generators.py
@@ -282,18 +282,26 @@
def UnconnectedPageGenerator(
site: BaseSite | None = None,
- total: int | None = None
+ total: int | None = None,
+ *,
+ strict: bool = False
) -> Iterable[pywikibot.page.Page]:
"""Iterate Page objects for all unconnected pages to a Wikibase repository.
- :param total: Maximum number of pages to retrieve in total
+ .. versionchanged::
+ The *strict* parameter was added.
+
:param site: Site for generator results.
+ :param total: Maximum number of pages to retrieve in total
+ :param strict: If ``True``, verify that each page still has no data
+ item before yielding it.
+ :raises ValueError: The given site does not have Wikibase repository
"""
if site is None:
site = pywikibot.Site()
if not site.data_repository():
raise ValueError('The given site does not have Wikibase repository.')
- return site.unconnected_pages(total=total)
+ return site.unconnected_pages(total=total, strict=strict)
def FileLinksGenerator(
diff --git a/pywikibot/site/_extensions.py b/pywikibot/site/_extensions.py
index e30f130..66d342b 100644
--- a/pywikibot/site/_extensions.py
+++ b/pywikibot/site/_extensions.py
@@ -9,6 +9,7 @@
from typing import TYPE_CHECKING, Protocol
import pywikibot
+from pywikibot.backports import Generator
from pywikibot.data import api
from pywikibot.echo import Notification
from pywikibot.exceptions import (
@@ -285,14 +286,41 @@
"""APISite mixin for WikibaseClient extension."""
@need_extension('WikibaseClient')
- def unconnected_pages(self, total=None):
+ def unconnected_pages(
+ self,
+ total: int | None = None,
+ *,
+ strict: bool = False
+ ) -> Generator[pywikibot.Page, None, None]:
"""Yield Page objects from Special:UnconnectedPages.
.. warning:: The retrieved pages may be connected in meantime.
+ To avoid this, use *strict* parameter to check.
- :param total: number of pages to return
+ .. versionchanged::
+ The *strict* parameter was added.
+
+ :param total: Maximum number of pages to return, or ``None`` for
+ all.
+ :param strict: If ``True``, verify that each page still has no
+ data item before yielding it.
"""
- return self.querypage('UnconnectedPages', total)
+ if total <= 0:
+ return
+
+ if not strict:
+ return self.querypage('UnconnectedPages', total)
+
+ count = 0
+ for page in self.querypage('UnconnectedPages'):
+ if total is not None and count >= total:
+ break
+
+ try:
+ page.data_item()
+ except NoPageError:
+ yield page
+ count += 1
class LinterMixin:
diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py
index b639797..9a651d7 100755
--- a/tests/pagegenerators_tests.py
+++ b/tests/pagegenerators_tests.py
@@ -27,7 +27,7 @@
PreloadingGenerator,
WikibaseItemFilterPageGenerator,
)
-from tests import join_data_path, unittest_print
+from tests import join_data_path
from tests.aspects import (
DefaultSiteTestCase,
DeprecationTestCase,
@@ -1687,26 +1687,16 @@
if not site:
self.skipTest('Site is not using a Wikibase repository')
- pages = list(pagegenerators.UnconnectedPageGenerator(self.site, 3))
+ pages = list(
+ pagegenerators.UnconnectedPageGenerator(self.site, 3, strict=True))
self.assertLessEqual(len(pages), 3)
pattern = (fr'Page \[\[({site.sitename}:|{site.code}:)-1\]\]'
r" doesn't exist\.")
- found = []
for page in pages:
- with self.subTest(page=page):
- try:
- page.data_item()
- except NoPageError as e:
- self.assertRegex(str(e), pattern)
- else:
- found.append(page)
- if found:
- unittest_print('connection found for ',
- ', '.join(str(p) for p in found))
-
- # assume that we have at least one unconnected page
- self.assertLess(len(found), 3)
+ with self.subTest(page=page), self.assertRaisesRegex(NoPageError,
+ pattern):
+ page.data_item()
def test_unconnected_without_repo(self) -> None:
"""Test that it raises a ValueError on sites without repository."""
diff --git a/tests/site_generators_tests.py b/tests/site_generators_tests.py
index 1deb37b..7fbf716 100755
--- a/tests/site_generators_tests.py
+++ b/tests/site_generators_tests.py
@@ -689,26 +689,15 @@
if not site:
self.skipTest('Site is not using a Wikibase repository')
- pages = list(self.site.unconnected_pages(total=3))
+ pages = list(self.site.unconnected_pages(total=3, strict=True))
self.assertLessEqual(len(pages), 3)
pattern = (fr'Page \[\[({site.sitename}:|{site.code}:)-1\]\]'
r" doesn't exist\.")
- found = []
for page in pages:
- with self.subTest(page=page):
- try:
- page.data_item()
- except NoPageError as e:
- self.assertRegex(str(e), pattern)
- else:
- found.append(page)
- if found:
- unittest_print('connection found for ',
- ', '.join(str(p) for p in found))
-
- # assume that we have at least one unconnected page
- self.assertLess(len(found), 3)
+ with self.subTest(page=page), self.assertRaisesRegex(NoPageError,
+ pattern):
+ page.data_item()
class TestSiteGeneratorsUsers(DefaultSiteTestCase):
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1178033?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings?usp=email
Gerrit-MessageType: merged
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Ib88712dfd88d92d585dce4739c3696d1e9e39ae1
Gerrit-Change-Number: 1178033
Gerrit-PatchSet: 2
Gerrit-Owner: Xqt <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
