Xqt has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1320125?usp=email )
Change subject: [bugfix] Avoid duplicate page references
......................................................................
[bugfix] Avoid duplicate page references
Merge sorted backlink and transclusion generators by page ID so a page
appearing in both is only yielded once. Use an order-preserving filter
when following redirects because that backlink stream is not sorted.
Bug: T405551
Change-Id: I30efd22dc10a42a394f91ce0a0739163f0ea0c1c
---
M pywikibot/page/_basepage.py
M pywikibot/site/_generators.py
M tests/site_generators_tests.py
3 files changed, 92 insertions(+), 16 deletions(-)
Approvals:
Xqt: Verified; Looks good to me, approved
diff --git a/pywikibot/page/_basepage.py b/pywikibot/page/_basepage.py
index 8067897..a4f6fdd 100644
--- a/pywikibot/page/_basepage.py
+++ b/pywikibot/page/_basepage.py
@@ -1055,6 +1055,10 @@
If you need a full list of referring pages, use
``pages = list(s.getReferences())``
+ .. version-changed:: 11.7
+ Duplicate pages are no longer yielded when backlinks and template
+ inclusions overlap.
+
:param follow_redirects: If True, also iterate pages that link to a
redirect pointing to the page.
:param with_template_inclusion: If True, also iterate pages where self
@@ -1063,7 +1067,7 @@
is used as a template.
:param filter_redirects: If True, only iterate redirects to self.
:param namespaces: Only iterate pages in these namespaces
- :param total: Iterate no more than this number of pages in total
+ :param total: Iterate no more than this number of unique pages in total
:param content: If True, retrieve the content of the current version
of each referring page (default False)
"""
diff --git a/pywikibot/site/_generators.py b/pywikibot/site/_generators.py
index 40dd40f..4888478 100644
--- a/pywikibot/site/_generators.py
+++ b/pywikibot/site/_generators.py
@@ -34,7 +34,7 @@
deprecated_signature,
is_ip_address,
)
-from pywikibot.tools.itertools import filter_unique
+from pywikibot.tools.itertools import filter_unique, union_generators
if typing.TYPE_CHECKING:
@@ -400,8 +400,13 @@
) -> Iterable[pywikibot.Page]:
"""Convenience method combining pagebacklinks and page_embeddedin.
+ .. version-changed:: 11.7
+ Duplicate pages are no longer yielded when backlinks and template
+ inclusions overlap.
+
:param namespaces: If present, only return links from the
namespaces in this list.
+ :param total: Maximum number of unique pages to retrieve in total.
:raises KeyError: A namespace identifier was not resolved
:raises TypeError: A namespace identifier has an inappropriate
type such as NoneType or bool
@@ -416,16 +421,24 @@
filter_redirects=filter_redirects,
namespaces=namespaces, total=total,
content=content)
- return itertools.islice(
- itertools.chain(
- self.pagebacklinks(
- page, follow_redirects=follow_redirects,
- filter_redirects=filter_redirects,
- namespaces=namespaces, content=content),
- self.page_embeddedin(
- page, filter_redirects=filter_redirects,
- namespaces=namespaces, content=content)
- ), total)
+ generators = (
+ self.pagebacklinks(
+ page, follow_redirects=follow_redirects,
+ filter_redirects=filter_redirects,
+ namespaces=namespaces, content=content),
+ self.page_embeddedin(
+ page, filter_redirects=filter_redirects,
+ namespaces=namespaces, content=content),
+ )
+ if follow_redirects:
+ # Following redirects chains multiple sorted backlink generators,
+ # so the resulting iterable itself is not necessarily sorted.
+ references = filter_unique(
+ itertools.chain(*generators), key=lambda item: item.pageid)
+ else:
+ references = union_generators(
+ *generators, key=lambda item: item.pageid)
+ return itertools.islice(references, total)
def pagelinks(
self,
diff --git a/tests/site_generators_tests.py b/tests/site_generators_tests.py
index 863dfae..0ef2ff6 100755
--- a/tests/site_generators_tests.py
+++ b/tests/site_generators_tests.py
@@ -37,6 +37,63 @@
}
+class TestPageReferences(TestCase):
+
+ """Offline tests for Site.pagereferences."""
+
+ family = 'wikipedia'
+ code = 'en'
+ dry = True
+
+ def setUp(self) -> None:
+ """Initialize the test site."""
+ super().setUp()
+ self.site = self.get_site()
+ self.target = pywikibot.Page(self.site, 'Target')
+
+ def _page(self, title: str, pageid: int) -> pywikibot.Page:
+ """Create a page with a known page ID."""
+ page = pywikibot.Page(self.site, title)
+ page._pageid = pageid
+ return page
+
+ def test_pagereferences_unique(self) -> None:
+ """Test overlapping sorted generators only yield unique pages."""
+ page_a = self._page('A', 1)
+ backlink_b = self._page('B', 3)
+ embedded_b = self._page('B', 3)
+ page_c = self._page('C', 4)
+
+ with (
+ patch.object(self.site, 'pagebacklinks',
+ return_value=iter((page_a, backlink_b))),
+ patch.object(self.site, 'page_embeddedin',
+ return_value=iter((embedded_b, page_c))),
+ ):
+ references = list(self.site.pagereferences(self.target, total=3))
+
+ self.assertEqual(references, [page_a, backlink_b, page_c])
+ self.assertIs(references[1], backlink_b)
+
+ def test_pagereferences_follow_redirects_unique(self) -> None:
+ """Test unsorted redirect backlinks only yield unique pages."""
+ backlink_c = self._page('C', 3)
+ backlink_a = self._page('A', 1)
+ embedded_a = self._page('A', 1)
+ embedded_b = self._page('B', 2)
+
+ with (
+ patch.object(self.site, 'pagebacklinks',
+ return_value=iter((backlink_c, backlink_a))),
+ patch.object(self.site, 'page_embeddedin',
+ return_value=iter((embedded_a, embedded_b))),
+ ):
+ references = list(self.site.pagereferences(
+ self.target, follow_redirects=True))
+
+ self.assertEqual(references, [backlink_c, backlink_a, embedded_b])
+
+
class TestSiteGenerators(DefaultSiteTestCase):
"""Test cases for Site methods."""
@@ -68,11 +125,13 @@
with skipping(ApiTimeoutError):
embedded = set(self.site.page_embeddedin(self.mainpage,
namespaces=[0]))
- refs = set(self.site.pagereferences(self.mainpage, namespaces=[0]))
+ refs = list(self.site.pagereferences(self.mainpage, namespaces=[0]))
+ unique_refs = set(refs)
- self.assertLessEqual(backlinks, refs)
- self.assertLessEqual(embedded, refs)
- self.assertEqual(refs, backlinks | embedded)
+ self.assertLength(refs, len(unique_refs))
+ self.assertLessEqual(backlinks, unique_refs)
+ self.assertLessEqual(embedded, unique_refs)
+ self.assertEqual(unique_refs, backlinks | embedded)
def test_backlinks(self) -> None:
"""Test Site.pagebacklinks."""
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1320125?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings?usp=email
Gerrit-MessageType: merged
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I30efd22dc10a42a394f91ce0a0739163f0ea0c1c
Gerrit-Change-Number: 1320125
Gerrit-PatchSet: 2
Gerrit-Owner: Mahveotm <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
