jenkins-bot has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1034059?usp=email )
Change subject: [IMPR] use CachedRequest for userinfo requests
......................................................................
[IMPR] use CachedRequest for userinfo requests
The expiry for userinfo is set to 1 day and can be changed within
user-config.py file. The second call for the same userinfo is upto
50 times faster.
Also create the site within its own worker thread. Now preload_sites.py
is up to 90 times faster than before and only half as fast than pwb 8.
Bug: T348925
Change-Id: I2bc6c375ee9b676e0a40a372362f37ee4a61ced8
---
M pywikibot/config.py
M pywikibot/scripts/preload_sites.py
M pywikibot/site/_apisite.py
3 files changed, 35 insertions(+), 12 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/config.py b/pywikibot/config.py
index 7f12bcc..b2f20fb 100644
--- a/pywikibot/config.py
+++ b/pywikibot/config.py
@@ -175,6 +175,8 @@
site_interface = 'APISite'
# number of days to cache namespaces, api configuration, etc.
API_config_expiry = 30
+# userinfo expiry
+API_uinfo_expiry = 1
# The maximum number of bytes which uses a GET request, if not positive
# it'll always use POST requests
diff --git a/pywikibot/scripts/preload_sites.py
b/pywikibot/scripts/preload_sites.py
index 8ff7d57..27a31f7 100755
--- a/pywikibot/scripts/preload_sites.py
+++ b/pywikibot/scripts/preload_sites.py
@@ -6,13 +6,18 @@
-worker:<num> The number of parallel tasks to be run. Default is the
number of processors on the machine
-Usage::
+**Usage:**
python pwb.py preload_sites [{<family>}] [-worker:{<num>}]
-To force preloading, change the global expiry value to 0::
+To force preloading, change the global expiry values to 0:
- python pwb.py -API_config_expiry:0 preload_sites [{<family>}]
+ python pwb.py -API_config_expiry:0 -API_uinfo_expiry:0 \
+ preload_sites [{<family>}]
+
+or run the :mod:`cache<scripts.maintenance.cache>` script previeously:
+
+ python pwb.py cache -delete
.. versionchanged:: 7.4
script was moved to the framework scripts folder.
@@ -57,7 +62,17 @@
def preload_family(family: str, executor: ThreadPoolExecutor) -> None:
- """Preload all sites of a single family file."""
+ """Preload all sites of a single family file.
+
+ .. versionchanged:: 9.2
+ use a separate worker thread for each site.
+ """
+
+ def create_page(code, family):
+ """Preload siteinfo and userinfo."""
+ site = pywikibot.Site(code, family)
+ pywikibot.Page(site, 'Main Page')
+
msg = 'Preloading sites of {} family{}'
pywikibot.info(msg.format(family, '...'))
@@ -65,14 +80,13 @@
for code in exceptions.get(family, []):
if code in codes:
codes.remove(code)
+
obsolete = Family.load(family).obsolete
futures = set()
for code in codes:
if code not in obsolete:
- site = pywikibot.Site(code, family)
- # page title does not care
- futures.add(executor.submit(pywikibot.Page, site, 'Main page'))
+ futures.add(executor.submit(create_page, code, family))
wait(futures)
pywikibot.info(msg.format(family, ' completed.'))
diff --git a/pywikibot/site/_apisite.py b/pywikibot/site/_apisite.py
index 729a7ec..0fe76de 100644
--- a/pywikibot/site/_apisite.py
+++ b/pywikibot/site/_apisite.py
@@ -648,6 +648,10 @@
.. seealso:: :api:`Userinfo`
.. versionchanged:: 8.0
Use API formatversion 2.
+ .. versionchanged:: 9.2
+ API call is made through :class:`data.api.CachedRequest` with
+ expiry is set in ``API_uinfo_expiry`` within
+ :ref:`Account settings`.
:return: A dict with the following keys and values:
@@ -661,11 +665,14 @@
"""
if not hasattr(self, '_userinfo'):
- uirequest = self.simple_request(
- action='query',
- meta='userinfo',
- uiprop='blockinfo|hasmsg|groups|rights|ratelimits',
- formatversion=2,
+ uirequest = self._request(
+ expiry=pywikibot.config.API_uinfo_expiry,
+ parameters={
+ 'action': 'query',
+ 'meta': 'userinfo',
+ 'uiprop': 'blockinfo|hasmsg|groups|rights|ratelimits',
+ 'formatversion': 2,
+ }
)
uidata = uirequest.submit()
assert 'query' in uidata, \
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1034059?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I2bc6c375ee9b676e0a40a372362f37ee4a61ced8
Gerrit-Change-Number: 1034059
Gerrit-PatchSet: 4
Gerrit-Owner: Xqt <[email protected]>
Gerrit-Reviewer: Xqt <[email protected]>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
_______________________________________________
Pywikibot-commits mailing list -- [email protected]
To unsubscribe send an email to [email protected]
