Copilot commented on code in PR #14509: URL: https://github.com/apache/cloudstack/pull/14509#discussion_r4398951375
########## tools/contribution/apache_list_mentions.py: ########## @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Apache mailing list name-mention counter (cached mbox downloads). + +What it does: +- Downloads monthly mbox archives from lists.apache.org ONLY if not already cached locally + (the current, still-incomplete month is always re-downloaded) +- Scans the email bodies (optionally excluding quoted lines and signatures) +- Counts how many times EACH provided name (and its aliases) is mentioned +- Prints counts in the SAME ORDER as your names file (no top-N / no ranking) + +Names file format (names.txt): + # comments allowed + Canonical Name + Canonical Name: alias1, alias2 + Canonical Name: alias1, alias2 | email1, email2 (emails optional; currently not used for mention counting) + +Example: + Jane Doe: Jane, JD + John Smith: John, J. Smith + +Usage: + python3 apache_list_mentions.py --list dev --domain cloudstack.apache.org --months 12 --names names.txt +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import email +import mailbox +import re +import sys +from email.utils import parseaddr +from pathlib import Path +from typing import Dict, Iterable, List, Tuple + +import requests + +MBOX_API = "https://lists.apache.org/api/mbox.lua?list={list}&domain={domain}&d={ym}"; +DEFAULT_CACHE_DIR = Path.home() / ".cache" / "apache-list-mentions" + +# Heuristics for cleaning bodies before counting mentions +QUOTE_LINE_RE = re.compile(r"^\s*>") # drop quoted lines (reply history) +SIG_CUTOFF_RE = re.compile(r"^\s*--\s*$") # signature separator + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + +def month_starts(end_utc: dt.datetime, months: int) -> Iterable[dt.date]: + """ + Yield month starts for the last N months ending at end_utc month (inclusive). + """ + y, m = end_utc.year, end_utc.month + for i in range(months): + mm = m - i + yy = y + while mm <= 0: + mm += 12 + yy -= 1 + yield dt.date(yy, mm, 1) + +def download_mbox(list_name: str, domain: str, ym: str, out_dir: Path) -> Path: + """ + Download a single YYYY-MM mbox archive from lists.apache.org and save to out_dir. + """ + url = MBOX_API.format(list=list_name, domain=domain, ym=ym) + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / f"{list_name}@{domain}-{ym}.mbox" + + headers = {"User-Agent": "apache-list-mentions/1.0"} + r = requests.get(url, headers=headers, timeout=120) + + if r.status_code == 404 or not r.content.strip(): + raise FileNotFoundError(f"No archive found for {ym}") + + r.raise_for_status() + # Write via a temp file so an interrupted download is never reused as a cached archive + tmp_path = out_path.with_suffix(".mbox.part") + tmp_path.write_bytes(r.content) + tmp_path.replace(out_path) + return out_path + Review Comment: This loads the entire monthly mbox into memory (`r.content`) before writing it. Mbox archives can be large, so consider using a streaming download (e.g., `stream=True` and iterating over `iter_content`) to write incrementally to the temp file to reduce peak memory usage and improve robustness. ########## tools/contribution/apache_list_mentions.py: ########## @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Apache mailing list name-mention counter (cached mbox downloads). + +What it does: +- Downloads monthly mbox archives from lists.apache.org ONLY if not already cached locally + (the current, still-incomplete month is always re-downloaded) +- Scans the email bodies (optionally excluding quoted lines and signatures) +- Counts how many times EACH provided name (and its aliases) is mentioned +- Prints counts in the SAME ORDER as your names file (no top-N / no ranking) + +Names file format (names.txt): + # comments allowed + Canonical Name + Canonical Name: alias1, alias2 + Canonical Name: alias1, alias2 | email1, email2 (emails optional; currently not used for mention counting) + +Example: + Jane Doe: Jane, JD + John Smith: John, J. Smith + +Usage: + python3 apache_list_mentions.py --list dev --domain cloudstack.apache.org --months 12 --names names.txt +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import email +import mailbox +import re +import sys +from email.utils import parseaddr +from pathlib import Path +from typing import Dict, Iterable, List, Tuple + +import requests + +MBOX_API = "https://lists.apache.org/api/mbox.lua?list={list}&domain={domain}&d={ym}"; +DEFAULT_CACHE_DIR = Path.home() / ".cache" / "apache-list-mentions" + +# Heuristics for cleaning bodies before counting mentions +QUOTE_LINE_RE = re.compile(r"^\s*>") # drop quoted lines (reply history) +SIG_CUTOFF_RE = re.compile(r"^\s*--\s*$") # signature separator + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + +def month_starts(end_utc: dt.datetime, months: int) -> Iterable[dt.date]: + """ + Yield month starts for the last N months ending at end_utc month (inclusive). + """ + y, m = end_utc.year, end_utc.month + for i in range(months): + mm = m - i + yy = y + while mm <= 0: + mm += 12 + yy -= 1 + yield dt.date(yy, mm, 1) + +def download_mbox(list_name: str, domain: str, ym: str, out_dir: Path) -> Path: + """ + Download a single YYYY-MM mbox archive from lists.apache.org and save to out_dir. + """ + url = MBOX_API.format(list=list_name, domain=domain, ym=ym) + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / f"{list_name}@{domain}-{ym}.mbox" + + headers = {"User-Agent": "apache-list-mentions/1.0"} + r = requests.get(url, headers=headers, timeout=120) + + if r.status_code == 404 or not r.content.strip(): + raise FileNotFoundError(f"No archive found for {ym}") + + r.raise_for_status() Review Comment: The `not r.content.strip()` check runs before `raise_for_status()`, which can misclassify server/proxy errors (e.g., 500/502 with empty body) as `FileNotFoundError`. Call `r.raise_for_status()` first (handling 404 explicitly if desired), and only treat an empty body as missing *after* a successful (2xx) response. ########## tools/contribution/github_org_contributions.py: ########## @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +GitHub org activity counters for a set of users (GraphQL). + +Counts (scoped to org-owned repos whose name matches --repo-pattern, default ^cloudstack): +- Pull requests created (opened) +- Issues created (opened) +- Pull request reviews + +Time window: +- If --from is NOT provided → defaults to the year before --to +- Windows longer than a year are split into yearly chunks (GitHub limitation) and summed +- A date-only --to (YYYY-MM-DD) includes that whole day + +Auth: + export GITHUB_TOKEN=ghp_... + +Examples: + python3 github_org_contributions.py --org apache --users octocat + python3 github_org_contributions.py --org apache --users octocat monalisa --from 2025-02-01 --to 2026-01-31 + +See README.md in this directory for details and caveats. +""" + +import argparse +import csv +import datetime as dt +import os +import re +import sys +from typing import Any, Dict, Iterable, List, Tuple + +import requests + +GQL_ENDPOINT = "https://api.github.com/graphql"; +# GitHub's upper limit for the *ContributionsByRepository fields +MAX_REPOSITORIES = 100 + +COUNTERS = { + "prs_created": "pullRequestContributionsByRepository", + "issues_created": "issueContributionsByRepository", + "pr_reviews": "pullRequestReviewContributionsByRepository", +} + + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + + +def iso_to_datetime(s: str, end_of_day: bool = False) -> dt.datetime: + if len(s) == 10: # YYYY-MM-DD + d = dt.datetime.fromisoformat(s).replace(tzinfo=dt.timezone.utc) + return d + dt.timedelta(days=1) if end_of_day else d + if s.endswith(("Z", "z")): # fromisoformat() only accepts "Z" from Python 3.11 + s = s[:-1] + "+00:00" + d = dt.datetime.fromisoformat(s) + if d.tzinfo is None: + d = d.replace(tzinfo=dt.timezone.utc) + return d.astimezone(dt.timezone.utc) + + +def shift_years(d: dt.datetime, years: int) -> dt.datetime: + try: + return d.replace(year=d.year + years) + except ValueError: # Feb 29 + return d.replace(year=d.year + years, day=28) + + +def year_chunks(from_dt: dt.datetime, to_dt: dt.datetime) -> Iterable[Tuple[dt.datetime, dt.datetime]]: + """ + Split [from_dt, to_dt] into consecutive windows of at most one year. + """ + start = from_dt + while start < to_dt: + end = min(shift_years(start, 1), to_dt) + yield start, end + start = end + + +def graphql(token: str, query: str, variables: Dict[str, Any]) -> Dict[str, Any]: + headers = { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + } + r = requests.post(GQL_ENDPOINT, json={"query": query, "variables": variables}, headers=headers, timeout=60) + r.raise_for_status() + payload = r.json() + if payload.get("errors"): + msgs = "; ".join(e.get("message", "Unknown error") for e in payload["errors"]) + raise RuntimeError(f"GraphQL error(s): {msgs}") + return payload["data"] + + +ORG_ID_QUERY = """ +query($org: String!) { + organization(login: $org) { id login } +} +""" + +USER_CONTRIB_QUERY = """ +query($login: String!, $orgId: ID!, $from: DateTime!, $to: DateTime!, $maxRepos: Int!) { + user(login: $login) { + login + contributionsCollection(organizationID: $orgId, from: $from, to: $to) { + pullRequestContributionsByRepository(maxRepositories: $maxRepos) { + repository { name owner { login } } + contributions { totalCount } + } + issueContributionsByRepository(maxRepositories: $maxRepos) { + repository { name owner { login } } + contributions { totalCount } + } + pullRequestReviewContributionsByRepository(maxRepositories: $maxRepos) { + repository { name owner { login } } + contributions { totalCount } + } + } + } +} +""" + + +def count_user(token: str, login: str, org_id: str, org_login: str, repo_rx: re.Pattern, + from_dt: dt.datetime, to_dt: dt.datetime) -> Dict[str, Any]: + """ + Sum the user's contributions to matching org repos over all yearly chunks of the window. + """ + counts = {k: 0 for k in COUNTERS} + repos = set() + user_login = login + for start, end in year_chunks(from_dt, to_dt): + data = graphql(token, USER_CONTRIB_QUERY, { + "login": login, + "orgId": org_id, + "from": start.isoformat(), + "to": end.isoformat(), + "maxRepos": MAX_REPOSITORIES, + }) + user = data.get("user") + if not user: + raise RuntimeError("User not found") Review Comment: This error message loses important context when multiple users are processed. Include the login in the message (e.g., `User not found: <login>`) so the failure is actionable in both console output and the CSV `error` column. ########## tools/contribution/apache_list_mentions.py: ########## @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Apache mailing list name-mention counter (cached mbox downloads). + +What it does: +- Downloads monthly mbox archives from lists.apache.org ONLY if not already cached locally + (the current, still-incomplete month is always re-downloaded) +- Scans the email bodies (optionally excluding quoted lines and signatures) +- Counts how many times EACH provided name (and its aliases) is mentioned +- Prints counts in the SAME ORDER as your names file (no top-N / no ranking) + +Names file format (names.txt): + # comments allowed + Canonical Name + Canonical Name: alias1, alias2 + Canonical Name: alias1, alias2 | email1, email2 (emails optional; currently not used for mention counting) + +Example: + Jane Doe: Jane, JD + John Smith: John, J. Smith + +Usage: + python3 apache_list_mentions.py --list dev --domain cloudstack.apache.org --months 12 --names names.txt +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import email +import mailbox +import re +import sys +from email.utils import parseaddr +from pathlib import Path +from typing import Dict, Iterable, List, Tuple + +import requests + +MBOX_API = "https://lists.apache.org/api/mbox.lua?list={list}&domain={domain}&d={ym}"; +DEFAULT_CACHE_DIR = Path.home() / ".cache" / "apache-list-mentions" + +# Heuristics for cleaning bodies before counting mentions +QUOTE_LINE_RE = re.compile(r"^\s*>") # drop quoted lines (reply history) +SIG_CUTOFF_RE = re.compile(r"^\s*--\s*$") # signature separator + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + +def month_starts(end_utc: dt.datetime, months: int) -> Iterable[dt.date]: + """ + Yield month starts for the last N months ending at end_utc month (inclusive). + """ + y, m = end_utc.year, end_utc.month + for i in range(months): + mm = m - i + yy = y + while mm <= 0: + mm += 12 + yy -= 1 + yield dt.date(yy, mm, 1) + +def download_mbox(list_name: str, domain: str, ym: str, out_dir: Path) -> Path: + """ + Download a single YYYY-MM mbox archive from lists.apache.org and save to out_dir. + """ + url = MBOX_API.format(list=list_name, domain=domain, ym=ym) + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / f"{list_name}@{domain}-{ym}.mbox" + + headers = {"User-Agent": "apache-list-mentions/1.0"} + r = requests.get(url, headers=headers, timeout=120) + + if r.status_code == 404 or not r.content.strip(): + raise FileNotFoundError(f"No archive found for {ym}") + + r.raise_for_status() + # Write via a temp file so an interrupted download is never reused as a cached archive + tmp_path = out_path.with_suffix(".mbox.part") + tmp_path.write_bytes(r.content) + tmp_path.replace(out_path) + return out_path + Review Comment: This loads the entire monthly mbox into memory (`r.content`) before writing it. Mbox archives can be large, so consider using a streaming download (e.g., `stream=True` and iterating over `iter_content`) to write incrementally to the temp file to reduce peak memory usage and improve robustness. ########## tools/contribution/apache_list_mentions.py: ########## @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Apache mailing list name-mention counter (cached mbox downloads). + +What it does: +- Downloads monthly mbox archives from lists.apache.org ONLY if not already cached locally + (the current, still-incomplete month is always re-downloaded) +- Scans the email bodies (optionally excluding quoted lines and signatures) +- Counts how many times EACH provided name (and its aliases) is mentioned +- Prints counts in the SAME ORDER as your names file (no top-N / no ranking) + +Names file format (names.txt): + # comments allowed + Canonical Name + Canonical Name: alias1, alias2 + Canonical Name: alias1, alias2 | email1, email2 (emails optional; currently not used for mention counting) + +Example: + Jane Doe: Jane, JD + John Smith: John, J. Smith + +Usage: + python3 apache_list_mentions.py --list dev --domain cloudstack.apache.org --months 12 --names names.txt +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import email +import mailbox +import re +import sys +from email.utils import parseaddr Review Comment: `parseaddr` is imported but not used anywhere in this file. Please remove it to keep the script minimal and avoid confusion about intended behavior. ########## tools/contribution/github_org_contributions.py: ########## @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +GitHub org activity counters for a set of users (GraphQL). + +Counts (scoped to org-owned repos whose name matches --repo-pattern, default ^cloudstack): +- Pull requests created (opened) +- Issues created (opened) +- Pull request reviews + +Time window: +- If --from is NOT provided → defaults to the year before --to +- Windows longer than a year are split into yearly chunks (GitHub limitation) and summed +- A date-only --to (YYYY-MM-DD) includes that whole day + +Auth: + export GITHUB_TOKEN=ghp_... + +Examples: + python3 github_org_contributions.py --org apache --users octocat + python3 github_org_contributions.py --org apache --users octocat monalisa --from 2025-02-01 --to 2026-01-31 + +See README.md in this directory for details and caveats. +""" + +import argparse +import csv +import datetime as dt +import os +import re +import sys +from typing import Any, Dict, Iterable, List, Tuple + +import requests + +GQL_ENDPOINT = "https://api.github.com/graphql"; +# GitHub's upper limit for the *ContributionsByRepository fields +MAX_REPOSITORIES = 100 + +COUNTERS = { + "prs_created": "pullRequestContributionsByRepository", + "issues_created": "issueContributionsByRepository", + "pr_reviews": "pullRequestReviewContributionsByRepository", +} + + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + + +def iso_to_datetime(s: str, end_of_day: bool = False) -> dt.datetime: + if len(s) == 10: # YYYY-MM-DD + d = dt.datetime.fromisoformat(s).replace(tzinfo=dt.timezone.utc) + return d + dt.timedelta(days=1) if end_of_day else d + if s.endswith(("Z", "z")): # fromisoformat() only accepts "Z" from Python 3.11 + s = s[:-1] + "+00:00" + d = dt.datetime.fromisoformat(s) + if d.tzinfo is None: + d = d.replace(tzinfo=dt.timezone.utc) + return d.astimezone(dt.timezone.utc) + + +def shift_years(d: dt.datetime, years: int) -> dt.datetime: + try: + return d.replace(year=d.year + years) + except ValueError: # Feb 29 + return d.replace(year=d.year + years, day=28) + + +def year_chunks(from_dt: dt.datetime, to_dt: dt.datetime) -> Iterable[Tuple[dt.datetime, dt.datetime]]: + """ + Split [from_dt, to_dt] into consecutive windows of at most one year. Review Comment: The implementation yields half-open ranges (`start < to_dt` and `end = min(..., to_dt)`), so the docstring is misleading by implying an inclusive end (`[from_dt, to_dt]`). Update the docstring to reflect `[from_dt, to_dt)` (end-exclusive) or clarify the exact inclusion semantics. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
