Copilot commented on code in PR #14509: URL: https://github.com/apache/cloudstack/pull/14509#discussion_r4400660009
########## tools/contribution/apache_list_mentions.py: ########## @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Apache mailing list name-mention counter (cached mbox downloads). + +What it does: +- Downloads monthly mbox archives from lists.apache.org ONLY if not already cached locally + (the current, still-incomplete month is always re-downloaded) +- Scans the email bodies (optionally excluding quoted lines and signatures) +- Counts how many times EACH provided name (and its aliases) is mentioned +- Prints counts in the SAME ORDER as your names file (no top-N / no ranking) + +Names file format (names.txt): + # comments allowed + Canonical Name + Canonical Name: alias1, alias2 + Canonical Name: alias1, alias2 | email1, email2 (emails optional; currently not used for mention counting) + +Example: + Jane Doe: Jane, JD + John Smith: John, J. Smith + +Usage: + python3 apache_list_mentions.py --list dev --domain cloudstack.apache.org --months 12 --names names.txt +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import email +import mailbox +import re +import sys +from pathlib import Path +from typing import Dict, Iterable, List, Tuple + +import requests + +MBOX_API = "https://lists.apache.org/api/mbox.lua?list={list}&domain={domain}&d={ym}"; +DEFAULT_CACHE_DIR = Path.home() / ".cache" / "apache-list-mentions" + +# Heuristics for cleaning bodies before counting mentions +QUOTE_LINE_RE = re.compile(r"^\s*>") # drop quoted lines (reply history) +SIG_CUTOFF_RE = re.compile(r"^\s*--\s*$") # signature separator + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + +def month_starts(end_utc: dt.datetime, months: int) -> Iterable[dt.date]: + """ + Yield month starts for the last N months ending at end_utc month (inclusive). + """ + y, m = end_utc.year, end_utc.month + for i in range(months): + mm = m - i + yy = y + while mm <= 0: + mm += 12 + yy -= 1 + yield dt.date(yy, mm, 1) + +def download_mbox(list_name: str, domain: str, ym: str, out_dir: Path) -> Path: + """ + Download a single YYYY-MM mbox archive from lists.apache.org and save to out_dir. + """ + url = MBOX_API.format(list=list_name, domain=domain, ym=ym) Review Comment: The mbox URL is constructed via string formatting with user-provided `--list` / `--domain`. For correctness and safer encoding (e.g., unexpected characters), prefer using `requests.get(..., params={...})` (or at least URL-encoding inputs) rather than interpolating directly into the query string. This reduces the risk of malformed URLs and makes the request construction more robust. ########## tools/contribution/apache_list_mentions.py: ########## @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Apache mailing list name-mention counter (cached mbox downloads). + +What it does: +- Downloads monthly mbox archives from lists.apache.org ONLY if not already cached locally + (the current, still-incomplete month is always re-downloaded) +- Scans the email bodies (optionally excluding quoted lines and signatures) +- Counts how many times EACH provided name (and its aliases) is mentioned +- Prints counts in the SAME ORDER as your names file (no top-N / no ranking) + +Names file format (names.txt): + # comments allowed + Canonical Name + Canonical Name: alias1, alias2 + Canonical Name: alias1, alias2 | email1, email2 (emails optional; currently not used for mention counting) + +Example: + Jane Doe: Jane, JD + John Smith: John, J. Smith + +Usage: + python3 apache_list_mentions.py --list dev --domain cloudstack.apache.org --months 12 --names names.txt +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import email +import mailbox +import re +import sys +from pathlib import Path +from typing import Dict, Iterable, List, Tuple + +import requests + +MBOX_API = "https://lists.apache.org/api/mbox.lua?list={list}&domain={domain}&d={ym}"; +DEFAULT_CACHE_DIR = Path.home() / ".cache" / "apache-list-mentions" + +# Heuristics for cleaning bodies before counting mentions +QUOTE_LINE_RE = re.compile(r"^\s*>") # drop quoted lines (reply history) +SIG_CUTOFF_RE = re.compile(r"^\s*--\s*$") # signature separator + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + +def month_starts(end_utc: dt.datetime, months: int) -> Iterable[dt.date]: + """ + Yield month starts for the last N months ending at end_utc month (inclusive). + """ + y, m = end_utc.year, end_utc.month + for i in range(months): + mm = m - i + yy = y + while mm <= 0: + mm += 12 + yy -= 1 + yield dt.date(yy, mm, 1) + +def download_mbox(list_name: str, domain: str, ym: str, out_dir: Path) -> Path: + """ + Download a single YYYY-MM mbox archive from lists.apache.org and save to out_dir. + """ + url = MBOX_API.format(list=list_name, domain=domain, ym=ym) + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / f"{list_name}@{domain}-{ym}.mbox" + + headers = {"User-Agent": "apache-list-mentions/1.0"} + with requests.get(url, headers=headers, timeout=120, stream=True) as r: + if r.status_code == 404: + raise FileNotFoundError(f"No archive found for {ym}") + r.raise_for_status() + + # Stream to a temp file so an interrupted download is never reused as a cached archive + tmp_path = out_path.with_suffix(".mbox.part") + has_content = False + try: + with tmp_path.open("wb") as f: + for chunk in r.iter_content(chunk_size=1024 * 1024): + has_content = has_content or bool(chunk.strip()) + f.write(chunk) + except BaseException: + tmp_path.unlink(missing_ok=True) + raise + + if not has_content: + tmp_path.unlink(missing_ok=True) + raise FileNotFoundError(f"No archive found for {ym}") + tmp_path.replace(out_path) + return out_path + +def strip_html(html: str) -> str: + """ + Very simple HTML->text stripping (good enough for mailing list bodies). + """ + html = re.sub(r"(?is)<(script|style).*?>.*?</\1>", " ", html) + html = re.sub(r"(?is)<.*?>", " ", html) + html = re.sub(r" | ", " ", html) + html = re.sub(r"&", "&", html) + html = re.sub(r"<", "<", html) + html = re.sub(r">", ">", html) + return html + +def extract_text(msg: email.message.Message) -> str: + """ + Prefer text/plain parts; fallback to text/html stripped; ignore attachments. + """ + if msg.is_multipart(): + plains: List[str] = [] + htmls: List[str] = [] + for part in msg.walk(): + disp = (part.get("Content-Disposition") or "").lower() + if "attachment" in disp: + continue + + ctype = part.get_content_type() + try: + raw = part.get_payload(decode=True) + if raw is None: + continue + text = raw.decode(part.get_content_charset() or "utf-8", "replace") + except Exception: + continue + + if ctype == "text/plain": + plains.append(text) + elif ctype == "text/html": + htmls.append(text) + + if plains: + return "\n".join(plains) + if htmls: + return strip_html("\n".join(htmls)) + return "" + + # non-multipart + ctype = msg.get_content_type() + try: + raw = msg.get_payload(decode=True) + if raw is None: + return "" + text = raw.decode(msg.get_content_charset() or "utf-8", "replace") + except Exception: + return "" + + return strip_html(text) if ctype == "text/html" else text + +def normalize_text(text: str, drop_quotes: bool, drop_signature: bool) -> str: + """ + Remove quoted lines and/or signatures to reduce false mention counts. + """ + lines = text.splitlines() + out: List[str] = [] + for ln in lines: + if drop_signature and SIG_CUTOFF_RE.match(ln): + break + if drop_quotes and QUOTE_LINE_RE.match(ln): + continue + out.append(ln) + return "\n".join(out) + +def parse_names_file(path: Path) -> Tuple[List[str], Dict[str, List[str]]]: + """ + Parse names file while preserving order. + + Supports: + Canonical Name + Canonical Name: alias1, alias2 + Canonical Name: alias1, alias2 | email1, email2 (emails ignored in this script) + + Returns: + ordered_canonicals: list of canonical names in file order + aliases_map: canonical -> [canonical + aliases] + """ + ordered: List[str] = [] + aliases_map: Dict[str, List[str]] = {} + + for raw in path.read_text(encoding="utf-8").splitlines(): + ln = raw.strip() + if not ln or ln.startswith("#"): + continue + + left = ln.split("|", 1)[0].strip() + + if ":" in left: + canon, aliases = left.split(":", 1) + canon = canon.strip() + alias_list = [a.strip() for a in aliases.split(",") if a.strip()] + variants = [canon] + [a for a in alias_list if a.lower() != canon.lower()] + else: + canon = left + variants = [canon] + + ordered.append(canon) + aliases_map[canon] = variants Review Comment: If the names file contains the same canonical name more than once, `ordered` will contain duplicates but `aliases_map` (and consequently `mention_rx`) will only keep the last entry. This can lead to confusing output (duplicate printed rows) and incorrect counting configuration for earlier duplicates. Consider detecting duplicates and either (a) raising a clear error with the duplicate name/line, or (b) merging aliases into an existing canonical entry while keeping the first occurrence’s position in `ordered`. ########## tools/contribution/apache_list_mentions.py: ########## @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Apache mailing list name-mention counter (cached mbox downloads). + +What it does: +- Downloads monthly mbox archives from lists.apache.org ONLY if not already cached locally + (the current, still-incomplete month is always re-downloaded) +- Scans the email bodies (optionally excluding quoted lines and signatures) +- Counts how many times EACH provided name (and its aliases) is mentioned +- Prints counts in the SAME ORDER as your names file (no top-N / no ranking) + +Names file format (names.txt): + # comments allowed + Canonical Name + Canonical Name: alias1, alias2 + Canonical Name: alias1, alias2 | email1, email2 (emails optional; currently not used for mention counting) + +Example: + Jane Doe: Jane, JD + John Smith: John, J. Smith + +Usage: + python3 apache_list_mentions.py --list dev --domain cloudstack.apache.org --months 12 --names names.txt +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import email +import mailbox +import re +import sys +from pathlib import Path +from typing import Dict, Iterable, List, Tuple + +import requests + +MBOX_API = "https://lists.apache.org/api/mbox.lua?list={list}&domain={domain}&d={ym}"; +DEFAULT_CACHE_DIR = Path.home() / ".cache" / "apache-list-mentions" + +# Heuristics for cleaning bodies before counting mentions +QUOTE_LINE_RE = re.compile(r"^\s*>") # drop quoted lines (reply history) +SIG_CUTOFF_RE = re.compile(r"^\s*--\s*$") # signature separator + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + +def month_starts(end_utc: dt.datetime, months: int) -> Iterable[dt.date]: + """ + Yield month starts for the last N months ending at end_utc month (inclusive). + """ + y, m = end_utc.year, end_utc.month + for i in range(months): + mm = m - i + yy = y + while mm <= 0: + mm += 12 + yy -= 1 + yield dt.date(yy, mm, 1) + +def download_mbox(list_name: str, domain: str, ym: str, out_dir: Path) -> Path: + """ + Download a single YYYY-MM mbox archive from lists.apache.org and save to out_dir. + """ + url = MBOX_API.format(list=list_name, domain=domain, ym=ym) + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / f"{list_name}@{domain}-{ym}.mbox" + + headers = {"User-Agent": "apache-list-mentions/1.0"} + with requests.get(url, headers=headers, timeout=120, stream=True) as r: Review Comment: The mbox URL is constructed via string formatting with user-provided `--list` / `--domain`. For correctness and safer encoding (e.g., unexpected characters), prefer using `requests.get(..., params={...})` (or at least URL-encoding inputs) rather than interpolating directly into the query string. This reduces the risk of malformed URLs and makes the request construction more robust. ########## tools/contribution/apache_list_mentions.py: ########## @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Apache mailing list name-mention counter (cached mbox downloads). + +What it does: +- Downloads monthly mbox archives from lists.apache.org ONLY if not already cached locally + (the current, still-incomplete month is always re-downloaded) +- Scans the email bodies (optionally excluding quoted lines and signatures) +- Counts how many times EACH provided name (and its aliases) is mentioned +- Prints counts in the SAME ORDER as your names file (no top-N / no ranking) + +Names file format (names.txt): + # comments allowed + Canonical Name + Canonical Name: alias1, alias2 + Canonical Name: alias1, alias2 | email1, email2 (emails optional; currently not used for mention counting) + +Example: + Jane Doe: Jane, JD + John Smith: John, J. Smith + +Usage: + python3 apache_list_mentions.py --list dev --domain cloudstack.apache.org --months 12 --names names.txt +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import email +import mailbox +import re +import sys +from pathlib import Path +from typing import Dict, Iterable, List, Tuple + +import requests + +MBOX_API = "https://lists.apache.org/api/mbox.lua?list={list}&domain={domain}&d={ym}"; +DEFAULT_CACHE_DIR = Path.home() / ".cache" / "apache-list-mentions" + +# Heuristics for cleaning bodies before counting mentions +QUOTE_LINE_RE = re.compile(r"^\s*>") # drop quoted lines (reply history) +SIG_CUTOFF_RE = re.compile(r"^\s*--\s*$") # signature separator + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + +def month_starts(end_utc: dt.datetime, months: int) -> Iterable[dt.date]: + """ + Yield month starts for the last N months ending at end_utc month (inclusive). + """ + y, m = end_utc.year, end_utc.month + for i in range(months): + mm = m - i + yy = y + while mm <= 0: + mm += 12 + yy -= 1 + yield dt.date(yy, mm, 1) + +def download_mbox(list_name: str, domain: str, ym: str, out_dir: Path) -> Path: + """ + Download a single YYYY-MM mbox archive from lists.apache.org and save to out_dir. + """ + url = MBOX_API.format(list=list_name, domain=domain, ym=ym) + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / f"{list_name}@{domain}-{ym}.mbox" + + headers = {"User-Agent": "apache-list-mentions/1.0"} + with requests.get(url, headers=headers, timeout=120, stream=True) as r: + if r.status_code == 404: + raise FileNotFoundError(f"No archive found for {ym}") + r.raise_for_status() + + # Stream to a temp file so an interrupted download is never reused as a cached archive + tmp_path = out_path.with_suffix(".mbox.part") + has_content = False + try: + with tmp_path.open("wb") as f: + for chunk in r.iter_content(chunk_size=1024 * 1024): + has_content = has_content or bool(chunk.strip()) + f.write(chunk) + except BaseException: + tmp_path.unlink(missing_ok=True) + raise + + if not has_content: + tmp_path.unlink(missing_ok=True) + raise FileNotFoundError(f"No archive found for {ym}") + tmp_path.replace(out_path) + return out_path + +def strip_html(html: str) -> str: + """ + Very simple HTML->text stripping (good enough for mailing list bodies). + """ + html = re.sub(r"(?is)<(script|style).*?>.*?</\1>", " ", html) + html = re.sub(r"(?is)<.*?>", " ", html) + html = re.sub(r" | ", " ", html) + html = re.sub(r"&", "&", html) + html = re.sub(r"<", "<", html) + html = re.sub(r">", ">", html) + return html + +def extract_text(msg: email.message.Message) -> str: + """ + Prefer text/plain parts; fallback to text/html stripped; ignore attachments. + """ + if msg.is_multipart(): + plains: List[str] = [] + htmls: List[str] = [] + for part in msg.walk(): + disp = (part.get("Content-Disposition") or "").lower() + if "attachment" in disp: + continue + + ctype = part.get_content_type() + try: + raw = part.get_payload(decode=True) + if raw is None: + continue + text = raw.decode(part.get_content_charset() or "utf-8", "replace") + except Exception: + continue + + if ctype == "text/plain": + plains.append(text) + elif ctype == "text/html": + htmls.append(text) + + if plains: + return "\n".join(plains) + if htmls: + return strip_html("\n".join(htmls)) + return "" + + # non-multipart + ctype = msg.get_content_type() + try: + raw = msg.get_payload(decode=True) + if raw is None: + return "" + text = raw.decode(msg.get_content_charset() or "utf-8", "replace") + except Exception: + return "" + + return strip_html(text) if ctype == "text/html" else text + +def normalize_text(text: str, drop_quotes: bool, drop_signature: bool) -> str: + """ + Remove quoted lines and/or signatures to reduce false mention counts. + """ + lines = text.splitlines() + out: List[str] = [] + for ln in lines: + if drop_signature and SIG_CUTOFF_RE.match(ln): + break + if drop_quotes and QUOTE_LINE_RE.match(ln): + continue + out.append(ln) + return "\n".join(out) + +def parse_names_file(path: Path) -> Tuple[List[str], Dict[str, List[str]]]: + """ + Parse names file while preserving order. + + Supports: + Canonical Name + Canonical Name: alias1, alias2 + Canonical Name: alias1, alias2 | email1, email2 (emails ignored in this script) + + Returns: + ordered_canonicals: list of canonical names in file order + aliases_map: canonical -> [canonical + aliases] + """ + ordered: List[str] = [] + aliases_map: Dict[str, List[str]] = {} + + for raw in path.read_text(encoding="utf-8").splitlines(): + ln = raw.strip() + if not ln or ln.startswith("#"): + continue + + left = ln.split("|", 1)[0].strip() + + if ":" in left: + canon, aliases = left.split(":", 1) + canon = canon.strip() + alias_list = [a.strip() for a in aliases.split(",") if a.strip()] + variants = [canon] + [a for a in alias_list if a.lower() != canon.lower()] + else: + canon = left + variants = [canon] + + ordered.append(canon) + aliases_map[canon] = variants + + return ordered, aliases_map + +def build_mention_regex(aliases_map: Dict[str, List[str]]) -> Dict[str, re.Pattern]: + """ + Build case-insensitive regex per canonical name that matches any alias as a whole token. + """ + out: Dict[str, re.Pattern] = {} + for canon, variants in aliases_map.items(): + pats: List[str] = [] + for v in variants: + v = v.strip() + if not v: + continue + v_esc = re.escape(v) + v_esc = re.sub(r"\\\s+", r"\\s+", v_esc) # flexible whitespace for multi-word names + pats.append(rf"(?<!\w){v_esc}(?!\w)") + out[canon] = re.compile("|".join(pats), re.IGNORECASE) if pats else re.compile(r"(?!x)x") + return out + +def main() -> int: + ap = argparse.ArgumentParser(description="Count mentions of provided names in Apache mailing list archives.") + ap.add_argument("--list", required=True, help="List name, e.g. dev, users") + ap.add_argument("--domain", required=True, help="List domain, e.g. cloudstack.apache.org") + ap.add_argument("--months", type=int, default=12, help="How many months back (default: 12)") + ap.add_argument("--cache-dir", default=str(DEFAULT_CACHE_DIR), + help=f"Cache directory for downloaded mbox files (default: {DEFAULT_CACHE_DIR})") + ap.add_argument("--names", required=True, help="Path to names file (canonical names + aliases)") + ap.add_argument("--include-quotes", action="store_true", help="Include quoted lines in counting (default: exclude)") + ap.add_argument("--include-signature", action="store_true", help="Include signature in counting (default: exclude)") + args = ap.parse_args() + + cache_dir = Path(args.cache_dir).expanduser() + end = utc_now() + current_ym = f"{end.year:04d}-{end.month:02d}" + + ordered_names, aliases_map = parse_names_file(Path(args.names)) + mention_rx = build_mention_regex(aliases_map) + + mentions = {canon: 0 for canon in ordered_names} + + downloaded = 0 + reused = 0 + missing = 0 + total_msgs = 0 + + for d in month_starts(end, args.months): + ym = f"{d.year:04d}-{d.month:02d}" + mbox_path = cache_dir / f"{args.list}@{args.domain}-{ym}.mbox" + + try: + # The current month is still receiving mail, so never trust its cached copy + if mbox_path.exists() and ym != current_ym: + reused += 1 + else: + mbox_path = download_mbox(args.list, args.domain, ym, cache_dir) + downloaded += 1 + + mbox = mailbox.mbox(mbox_path) + for msg in mbox: + total_msgs += 1 + body = extract_text(msg) + body = normalize_text( + body, + drop_quotes=not args.include_quotes, + drop_signature=not args.include_signature, + ) + for canon in ordered_names: + mentions[canon] += len(mention_rx[canon].findall(body)) Review Comment: This is O(messages × names) regex scans, which can become slow for large name lists and/or large archives. A more scalable approach is to build one combined regex (e.g., alternation with named groups per canonical) and run a single `finditer` per message, then attribute matches back to canonicals. That reduces per-message work from N regex passes to 1 pass. ########## tools/contribution/github_org_contributions.py: ########## @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +GitHub org activity counters for a set of users (GraphQL). + +Counts (scoped to org-owned repos whose name matches --repo-pattern, default ^cloudstack): +- Pull requests created (opened) +- Issues created (opened) +- Pull request reviews + +Time window: +- If --from is NOT provided → defaults to the year before --to +- Windows longer than a year are split into yearly chunks (GitHub limitation) and summed +- A date-only --to (YYYY-MM-DD) includes that whole day + +Auth: + export GITHUB_TOKEN=ghp_... + +Examples: + python3 github_org_contributions.py --org apache --users octocat + python3 github_org_contributions.py --org apache --users octocat monalisa --from 2025-02-01 --to 2026-01-31 + +See README.md in this directory for details and caveats. +""" + +import argparse +import csv +import datetime as dt +import os +import re +import sys +from typing import Any, Dict, Iterable, List, Tuple + +import requests + +GQL_ENDPOINT = "https://api.github.com/graphql"; +# GitHub's upper limit for the *ContributionsByRepository fields +MAX_REPOSITORIES = 100 + +COUNTERS = { + "prs_created": "pullRequestContributionsByRepository", + "issues_created": "issueContributionsByRepository", + "pr_reviews": "pullRequestReviewContributionsByRepository", +} + + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + + +def iso_to_datetime(s: str, end_of_day: bool = False) -> dt.datetime: + if len(s) == 10: # YYYY-MM-DD + d = dt.datetime.fromisoformat(s).replace(tzinfo=dt.timezone.utc) + return d + dt.timedelta(days=1) if end_of_day else d + if s.endswith(("Z", "z")): # fromisoformat() only accepts "Z" from Python 3.11 + s = s[:-1] + "+00:00" + d = dt.datetime.fromisoformat(s) + if d.tzinfo is None: + d = d.replace(tzinfo=dt.timezone.utc) + return d.astimezone(dt.timezone.utc) + + +def shift_years(d: dt.datetime, years: int) -> dt.datetime: + try: + return d.replace(year=d.year + years) + except ValueError: # Feb 29 + return d.replace(year=d.year + years, day=28) + + +def year_chunks(from_dt: dt.datetime, to_dt: dt.datetime) -> Iterable[Tuple[dt.datetime, dt.datetime]]: + """ + Split [from_dt, to_dt) into consecutive end-exclusive windows of at most one year. + """ + start = from_dt + while start < to_dt: + end = min(shift_years(start, 1), to_dt) + yield start, end + start = end + + +def graphql(token: str, query: str, variables: Dict[str, Any]) -> Dict[str, Any]: + headers = { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + } + r = requests.post(GQL_ENDPOINT, json={"query": query, "variables": variables}, headers=headers, timeout=60) + r.raise_for_status() + payload = r.json() + if payload.get("errors"): + msgs = "; ".join(e.get("message", "Unknown error") for e in payload["errors"]) + raise RuntimeError(f"GraphQL error(s): {msgs}") + return payload["data"] + + +ORG_ID_QUERY = """ +query($org: String!) { + organization(login: $org) { id login } +} +""" + +USER_CONTRIB_QUERY = """ +query($login: String!, $orgId: ID!, $from: DateTime!, $to: DateTime!, $maxRepos: Int!) { + user(login: $login) { + login + contributionsCollection(organizationID: $orgId, from: $from, to: $to) { + pullRequestContributionsByRepository(maxRepositories: $maxRepos) { + repository { name owner { login } } + contributions { totalCount } + } + issueContributionsByRepository(maxRepositories: $maxRepos) { + repository { name owner { login } } + contributions { totalCount } + } + pullRequestReviewContributionsByRepository(maxRepositories: $maxRepos) { + repository { name owner { login } } + contributions { totalCount } + } + } + } +} +""" + + +def count_user(token: str, login: str, org_id: str, org_login: str, repo_rx: re.Pattern, + from_dt: dt.datetime, to_dt: dt.datetime) -> Dict[str, Any]: + """ + Sum the user's contributions to matching org repos over all yearly chunks of the window. + """ + counts = {k: 0 for k in COUNTERS} + repos = set() + user_login = login + for start, end in year_chunks(from_dt, to_dt): + data = graphql(token, USER_CONTRIB_QUERY, { + "login": login, + "orgId": org_id, + "from": start.isoformat(), + "to": end.isoformat(), + "maxRepos": MAX_REPOSITORIES, + }) + user = data.get("user") + if not user: + raise RuntimeError(f"User not found: {login}") + user_login = user["login"] + + cc = user["contributionsCollection"] + for key, field in COUNTERS.items(): + entries = cc[field] + if len(entries) >= MAX_REPOSITORIES: + print(f"WARN: {login}: {field} hit the {MAX_REPOSITORIES} repository limit for " + f"{start.date()} -> {end.date()}; counts may be incomplete", file=sys.stderr) + for e in entries: + repo = e["repository"] + if repo["owner"]["login"].lower() != org_login.lower() or not repo_rx.search(repo["name"]): + continue + counts[key] += e["contributions"]["totalCount"] + repos.add(repo["name"]) + + return {"user": user_login, **counts, "repos": " ".join(sorted(repos)), "error": ""} + + +def main() -> int: + ap = argparse.ArgumentParser(description="Count PRs, issues, and PR reviews for GitHub users within an org.") + ap.add_argument("--org", required=True, help="GitHub organization login (e.g. apache)") + ap.add_argument("--users", nargs="+", required=True, help="GitHub usernames") + ap.add_argument("--repo-pattern", default="^cloudstack", + help="Regex matched against repo names; only matching repos are counted (default: ^cloudstack). " + "Use '.' to count all org repos") + ap.add_argument("--from", dest="from_", default=None, + help="Start date (YYYY-MM-DD or ISO). Default: one year before --to") + ap.add_argument("--to", default=None, + help="End date (YYYY-MM-DD, inclusive, or ISO). Default: now (UTC)") + ap.add_argument("--csv", default=None, help="Optional CSV output path") + args = ap.parse_args() + + token = os.environ.get("GITHUB_TOKEN") + if not token: + print("ERROR: GITHUB_TOKEN is not set", file=sys.stderr) + return 2 + + try: + repo_rx = re.compile(args.repo_pattern, re.IGNORECASE) + except re.error as e: + print(f"ERROR: invalid --repo-pattern: {e}", file=sys.stderr) + return 2 + + try: + to_dt = iso_to_datetime(args.to, end_of_day=True) if args.to else utc_now() + from_dt = iso_to_datetime(args.from_) if args.from_ else shift_years(to_dt, -1) + except ValueError as e: + print(f"ERROR: invalid date: {e}", file=sys.stderr) + return 2 + + if to_dt <= from_dt: + print("ERROR: --to must be after --from", file=sys.stderr) + return 2 + + try: + org = graphql(token, ORG_ID_QUERY, {"org": args.org}).get("organization") + except Exception as e: + print(f"ERROR: failed to look up organization '{args.org}': {e}", file=sys.stderr) + return 2 + if not org: + print(f"ERROR: Organization '{args.org}' not found or inaccessible", file=sys.stderr) + return 2 + + rows: List[Dict[str, Any]] = [] + + for u in args.users: + try: + rows.append(count_user(token, u, org["id"], org["login"], repo_rx, from_dt, to_dt)) + except Exception as e: + rows.append({"user": u, **{k: None for k in COUNTERS}, "repos": "", "error": str(e)}) + + headers = ["user", *COUNTERS, "repos", "error"] + + # Print table + colw = {h: len(h) for h in headers} + for r in rows: + for h in headers: + colw[h] = max(colw[h], len("" if r[h] is None else str(r[h]))) + + def fmt(r): + return " ".join(("" if r[h] is None else str(r[h])).ljust(colw[h]) for h in headers).rstrip() + + print(f"Org: {org['login']} (repos matching: {args.repo_pattern})") + print(f"Window (UTC): {from_dt.isoformat()} -> {to_dt.isoformat()}") Review Comment: When `--to` is provided as a date-only value, the implementation intentionally converts it to an end-exclusive boundary by adding one day (`end_of_day=True`). Printing `to_dt` as-is can therefore look like the window ends on the following day at 00:00Z, which conflicts with the help text saying date-only `--to` is inclusive. Consider adjusting the printed window to reflect inclusivity (e.g., show the original `--to` input, or explicitly print `[from, to)` semantics). -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
