Copilot commented on code in PR #14509: URL: https://github.com/apache/cloudstack/pull/14509#discussion_r4400353495
########## tools/contribution/apache_list_mentions.py: ########## @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Apache mailing list name-mention counter (cached mbox downloads). + +What it does: +- Downloads monthly mbox archives from lists.apache.org ONLY if not already cached locally + (the current, still-incomplete month is always re-downloaded) +- Scans the email bodies (optionally excluding quoted lines and signatures) +- Counts how many times EACH provided name (and its aliases) is mentioned +- Prints counts in the SAME ORDER as your names file (no top-N / no ranking) + +Names file format (names.txt): + # comments allowed + Canonical Name + Canonical Name: alias1, alias2 + Canonical Name: alias1, alias2 | email1, email2 (emails optional; currently not used for mention counting) + +Example: + Jane Doe: Jane, JD + John Smith: John, J. Smith + +Usage: + python3 apache_list_mentions.py --list dev --domain cloudstack.apache.org --months 12 --names names.txt +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import email +import mailbox +import re +import sys +from pathlib import Path +from typing import Dict, Iterable, List, Tuple + +import requests + +MBOX_API = "https://lists.apache.org/api/mbox.lua?list={list}&domain={domain}&d={ym}"; +DEFAULT_CACHE_DIR = Path.home() / ".cache" / "apache-list-mentions" + +# Heuristics for cleaning bodies before counting mentions +QUOTE_LINE_RE = re.compile(r"^\s*>") # drop quoted lines (reply history) +SIG_CUTOFF_RE = re.compile(r"^\s*--\s*$") # signature separator + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + +def month_starts(end_utc: dt.datetime, months: int) -> Iterable[dt.date]: + """ + Yield month starts for the last N months ending at end_utc month (inclusive). + """ + y, m = end_utc.year, end_utc.month + for i in range(months): + mm = m - i + yy = y + while mm <= 0: + mm += 12 + yy -= 1 + yield dt.date(yy, mm, 1) + +def download_mbox(list_name: str, domain: str, ym: str, out_dir: Path) -> Path: + """ + Download a single YYYY-MM mbox archive from lists.apache.org and save to out_dir. + """ + url = MBOX_API.format(list=list_name, domain=domain, ym=ym) + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / f"{list_name}@{domain}-{ym}.mbox" + + headers = {"User-Agent": "apache-list-mentions/1.0"} + with requests.get(url, headers=headers, timeout=120, stream=True) as r: + if r.status_code == 404: + raise FileNotFoundError(f"No archive found for {ym}") + r.raise_for_status() + + # Stream to a temp file so an interrupted download is never reused as a cached archive + tmp_path = out_path.with_suffix(".mbox.part") + has_content = False + try: + with tmp_path.open("wb") as f: + for chunk in r.iter_content(chunk_size=1024 * 1024): + has_content = has_content or bool(chunk.strip()) + f.write(chunk) + except BaseException: + tmp_path.unlink(missing_ok=True) + raise + + if not has_content: + tmp_path.unlink(missing_ok=True) + raise FileNotFoundError(f"No archive found for {ym}") + tmp_path.replace(out_path) + return out_path + +def strip_html(html: str) -> str: + """ + Very simple HTML->text stripping (good enough for mailing list bodies). + """ + html = re.sub(r"(?is)<(script|style).*?>.*?</\1>", " ", html) + html = re.sub(r"(?is)<.*?>", " ", html) + html = re.sub(r" | ", " ", html) + html = re.sub(r"&", "&", html) + html = re.sub(r"<", "<", html) + html = re.sub(r">", ">", html) + return html + +def extract_text(msg: email.message.Message) -> str: + """ + Prefer text/plain parts; fallback to text/html stripped; ignore attachments. + """ + if msg.is_multipart(): + plains: List[str] = [] + htmls: List[str] = [] + for part in msg.walk(): + disp = (part.get("Content-Disposition") or "").lower() + if "attachment" in disp: + continue + + ctype = part.get_content_type() + try: + raw = part.get_payload(decode=True) + if raw is None: + continue + text = raw.decode(part.get_content_charset() or "utf-8", "replace") + except Exception: + continue + + if ctype == "text/plain": + plains.append(text) + elif ctype == "text/html": + htmls.append(text) + + if plains: + return "\n".join(plains) + if htmls: + return strip_html("\n".join(htmls)) + return "" + + # non-multipart + ctype = msg.get_content_type() + try: + raw = msg.get_payload(decode=True) + if raw is None: + return "" + text = raw.decode(msg.get_content_charset() or "utf-8", "replace") + except Exception: + return "" + + return strip_html(text) if ctype == "text/html" else text + +def normalize_text(text: str, drop_quotes: bool, drop_signature: bool) -> str: + """ + Remove quoted lines and/or signatures to reduce false mention counts. + """ + lines = text.splitlines() + out: List[str] = [] + for ln in lines: + if drop_signature and SIG_CUTOFF_RE.match(ln): + break + if drop_quotes and QUOTE_LINE_RE.match(ln): + continue + out.append(ln) + return "\n".join(out) + +def parse_names_file(path: Path) -> Tuple[List[str], Dict[str, List[str]]]: + """ + Parse names file while preserving order. + + Supports: + Canonical Name + Canonical Name: alias1, alias2 + Canonical Name: alias1, alias2 | email1, email2 (emails ignored in this script) + + Returns: + ordered_canonicals: list of canonical names in file order + aliases_map: canonical -> [canonical + aliases] + """ + ordered: List[str] = [] + aliases_map: Dict[str, List[str]] = {} + + for raw in path.read_text(encoding="utf-8").splitlines(): + ln = raw.strip() + if not ln or ln.startswith("#"): + continue + + left = ln.split("|", 1)[0].strip() + + if ":" in left: + canon, aliases = left.split(":", 1) + canon = canon.strip() + als = [a.strip() for a in aliases.split(",") if a.strip()] + variants = [canon] + [a for a in als if a.lower() != canon.lower()] + else: + canon = left + variants = [canon] + + ordered.append(canon) + aliases_map[canon] = variants Review Comment: If the names file contains the same canonical name more than once, `ordered` will contain duplicates but `aliases_map` and `mentions` (a dict keyed by canonical) will silently overwrite counts/aliases. This produces inconsistent output (duplicate lines with identical counts and only the last alias set used). Consider detecting duplicates during parsing and raising a clear error (ideally including the offending canonical name and line content/number) or explicitly defining merge behavior. ########## tools/contribution/apache_list_mentions.py: ########## @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Apache mailing list name-mention counter (cached mbox downloads). + +What it does: +- Downloads monthly mbox archives from lists.apache.org ONLY if not already cached locally + (the current, still-incomplete month is always re-downloaded) +- Scans the email bodies (optionally excluding quoted lines and signatures) +- Counts how many times EACH provided name (and its aliases) is mentioned +- Prints counts in the SAME ORDER as your names file (no top-N / no ranking) + +Names file format (names.txt): + # comments allowed + Canonical Name + Canonical Name: alias1, alias2 + Canonical Name: alias1, alias2 | email1, email2 (emails optional; currently not used for mention counting) + +Example: + Jane Doe: Jane, JD + John Smith: John, J. Smith + +Usage: + python3 apache_list_mentions.py --list dev --domain cloudstack.apache.org --months 12 --names names.txt +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import email +import mailbox +import re +import sys +from pathlib import Path +from typing import Dict, Iterable, List, Tuple + +import requests + +MBOX_API = "https://lists.apache.org/api/mbox.lua?list={list}&domain={domain}&d={ym}"; +DEFAULT_CACHE_DIR = Path.home() / ".cache" / "apache-list-mentions" + +# Heuristics for cleaning bodies before counting mentions +QUOTE_LINE_RE = re.compile(r"^\s*>") # drop quoted lines (reply history) +SIG_CUTOFF_RE = re.compile(r"^\s*--\s*$") # signature separator + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + +def month_starts(end_utc: dt.datetime, months: int) -> Iterable[dt.date]: + """ + Yield month starts for the last N months ending at end_utc month (inclusive). + """ + y, m = end_utc.year, end_utc.month + for i in range(months): + mm = m - i + yy = y + while mm <= 0: + mm += 12 + yy -= 1 + yield dt.date(yy, mm, 1) + +def download_mbox(list_name: str, domain: str, ym: str, out_dir: Path) -> Path: + """ + Download a single YYYY-MM mbox archive from lists.apache.org and save to out_dir. + """ + url = MBOX_API.format(list=list_name, domain=domain, ym=ym) + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / f"{list_name}@{domain}-{ym}.mbox" + + headers = {"User-Agent": "apache-list-mentions/1.0"} + with requests.get(url, headers=headers, timeout=120, stream=True) as r: + if r.status_code == 404: + raise FileNotFoundError(f"No archive found for {ym}") + r.raise_for_status() + + # Stream to a temp file so an interrupted download is never reused as a cached archive + tmp_path = out_path.with_suffix(".mbox.part") + has_content = False + try: + with tmp_path.open("wb") as f: + for chunk in r.iter_content(chunk_size=1024 * 1024): + has_content = has_content or bool(chunk.strip()) + f.write(chunk) + except BaseException: + tmp_path.unlink(missing_ok=True) + raise + + if not has_content: + tmp_path.unlink(missing_ok=True) + raise FileNotFoundError(f"No archive found for {ym}") + tmp_path.replace(out_path) + return out_path + +def strip_html(html: str) -> str: + """ + Very simple HTML->text stripping (good enough for mailing list bodies). + """ + html = re.sub(r"(?is)<(script|style).*?>.*?</\1>", " ", html) + html = re.sub(r"(?is)<.*?>", " ", html) + html = re.sub(r" | ", " ", html) + html = re.sub(r"&", "&", html) + html = re.sub(r"<", "<", html) + html = re.sub(r">", ">", html) + return html + +def extract_text(msg: email.message.Message) -> str: + """ + Prefer text/plain parts; fallback to text/html stripped; ignore attachments. + """ + if msg.is_multipart(): + plains: List[str] = [] + htmls: List[str] = [] + for part in msg.walk(): + disp = (part.get("Content-Disposition") or "").lower() + if "attachment" in disp: + continue + + ctype = part.get_content_type() + try: + raw = part.get_payload(decode=True) + if raw is None: + continue + text = raw.decode(part.get_content_charset() or "utf-8", "replace") + except Exception: + continue + + if ctype == "text/plain": + plains.append(text) + elif ctype == "text/html": + htmls.append(text) + + if plains: + return "\n".join(plains) + if htmls: + return strip_html("\n".join(htmls)) + return "" + + # non-multipart + ctype = msg.get_content_type() + try: + raw = msg.get_payload(decode=True) + if raw is None: + return "" + text = raw.decode(msg.get_content_charset() or "utf-8", "replace") + except Exception: + return "" + + return strip_html(text) if ctype == "text/html" else text + +def normalize_text(text: str, drop_quotes: bool, drop_signature: bool) -> str: + """ + Remove quoted lines and/or signatures to reduce false mention counts. + """ + lines = text.splitlines() + out: List[str] = [] + for ln in lines: + if drop_signature and SIG_CUTOFF_RE.match(ln): + break + if drop_quotes and QUOTE_LINE_RE.match(ln): + continue + out.append(ln) + return "\n".join(out) + +def parse_names_file(path: Path) -> Tuple[List[str], Dict[str, List[str]]]: + """ + Parse names file while preserving order. + + Supports: + Canonical Name + Canonical Name: alias1, alias2 + Canonical Name: alias1, alias2 | email1, email2 (emails ignored in this script) + + Returns: + ordered_canonicals: list of canonical names in file order + aliases_map: canonical -> [canonical + aliases] + """ + ordered: List[str] = [] + aliases_map: Dict[str, List[str]] = {} + + for raw in path.read_text(encoding="utf-8").splitlines(): + ln = raw.strip() + if not ln or ln.startswith("#"): + continue + + left = ln.split("|", 1)[0].strip() + + if ":" in left: + canon, aliases = left.split(":", 1) + canon = canon.strip() + als = [a.strip() for a in aliases.split(",") if a.strip()] + variants = [canon] + [a for a in als if a.lower() != canon.lower()] + else: + canon = left + variants = [canon] + + ordered.append(canon) + aliases_map[canon] = variants + + return ordered, aliases_map + +def build_mention_regex(aliases_map: Dict[str, List[str]]) -> Dict[str, re.Pattern]: + """ + Build case-insensitive regex per canonical name that matches any alias as a whole token. + """ + out: Dict[str, re.Pattern] = {} + for canon, variants in aliases_map.items(): + pats: List[str] = [] + for v in variants: + v = v.strip() + if not v: + continue + v_esc = re.escape(v) + v_esc = re.sub(r"\\\s+", r"\\s+", v_esc) # flexible whitespace for multi-word names + pats.append(rf"(?<!\w){v_esc}(?!\w)") + out[canon] = re.compile("|".join(pats), re.IGNORECASE) if pats else re.compile(r"(?!x)x") + return out + +def main() -> int: + ap = argparse.ArgumentParser(description="Count mentions of provided names in Apache mailing list archives.") + ap.add_argument("--list", required=True, help="List name, e.g. dev, users") + ap.add_argument("--domain", required=True, help="List domain, e.g. cloudstack.apache.org") + ap.add_argument("--months", type=int, default=12, help="How many months back (default: 12)") + ap.add_argument("--cache-dir", default=str(DEFAULT_CACHE_DIR), + help=f"Cache directory for downloaded mbox files (default: {DEFAULT_CACHE_DIR})") + ap.add_argument("--names", required=True, help="Path to names file (canonical names + aliases)") + ap.add_argument("--include-quotes", action="store_true", help="Include quoted lines in counting (default: exclude)") + ap.add_argument("--include-signature", action="store_true", help="Include signature in counting (default: exclude)") + args = ap.parse_args() + + cache_dir = Path(args.cache_dir).expanduser() + end = utc_now() + current_ym = f"{end.year:04d}-{end.month:02d}" + + ordered_names, aliases_map = parse_names_file(Path(args.names)) + mention_rx = build_mention_regex(aliases_map) + + mentions = {canon: 0 for canon in ordered_names} + + downloaded = 0 + reused = 0 + missing = 0 + total_msgs = 0 + + for d in month_starts(end, args.months): + ym = f"{d.year:04d}-{d.month:02d}" + mbox_path = cache_dir / f"{args.list}@{args.domain}-{ym}.mbox" + + try: + # The current month is still receiving mail, so never trust its cached copy + if mbox_path.exists() and ym != current_ym: + reused += 1 + else: + mbox_path = download_mbox(args.list, args.domain, ym, cache_dir) + downloaded += 1 + + mbox = mailbox.mbox(mbox_path) + for msg in mbox: + total_msgs += 1 + body = extract_text(msg) + body = normalize_text( + body, + drop_quotes=not args.include_quotes, + drop_signature=not args.include_signature, + ) + for canon in ordered_names: + mentions[canon] += len(mention_rx[canon].findall(body)) Review Comment: This scales as O(messages × names) full-text regex scans (each `findall` re-scans the entire body). With many names and larger month ranges, this can become very slow. A more efficient approach is to compile a single combined regex for all aliases (e.g., using named groups or an alias→canonical mapping) and do one `finditer` pass per message, incrementing per-canonical counters based on which group matched. ########## tools/contribution/README.md: ########## @@ -0,0 +1,117 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + --> + +# Contribution Evaluation Scripts + +Helper scripts to gather activity numbers for a set of contributors, e.g. as +input when discussing committer or PMC nominations. The numbers are only a +rough signal; read the actual contributions before drawing conclusions. + +| Script | Data source | What it counts | +|---|---|---| +| `github_org_contributions.py` | GitHub GraphQL API | PRs opened, issues opened and PR reviews per user, in CloudStack repos (by default) of an org | +| `apache_list_mentions.py` | lists.apache.org mbox archives | How often each name (or alias) appears in mailing list message bodies | + +## Requirements + +- Python 3.8+ +- [`requests`](https://pypi.org/project/requests/): `pip install requests` + +## github_org_contributions.py + +Requires a GitHub token in `GITHUB_TOKEN`. A classic token with no scopes (or a +fine-grained token with public read-only access) is enough for public repos. +If the GitHub CLI is logged in, `export GITHUB_TOKEN=$(gh auth token)` works too. + +```bash +export GITHUB_TOKEN=<your-token> +python3 github_org_contributions.py --org apache --users octocat monalisa +python3 github_org_contributions.py --org apache --users octocat \ + --from 2023-01-01 --to 2025-12-31 --csv contributions.csv +``` + +| Option | Description | +|---|---| +| `--org` | GitHub organization login (required) | +| `--users` | One or more GitHub usernames, space separated (required) | +| `--repo-pattern` | Regex matched (case-insensitively) against repo names; only matching repos are counted. Default: `^cloudstack`. Use `.` for all org repos | +| `--from` | Start, `YYYY-MM-DD` or ISO 8601. Default: one year before `--to` | +| `--to` | End, `YYYY-MM-DD` (inclusive) or ISO 8601. Default: now (UTC) | +| `--csv` | Also write the table to this CSV file | + +Notes: + +- Only repos owned by `--org` whose name matches `--repo-pattern` are counted. + The default `^cloudstack` covers `apache/cloudstack`, `cloudstack-documentation`, + `cloudstack-www`, `cloudstack-cloudmonkey`, `cloudstack-go`, + `cloudstack-terraform-provider` and the other `cloudstack-*` repos. The `repos` + column lists the repos that contributed to the counts. +- Dates without a time are in UTC; `--from` starts at the beginning of that day + and `--to` includes the whole day. +- GitHub limits a query to one year, so longer windows are split into yearly + chunks and summed. Review Comment: The GitHub script also has a hard per-counter per-year cap due to `maxRepositories` (100), and it warns when the limit is hit (`counts may be incomplete`). It would be helpful to document this limitation here as well (what triggers the warning and that counts can be underreported for users contributing across many repos within a chunk). ########## tools/contribution/github_org_contributions.py: ########## @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +GitHub org activity counters for a set of users (GraphQL). + +Counts (scoped to org-owned repos whose name matches --repo-pattern, default ^cloudstack): +- Pull requests created (opened) +- Issues created (opened) +- Pull request reviews + +Time window: +- If --from is NOT provided → defaults to the year before --to +- Windows longer than a year are split into yearly chunks (GitHub limitation) and summed +- A date-only --to (YYYY-MM-DD) includes that whole day + +Auth: + export GITHUB_TOKEN=ghp_... + +Examples: + python3 github_org_contributions.py --org apache --users octocat + python3 github_org_contributions.py --org apache --users octocat monalisa --from 2025-02-01 --to 2026-01-31 + +See README.md in this directory for details and caveats. +""" + +import argparse +import csv +import datetime as dt +import os +import re +import sys +from typing import Any, Dict, Iterable, List, Tuple + +import requests + +GQL_ENDPOINT = "https://api.github.com/graphql"; +# GitHub's upper limit for the *ContributionsByRepository fields +MAX_REPOSITORIES = 100 + +COUNTERS = { + "prs_created": "pullRequestContributionsByRepository", + "issues_created": "issueContributionsByRepository", + "pr_reviews": "pullRequestReviewContributionsByRepository", +} + + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + + +def iso_to_datetime(s: str, end_of_day: bool = False) -> dt.datetime: + if len(s) == 10: # YYYY-MM-DD + d = dt.datetime.fromisoformat(s).replace(tzinfo=dt.timezone.utc) + return d + dt.timedelta(days=1) if end_of_day else d + if s.endswith(("Z", "z")): # fromisoformat() only accepts "Z" from Python 3.11 + s = s[:-1] + "+00:00" + d = dt.datetime.fromisoformat(s) + if d.tzinfo is None: + d = d.replace(tzinfo=dt.timezone.utc) + return d.astimezone(dt.timezone.utc) + + +def shift_years(d: dt.datetime, years: int) -> dt.datetime: + try: + return d.replace(year=d.year + years) + except ValueError: # Feb 29 + return d.replace(year=d.year + years, day=28) + + +def year_chunks(from_dt: dt.datetime, to_dt: dt.datetime) -> Iterable[Tuple[dt.datetime, dt.datetime]]: + """ + Split [from_dt, to_dt) into consecutive end-exclusive windows of at most one year. + """ + start = from_dt + while start < to_dt: + end = min(shift_years(start, 1), to_dt) + yield start, end + start = end + + +def graphql(token: str, query: str, variables: Dict[str, Any]) -> Dict[str, Any]: + headers = { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + } + r = requests.post(GQL_ENDPOINT, json={"query": query, "variables": variables}, headers=headers, timeout=60) + r.raise_for_status() + payload = r.json() + if payload.get("errors"): + msgs = "; ".join(e.get("message", "Unknown error") for e in payload["errors"]) + raise RuntimeError(f"GraphQL error(s): {msgs}") + return payload["data"] + + +ORG_ID_QUERY = """ +query($org: String!) { + organization(login: $org) { id login } +} +""" + +USER_CONTRIB_QUERY = """ +query($login: String!, $orgId: ID!, $from: DateTime!, $to: DateTime!, $maxRepos: Int!) { + user(login: $login) { + login + contributionsCollection(organizationID: $orgId, from: $from, to: $to) { + pullRequestContributionsByRepository(maxRepositories: $maxRepos) { + repository { name owner { login } } + contributions { totalCount } + } + issueContributionsByRepository(maxRepositories: $maxRepos) { + repository { name owner { login } } + contributions { totalCount } + } + pullRequestReviewContributionsByRepository(maxRepositories: $maxRepos) { + repository { name owner { login } } + contributions { totalCount } + } + } + } +} +""" + + +def count_user(token: str, login: str, org_id: str, org_login: str, repo_rx: re.Pattern, + from_dt: dt.datetime, to_dt: dt.datetime) -> Dict[str, Any]: + """ + Sum the user's contributions to matching org repos over all yearly chunks of the window. + """ + counts = {k: 0 for k in COUNTERS} + repos = set() + user_login = login + for start, end in year_chunks(from_dt, to_dt): + data = graphql(token, USER_CONTRIB_QUERY, { + "login": login, + "orgId": org_id, + "from": start.isoformat(), + "to": end.isoformat(), + "maxRepos": MAX_REPOSITORIES, + }) + user = data.get("user") + if not user: + raise RuntimeError(f"User not found: {login}") + user_login = user["login"] + + cc = user["contributionsCollection"] + for key, field in COUNTERS.items(): + entries = cc[field] + if len(entries) >= MAX_REPOSITORIES: Review Comment: Given `maxRepositories` is set to `MAX_REPOSITORIES`, the returned list length cannot exceed that value, so `>=` is effectively the same as `==` here. Using `== MAX_REPOSITORIES` more clearly communicates the intent (possible truncation at the limit) and avoids implying that values greater than the limit are possible. ########## tools/contribution/github_org_contributions.py: ########## @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +GitHub org activity counters for a set of users (GraphQL). + +Counts (scoped to org-owned repos whose name matches --repo-pattern, default ^cloudstack): +- Pull requests created (opened) +- Issues created (opened) +- Pull request reviews + +Time window: +- If --from is NOT provided → defaults to the year before --to +- Windows longer than a year are split into yearly chunks (GitHub limitation) and summed +- A date-only --to (YYYY-MM-DD) includes that whole day + +Auth: + export GITHUB_TOKEN=ghp_... + +Examples: + python3 github_org_contributions.py --org apache --users octocat + python3 github_org_contributions.py --org apache --users octocat monalisa --from 2025-02-01 --to 2026-01-31 + +See README.md in this directory for details and caveats. +""" + +import argparse +import csv +import datetime as dt +import os +import re +import sys +from typing import Any, Dict, Iterable, List, Tuple + +import requests + +GQL_ENDPOINT = "https://api.github.com/graphql"; +# GitHub's upper limit for the *ContributionsByRepository fields +MAX_REPOSITORIES = 100 + +COUNTERS = { + "prs_created": "pullRequestContributionsByRepository", + "issues_created": "issueContributionsByRepository", + "pr_reviews": "pullRequestReviewContributionsByRepository", +} + + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + + +def iso_to_datetime(s: str, end_of_day: bool = False) -> dt.datetime: + if len(s) == 10: # YYYY-MM-DD + d = dt.datetime.fromisoformat(s).replace(tzinfo=dt.timezone.utc) + return d + dt.timedelta(days=1) if end_of_day else d + if s.endswith(("Z", "z")): # fromisoformat() only accepts "Z" from Python 3.11 + s = s[:-1] + "+00:00" + d = dt.datetime.fromisoformat(s) + if d.tzinfo is None: + d = d.replace(tzinfo=dt.timezone.utc) + return d.astimezone(dt.timezone.utc) + + +def shift_years(d: dt.datetime, years: int) -> dt.datetime: + try: + return d.replace(year=d.year + years) + except ValueError: # Feb 29 + return d.replace(year=d.year + years, day=28) + + +def year_chunks(from_dt: dt.datetime, to_dt: dt.datetime) -> Iterable[Tuple[dt.datetime, dt.datetime]]: + """ + Split [from_dt, to_dt) into consecutive end-exclusive windows of at most one year. + """ + start = from_dt + while start < to_dt: + end = min(shift_years(start, 1), to_dt) + yield start, end + start = end + + +def graphql(token: str, query: str, variables: Dict[str, Any]) -> Dict[str, Any]: + headers = { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + } + r = requests.post(GQL_ENDPOINT, json={"query": query, "variables": variables}, headers=headers, timeout=60) + r.raise_for_status() + payload = r.json() + if payload.get("errors"): + msgs = "; ".join(e.get("message", "Unknown error") for e in payload["errors"]) + raise RuntimeError(f"GraphQL error(s): {msgs}") + return payload["data"] + + +ORG_ID_QUERY = """ +query($org: String!) { + organization(login: $org) { id login } +} +""" + +USER_CONTRIB_QUERY = """ +query($login: String!, $orgId: ID!, $from: DateTime!, $to: DateTime!, $maxRepos: Int!) { + user(login: $login) { + login + contributionsCollection(organizationID: $orgId, from: $from, to: $to) { + pullRequestContributionsByRepository(maxRepositories: $maxRepos) { + repository { name owner { login } } + contributions { totalCount } + } + issueContributionsByRepository(maxRepositories: $maxRepos) { + repository { name owner { login } } + contributions { totalCount } + } + pullRequestReviewContributionsByRepository(maxRepositories: $maxRepos) { + repository { name owner { login } } + contributions { totalCount } + } + } + } +} +""" + + +def count_user(token: str, login: str, org_id: str, org_login: str, repo_rx: re.Pattern, + from_dt: dt.datetime, to_dt: dt.datetime) -> Dict[str, Any]: + """ + Sum the user's contributions to matching org repos over all yearly chunks of the window. + """ + counts = {k: 0 for k in COUNTERS} + repos = set() + user_login = login + for start, end in year_chunks(from_dt, to_dt): + data = graphql(token, USER_CONTRIB_QUERY, { + "login": login, + "orgId": org_id, + "from": start.isoformat(), + "to": end.isoformat(), + "maxRepos": MAX_REPOSITORIES, + }) + user = data.get("user") + if not user: + raise RuntimeError(f"User not found: {login}") + user_login = user["login"] + + cc = user["contributionsCollection"] + for key, field in COUNTERS.items(): + entries = cc[field] + if len(entries) >= MAX_REPOSITORIES: + print(f"WARN: {login}: {field} hit the {MAX_REPOSITORIES} repository limit for " + f"{start.date()} -> {end.date()}; counts may be incomplete", file=sys.stderr) + for e in entries: + repo = e["repository"] + if repo["owner"]["login"].lower() != org_login.lower() or not repo_rx.search(repo["name"]): + continue + counts[key] += e["contributions"]["totalCount"] + repos.add(repo["name"]) + + return {"user": user_login, **counts, "repos": " ".join(sorted(repos)), "error": ""} + + +def main() -> int: + ap = argparse.ArgumentParser(description="Count PRs, issues, and PR reviews for GitHub users within an org.") + ap.add_argument("--org", required=True, help="GitHub organization login (e.g. apache)") + ap.add_argument("--users", nargs="+", required=True, help="GitHub usernames") + ap.add_argument("--repo-pattern", default="^cloudstack", + help="Regex matched against repo names; only matching repos are counted (default: ^cloudstack). " + "Use '.' to count all org repos") + ap.add_argument("--from", dest="from_", default=None, + help="Start date (YYYY-MM-DD or ISO). Default: one year before --to") + ap.add_argument("--to", default=None, + help="End date (YYYY-MM-DD, inclusive, or ISO). Default: now (UTC)") + ap.add_argument("--csv", default=None, help="Optional CSV output path") + args = ap.parse_args() + + token = os.environ.get("GITHUB_TOKEN") + if not token: + print("ERROR: GITHUB_TOKEN is not set", file=sys.stderr) + return 2 + + try: + repo_rx = re.compile(args.repo_pattern, re.IGNORECASE) + except re.error as e: + print(f"ERROR: invalid --repo-pattern: {e}", file=sys.stderr) + return 2 + + try: + to_dt = iso_to_datetime(args.to, end_of_day=True) if args.to else utc_now() + from_dt = iso_to_datetime(args.from_) if args.from_ else shift_years(to_dt, -1) + except ValueError as e: + print(f"ERROR: invalid date: {e}", file=sys.stderr) + return 2 + + if to_dt <= from_dt: + print("ERROR: --to must be after --from", file=sys.stderr) + return 2 + + try: + org = graphql(token, ORG_ID_QUERY, {"org": args.org}).get("organization") + except Exception as e: + print(f"ERROR: failed to look up organization '{args.org}': {e}", file=sys.stderr) + return 2 + if not org: + print(f"ERROR: Organization '{args.org}' not found or inaccessible", file=sys.stderr) + return 2 + + rows: List[Dict[str, Any]] = [] + + for u in args.users: + try: + rows.append(count_user(token, u, org["id"], org["login"], repo_rx, from_dt, to_dt)) + except Exception as e: + rows.append({"user": u, **{k: None for k in COUNTERS}, "repos": "", "error": str(e)}) + + headers = ["user", *COUNTERS, "repos", "error"] + + # Print table + colw = {h: len(h) for h in headers} + for r in rows: + for h in headers: + colw[h] = max(colw[h], len("" if r[h] is None else str(r[h]))) + + def fmt(r): + return " ".join(("" if r[h] is None else str(r[h])).ljust(colw[h]) for h in headers).rstrip() + + print(f"Org: {org['login']} (repos matching: {args.repo_pattern})") + print(f"Window (UTC): {from_dt.isoformat()} -> {to_dt.isoformat()}") Review Comment: When the user passes a date-only `--to`, `to_dt` is intentionally advanced by one day (end-exclusive) to include the whole day. Printing `to_dt.isoformat()` therefore shows the next day at 00:00:00Z, which is confusing and appears to contradict the help text ('inclusive'). Consider formatting the window explicitly as end-exclusive (e.g., `Window (UTC) [from, to): ...`) or printing an additional human-friendly line that shows the inclusive end date when `--to` is date-only. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
