diff --git a/.github/workflows/issue-difficulty.yml b/.github/workflows/issue-difficulty.yml new file mode 100644 index 0000000..4ffb781 --- /dev/null +++ b/.github/workflows/issue-difficulty.yml @@ -0,0 +1,61 @@ +name: Issue difficulty + +# Every issue needs a `difficulty: hard|medium|low` label so that the PR +# closing it can be scored on the leaderboard. This workflow creates those +# labels if they're missing and marks any issue that lacks one with +# `needs difficulty`, so untriaged issues are easy to find: +# https://github.com/secfoo-com/secfoo/issues?q=is:open+label:"needs+difficulty" + +on: + issues: + types: [opened, reopened, labeled, unlabeled] + workflow_dispatch: + inputs: + backfill: + description: "Flag every open issue that has no difficulty label" + type: boolean + default: true + +permissions: read-all + +jobs: + triage: + name: Check difficulty label + # Ignore the events this workflow causes itself (adding/removing + # `needs difficulty`), so it can't loop. + if: github.event_name == 'workflow_dispatch' || github.event.label.name != 'needs difficulty' + runs-on: ubuntu-latest + permissions: + issues: write + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + + steps: + - name: Ensure the labels exist + run: | + gh label create "difficulty: hard" --force --color b60205 --description "Leaderboard: 10 points when the PR closing this is merged" + gh label create "difficulty: medium" --force --color d93f0b --description "Leaderboard: 6 points when the PR closing this is merged" + gh label create "difficulty: low" --force --color 0e8a16 --description "Leaderboard: 3 points when the PR closing this is merged" + gh label create "needs difficulty" --force --color fbca04 --description "Maintainer: add difficulty: hard, medium or low so this can be scored" + + - name: Flag or clear this issue + if: github.event_name == 'issues' + env: + ISSUE: ${{ github.event.issue.number }} + run: | + if gh issue view "$ISSUE" --json labels --jq '.labels[].name' | grep -qiE '^difficulty'; then + gh issue edit "$ISSUE" --remove-label "needs difficulty" || true + else + gh issue edit "$ISSUE" --add-label "needs difficulty" + fi + + - name: Backfill open issues + if: github.event_name == 'workflow_dispatch' && inputs.backfill + run: | + gh issue list --state open --limit 500 --json number,labels \ + --jq '.[] | select(([.labels[].name] | map(test("^difficulty"; "i")) | any) | not) | .number' \ + | while read -r number; do + echo "#$number needs a difficulty label" + gh issue edit "$number" --add-label "needs difficulty" + done diff --git a/.github/workflows/leaderboard.yml b/.github/workflows/leaderboard.yml new file mode 100644 index 0000000..c9da72e --- /dev/null +++ b/.github/workflows/leaderboard.yml @@ -0,0 +1,93 @@ +name: Leaderboard + +# Scores merged PRs by the difficulty of the issues they close and publishes +# the result to the `leaderboard` branch (LEADERBOARD.md, leaderboard.json, +# badge.json). Rules live in scripts/leaderboard.py and CONTRIBUTING.md. +# +# main is branch-protected, so the generated files go to their own branch +# instead: https://github.com/secfoo-com/secfoo/blob/leaderboard/LEADERBOARD.md + +on: + pull_request_target: + # pull_request_target (not pull_request) so PRs from forks -- which is + # most hackathon PRs -- get a token that can comment and push. Safe + # because this workflow never checks out or runs the PR's code; it only + # reads merge metadata through the API. + types: [closed] + branches: [main] + issues: + # Re-score when a maintainer adds a difficulty label after the fact. + types: [labeled, unlabeled] + schedule: + # Every 6 hours, as a catch-all. + - cron: "15 */6 * * *" + workflow_dispatch: + +permissions: read-all + +# One publish at a time so two merges in quick succession don't race on the +# leaderboard branch. +concurrency: + group: leaderboard + cancel-in-progress: false + +jobs: + publish: + name: Score and publish + if: >- + (github.event_name != 'pull_request_target' && github.event_name != 'issues') + || (github.event_name == 'pull_request_target' && github.event.pull_request.merged == true) + || (github.event_name == 'issues' && startsWith(github.event.label.name, 'difficulty')) + runs-on: ubuntu-latest + permissions: + contents: write # push to the leaderboard branch + pull-requests: write # comment the award on the merged PR + env: + # Optional tuning. LEADERBOARD_SINCE counts only PRs merged at/after an + # ISO-8601 instant (e.g. "2026-11-08T00:00:00Z" for hackathon-only); + # LEADERBOARD_EXCLUDE is a comma-separated list of logins to leave off + # (e.g. maintainers). + LEADERBOARD_SINCE: "" + LEADERBOARD_EXCLUDE: "" + + steps: + - name: Checkout default branch + # No `ref:` on purpose -- for pull_request_target this is the base + # branch, never the PR head. Credentials stay so the push step works. + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Build leaderboard + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MERGED_PR: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.number || '' }} + run: | + args=(--repo "$GITHUB_REPOSITORY" --out build/leaderboard) + [ -n "$LEADERBOARD_SINCE" ] && args+=(--since "$LEADERBOARD_SINCE") + [ -n "$LEADERBOARD_EXCLUDE" ] && args+=(--exclude "$LEADERBOARD_EXCLUDE") + [ -n "$MERGED_PR" ] && args+=(--announce-pr "$MERGED_PR") + python3 scripts/leaderboard.py "${args[@]}" + + - name: Publish to the leaderboard branch + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + if git ls-remote --exit-code --heads origin leaderboard >/dev/null; then + git fetch --depth=1 origin leaderboard + git checkout -B leaderboard origin/leaderboard + else + # First run: start the branch empty so it carries only the + # generated files, not a copy of main. + git checkout --orphan leaderboard + git rm -rfq --cached . + git clean -fdxq -e build + printf '# secfoo leaderboard branch\n\nGenerated by the Leaderboard workflow on main. See LEADERBOARD.md.\n' > README.md + git add README.md + fi + cp build/leaderboard/LEADERBOARD.md build/leaderboard/leaderboard.json build/leaderboard/badge.json . + git add LEADERBOARD.md leaderboard.json badge.json + if git diff --cached --quiet; then + echo "Leaderboard unchanged; nothing to publish." + exit 0 + fi + git commit -q -m "leaderboard: update $(date -u +%Y-%m-%dT%H:%MZ)" + git push origin leaderboard diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ddc2c8b..06a740b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,6 +43,39 @@ fine with the generic activity page (`activities/detail.html`). - Explain the *why* in the PR description, not just the what — what broke, or what was missing, that this fixes. +## Hackathon points + +Merged PRs earn points on the +[contributor leaderboard](https://github.com/secfoo-com/secfoo/blob/leaderboard/LEADERBOARD.md), +based on the difficulty of the issue they close: + +| Issue label | Points | +|----------------------|-------:| +| `difficulty: hard` | 10 | +| `difficulty: medium` | 6 | +| `difficulty: low` | 3 | + +| Level | Points needed | +|---------|--------------:| +| Level 1 | 100 | +| Level 2 | 50 | +| Level 3 | 25 | + +How it works: + +- Maintainers put one `difficulty:` label on every issue. Issues without + one carry `needs difficulty` until that's done; if you pick one up, ask + for a difficulty in the issue before you start so you know its value. +- Link the issue from your PR (`Fixes #123` in the description, or the + "Development" sidebar). Only issues GitHub records as closed by the + merged PR are scored. A PR that closes several issues scores each one. +- Each issue is scored once, for the first merged PR that closes it. +- The board is rebuilt from scratch after every merge and every label + change, so an issue labelled after your PR merged still counts. The + workflow leaves a comment on your PR with the points it earned. +- Scoring lives in `scripts/leaderboard.py`; the rules above are the + constants at the top of that file. + ## Reporting bugs Open a GitHub issue with the exact command you ran, what you expected, diff --git a/README.md b/README.md index 800594d..9b9a7c4 100644 --- a/README.md +++ b/README.md @@ -148,5 +148,12 @@ PRs welcome — see [CONTRIBUTING.md](CONTRIBUTING.md) and [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md). Security issues go through [SECURITY.md](SECURITY.md), not a public issue. +Hackathon? Issues tagged [`hackathon`](https://github.com/secfoo-com/secfoo/issues?q=is%3Aopen+label%3Ahackathon) +are open for contribution, and merged PRs earn points on the +[contributor leaderboard](https://github.com/secfoo-com/secfoo/blob/leaderboard/LEADERBOARD.md) +(10 / 6 / 3 for hard / medium / low issues; levels at 25, 50 and 100). + +Hackathon leaderboard + Choose a model for one assessment with `secfoo run --skill sast --agent claude --model `, or set a per-agent default in [configuration](docs/REFERENCE.md#configuration-secfooconfigtoml). diff --git a/scripts/leaderboard.py b/scripts/leaderboard.py new file mode 100644 index 0000000..b80f13a --- /dev/null +++ b/scripts/leaderboard.py @@ -0,0 +1,411 @@ +"""Builds the contributor leaderboard for the secfoo hackathon. + +Scoring rules (see CONTRIBUTING.md, "Hackathon points"): + +* A merged PR earns points for every issue it closes, based on the issue's + difficulty label: ``difficulty: hard`` = 10, ``difficulty: medium`` = 6, + ``difficulty: low`` = 3. +* Each issue is scored once, for the first merged PR that closed it. +* Contributors reach Level 3 at 25 points, Level 2 at 50, Level 1 at 100. +* Merged PRs that close an issue without a difficulty label earn nothing yet; + they're listed under "Needs a difficulty label" so a maintainer can add one. + The board recomputes from scratch on every run, so late labels count. + +Run by .github/workflows/leaderboard.yml, which publishes the output to the +``leaderboard`` branch. Local use (needs a token with read access): + + GITHUB_TOKEN=... python scripts/leaderboard.py --repo secfoo-com/secfoo --out build/leaderboard + +Offline use with a saved fixture (the same shape ``--dump-prs`` writes): + + python scripts/leaderboard.py --repo secfoo-com/secfoo --from-json prs.json --out build/leaderboard +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +# Points per difficulty label. Labels are matched case-insensitively as +# "difficulty: hard", "difficulty/hard" or "difficulty-hard". +POINTS = {"hard": 10, "medium": 6, "low": 3} +DIFFICULTY_RE = re.compile(r"^difficulty\s*[:/-]\s*(hard|medium|low)$", re.IGNORECASE) + +# Highest level first. A contributor holds the first level whose threshold +# they meet; below the lowest threshold they have no level yet. +LEVELS = [ + {"name": "Level 1", "min_points": 100}, + {"name": "Level 2", "min_points": 50}, + {"name": "Level 3", "min_points": 25}, +] + +GRAPHQL_URL = "https://api.github.com/graphql" +PR_QUERY = """ +query($owner: String!, $name: String!, $after: String) { + repository(owner: $owner, name: $name) { + pullRequests(states: MERGED, first: 100, after: $after, + orderBy: {field: CREATED_AT, direction: ASC}) { + pageInfo { hasNextPage endCursor } + nodes { + number title url mergedAt + author { login __typename } + closingIssuesReferences(first: 20) { + nodes { + number title url state + labels(first: 30) { nodes { name } } + } + } + } + } + } +} +""" + + +# --------------------------------------------------------------------------- # +# GitHub API +# --------------------------------------------------------------------------- # + +def _token() -> str: + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if not token: + sys.exit("GITHUB_TOKEN (or GH_TOKEN) is required to query GitHub") + return token + + +def _request(url: str, payload: dict | None = None, method: str = "POST") -> dict: + data = json.dumps(payload).encode("utf-8") if payload is not None else None + request = urllib.request.Request(url, data=data, method=method) + request.add_header("Authorization", f"Bearer {_token()}") + request.add_header("Accept", "application/vnd.github+json") + request.add_header("Content-Type", "application/json") + request.add_header("User-Agent", "secfoo-leaderboard") + try: + with urllib.request.urlopen(request, timeout=60) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + body = error.read().decode("utf-8", "replace") + sys.exit(f"GitHub API {error.code} for {url}: {body[:500]}") + + +def fetch_merged_prs(repo: str) -> list[dict]: + """Every merged PR with the issues GitHub records it as closing.""" + owner, name = repo.split("/", 1) + prs: list[dict] = [] + after = None + while True: + result = _request(GRAPHQL_URL, {"query": PR_QUERY, "variables": {"owner": owner, "name": name, "after": after}}) + if "errors" in result: + sys.exit(f"GraphQL errors: {json.dumps(result['errors'], indent=2)}") + connection = result["data"]["repository"]["pullRequests"] + for node in connection["nodes"]: + author = node.get("author") or {} + prs.append({ + "number": node["number"], + "title": node["title"], + "url": node["url"], + "merged_at": node["mergedAt"], + "author": author.get("login") or "ghost", + "author_type": author.get("__typename") or "User", + "issues": [ + { + "number": issue["number"], + "title": issue["title"], + "url": issue["url"], + "labels": [label["name"] for label in issue["labels"]["nodes"]], + } + for issue in node["closingIssuesReferences"]["nodes"] + ], + }) + if not connection["pageInfo"]["hasNextPage"]: + return prs + after = connection["pageInfo"]["endCursor"] + + +def post_pr_comment(repo: str, pr_number: int, body: str) -> None: + _request(f"https://api.github.com/repos/{repo}/issues/{pr_number}/comments", {"body": body}) + + +# --------------------------------------------------------------------------- # +# Scoring (pure; covered by tests/test_leaderboard.py) +# --------------------------------------------------------------------------- # + +def difficulty_of(labels: list[str]) -> str | None: + """The issue's difficulty, or None. If several labels disagree, the + highest wins (and the board flags the issue for cleanup).""" + found = {match.group(1).lower() for label in labels if (match := DIFFICULTY_RE.match(label.strip()))} + for difficulty in ("hard", "medium", "low"): + if difficulty in found: + return difficulty + return None + + +def level_for(points: int) -> str | None: + for level in LEVELS: + if points >= level["min_points"]: + return level["name"] + return None + + +def next_level_for(points: int) -> dict | None: + """The lowest level the contributor hasn't reached yet, or None at the top.""" + for level in reversed(LEVELS): + if points < level["min_points"]: + return level + return None + + +def is_bot(pr: dict) -> bool: + return pr.get("author_type") == "Bot" or pr["author"].endswith("[bot]") + + +def score(prs: list[dict], repo: str, *, since: str | None = None, exclude: set[str] | None = None) -> dict: + exclude = {login.lower() for login in (exclude or set())} + eligible = [ + pr for pr in prs + if not is_bot(pr) + and pr["author"].lower() not in exclude + and (since is None or pr["merged_at"] >= since) + ] + eligible.sort(key=lambda pr: (pr["merged_at"], pr["number"])) + + contributors: dict[str, dict] = {} + awards: list[dict] = [] + needs_difficulty: list[dict] = [] + seen_issues: set[int] = set() + + def contributor(login: str) -> dict: + return contributors.setdefault(login, { + "login": login, + "points": 0, + "counts": {"hard": 0, "medium": 0, "low": 0}, + "merged_prs": 0, + "awards": [], + }) + + for pr in eligible: + entry = contributor(pr["author"]) + entry["merged_prs"] += 1 + for issue in pr["issues"]: + if issue["number"] in seen_issues: + continue # already credited to an earlier PR + seen_issues.add(issue["number"]) + difficulty = difficulty_of(issue["labels"]) + record = { + "pr": pr["number"], + "pr_url": pr["url"], + "issue": issue["number"], + "issue_url": issue["url"], + "issue_title": issue["title"], + "contributor": pr["author"], + "merged_at": pr["merged_at"], + } + if difficulty is None: + needs_difficulty.append(record) + continue + record.update({"difficulty": difficulty, "points": POINTS[difficulty]}) + entry["points"] += POINTS[difficulty] + entry["counts"][difficulty] += 1 + entry["awards"].append(record) + awards.append(record) + + ranked = sorted( + contributors.values(), + key=lambda c: (-c["points"], -c["counts"]["hard"], -c["counts"]["medium"], -c["merged_prs"], c["login"].lower()), + ) + for rank, entry in enumerate(ranked, start=1): + entry["rank"] = rank + entry["level"] = level_for(entry["points"]) + + return { + "repo": repo, + "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "since": since, + "points": POINTS, + "levels": LEVELS, + "contributors": ranked, + "awards": sorted(awards, key=lambda a: a["merged_at"], reverse=True), + "needs_difficulty": sorted(needs_difficulty, key=lambda a: a["merged_at"], reverse=True), + } + + +# --------------------------------------------------------------------------- # +# Rendering +# --------------------------------------------------------------------------- # + +def _when(iso: str) -> str: + return iso[:10] + + +def render_markdown(board: dict) -> str: + repo = board["repo"] + out: list[str] = [] + out.append("# secfoo contributor leaderboard\n") + out.append( + f"_Updated {board['generated_at'].replace('T', ' ').rstrip('Z')} UTC by " + f"[`leaderboard.yml`](https://github.com/{repo}/blob/main/.github/workflows/leaderboard.yml). " + "Generated file; edits here are overwritten._\n" + ) + if board["since"]: + out.append(f"_Counting PRs merged on or after {_when(board['since'])}._\n") + + out.append("## Standings\n") + scored = [c for c in board["contributors"] if c["points"] > 0] + if not scored: + out.append("No points awarded yet. Merge a PR that closes an issue with a `difficulty:` label to get on the board.\n") + else: + out.append("| # | Contributor | Points | Level | Hard | Medium | Low | Merged PRs |") + out.append("|---:|---|---:|---|---:|---:|---:|---:|") + for c in scored: + out.append( + f"| {c['rank']} | [@{c['login']}](https://github.com/{c['login']}) | **{c['points']}** | " + f"{c['level'] or '—'} | {c['counts']['hard']} | {c['counts']['medium']} | {c['counts']['low']} | {c['merged_prs']} |" + ) + out.append("") + unscored = [c for c in board["contributors"] if c["points"] == 0] + if unscored: + names = ", ".join(f"[@{c['login']}](https://github.com/{c['login']})" for c in unscored) + out.append(f"Merged PRs but no scored issues yet: {names}.\n") + + out.append("## How points work\n") + out.append("Points come from the `difficulty:` label on the issue a merged PR closes " + "(link it with `Fixes #123` in the PR description or the sidebar).\n") + out.append("| Issue label | Points |") + out.append("|---|---:|") + for difficulty in ("hard", "medium", "low"): + out.append(f"| `difficulty: {difficulty}` | {POINTS[difficulty]} |") + out.append("") + out.append("| Level | Points needed |") + out.append("|---|---:|") + for level in LEVELS: + out.append(f"| {level['name']} | {level['min_points']} |") + out.append("") + out.append("Each issue is scored once, for the first PR merged that closes it. " + "Bots are excluded. Full rules: " + f"[CONTRIBUTING.md](https://github.com/{repo}/blob/main/CONTRIBUTING.md#hackathon-points).\n") + + out.append("## Recent awards\n") + if not board["awards"]: + out.append("None yet.\n") + else: + out.append("| Merged | Contributor | PR | Issue | Difficulty | Points |") + out.append("|---|---|---|---|---|---:|") + for a in board["awards"][:25]: + out.append( + f"| {_when(a['merged_at'])} | [@{a['contributor']}](https://github.com/{a['contributor']}) | " + f"[#{a['pr']}]({a['pr_url']}) | [#{a['issue']}]({a['issue_url']}) {a['issue_title']} | " + f"{a['difficulty']} | +{a['points']} |" + ) + out.append("") + + out.append("## Needs a difficulty label\n") + if not board["needs_difficulty"]: + out.append("Every closed issue is labelled. Nothing to do.\n") + else: + out.append("These merged PRs closed an issue that has no `difficulty:` label, so they " + "haven't been scored. A maintainer adds the label and the next run credits them.\n") + out.append("| Merged | Contributor | PR | Issue |") + out.append("|---|---|---|---|") + for a in board["needs_difficulty"]: + out.append( + f"| {_when(a['merged_at'])} | [@{a['contributor']}](https://github.com/{a['contributor']}) | " + f"[#{a['pr']}]({a['pr_url']}) | [#{a['issue']}]({a['issue_url']}) {a['issue_title']} |" + ) + out.append("") + return "\n".join(out) + + +def render_badge(board: dict) -> dict: + """shields.io endpoint schema: https://shields.io/badges/endpoint-badge""" + scored = [c for c in board["contributors"] if c["points"] > 0] + if not scored: + message, color = "no points yet", "lightgrey" + else: + leader = scored[0] + message = f"{leader['login']} leads with {leader['points']} pts" + color = "brightgreen" + return {"schemaVersion": 1, "label": "hackathon leaderboard", "message": message, "color": color} + + +def render_pr_comment(board: dict, pr_number: int) -> str | None: + """Comment for a just-merged PR, or None if it closed no issues.""" + awarded = [a for a in board["awards"] if a["pr"] == pr_number] + pending = [a for a in board["needs_difficulty"] if a["pr"] == pr_number] + if not awarded and not pending: + return None + repo = board["repo"] + lines = ["### Leaderboard update", ""] + for a in awarded: + lines.append(f"- Closed #{a['issue']} (`difficulty: {a['difficulty']}`): **+{a['points']} points**") + for a in pending: + lines.append(f"- Closed #{a['issue']}, which has no `difficulty:` label yet, so it isn't scored. " + "A maintainer can add `difficulty: hard`, `medium` or `low` and the board recomputes automatically.") + login = (awarded or pending)[0]["contributor"] + entry = next((c for c in board["contributors"] if c["login"] == login), None) + if entry and entry["points"] > 0: + standing = f"{login} now has **{entry['points']} points** (rank #{entry['rank']}" + standing += f", {entry['level']})." if entry["level"] else ")." + nxt = next_level_for(entry["points"]) + if nxt: + standing += f" {nxt['min_points'] - entry['points']} more to reach {nxt['name']}." + lines += ["", standing] + lines += ["", f"[Full leaderboard](https://github.com/{repo}/blob/leaderboard/LEADERBOARD.md)"] + return "\n".join(lines) + + +# --------------------------------------------------------------------------- # +# CLI +# --------------------------------------------------------------------------- # + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--repo", required=True, help="owner/name, e.g. secfoo-com/secfoo") + parser.add_argument("--out", type=Path, default=Path("build/leaderboard"), help="output directory") + parser.add_argument("--since", help="only count PRs merged at/after this ISO-8601 instant") + parser.add_argument("--exclude", default="", help="comma-separated logins to leave off the board") + parser.add_argument("--from-json", type=Path, help="read merged PRs from this file instead of GitHub") + parser.add_argument("--dump-prs", type=Path, help="also save the fetched merged PRs here (fixture for --from-json)") + parser.add_argument("--announce-pr", type=int, metavar="N", help="comment the award on merged PR #N") + parser.add_argument("--dry-run", action="store_true", help="print the PR comment instead of posting it") + args = parser.parse_args(argv) + + if args.from_json: + prs = json.loads(args.from_json.read_text(encoding="utf-8")) + else: + prs = fetch_merged_prs(args.repo) + if args.dump_prs: + args.dump_prs.parent.mkdir(parents=True, exist_ok=True) + args.dump_prs.write_text(json.dumps(prs, indent=2) + "\n", encoding="utf-8") + + exclude = {login.strip() for login in args.exclude.split(",") if login.strip()} + board = score(prs, args.repo, since=args.since or None, exclude=exclude) + + args.out.mkdir(parents=True, exist_ok=True) + (args.out / "LEADERBOARD.md").write_text(render_markdown(board), encoding="utf-8") + (args.out / "leaderboard.json").write_text(json.dumps(board, indent=2) + "\n", encoding="utf-8") + (args.out / "badge.json").write_text(json.dumps(render_badge(board)) + "\n", encoding="utf-8") + scored = sum(1 for c in board["contributors"] if c["points"] > 0) + print(f"{len(prs)} merged PRs, {len(board['awards'])} awards, {scored} contributors on the board, " + f"{len(board['needs_difficulty'])} issues need a difficulty label -> {args.out}") + + if args.announce_pr: + comment = render_pr_comment(board, args.announce_pr) + if comment is None: + print(f"PR #{args.announce_pr} closed no issues; nothing to announce") + elif args.dry_run: + print(comment) + else: + post_pr_comment(args.repo, args.announce_pr, comment) + print(f"commented on PR #{args.announce_pr}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_leaderboard.py b/tests/test_leaderboard.py new file mode 100644 index 0000000..4594c6b --- /dev/null +++ b/tests/test_leaderboard.py @@ -0,0 +1,161 @@ +"""Scoring rules for scripts/leaderboard.py (the hackathon leaderboard).""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "leaderboard.py" +spec = importlib.util.spec_from_file_location("leaderboard", SCRIPT) +leaderboard = importlib.util.module_from_spec(spec) +spec.loader.exec_module(leaderboard) + +REPO = "secfoo-com/secfoo" + + +def pr(number, author, merged_at, issues=(), author_type="User"): + return { + "number": number, + "title": f"PR {number}", + "url": f"https://github.com/{REPO}/pull/{number}", + "merged_at": merged_at, + "author": author, + "author_type": author_type, + "issues": list(issues), + } + + +def issue(number, *labels): + return { + "number": number, + "title": f"Issue {number}", + "url": f"https://github.com/{REPO}/issues/{number}", + "labels": list(labels), + } + + +@pytest.mark.parametrize( + ("labels", "expected"), + [ + (["difficulty: hard"], "hard"), + (["Difficulty: Medium", "bug"], "medium"), + (["difficulty/low"], "low"), + (["difficulty-hard"], "hard"), + (["hard", "enhancement"], None), # bare words don't count + (["difficulty: low", "difficulty: hard"], "hard"), # conflict -> highest + ([], None), + ], +) +def test_difficulty_label_parsing(labels, expected): + assert leaderboard.difficulty_of(labels) == expected + + +def test_points_and_levels(): + assert leaderboard.POINTS == {"hard": 10, "medium": 6, "low": 3} + assert leaderboard.level_for(0) is None + assert leaderboard.level_for(24) is None + assert leaderboard.level_for(25) == "Level 3" + assert leaderboard.level_for(50) == "Level 2" + assert leaderboard.level_for(99) == "Level 2" + assert leaderboard.level_for(100) == "Level 1" + assert leaderboard.next_level_for(30) == {"name": "Level 2", "min_points": 50} + assert leaderboard.next_level_for(150) is None + + +def test_points_are_summed_per_contributor_and_ranked(): + prs = [ + pr(1, "alice", "2026-11-08T10:00:00Z", [issue(10, "difficulty: hard")]), + pr(2, "bob", "2026-11-08T11:00:00Z", [issue(11, "difficulty: low"), issue(12, "difficulty: medium")]), + pr(3, "alice", "2026-11-09T09:00:00Z", [issue(13, "difficulty: medium")]), + ] + board = leaderboard.score(prs, REPO) + by_login = {c["login"]: c for c in board["contributors"]} + assert by_login["alice"]["points"] == 16 + assert by_login["alice"]["counts"] == {"hard": 1, "medium": 1, "low": 0} + assert by_login["alice"]["merged_prs"] == 2 + assert by_login["bob"]["points"] == 9 + assert [c["login"] for c in board["contributors"]] == ["alice", "bob"] + assert by_login["alice"]["rank"] == 1 and by_login["bob"]["rank"] == 2 + assert by_login["alice"]["level"] is None # 16 < 25 + + +def test_each_issue_is_scored_once_for_the_first_merged_pr(): + prs = [ + pr(2, "bob", "2026-11-08T12:00:00Z", [issue(10, "difficulty: hard")]), + pr(1, "alice", "2026-11-08T10:00:00Z", [issue(10, "difficulty: hard")]), # merged earlier + ] + board = leaderboard.score(prs, REPO) + by_login = {c["login"]: c for c in board["contributors"]} + assert by_login["alice"]["points"] == 10 + assert by_login["bob"]["points"] == 0 + assert len(board["awards"]) == 1 + + +def test_unlabelled_issues_are_flagged_not_scored(): + prs = [pr(1, "alice", "2026-11-08T10:00:00Z", [issue(10, "enhancement", "hackathon")])] + board = leaderboard.score(prs, REPO) + assert board["contributors"][0]["points"] == 0 + assert board["awards"] == [] + assert [(a["pr"], a["issue"], a["contributor"]) for a in board["needs_difficulty"]] == [(1, 10, "alice")] + + +def test_bots_excluded_and_since_and_exclude_filters(): + prs = [ + pr(1, "dependabot[bot]", "2026-11-08T10:00:00Z", [issue(10, "difficulty: hard")], author_type="Bot"), + pr(2, "maintainer", "2026-11-08T10:00:00Z", [issue(11, "difficulty: hard")]), + pr(3, "alice", "2026-11-01T10:00:00Z", [issue(12, "difficulty: hard")]), # before --since + pr(4, "alice", "2026-11-09T10:00:00Z", [issue(13, "difficulty: low")]), + ] + board = leaderboard.score(prs, REPO, since="2026-11-08T00:00:00Z", exclude={"Maintainer"}) + assert [(c["login"], c["points"]) for c in board["contributors"]] == [("alice", 3)] + + +def test_levels_reached(): + issues = [issue(100 + i, "difficulty: hard") for i in range(10)] + board = leaderboard.score([pr(1, "alice", "2026-11-08T10:00:00Z", issues)], REPO) + assert board["contributors"][0]["points"] == 100 + assert board["contributors"][0]["level"] == "Level 1" + + +def test_markdown_badge_and_comment_render(): + prs = [ + pr(1, "alice", "2026-11-08T10:00:00Z", [issue(10, "difficulty: hard")]), + pr(2, "bob", "2026-11-08T11:00:00Z", [issue(11, "enhancement")]), + ] + board = leaderboard.score(prs, REPO) + + md = leaderboard.render_markdown(board) + assert "| 1 | [@alice](https://github.com/alice) | **10** |" in md + assert "## Needs a difficulty label" in md + assert "[#2](https://github.com/secfoo-com/secfoo/pull/2) | [#11]" in md + + badge = leaderboard.render_badge(board) + assert badge["schemaVersion"] == 1 + assert badge["message"] == "alice leads with 10 pts" + + comment = leaderboard.render_pr_comment(board, 1) + assert "+10 points" in comment + assert "alice now has **10 points** (rank #1)." in comment + assert "15 more to reach Level 3" in comment + + pending = leaderboard.render_pr_comment(board, 2) + assert "no `difficulty:` label yet" in pending + + assert leaderboard.render_pr_comment(board, 999) is None + + +def test_cli_offline_writes_outputs(tmp_path): + fixture = tmp_path / "prs.json" + fixture.write_text( + __import__("json").dumps([pr(1, "alice", "2026-11-08T10:00:00Z", [issue(10, "difficulty: medium")])]), + encoding="utf-8", + ) + out = tmp_path / "out" + assert leaderboard.main(["--repo", REPO, "--from-json", str(fixture), "--out", str(out), + "--announce-pr", "1", "--dry-run"]) == 0 + assert (out / "LEADERBOARD.md").exists() + assert (out / "badge.json").exists() + data = __import__("json").loads((out / "leaderboard.json").read_text(encoding="utf-8")) + assert data["contributors"][0]["points"] == 6