85 lines
3.3 KiB
Python
85 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
|
"""extract_github_repo_census.py
|
|
|
|
Extract the 15 most-starred GitHub repositories (>=10000 stars) into a TSV
|
|
table. Single live API query (unauthenticated search), one call.
|
|
|
|
Source:
|
|
https://api.github.com/search/repositories?q=stars:>=10000&per_page=15&sort=stars&order=desc
|
|
|
|
Outputs (same dir as this script):
|
|
github-repo-census-2026-11-02.tsv ranked table (rank, repo, owner, description,
|
|
stars, forks, language, url)
|
|
github-repo-census-2026-11-02.raw.json raw API reply
|
|
github-repo-census-2026-11-02.manifest.json extract manifest
|
|
"""
|
|
import csv
|
|
import json
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
OUT = Path(__file__).resolve().parent
|
|
TSV = OUT / "github-repo-census-2026-11-02.tsv"
|
|
RAW = OUT / "github-repo-census-2026-11-02.raw.json"
|
|
MAN = OUT / "github-repo-census-2026-11-02.manifest.json"
|
|
|
|
SOURCE = ("https://api.github.com/search/repositories"
|
|
"?q=stars:>=10000&per_page=15&sort=stars&order=desc")
|
|
HEADERS = {"User-Agent": "ledger-wizard/1.0",
|
|
"Accept": "application/vnd.github+json"}
|
|
|
|
|
|
def fetch():
|
|
req = urllib.request.Request(SOURCE, headers=HEADERS)
|
|
with urllib.request.urlopen(req, timeout=30) as r:
|
|
return json.loads(r.read(4 * 1024 * 1024))
|
|
|
|
|
|
def main():
|
|
api = fetch()
|
|
items = sorted(api.get("items", []),
|
|
key=lambda i: i.get("stargazers_count", 0), reverse=True)[:15]
|
|
rows = []
|
|
for i, it in enumerate(items, 1):
|
|
rows.append({
|
|
"rank": i,
|
|
"repo": it.get("name", ""),
|
|
"owner": (it.get("owner") or {}).get("login", ""),
|
|
"description": (it.get("description") or "").strip(),
|
|
"stars": it.get("stargazers_count", 0),
|
|
"forks": it.get("forks_count", 0),
|
|
"language": (it.get("language") or "").strip(),
|
|
"url": it.get("html_url", ""),
|
|
})
|
|
with TSV.open("w", encoding="utf-8", newline="") as f:
|
|
w = csv.DictWriter(f,
|
|
fieldnames=["rank", "repo", "owner", "description",
|
|
"stars", "forks", "language", "url"],
|
|
delimiter="\t", lineterminator="\n")
|
|
w.writeheader()
|
|
w.writerows(rows)
|
|
RAW.write_text(json.dumps(api, indent=1, ensure_ascii=False), encoding="utf-8")
|
|
manifest = {
|
|
"schema": "github-repo-census-v1",
|
|
"source": SOURCE,
|
|
"extracted_at_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"generated_by": "extract_github_repo_census.py (urllib -> api.github.com)",
|
|
"rows": len(rows),
|
|
"columns": ["rank", "repo", "owner", "description", "stars", "forks",
|
|
"language", "url"],
|
|
"verification": ("CHECK github-repo-census-2026-11-02.sh "
|
|
"(independent re-derivation from raw.json; exit 0 iff equal)"),
|
|
"total_count": api.get("total_count"),
|
|
}
|
|
MAN.write_text(json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
print("API total_count:", api.get("total_count"))
|
|
print("rows written:", len(rows))
|
|
for r in rows[:3]:
|
|
print(r["repo"], r["stars"], r["owner"], "|", r["language"])
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|