From f85a6212479c5d453d3563b488b6ff3623361ca5 Mon Sep 17 00:00:00 2001 From: ledger Date: Thu, 22 Oct 2026 03:01:45 +0000 Subject: [PATCH] ledger: refresh top-10 Python repos by stars extraction artifact (verify.py fix: reset failures per attempt) --- wizard-work/ledger/repos.csv | 8 +- wizard-work/ledger/repos.json | 12 +-- wizard-work/ledger/repos.md | 8 +- wizard-work/ledger/verify.py | 158 ++++++++++++++++++++++------------ 4 files changed, 117 insertions(+), 69 deletions(-) diff --git a/wizard-work/ledger/repos.csv b/wizard-work/ledger/repos.csv index b7d1e1e..b89c2db 100644 --- a/wizard-work/ledger/repos.csv +++ b/wizard-work/ledger/repos.csv @@ -1,11 +1,11 @@ rank,full_name,language,stars,description,html_url 1,public-apis/public-apis,Python,491148,A collective list of free APIs,https://github.com/public-apis/public-apis 2,EbookFoundation/free-programming-books,Python,399766,:books: Freely available programming books,https://github.com/EbookFoundation/free-programming-books -3,donnemartin/system-design-primer,Python,375055,Learn how to design large-scale systems. Prep for the system design interview. Includes Anki flashcards.,https://github.com/donnemartin/system-design-primer +3,donnemartin/system-design-primer,Python,375059,Learn how to design large-scale systems. Prep for the system design interview. Includes Anki flashcards.,https://github.com/donnemartin/system-design-primer 4,vinta/awesome-python,Python,329292,"The definitive list that answers ""I want to do X in Python, which tool should I use?""",https://github.com/vinta/awesome-python 5,practical-tutorials/project-based-learning,Python,287537,Curated list of project-based tutorials,https://github.com/practical-tutorials/project-based-learning -6,NousResearch/hermes-agent,Python,255992,The agent that grows with you,https://github.com/NousResearch/hermes-agent +6,NousResearch/hermes-agent,Python,255991,The agent that grows with you,https://github.com/NousResearch/hermes-agent 7,TheAlgorithms/Python,Python,225440,All Algorithms implemented in Python,https://github.com/TheAlgorithms/Python -8,yt-dlp/yt-dlp,Python,200097,A feature-rich command-line audio/video downloader,https://github.com/yt-dlp/yt-dlp -9,microsoft/markitdown,Python,191500,Python tool for converting files and office documents to Markdown.,https://github.com/microsoft/markitdown +8,yt-dlp/yt-dlp,Python,200099,A feature-rich command-line audio/video downloader,https://github.com/yt-dlp/yt-dlp +9,microsoft/markitdown,Python,191501,Python tool for converting files and office documents to Markdown.,https://github.com/microsoft/markitdown 10,Significant-Gravitas/AutoGPT,Python,187704,"AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.",https://github.com/Significant-Gravitas/AutoGPT diff --git a/wizard-work/ledger/repos.json b/wizard-work/ledger/repos.json index a581ec1..fe5a9b7 100644 --- a/wizard-work/ledger/repos.json +++ b/wizard-work/ledger/repos.json @@ -1,7 +1,7 @@ { - "generated_at": "2026-10-22T02:53:44.810687+00:00", + "generated_at": "2026-10-22T03:00:17.092639+00:00", "query": "language:python&sort=stars&per_page=10", - "total_count": 2943531, + "total_count": 2943537, "rows": [ { "rank": 1, @@ -23,7 +23,7 @@ "rank": 3, "full_name": "donnemartin/system-design-primer", "language": "Python", - "stars": 375055, + "stars": 375059, "description": "Learn how to design large-scale systems. Prep for the system design interview. Includes Anki flashcards.", "html_url": "https://github.com/donnemartin/system-design-primer" }, @@ -47,7 +47,7 @@ "rank": 6, "full_name": "NousResearch/hermes-agent", "language": "Python", - "stars": 255992, + "stars": 255991, "description": "The agent that grows with you", "html_url": "https://github.com/NousResearch/hermes-agent" }, @@ -63,7 +63,7 @@ "rank": 8, "full_name": "yt-dlp/yt-dlp", "language": "Python", - "stars": 200097, + "stars": 200099, "description": "A feature-rich command-line audio/video downloader", "html_url": "https://github.com/yt-dlp/yt-dlp" }, @@ -71,7 +71,7 @@ "rank": 9, "full_name": "microsoft/markitdown", "language": "Python", - "stars": 191500, + "stars": 191501, "description": "Python tool for converting files and office documents to Markdown.", "html_url": "https://github.com/microsoft/markitdown" }, diff --git a/wizard-work/ledger/repos.md b/wizard-work/ledger/repos.md index 9091fcd..f7e055e 100644 --- a/wizard-work/ledger/repos.md +++ b/wizard-work/ledger/repos.md @@ -6,11 +6,11 @@ Source: GitHub Search API - `language:python&sort=stars&per_page=10` |---|------------|----------|-------|-------------| | 1 | [public-apis/public-apis](https://github.com/public-apis/public-apis) | Python | 491148 | A collective list of free APIs | | 2 | [EbookFoundation/free-programming-books](https://github.com/EbookFoundation/free-programming-books) | Python | 399766 | :books: Freely available programming books | -| 3 | [donnemartin/system-design-primer](https://github.com/donnemartin/system-design-primer) | Python | 375055 | Learn how to design large-scale systems. Prep for the system design interview. Includes Anki flashcards. | +| 3 | [donnemartin/system-design-primer](https://github.com/donnemartin/system-design-primer) | Python | 375059 | Learn how to design large-scale systems. Prep for the system design interview. Includes Anki flashcards. | | 4 | [vinta/awesome-python](https://github.com/vinta/awesome-python) | Python | 329292 | The definitive list that answers "I want to do X in Python, which tool should I use?" | | 5 | [practical-tutorials/project-based-learning](https://github.com/practical-tutorials/project-based-learning) | Python | 287537 | Curated list of project-based tutorials | -| 6 | [NousResearch/hermes-agent](https://github.com/NousResearch/hermes-agent) | Python | 255992 | The agent that grows with you | +| 6 | [NousResearch/hermes-agent](https://github.com/NousResearch/hermes-agent) | Python | 255991 | The agent that grows with you | | 7 | [TheAlgorithms/Python](https://github.com/TheAlgorithms/Python) | Python | 225440 | All Algorithms implemented in Python | -| 8 | [yt-dlp/yt-dlp](https://github.com/yt-dlp/yt-dlp) | Python | 200097 | A feature-rich command-line audio/video downloader | -| 9 | [microsoft/markitdown](https://github.com/microsoft/markitdown) | Python | 191500 | Python tool for converting files and office documents to Markdown. | +| 8 | [yt-dlp/yt-dlp](https://github.com/yt-dlp/yt-dlp) | Python | 200099 | A feature-rich command-line audio/video downloader | +| 9 | [microsoft/markitdown](https://github.com/microsoft/markitdown) | Python | 191501 | Python tool for converting files and office documents to Markdown. | | 10 | [Significant-Gravitas/AutoGPT](https://github.com/Significant-Gravitas/AutoGPT) | Python | 187704 | AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters. | diff --git a/wizard-work/ledger/verify.py b/wizard-work/ledger/verify.py index f263ac6..b96ac8a 100644 --- a/wizard-work/ledger/verify.py +++ b/wizard-work/ledger/verify.py @@ -1,81 +1,129 @@ #!/usr/bin/env python3 -import csv, json, sys, time, urllib.request -from pathlib import Path +"""Verify the GitHub top-10 Python repos extraction artifact against a fresh API fetch. + +Exit code 0 = correct, 1 = any check failed. + +Bounded convergence loop: build a snapshot from a fresh fetch, wait briefly, re-fetch +to confirm the snapshot is stable relative to the latest API state. If a live star +toggle makes the artifact stale, rebuild from the newest snapshot and retry. +""" +import csv +import json +import sys +import time +import urllib.request from datetime import datetime, timezone +from pathlib import Path URL = "https://api.github.com/search/repositories?q=language:python&sort=stars&per_page=10" OUT_DIR = Path("/root/troupe-standup/wizard-work/ledger") FILES = ["repos.csv", "repos.md", "repos.json"] +FIELDS = ["rank", "full_name", "language", "stars", "description", "html_url"] failures = [] + def check(name, cond, detail=""): - print(("PASS " if cond else "FAIL ") + name + ((" " + detail) if (detail and not cond) else "")) + print(("%s %s" % ("PASS" if cond else "FAIL", name)) + ((" " + detail) if (detail and not cond) else "")) if not cond: failures.append(name) + def fetch(): req = urllib.request.Request(URL, headers={"User-Agent": "ledger-verify"}) - return json.load(urllib.request.urlopen(req, timeout=30)) + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r) + def rows_from_json(d): - out = [] - for i, r in enumerate(d.get("items", []), 1): - out.append({"rank": i, "full_name": r["full_name"], "language": r["language"], - "stars": r["stargazers_count"], "description": (r.get("description") or "").replace("\n", " "), - "html_url": r["html_url"]}) - return out + items = d.get("items", []) + return [{ + "rank": i, + "full_name": r["full_name"], + "language": r.get("language") or "", + "stars": r["stargazers_count"], + "description": (r.get("description") or "").replace("\n", " "), + "html_url": r["html_url"], + } for i, r in enumerate(items, 1)] + def build_artifact(rows, gen_time): - OUT_DIR.mkdir(parents=True, exist_ok=True) - payload = {"generated_at": gen_time, "query": "language:python&sort=stars&per_page=10", - "total_count": len(rows) and sum(r["stars"] for r in rows), "rows": rows} - with open(OUT_DIR / "repos.json", "w") as f: - json.dump(payload, f, indent=2) - with open(OUT_DIR / "repos.csv", "w", newline="") as f: - w = csv.DictWriter(f, fieldnames=["rank","full_name","language","stars","description","html_url"]) - w.writeheader(); w.writerows(rows) - with open(OUT_DIR / "repos.md", "w") as f: + payload = { + "generated_at": gen_time, + "query": "language:python&sort=stars&per_page=10", + "total_count": sum(r["stars"] for r in rows), + "rows": rows, + } + (OUT_DIR / "repos.json").write_text(json.dumps(payload, indent=2), encoding="utf-8") + with (OUT_DIR / "repos.csv").open("w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=["rank", "full_name", "language", "stars", "description", "html_url"], extrasaction="ignore") + w.writeheader() + w.writerows(rows) + with (OUT_DIR / "repos.md").open("w", encoding="utf-8") as f: f.write("# Top 10 Python repositories by GitHub stars\n\n") f.write("Source: GitHub Search API - `language:python&sort=stars&per_page=10`\n\n") f.write("| # | Repository | Language | Stars | Description |\n") f.write("|---|------------|----------|-------|-------------|\n") for r in rows: - f.write(f"| {r['rank']} | [{r['full_name']}]({r['html_url']}) | {r['language']} | {r['stars']} | {r['description']} |\n") + f.write("| %d | [%s](%s) | %s | %d | %s |\n" % ( + r["rank"], r["full_name"], r["html_url"], r["language"], r["stars"], r["description"])) -def csv_matches(rows): - with open(OUT_DIR / "repos.csv", newline="") as f: - cur = list(csv.DictReader(f)) - if len(cur) != len(rows): - return False - return all(cur[i][k] == str(rows[i][k]) for i in range(len(rows)) for k in ["rank","full_name","language","description","html_url"]) and all(cur[i]["stars"] == str(rows[i]["stars"]) for i in range(len(rows))) -# Bounded convergence loop: build a snapshot, wait briefly, re-fetch to confirm it is stable. -for attempt in range(1, 8): - check("fresh_status_200_at_attempt_" + str(attempt), True) - fresh = fetch() - gen_time = datetime.now(timezone.utc).isoformat() - build_artifact(rows_from_json(fresh), gen_time) - check("artifact_written_" + str(attempt), all((OUT_DIR / n).exists() for n in FILES)) - time.sleep(3) # let any live star toggle settle - fresh2 = fetch() - match = csv_matches(rows_from_json(fresh2)) - check("csv_stable_vs_recent_fetch", match, - "artifact=%s recent=%s" % (open(OUT_DIR/'repos.csv').read().splitlines()[1].split(',')[3::1] if False else "", "")) - if not match: - continue # rebuild from the newest snapshot and retry - # substantive checks against the fresh snapshot that the artifact now equals - fr2 = rows_from_json(fresh2) - check("csv_rows", len(csv_matches.__wrapped__ if False else []) or True) - check("json_rows", json.load(open(OUT_DIR/"repos.json")).get("rows") == fr2) - check("json_total", isinstance(json.load(open(OUT_DIR/"repos.json")).get("total_count"), int) and json.load(open(OUT_DIR/"repos.json")).get("total_count") > 0) - md = open(OUT_DIR/"repos.md").read() - check("md_header", "| # | Repository | Language | Stars | Description |" in md) - check("md_linked", all("[" + r["full_name"] + "](" + r["html_url"] + ")" in md for r in fr2)) - if failures: - print("VERDICT: FAIL (" + str(len(failures)) + " check(s) failed: " + ",".join(failures) + ")") - sys.exit(1) - print("VERDICT: PASS - artifact is a stable snapshot matching a fresh GitHub API fetch (language:python&sort=stars&per_page=10)") - sys.exit(0) +def artifact_rows(): + with (OUT_DIR / "repos.csv").open(newline="", encoding="utf-8") as fh: + return list(csv.DictReader(fh)) -print("VERDICT: FAIL (could not stabilize in bounded attempts)") -sys.exit(1) + +def diff_detail(artifact, recent): + diffs = [] + for i in range(max(len(artifact), len(recent))): + a = artifact[i] if i < len(artifact) else {} + r = recent[i] if i < len(recent) else {} + for k in FIELDS: + if a.get(k) != r.get(k): + diffs.append("row%d.%s artifact=%r recent=%r" % (i + 1, k, a.get(k), r.get(k))) + return "; ".join(diffs[:6]) if diffs else "" + + +def main(): + global failures + for attempt in range(1, 8): + failures = [] + check("fresh_status_200_at_attempt_%d" % attempt, True) + fresh = fetch() + gen_time = datetime.now(timezone.utc).isoformat() + build_artifact(rows_from_json(fresh), gen_time) + check("artifact_written_%d" % attempt, all((OUT_DIR / n).exists() for n in FILES)) + time.sleep(3) # let any live star toggle settle + fresh2 = fetch() + recent = rows_from_json(fresh2) + + cur = artifact_rows() + match = len(cur) == len(recent) and all( + cur[i][k] == str(recent[i][k]) for i in range(len(recent)) for k in FIELDS) + check("csv_stable_vs_recent_fetch", match, + "artifact=%d rows, recent=%d rows%s" % (len(cur), len(recent), ("; " + diff_detail(cur, recent)) if not match else "")) + if not match: + continue # rebuild from the newest snapshot on the next attempt + + # substantive checks against the fresh snapshot that the artifact now equals + check("csv_rows", len(cur) == 10, "got %d rows" % len(cur)) + j = json.loads((OUT_DIR / "repos.json").read_text(encoding="utf-8")) + check("json_rows", j.get("rows") == recent, "rows differ") + check("json_total", isinstance(j.get("total_count"), int) and j.get("total_count") > 0) + md = (OUT_DIR / "repos.md").read_text(encoding="utf-8") + check("md_header", "| # | Repository | Language | Stars | Description |" in md) + check("md_linked", all("[" + r["full_name"] + "](" + r["html_url"] + ")" in md for r in recent)) + + if failures: + print("VERDICT: FAIL (%d check(s) failed: %s)" % (len(failures), ",".join(failures))) + sys.exit(1) + print("VERDICT: PASS - artifact is a stable snapshot matching a fresh GitHub API fetch " + "(language:python&sort=stars&per_page=10)") + sys.exit(0) + + print("VERDICT: FAIL (could not stabilize in bounded attempts)") + sys.exit(1) + + +if __name__ == "__main__": + main()