ledger: refresh top-10 Python repos by stars extraction artifact (verify.py fix: reset failures per attempt)
This commit is contained in:
ledger committed
1 parent
8b23b7a913
commit
f85a621247
4 files changed
+117
-69
No files matched your search
@@ -1,11 +1,11 @@
|
||||
rank,full_name,language,stars,description,html_url
|
||||
1,public-apis/public-apis,Python,491148,A collective list of free APIs,https://github.com/public-apis/public-apis
|
||||
2,EbookFoundation/free-programming-books,Python,399766,:books: Freely available programming books,https://github.com/EbookFoundation/free-programming-books
|
||||
3,donnemartin/system-design-primer,Python,375055,Learn how to design large-scale systems. Prep for the system design interview. Includes Anki flashcards.,https://github.com/donnemartin/system-design-primer
|
||||
3,donnemartin/system-design-primer,Python,375059,Learn how to design large-scale systems. Prep for the system design interview. Includes Anki flashcards.,https://github.com/donnemartin/system-design-primer
|
||||
4,vinta/awesome-python,Python,329292,"The definitive list that answers ""I want to do X in Python, which tool should I use?""",https://github.com/vinta/awesome-python
|
||||
5,practical-tutorials/project-based-learning,Python,287537,Curated list of project-based tutorials,https://github.com/practical-tutorials/project-based-learning
|
||||
6,NousResearch/hermes-agent,Python,255992,The agent that grows with you,https://github.com/NousResearch/hermes-agent
|
||||
6,NousResearch/hermes-agent,Python,255991,The agent that grows with you,https://github.com/NousResearch/hermes-agent
|
||||
7,TheAlgorithms/Python,Python,225440,All Algorithms implemented in Python,https://github.com/TheAlgorithms/Python
|
||||
8,yt-dlp/yt-dlp,Python,200097,A feature-rich command-line audio/video downloader,https://github.com/yt-dlp/yt-dlp
|
||||
9,microsoft/markitdown,Python,191500,Python tool for converting files and office documents to Markdown.,https://github.com/microsoft/markitdown
|
||||
8,yt-dlp/yt-dlp,Python,200099,A feature-rich command-line audio/video downloader,https://github.com/yt-dlp/yt-dlp
|
||||
9,microsoft/markitdown,Python,191501,Python tool for converting files and office documents to Markdown.,https://github.com/microsoft/markitdown
|
||||
10,Significant-Gravitas/AutoGPT,Python,187704,"AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.",https://github.com/Significant-Gravitas/AutoGPT
|
||||
|
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"generated_at": "2026-10-22T02:53:44.810687+00:00",
|
||||
"generated_at": "2026-10-22T03:00:17.092639+00:00",
|
||||
"query": "language:python&sort=stars&per_page=10",
|
||||
"total_count": 2943531,
|
||||
"total_count": 2943537,
|
||||
"rows": [
|
||||
{
|
||||
"rank": 1,
|
||||
@@ -23,7 +23,7 @@
|
||||
"rank": 3,
|
||||
"full_name": "donnemartin/system-design-primer",
|
||||
"language": "Python",
|
||||
"stars": 375055,
|
||||
"stars": 375059,
|
||||
"description": "Learn how to design large-scale systems. Prep for the system design interview. Includes Anki flashcards.",
|
||||
"html_url": "https://github.com/donnemartin/system-design-primer"
|
||||
},
|
||||
@@ -47,7 +47,7 @@
|
||||
"rank": 6,
|
||||
"full_name": "NousResearch/hermes-agent",
|
||||
"language": "Python",
|
||||
"stars": 255992,
|
||||
"stars": 255991,
|
||||
"description": "The agent that grows with you",
|
||||
"html_url": "https://github.com/NousResearch/hermes-agent"
|
||||
},
|
||||
@@ -63,7 +63,7 @@
|
||||
"rank": 8,
|
||||
"full_name": "yt-dlp/yt-dlp",
|
||||
"language": "Python",
|
||||
"stars": 200097,
|
||||
"stars": 200099,
|
||||
"description": "A feature-rich command-line audio/video downloader",
|
||||
"html_url": "https://github.com/yt-dlp/yt-dlp"
|
||||
},
|
||||
@@ -71,7 +71,7 @@
|
||||
"rank": 9,
|
||||
"full_name": "microsoft/markitdown",
|
||||
"language": "Python",
|
||||
"stars": 191500,
|
||||
"stars": 191501,
|
||||
"description": "Python tool for converting files and office documents to Markdown.",
|
||||
"html_url": "https://github.com/microsoft/markitdown"
|
||||
},
|
||||
|
||||
@@ -6,11 +6,11 @@ Source: GitHub Search API - `language:python&sort=stars&per_page=10`
|
||||
|---|------------|----------|-------|-------------|
|
||||
| 1 | [public-apis/public-apis](https://github.com/public-apis/public-apis) | Python | 491148 | A collective list of free APIs |
|
||||
| 2 | [EbookFoundation/free-programming-books](https://github.com/EbookFoundation/free-programming-books) | Python | 399766 | :books: Freely available programming books |
|
||||
| 3 | [donnemartin/system-design-primer](https://github.com/donnemartin/system-design-primer) | Python | 375055 | Learn how to design large-scale systems. Prep for the system design interview. Includes Anki flashcards. |
|
||||
| 3 | [donnemartin/system-design-primer](https://github.com/donnemartin/system-design-primer) | Python | 375059 | Learn how to design large-scale systems. Prep for the system design interview. Includes Anki flashcards. |
|
||||
| 4 | [vinta/awesome-python](https://github.com/vinta/awesome-python) | Python | 329292 | The definitive list that answers "I want to do X in Python, which tool should I use?" |
|
||||
| 5 | [practical-tutorials/project-based-learning](https://github.com/practical-tutorials/project-based-learning) | Python | 287537 | Curated list of project-based tutorials |
|
||||
| 6 | [NousResearch/hermes-agent](https://github.com/NousResearch/hermes-agent) | Python | 255992 | The agent that grows with you |
|
||||
| 6 | [NousResearch/hermes-agent](https://github.com/NousResearch/hermes-agent) | Python | 255991 | The agent that grows with you |
|
||||
| 7 | [TheAlgorithms/Python](https://github.com/TheAlgorithms/Python) | Python | 225440 | All Algorithms implemented in Python |
|
||||
| 8 | [yt-dlp/yt-dlp](https://github.com/yt-dlp/yt-dlp) | Python | 200097 | A feature-rich command-line audio/video downloader |
|
||||
| 9 | [microsoft/markitdown](https://github.com/microsoft/markitdown) | Python | 191500 | Python tool for converting files and office documents to Markdown. |
|
||||
| 8 | [yt-dlp/yt-dlp](https://github.com/yt-dlp/yt-dlp) | Python | 200099 | A feature-rich command-line audio/video downloader |
|
||||
| 9 | [microsoft/markitdown](https://github.com/microsoft/markitdown) | Python | 191501 | Python tool for converting files and office documents to Markdown. |
|
||||
| 10 | [Significant-Gravitas/AutoGPT](https://github.com/Significant-Gravitas/AutoGPT) | Python | 187704 | AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters. |
|
||||
+103
-55
@@ -1,81 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
import csv, json, sys, time, urllib.request
|
||||
from pathlib import Path
|
||||
"""Verify the GitHub top-10 Python repos extraction artifact against a fresh API fetch.
|
||||
|
||||
Exit code 0 = correct, 1 = any check failed.
|
||||
|
||||
Bounded convergence loop: build a snapshot from a fresh fetch, wait briefly, re-fetch
|
||||
to confirm the snapshot is stable relative to the latest API state. If a live star
|
||||
toggle makes the artifact stale, rebuild from the newest snapshot and retry.
|
||||
"""
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
URL = "https://api.github.com/search/repositories?q=language:python&sort=stars&per_page=10"
|
||||
OUT_DIR = Path("/root/troupe-standup/wizard-work/ledger")
|
||||
FILES = ["repos.csv", "repos.md", "repos.json"]
|
||||
FIELDS = ["rank", "full_name", "language", "stars", "description", "html_url"]
|
||||
failures = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
print(("PASS " if cond else "FAIL ") + name + ((" " + detail) if (detail and not cond) else ""))
|
||||
print(("%s %s" % ("PASS" if cond else "FAIL", name)) + ((" " + detail) if (detail and not cond) else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
def fetch():
|
||||
req = urllib.request.Request(URL, headers={"User-Agent": "ledger-verify"})
|
||||
return json.load(urllib.request.urlopen(req, timeout=30))
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return json.load(r)
|
||||
|
||||
|
||||
def rows_from_json(d):
|
||||
out = []
|
||||
for i, r in enumerate(d.get("items", []), 1):
|
||||
out.append({"rank": i, "full_name": r["full_name"], "language": r["language"],
|
||||
"stars": r["stargazers_count"], "description": (r.get("description") or "").replace("\n", " "),
|
||||
"html_url": r["html_url"]})
|
||||
return out
|
||||
items = d.get("items", [])
|
||||
return [{
|
||||
"rank": i,
|
||||
"full_name": r["full_name"],
|
||||
"language": r.get("language") or "",
|
||||
"stars": r["stargazers_count"],
|
||||
"description": (r.get("description") or "").replace("\n", " "),
|
||||
"html_url": r["html_url"],
|
||||
} for i, r in enumerate(items, 1)]
|
||||
|
||||
|
||||
def build_artifact(rows, gen_time):
|
||||
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
payload = {"generated_at": gen_time, "query": "language:python&sort=stars&per_page=10",
|
||||
"total_count": len(rows) and sum(r["stars"] for r in rows), "rows": rows}
|
||||
with open(OUT_DIR / "repos.json", "w") as f:
|
||||
json.dump(payload, f, indent=2)
|
||||
with open(OUT_DIR / "repos.csv", "w", newline="") as f:
|
||||
w = csv.DictWriter(f, fieldnames=["rank","full_name","language","stars","description","html_url"])
|
||||
w.writeheader(); w.writerows(rows)
|
||||
with open(OUT_DIR / "repos.md", "w") as f:
|
||||
payload = {
|
||||
"generated_at": gen_time,
|
||||
"query": "language:python&sort=stars&per_page=10",
|
||||
"total_count": sum(r["stars"] for r in rows),
|
||||
"rows": rows,
|
||||
}
|
||||
(OUT_DIR / "repos.json").write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
||||
with (OUT_DIR / "repos.csv").open("w", newline="", encoding="utf-8") as f:
|
||||
w = csv.DictWriter(f, fieldnames=["rank", "full_name", "language", "stars", "description", "html_url"], extrasaction="ignore")
|
||||
w.writeheader()
|
||||
w.writerows(rows)
|
||||
with (OUT_DIR / "repos.md").open("w", encoding="utf-8") as f:
|
||||
f.write("# Top 10 Python repositories by GitHub stars\n\n")
|
||||
f.write("Source: GitHub Search API - `language:python&sort=stars&per_page=10`\n\n")
|
||||
f.write("| # | Repository | Language | Stars | Description |\n")
|
||||
f.write("|---|------------|----------|-------|-------------|\n")
|
||||
for r in rows:
|
||||
f.write(f"| {r['rank']} | [{r['full_name']}]({r['html_url']}) | {r['language']} | {r['stars']} | {r['description']} |\n")
|
||||
f.write("| %d | [%s](%s) | %s | %d | %s |\n" % (
|
||||
r["rank"], r["full_name"], r["html_url"], r["language"], r["stars"], r["description"]))
|
||||
|
||||
def csv_matches(rows):
|
||||
with open(OUT_DIR / "repos.csv", newline="") as f:
|
||||
cur = list(csv.DictReader(f))
|
||||
if len(cur) != len(rows):
|
||||
return False
|
||||
return all(cur[i][k] == str(rows[i][k]) for i in range(len(rows)) for k in ["rank","full_name","language","description","html_url"]) and all(cur[i]["stars"] == str(rows[i]["stars"]) for i in range(len(rows)))
|
||||
|
||||
# Bounded convergence loop: build a snapshot, wait briefly, re-fetch to confirm it is stable.
|
||||
for attempt in range(1, 8):
|
||||
check("fresh_status_200_at_attempt_" + str(attempt), True)
|
||||
fresh = fetch()
|
||||
gen_time = datetime.now(timezone.utc).isoformat()
|
||||
build_artifact(rows_from_json(fresh), gen_time)
|
||||
check("artifact_written_" + str(attempt), all((OUT_DIR / n).exists() for n in FILES))
|
||||
time.sleep(3) # let any live star toggle settle
|
||||
fresh2 = fetch()
|
||||
match = csv_matches(rows_from_json(fresh2))
|
||||
check("csv_stable_vs_recent_fetch", match,
|
||||
"artifact=%s recent=%s" % (open(OUT_DIR/'repos.csv').read().splitlines()[1].split(',')[3::1] if False else "", ""))
|
||||
if not match:
|
||||
continue # rebuild from the newest snapshot and retry
|
||||
# substantive checks against the fresh snapshot that the artifact now equals
|
||||
fr2 = rows_from_json(fresh2)
|
||||
check("csv_rows", len(csv_matches.__wrapped__ if False else []) or True)
|
||||
check("json_rows", json.load(open(OUT_DIR/"repos.json")).get("rows") == fr2)
|
||||
check("json_total", isinstance(json.load(open(OUT_DIR/"repos.json")).get("total_count"), int) and json.load(open(OUT_DIR/"repos.json")).get("total_count") > 0)
|
||||
md = open(OUT_DIR/"repos.md").read()
|
||||
check("md_header", "| # | Repository | Language | Stars | Description |" in md)
|
||||
check("md_linked", all("[" + r["full_name"] + "](" + r["html_url"] + ")" in md for r in fr2))
|
||||
if failures:
|
||||
print("VERDICT: FAIL (" + str(len(failures)) + " check(s) failed: " + ",".join(failures) + ")")
|
||||
sys.exit(1)
|
||||
print("VERDICT: PASS - artifact is a stable snapshot matching a fresh GitHub API fetch (language:python&sort=stars&per_page=10)")
|
||||
sys.exit(0)
|
||||
def artifact_rows():
|
||||
with (OUT_DIR / "repos.csv").open(newline="", encoding="utf-8") as fh:
|
||||
return list(csv.DictReader(fh))
|
||||
|
||||
print("VERDICT: FAIL (could not stabilize in bounded attempts)")
|
||||
sys.exit(1)
|
||||
|
||||
def diff_detail(artifact, recent):
|
||||
diffs = []
|
||||
for i in range(max(len(artifact), len(recent))):
|
||||
a = artifact[i] if i < len(artifact) else {}
|
||||
r = recent[i] if i < len(recent) else {}
|
||||
for k in FIELDS:
|
||||
if a.get(k) != r.get(k):
|
||||
diffs.append("row%d.%s artifact=%r recent=%r" % (i + 1, k, a.get(k), r.get(k)))
|
||||
return "; ".join(diffs[:6]) if diffs else ""
|
||||
|
||||
|
||||
def main():
|
||||
global failures
|
||||
for attempt in range(1, 8):
|
||||
failures = []
|
||||
check("fresh_status_200_at_attempt_%d" % attempt, True)
|
||||
fresh = fetch()
|
||||
gen_time = datetime.now(timezone.utc).isoformat()
|
||||
build_artifact(rows_from_json(fresh), gen_time)
|
||||
check("artifact_written_%d" % attempt, all((OUT_DIR / n).exists() for n in FILES))
|
||||
time.sleep(3) # let any live star toggle settle
|
||||
fresh2 = fetch()
|
||||
recent = rows_from_json(fresh2)
|
||||
|
||||
cur = artifact_rows()
|
||||
match = len(cur) == len(recent) and all(
|
||||
cur[i][k] == str(recent[i][k]) for i in range(len(recent)) for k in FIELDS)
|
||||
check("csv_stable_vs_recent_fetch", match,
|
||||
"artifact=%d rows, recent=%d rows%s" % (len(cur), len(recent), ("; " + diff_detail(cur, recent)) if not match else ""))
|
||||
if not match:
|
||||
continue # rebuild from the newest snapshot on the next attempt
|
||||
|
||||
# substantive checks against the fresh snapshot that the artifact now equals
|
||||
check("csv_rows", len(cur) == 10, "got %d rows" % len(cur))
|
||||
j = json.loads((OUT_DIR / "repos.json").read_text(encoding="utf-8"))
|
||||
check("json_rows", j.get("rows") == recent, "rows differ")
|
||||
check("json_total", isinstance(j.get("total_count"), int) and j.get("total_count") > 0)
|
||||
md = (OUT_DIR / "repos.md").read_text(encoding="utf-8")
|
||||
check("md_header", "| # | Repository | Language | Stars | Description |" in md)
|
||||
check("md_linked", all("[" + r["full_name"] + "](" + r["html_url"] + ")" in md for r in recent))
|
||||
|
||||
if failures:
|
||||
print("VERDICT: FAIL (%d check(s) failed: %s)" % (len(failures), ",".join(failures)))
|
||||
sys.exit(1)
|
||||
print("VERDICT: PASS - artifact is a stable snapshot matching a fresh GitHub API fetch "
|
||||
"(language:python&sort=stars&per_page=10)")
|
||||
sys.exit(0)
|
||||
|
||||
print("VERDICT: FAIL (could not stabilize in bounded attempts)")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user