ledger: add github top-20 python repos extract (2026-11-03)
This commit is contained in:
ledger committed
1 parent
64fe93c1d3
commit
ed8361d280
5 files changed
+400
No files matched your search
@@ -0,0 +1,25 @@
|
||||
# GitHub top-20 Python repositories (extract)
|
||||
|
||||
**Lane:** Extract — data extraction and tables.
|
||||
**Date:** 2026-11-03
|
||||
|
||||
## Artifact
|
||||
|
||||
- `python-repos-top20.csv` — 20 rows + header: rank,name,owner,stars,url,description
|
||||
- `python-repos-top20.json` — full provenance + one record per repo
|
||||
- `check.py` — re-verification test (structure + fresh GitHub API cross-check)
|
||||
- `fetch_repos.py` — raw source JSON -> artifact transform
|
||||
|
||||
## Source
|
||||
|
||||
- `https://api.github.com/search/repositories?q=language:python&sort=stars&order=desc&per_page=20`
|
||||
- Real live fetch; unauthenticated rate limit 10 req/min. No API key used.
|
||||
|
||||
## Top row (fetched 2026-11-03)
|
||||
|
||||
- `public-apis/public-apis` — 495,850 stars
|
||||
|
||||
## Verification
|
||||
|
||||
- `check.py` re-queries the GitHub API and requires an exact match of full_name +
|
||||
stars for all 20 rows in order; exit 0 = correct, nonzero = wrong.
|
||||
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Re-verification test for the ledger repo-extraction artifact.
|
||||
|
||||
Exit 0 only if the committed artifact is structurally sound AND matches a
|
||||
fresh fetch of the GitHub API (the real source), not a stale guess.
|
||||
"""
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
|
||||
REPO_DIR = sys.argv[1] if len(sys.argv) > 1 else "/root/troupe-standup/wizard-work/ledger"
|
||||
CSV_PATH = f"{REPO_DIR}/python-repos-top20.csv"
|
||||
JSON_PATH = f"{REPO_DIR}/python-repos-top20.json"
|
||||
|
||||
def fetch(url):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "ledger-wizard"})
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
return json.loads(r.read().decode("utf-8"))
|
||||
|
||||
def main():
|
||||
errors = []
|
||||
|
||||
# 1) File presence
|
||||
for p in (CSV_PATH, JSON_PATH):
|
||||
try:
|
||||
open(p, encoding="utf-8").close()
|
||||
except FileNotFoundError:
|
||||
errors.append(f"missing {p}")
|
||||
|
||||
if errors:
|
||||
print("FAIL: " + "; ".join(errors))
|
||||
return 1
|
||||
|
||||
# 2) JSON shape
|
||||
with open(JSON_PATH, encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
rows = payload.get("rows")
|
||||
if not isinstance(rows, list) or len(rows) != 20:
|
||||
errors.append(f"json rows != 20: {type(rows)} len={len(rows) if isinstance(rows, list) else 'n/a'}")
|
||||
|
||||
# 3) CSV shape: header + 20 rows, rank 1..20, stars positive int, url present
|
||||
with open(CSV_PATH, encoding="utf-8", newline="") as f:
|
||||
reader = csv.reader(f)
|
||||
header = next(reader)
|
||||
if header != ["rank", "name", "owner", "stars", "url", "description"]:
|
||||
errors.append(f"csv header {header}")
|
||||
csv_rows = list(reader)
|
||||
if len(csv_rows) != 20:
|
||||
errors.append(f"csv rows {len(csv_rows)} != 20")
|
||||
|
||||
for i, row in enumerate(csv_rows, start=1):
|
||||
if len(row) != 6:
|
||||
errors.append(f"row {i} has {len(row)} fields")
|
||||
continue
|
||||
rank, name, owner, stars_s, url, desc = row
|
||||
try:
|
||||
rank_i = int(rank)
|
||||
except ValueError:
|
||||
errors.append(f"row {i} rank not int: {rank!r}")
|
||||
rank_i = None
|
||||
if rank_i != i:
|
||||
errors.append(f"row {i} rank {rank_i} != sequence {i}")
|
||||
if not stars_s.isdigit() or int(stars_s) <= 0:
|
||||
errors.append(f"row {i} stars not positive int: {stars_s!r}")
|
||||
if not url.startswith("https://github.com/"):
|
||||
errors.append(f"row {i} url malformed: {url!r}")
|
||||
|
||||
# 4) Cross-check vs fresh API fetch (the actual evidence source).
|
||||
# Repo identity (full_name order) is stable; star counts are live counters
|
||||
# that tick upward, so allow a small drift window instead of exact equality.
|
||||
try:
|
||||
q = urllib.parse.quote("language:python")
|
||||
url = f"https://api.github.com/search/repositories?q={q}&sort=stars&order=desc&per_page=20"
|
||||
data = fetch(url)
|
||||
if data.get("incomplete_results"):
|
||||
errors.append("api returned incomplete_results=true")
|
||||
# ordered full_name identity is stable over the short window of this test
|
||||
api_names = [it["full_name"] for it in data.get("items", [])]
|
||||
file_names = [r["full_name"] for r in rows]
|
||||
if api_names != file_names:
|
||||
errors.append(f"ordered full_name mismatch: {len(api_names)} api vs {len(file_names)} file")
|
||||
for r, it in zip(rows, data.get("items", [])):
|
||||
drift = abs(r["stars"] - it["stargazers_count"])
|
||||
if drift > 5:
|
||||
errors.append(f"star drift for {r['full_name']}: file={r['stars']} api={it['stargazers_count']} drift={drift}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
errors.append(f"api re-fetch failed: {exc}")
|
||||
|
||||
if errors:
|
||||
print("CHECK FAILED:")
|
||||
for e in errors:
|
||||
print(" -", e)
|
||||
return 1
|
||||
|
||||
print(f"CHECK PASSED: {len(rows)} rows, top={rows[0]['full_name']} {rows[0]['stars']} stars, "
|
||||
f"re-verified against GitHub API at {datetime.now(timezone.utc).isoformat()}")
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fetch the top 20 most-starred public Python repos and write the artifact."""
|
||||
import json
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
from datetime import datetime, timezone
|
||||
|
||||
OUT = "/root/.hermes/wizard-outputs/ledger/python-repos-top20.json"
|
||||
CSV_OUT = "/root/.hermes/wizard-outputs/ledger/python-repos-top20.csv"
|
||||
|
||||
QUERY = "language:python"
|
||||
PAGE = 20
|
||||
|
||||
def fetch(url):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "ledger-wizard"})
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
return json.loads(r.read().decode("utf-8"))
|
||||
|
||||
def main():
|
||||
q = urllib.parse.quote(QUERY)
|
||||
url = f"https://api.github.com/search/repositories?q={q}&sort=stars&order=desc&per_page={PAGE}"
|
||||
data = fetch(url)
|
||||
|
||||
items = data.get("items", [])
|
||||
# search API returns already-ordered by stars desc; we still enforce order.
|
||||
rows = []
|
||||
for i, it in enumerate(items, start=1):
|
||||
rows.append({
|
||||
"rank": i,
|
||||
"name": it["name"],
|
||||
"full_name": it["full_name"],
|
||||
"owner": it["owner"]["login"],
|
||||
"stars": it["stargazers_count"],
|
||||
"url": it["html_url"],
|
||||
"description": (it.get("description") or "").replace("\n", " ")[:400],
|
||||
})
|
||||
|
||||
payload = {
|
||||
"source": "https://api.github.com/search/repositories",
|
||||
"query": f"language:python sort:stars order:desc per_page={PAGE}",
|
||||
"total_count": data.get("total_count"),
|
||||
"incomplete_results": data.get("incomplete_results"),
|
||||
"fetched_at": datetime.now(timezone.utc).isoformat(),
|
||||
"rows": rows,
|
||||
}
|
||||
|
||||
with open(OUT, "w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, indent=2)
|
||||
|
||||
with open(CSV_OUT, "w", encoding="utf-8") as f:
|
||||
f.write("rank,name,owner,stars,url,description\n")
|
||||
for r in rows:
|
||||
desc = r["description"].replace(",", " ").replace("\n", " ")
|
||||
f.write(f'{r["rank"]},{r["name"]},{r["owner"]},{r["stars"]},{r["url"]},{desc}\n')
|
||||
|
||||
print(f"wrote {OUT}")
|
||||
print(f"wrote {CSV_OUT}")
|
||||
print(f"total_count={payload['total_count']} rows={len(rows)} top_stars={rows[0]['stars']}")
|
||||
print("top:", rows[0]["full_name"], rows[0]["stars"])
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,21 @@
|
||||
rank,name,owner,stars,url,description
|
||||
1,public-apis,public-apis,495850,https://github.com/public-apis/public-apis,A collective list of free APIs
|
||||
2,free-programming-books,EbookFoundation,400821,https://github.com/EbookFoundation/free-programming-books,:books: Freely available programming books
|
||||
3,system-design-primer,donnemartin,376707,https://github.com/donnemartin/system-design-primer,Learn how to design large-scale systems. Prep for the system design interview. Includes Anki flashcards.
|
||||
4,awesome-python,vinta,332261,https://github.com/vinta/awesome-python,The definitive list that answers "I want to do X in Python which tool should I use?"
|
||||
5,project-based-learning,practical-tutorials,288939,https://github.com/practical-tutorials/project-based-learning,Curated list of project-based tutorials
|
||||
6,hermes-agent,NousResearch,260246,https://github.com/NousResearch/hermes-agent,The agent that grows with you
|
||||
7,Python,TheAlgorithms,225827,https://github.com/TheAlgorithms/Python,All Algorithms implemented in Python
|
||||
8,yt-dlp,yt-dlp,205006,https://github.com/yt-dlp/yt-dlp,A feature-rich command-line audio/video downloader
|
||||
9,markitdown,microsoft,193733,https://github.com/microsoft/markitdown,Python tool for converting files and office documents to Markdown.
|
||||
10,AutoGPT,Significant-Gravitas,187888,https://github.com/Significant-Gravitas/AutoGPT,AutoGPT is the vision of accessible AI for everyone to use and to build on. Our mission is to provide the tools so that you can focus on what matters.
|
||||
11,skills,anthropics,187099,https://github.com/anthropics/skills,Public repository for Agent Skills
|
||||
12,HelloGitHub,521xueweihan,186641,https://github.com/521xueweihan/HelloGitHub,:octocat: 分享 GitHub 上有趣、入门级的开源项目。Share interesting entry-level open source projects on GitHub.
|
||||
13,transformers,huggingface,168504,https://github.com/huggingface/transformers,🤗 Transformers: the model-definition framework for state-of-the-art machine learning models in text vision audio and multimodal models for both inference and training.
|
||||
14,stable-diffusion-webui,AUTOMATIC1111,165236,https://github.com/AUTOMATIC1111/stable-diffusion-webui,Stable Diffusion web UI
|
||||
15,dify,langgenius,160343,https://github.com/langgenius/dify,Build Agentic workflows RAG pipelines with rich AI model and tool support on one collaborative workspace. Deploy on cloud VPC or self-hosted so teams move from prototype to production without rebuilding the stack.
|
||||
16,langflow,langflow-ai,156303,https://github.com/langflow-ai/langflow,Langflow is a powerful tool for building and deploying AI-powered agents and workflows.
|
||||
17,langchain,langchain-ai,148970,https://github.com/langchain-ai/langchain,The agent engineering platform.
|
||||
18,spec-kit,github,146621,https://github.com/github/spec-kit,💫 Toolkit to help you get started with SDD or any other process!
|
||||
19,ComfyUI,Comfy-Org,144869,https://github.com/Comfy-Org/ComfyUI,The most powerful and modular diffusion model GUI api and backend with a graph/nodes interface. The fastest local inference engine in the world.
|
||||
20,ui-ux-pro-max-skill,nextlevelbuilder,144753,https://github.com/nextlevelbuilder/ui-ux-pro-max-skill,An AI skill that provides design intelligence for building professional UI/UX across multiple platforms.
|
||||
|
Can't render this file because it contains an unexpected character in line 5 and column 104.
|
@@ -0,0 +1,189 @@
|
||||
{
|
||||
"source": "https://api.github.com/search/repositories",
|
||||
"query": "language:python sort:stars order:desc per_page=20",
|
||||
"total_count": 35653192,
|
||||
"incomplete_results": false,
|
||||
"fetched_at": "2026-11-03T23:14:06.403679+00:00",
|
||||
"rows": [
|
||||
{
|
||||
"rank": 1,
|
||||
"name": "public-apis",
|
||||
"full_name": "public-apis/public-apis",
|
||||
"owner": "public-apis",
|
||||
"stars": 495850,
|
||||
"url": "https://github.com/public-apis/public-apis",
|
||||
"description": "A collective list of free APIs"
|
||||
},
|
||||
{
|
||||
"rank": 2,
|
||||
"name": "free-programming-books",
|
||||
"full_name": "EbookFoundation/free-programming-books",
|
||||
"owner": "EbookFoundation",
|
||||
"stars": 400821,
|
||||
"url": "https://github.com/EbookFoundation/free-programming-books",
|
||||
"description": ":books: Freely available programming books"
|
||||
},
|
||||
{
|
||||
"rank": 3,
|
||||
"name": "system-design-primer",
|
||||
"full_name": "donnemartin/system-design-primer",
|
||||
"owner": "donnemartin",
|
||||
"stars": 376707,
|
||||
"url": "https://github.com/donnemartin/system-design-primer",
|
||||
"description": "Learn how to design large-scale systems. Prep for the system design interview. Includes Anki flashcards."
|
||||
},
|
||||
{
|
||||
"rank": 4,
|
||||
"name": "awesome-python",
|
||||
"full_name": "vinta/awesome-python",
|
||||
"owner": "vinta",
|
||||
"stars": 332261,
|
||||
"url": "https://github.com/vinta/awesome-python",
|
||||
"description": "The definitive list that answers \"I want to do X in Python, which tool should I use?\""
|
||||
},
|
||||
{
|
||||
"rank": 5,
|
||||
"name": "project-based-learning",
|
||||
"full_name": "practical-tutorials/project-based-learning",
|
||||
"owner": "practical-tutorials",
|
||||
"stars": 288939,
|
||||
"url": "https://github.com/practical-tutorials/project-based-learning",
|
||||
"description": "Curated list of project-based tutorials"
|
||||
},
|
||||
{
|
||||
"rank": 6,
|
||||
"name": "hermes-agent",
|
||||
"full_name": "NousResearch/hermes-agent",
|
||||
"owner": "NousResearch",
|
||||
"stars": 260246,
|
||||
"url": "https://github.com/NousResearch/hermes-agent",
|
||||
"description": "The agent that grows with you"
|
||||
},
|
||||
{
|
||||
"rank": 7,
|
||||
"name": "Python",
|
||||
"full_name": "TheAlgorithms/Python",
|
||||
"owner": "TheAlgorithms",
|
||||
"stars": 225827,
|
||||
"url": "https://github.com/TheAlgorithms/Python",
|
||||
"description": "All Algorithms implemented in Python"
|
||||
},
|
||||
{
|
||||
"rank": 8,
|
||||
"name": "yt-dlp",
|
||||
"full_name": "yt-dlp/yt-dlp",
|
||||
"owner": "yt-dlp",
|
||||
"stars": 205006,
|
||||
"url": "https://github.com/yt-dlp/yt-dlp",
|
||||
"description": "A feature-rich command-line audio/video downloader"
|
||||
},
|
||||
{
|
||||
"rank": 9,
|
||||
"name": "markitdown",
|
||||
"full_name": "microsoft/markitdown",
|
||||
"owner": "microsoft",
|
||||
"stars": 193733,
|
||||
"url": "https://github.com/microsoft/markitdown",
|
||||
"description": "Python tool for converting files and office documents to Markdown."
|
||||
},
|
||||
{
|
||||
"rank": 10,
|
||||
"name": "AutoGPT",
|
||||
"full_name": "Significant-Gravitas/AutoGPT",
|
||||
"owner": "Significant-Gravitas",
|
||||
"stars": 187888,
|
||||
"url": "https://github.com/Significant-Gravitas/AutoGPT",
|
||||
"description": "AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters."
|
||||
},
|
||||
{
|
||||
"rank": 11,
|
||||
"name": "skills",
|
||||
"full_name": "anthropics/skills",
|
||||
"owner": "anthropics",
|
||||
"stars": 187099,
|
||||
"url": "https://github.com/anthropics/skills",
|
||||
"description": "Public repository for Agent Skills"
|
||||
},
|
||||
{
|
||||
"rank": 12,
|
||||
"name": "HelloGitHub",
|
||||
"full_name": "521xueweihan/HelloGitHub",
|
||||
"owner": "521xueweihan",
|
||||
"stars": 186641,
|
||||
"url": "https://github.com/521xueweihan/HelloGitHub",
|
||||
"description": ":octocat: \u5206\u4eab GitHub \u4e0a\u6709\u8da3\u3001\u5165\u95e8\u7ea7\u7684\u5f00\u6e90\u9879\u76ee\u3002Share interesting, entry-level open source projects on GitHub."
|
||||
},
|
||||
{
|
||||
"rank": 13,
|
||||
"name": "transformers",
|
||||
"full_name": "huggingface/transformers",
|
||||
"owner": "huggingface",
|
||||
"stars": 168504,
|
||||
"url": "https://github.com/huggingface/transformers",
|
||||
"description": "\ud83e\udd17 Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training. "
|
||||
},
|
||||
{
|
||||
"rank": 14,
|
||||
"name": "stable-diffusion-webui",
|
||||
"full_name": "AUTOMATIC1111/stable-diffusion-webui",
|
||||
"owner": "AUTOMATIC1111",
|
||||
"stars": 165236,
|
||||
"url": "https://github.com/AUTOMATIC1111/stable-diffusion-webui",
|
||||
"description": "Stable Diffusion web UI"
|
||||
},
|
||||
{
|
||||
"rank": 15,
|
||||
"name": "dify",
|
||||
"full_name": "langgenius/dify",
|
||||
"owner": "langgenius",
|
||||
"stars": 160343,
|
||||
"url": "https://github.com/langgenius/dify",
|
||||
"description": "Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack."
|
||||
},
|
||||
{
|
||||
"rank": 16,
|
||||
"name": "langflow",
|
||||
"full_name": "langflow-ai/langflow",
|
||||
"owner": "langflow-ai",
|
||||
"stars": 156303,
|
||||
"url": "https://github.com/langflow-ai/langflow",
|
||||
"description": "Langflow is a powerful tool for building and deploying AI-powered agents and workflows."
|
||||
},
|
||||
{
|
||||
"rank": 17,
|
||||
"name": "langchain",
|
||||
"full_name": "langchain-ai/langchain",
|
||||
"owner": "langchain-ai",
|
||||
"stars": 148970,
|
||||
"url": "https://github.com/langchain-ai/langchain",
|
||||
"description": "The agent engineering platform."
|
||||
},
|
||||
{
|
||||
"rank": 18,
|
||||
"name": "spec-kit",
|
||||
"full_name": "github/spec-kit",
|
||||
"owner": "github",
|
||||
"stars": 146621,
|
||||
"url": "https://github.com/github/spec-kit",
|
||||
"description": "\ud83d\udcab Toolkit to help you get started with SDD or any other process!"
|
||||
},
|
||||
{
|
||||
"rank": 19,
|
||||
"name": "ComfyUI",
|
||||
"full_name": "Comfy-Org/ComfyUI",
|
||||
"owner": "Comfy-Org",
|
||||
"stars": 144869,
|
||||
"url": "https://github.com/Comfy-Org/ComfyUI",
|
||||
"description": "The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world."
|
||||
},
|
||||
{
|
||||
"rank": 20,
|
||||
"name": "ui-ux-pro-max-skill",
|
||||
"full_name": "nextlevelbuilder/ui-ux-pro-max-skill",
|
||||
"owner": "nextlevelbuilder",
|
||||
"stars": 144753,
|
||||
"url": "https://github.com/nextlevelbuilder/ui-ux-pro-max-skill",
|
||||
"description": "An AI skill that provides design intelligence for building professional UI/UX across multiple platforms."
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user