sol: sol-lane fileclass manifest 2026-11-04 (456 files, data/script/text)
This commit is contained in:
sol committed
1 parent
0a6b330a63
commit
d1f4975b14
5 files changed
+6682
No files matched your search
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
# CHECK -- sol: sol-lane fileclass 2026-11-04 independent verifier.
|
||||
# Exit 0 iff the committed manifest re-derives from the frozen seed and every
|
||||
# frozen path still exists as a real, non-ignored file in the repo.
|
||||
set -u
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$SCRIPT_DIR" || exit 2
|
||||
|
||||
exec python3 verify_sol_fileclass_2026_11_04.py
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the sol-lane file-class manifest from the raw output files.
|
||||
|
||||
Classifies every file under /root/.hermes/wizard-outputs/sol/ by extension
|
||||
into the rubric categories (text|data|media|script|doc), writes a frozen seed
|
||||
and a committed manifest into wizard-work/sol/, plus an independent verifier.
|
||||
|
||||
Categories (extension rubric):
|
||||
text md txt rst adoc org log
|
||||
data json csv tsv yaml yml toml sqlite db ipynb
|
||||
media mp3 wav mp4 webm png jpg jpeg gif webp svg
|
||||
script py sh bash zsh ps1 bat
|
||||
doc pdf doc docx ppt pptx xls xlsx
|
||||
Content-sniffing fallback applies only for an extension the ruler has no entry.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
RAW = Path("/root/troupe-standup/wizard-work/sol")
|
||||
REPO = Path("/root/troupe-standup")
|
||||
OUT = REPO / "wizard-work" / "sol"
|
||||
SEED_NAME = "sol-lane-fileclass-seed-2026-11-04.json"
|
||||
MANIFEST_NAME = "sol-lane-fileclass-2026-11-04.json"
|
||||
|
||||
RULER = {
|
||||
"md": "text", "txt": "text", "rst": "text", "adoc": "text", "org": "text", "log": "text",
|
||||
"json": "data", "csv": "data", "tsv": "data", "yaml": "data", "yml": "data", "toml": "data",
|
||||
"sqlite": "data", "db": "data", "ipynb": "data",
|
||||
"mp3": "media", "wav": "media", "mp4": "media", "webm": "media", "png": "media",
|
||||
"jpg": "media", "jpeg": "media", "gif": "media", "webp": "media", "svg": "media",
|
||||
"py": "script", "sh": "script", "bash": "script", "zsh": "script", "ps1": "script", "bat": "script",
|
||||
"pdf": "doc", "doc": "doc", "docx": "doc", "ppt": "doc", "pptx": "doc", "xls": "doc", "xlsx": "doc",
|
||||
}
|
||||
# Known binary magic prefixes; used only for extensions the ruler has no entry.
|
||||
MAGICS = {
|
||||
"png": b"\x89PNG", "gif": b"GIF8", "jpg": b"\xff\xf0", "webp": b"WEBP", "svg": b"<?xml",
|
||||
"zip/pptx": b"PK\x03\x04", "db": b"SQLite", "sqlite": b"SQLite",
|
||||
}
|
||||
MAGIC_OUT = {
|
||||
"png": "media", "gif": "media", "jpg": "media", "webp": "media", "svg": "text",
|
||||
"zip/pptx": "doc", "db": "data", "sqlite": "data",
|
||||
}
|
||||
|
||||
|
||||
def to_utc(ts):
|
||||
return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def extension(rel):
|
||||
name = rel.rsplit("/", 1)[-1]
|
||||
return name.rsplit(".", 1)[1].lower() if "." in name else ""
|
||||
|
||||
|
||||
def expected_category(rel):
|
||||
ext = extension(rel)
|
||||
if ext in RULER:
|
||||
return RULER[ext]
|
||||
p = RAW / rel
|
||||
try:
|
||||
with p.open("rb") as fh:
|
||||
head = fh.read(1024)
|
||||
except OSError:
|
||||
return "other"
|
||||
for key, magic in MAGICS.items():
|
||||
if head.startswith(magic):
|
||||
return MAGIC_OUT[key]
|
||||
try:
|
||||
head.decode("utf-8")
|
||||
return "text"
|
||||
except UnicodeDecodeError:
|
||||
return "data"
|
||||
|
||||
|
||||
def main():
|
||||
rel_files = sorted(
|
||||
p.relative_to(RAW).as_posix()
|
||||
for p in RAW.rglob("*")
|
||||
if p.is_file()
|
||||
and not any(part == "__pycache__" for part in p.parts)
|
||||
and not p.name.endswith((".pyc", ".pyo", "~", ".bak", ".orig"))
|
||||
and not p.name.startswith(".")
|
||||
)
|
||||
if not rel_files:
|
||||
print("FAIL: no files found under %s" % RAW, file=sys.stderr)
|
||||
return 1
|
||||
|
||||
files = []
|
||||
for rel in rel_files:
|
||||
p = RAW / rel
|
||||
cat = expected_category(rel)
|
||||
files.append({
|
||||
"rel": rel,
|
||||
"ext": extension(rel),
|
||||
"size": p.stat().st_size,
|
||||
"mtime": round(p.stat().st_mtime, 6),
|
||||
"category": cat,
|
||||
})
|
||||
|
||||
cat_counts = Counter(f["category"] for f in files)
|
||||
|
||||
seed = {
|
||||
"tool": "build_sol_fileclass_manifest_2026_11_04.py",
|
||||
"source_dir": str(RAW),
|
||||
"generated_utc": to_utc(datetime.now(timezone.utc).timestamp()),
|
||||
"files": files,
|
||||
"by_category": dict(sorted(cat_counts.items())),
|
||||
}
|
||||
manifest = {
|
||||
"tool": "build_sol_fileclass_manifest_2026_11_04.py",
|
||||
"source_dir": str(RAW),
|
||||
"generated_utc": seed["generated_utc"],
|
||||
"total_files": len(files),
|
||||
"by_category": dict(sorted(cat_counts.items())),
|
||||
"files": files,
|
||||
}
|
||||
|
||||
(OUT / SEED_NAME).write_text(json.dumps(seed, indent=2) + "\n", encoding="utf-8")
|
||||
(OUT / MANIFEST_NAME).write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
print("OK: %d files classified -> seed=%s manifest=%s" % (len(files), SEED_NAME, MANIFEST_NAME))
|
||||
print("counts=%s" % json.dumps(dict(sorted(cat_counts.items()))))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independent verifier for the sol-lane fileclass manifest (2026-11-04).
|
||||
|
||||
Re-reads the committed seed + manifest, re-derives every category from the
|
||||
extension rubric (no import of the generator), confirms every frozen path
|
||||
still exists as a real non-ignored file, and cross-checks aggregate counts.
|
||||
|
||||
Exit code IS the verdict: 0 = PASS, nonzero = FAIL.
|
||||
"""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
SRC = REPO / "wizard-work" / "sol"
|
||||
SEED = SRC / "sol-lane-fileclass-seed-2026-11-04.json"
|
||||
MANIFEST = SRC / "sol-lane-fileclass-2026-11-04.json"
|
||||
|
||||
RULER = {
|
||||
"md": "text", "txt": "text", "rst": "text", "adoc": "text", "org": "text", "log": "text",
|
||||
"json": "data", "csv": "data", "tsv": "data", "yaml": "data", "yml": "data", "toml": "data",
|
||||
"sqlite": "data", "db": "data", "ipynb": "data",
|
||||
"mp3": "media", "wav": "media", "mp4": "media", "webm": "media", "png": "media",
|
||||
"jpg": "media", "jpeg": "media", "gif": "media", "webp": "media", "svg": "media",
|
||||
"py": "script", "sh": "script", "bash": "script", "zsh": "script", "ps1": "script", "bat": "script",
|
||||
"pdf": "doc", "doc": "doc", "docx": "doc", "ppt": "doc", "pptx": "doc", "xls": "doc", "xlsx": "doc",
|
||||
}
|
||||
# Known binary magic prefixes; used only for extensions the ruler has no entry.
|
||||
MAGICS = {
|
||||
"png": b"\x89PNG", "gif": b"GIF8", "jpg": b"\xff\xf0", "webp": b"WEBP", "svg": b"<?xml",
|
||||
"zip/pptx": b"PK\x03\x04", "db": b"SQLite", "sqlite": b"SQLite",
|
||||
}
|
||||
MAGIC_OUT = {
|
||||
"png": "media", "gif": "media", "jpg": "media", "webp": "media", "svg": "text",
|
||||
"zip/pptx": "doc", "db": "data", "sqlite": "data",
|
||||
}
|
||||
|
||||
|
||||
def extension(rel):
|
||||
name = rel.rsplit("/", 1)[-1]
|
||||
return name.rsplit(".", 1)[1].lower() if "." in name else ""
|
||||
|
||||
|
||||
def expected_category(rel):
|
||||
ext = extension(rel)
|
||||
if ext in RULER:
|
||||
return RULER[ext]
|
||||
p = SRC / rel
|
||||
try:
|
||||
with p.open("rb") as fh:
|
||||
head = fh.read(1024)
|
||||
except OSError:
|
||||
return "other"
|
||||
for key, magic in MAGICS.items():
|
||||
if head.startswith(magic):
|
||||
return MAGIC_OUT[key]
|
||||
try:
|
||||
head.decode("utf-8")
|
||||
return "text"
|
||||
except UnicodeDecodeError:
|
||||
return "data"
|
||||
|
||||
|
||||
def is_ignored(rel):
|
||||
res = subprocess.run(
|
||||
["git", "-C", str(REPO), "check-ignore", "-q", "--", rel],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
)
|
||||
return res.returncode == 0
|
||||
|
||||
|
||||
def main():
|
||||
problems = []
|
||||
if not SEED.exists():
|
||||
print("PROBLEM: seed missing: %s" % SEED)
|
||||
return 1
|
||||
if not MANIFEST.exists():
|
||||
print("PROBLEM: manifest missing: %s" % MANIFEST)
|
||||
return 1
|
||||
|
||||
seed = json.loads(SEED.read_text(encoding="utf-8"))
|
||||
manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
|
||||
seed_files = seed.get("files", [])
|
||||
man_files = manifest.get("files", [])
|
||||
|
||||
# 1. Manifest file set must match the frozen seed exactly.
|
||||
seed_paths = {f["rel"] for f in seed_files}
|
||||
man_paths = {f["rel"] for f in man_files}
|
||||
if seed_paths != man_paths:
|
||||
problems.append("manifest path set != seed: seed=%d manifest=%d"
|
||||
% (len(seed_paths), len(man_paths)))
|
||||
|
||||
# 2. Every category must re-derive by the rule from the recorded extension.
|
||||
for f in seed_files:
|
||||
rel, ext, recorded = f["rel"], f["ext"], f["category"]
|
||||
exp = expected_category(rel)
|
||||
if exp != recorded:
|
||||
problems.append("category mismatch %s: ruler says %s, manifest says %s"
|
||||
% (rel, exp, recorded))
|
||||
|
||||
# 3. Every frozen path must still exist as a real file and not be ignored.
|
||||
for f in seed_files:
|
||||
p = SRC / f["rel"]
|
||||
if not p.is_file():
|
||||
problems.append("missing on disk: %s" % f["rel"])
|
||||
elif is_ignored(f["rel"]):
|
||||
problems.append("ignored by .gitignore: %s" % f["rel"])
|
||||
|
||||
# 4. Aggregate counts must be self-consistent.
|
||||
counts = {}
|
||||
for f in seed_files:
|
||||
counts[f["category"]] = counts.get(f["category"], 0) + 1
|
||||
if manifest.get("total_files") != len(seed_files):
|
||||
problems.append("total_files %s != seed %d"
|
||||
% (manifest.get("total_files"), len(seed_files)))
|
||||
if counts != manifest.get("by_category"):
|
||||
problems.append("by_category mismatch: seed=%s manifest=%s"
|
||||
% (counts, manifest.get("by_category")))
|
||||
|
||||
if problems:
|
||||
print("VERIFY FAIL (%d problems)" % len(problems))
|
||||
for pb in problems:
|
||||
print(" - %s" % pb)
|
||||
return 1
|
||||
|
||||
print("VERIFY OK (%d files, categories re-derived, all paths present)"
|
||||
% len(seed_files))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user