90 lines
2.6 KiB
Python
90 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Verifier for ledger-extract.tsv.
|
|
|
|
Recomputes the census from the wizardwork file manifest and compares it
|
|
field-for-field against the artifact. Exit code 0 = artifact correct,
|
|
nonzero = wrong.
|
|
"""
|
|
import csv
|
|
import statistics
|
|
import sys
|
|
from collections import defaultdict
|
|
|
|
MANIFEST = "/root/troupe-standup/wizard-work/ledger/wizardwork-file-manifest.tsv"
|
|
OUT = "/root/troupe-standup/wizard-work/ledger/ledger-extract.tsv"
|
|
HEADER = ["extension", "file_count", "bytes_total", "bytes_mean",
|
|
"bytes_median", "bytes_min", "bytes_max"]
|
|
|
|
|
|
def norm_ext(ext):
|
|
return "(no ext)" if ext == "" else ext
|
|
|
|
|
|
def expected_rows():
|
|
counts = defaultdict(int)
|
|
sizes = defaultdict(list)
|
|
with open(MANIFEST, newline="") as f:
|
|
for line in f:
|
|
parts = line.rstrip("\n").split("\t")
|
|
if len(parts) < 3:
|
|
continue
|
|
try:
|
|
size = int(parts[1])
|
|
except ValueError:
|
|
continue
|
|
ext = norm_ext(parts[2])
|
|
counts[ext] += 1
|
|
sizes[ext].append(size)
|
|
rows = []
|
|
for ext, szs in sizes.items():
|
|
total = sum(szs)
|
|
rows.append((ext, counts[ext], total,
|
|
total / counts[ext], statistics.median(szs),
|
|
min(szs), max(szs)))
|
|
rows.sort(key=lambda r: (-r[1], r[0]))
|
|
return rows
|
|
|
|
|
|
def main():
|
|
expected = expected_rows()
|
|
try:
|
|
with open(OUT, newline="") as f:
|
|
act = list(csv.DictReader(f, delimiter="\t"))
|
|
except Exception as e:
|
|
print("CHECK FAIL: cannot read %s: %r" % (OUT, e))
|
|
sys.exit(1)
|
|
|
|
if not act:
|
|
print("CHECK FAIL: artifact is empty")
|
|
sys.exit(1)
|
|
|
|
if list(act[0].keys()) != HEADER:
|
|
print("CHECK FAIL: header %r" % (list(act[0].keys()),))
|
|
sys.exit(1)
|
|
|
|
if len(act) != len(expected):
|
|
print("CHECK FAIL: row count %d != expected %d"
|
|
% (len(act), len(expected)))
|
|
sys.exit(1)
|
|
|
|
for i, (a, e) in enumerate(zip(act, expected), 1):
|
|
ext, cnt, total, mean, med, mn, mx = e
|
|
exp = {"extension": ext,
|
|
"file_count": str(cnt),
|
|
"bytes_total": str(total),
|
|
"bytes_mean": "%.1f" % mean,
|
|
"bytes_median": "%.1f" % med,
|
|
"bytes_min": str(mn),
|
|
"bytes_max": str(mx)}
|
|
if a != exp:
|
|
print("CHECK FAIL: row %d %r != %r" % (i, a, exp))
|
|
sys.exit(1)
|
|
|
|
print("CHECK PASS: %d extension rows | top=%s count=%s"
|
|
% (len(act), expected[0][0], expected[0][1]))
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|