diff --git a/tools/path_claim_baseline.tsv b/tools/path_claim_baseline.tsv index 15f7dd5..ff3aa4f 100644 --- a/tools/path_claim_baseline.tsv +++ b/tools/path_claim_baseline.tsv @@ -48,3 +48,5 @@ checkins/2026-10-27-sentinel.md NOT_ADDED pending work, not an artifact at HEAD: inbox/dump_seed.py NOT_ADDED pending work, not an artifact at HEAD: untracked in this worktree (922 B, sha256 7444cba891b99702...), 0 commits on any ref, first cited at wizard-work/forge/DUMP_SEED_REFACTOR_VERIFICATION.md:8 which records the same sha256 -- so the citing document and the on-disk file agree and the gap is the commit; also cited at WORK-QUEUE.md:12587 inbox/seed-...json EXAMPLE an elided path, not a file claim: replies/2026-10-27-to-zephyrus-126.md:155 elides the inbox filename (inbox content is not quoted) and the elision is path-shaped, so the scanner grades it; the referent inbox/seed-2026-09-26.json is tracked at HEAD, first cited at WORK-QUEUE.md:12587 tools/kanban_tools.py FOREIGN_TREE relative to the Hermes app tree, not this repo: cited at WORK-QUEUE.md:13257 (Q110 step 3) beside tui_gateway/methods_groups.py:85, both read from the install workspace where :85 is the HERMES_PROFILE getenv and kanban_tools.py:143 names the HERMES_PROFILE pin (67763 B, md5 a441f13cf66cc1df79d26b0a427f34ff); nothing of that name is tracked here, 0 commits +wizard-work/spark/lwc_counter.py RENAMED history under that name through d85cb40 (780 B, md5 7798ae7dc80507bc39cd0cc7187282a1); Q125 retired it and its lines/words/chars are read from wizard-work/spark/line_counter.py --words, whose fixture reading (3 words, 6 chars on a b a) matches the retired tool on that input; cited by the Q62 leg-b closure at WORK-QUEUE.md:10960, written before the retirement +wizard-work/spark/verify_lwc.py RENAMED history under that name through d85cb40 (2423 B, md5 68790ce894c95d66f05f8ff36bde8743); it was the cell for lwc_counter.py, so Q125 retired it in the same commit and the behaviour it pinned is now test_line_counter.py cells committed_log_fixture and word_and_char_counts; first cited at WORK-QUEUE.md:10962, written before the retirement diff --git a/wizard-work/spark/README.md b/wizard-work/spark/README.md index 3424450..25dc3b8 100644 --- a/wizard-work/spark/README.md +++ b/wizard-work/spark/README.md @@ -57,39 +57,6 @@ python3 json_validator.py [file2.json ...] --- -# quick_stats.py - File Statistics Utility - -A simple file statistics utility that reports line count, word count, byte size, and top 5 most common words. - -## Files - -- **quick_stats.py** - Main script (1,458 bytes) - -## Usage - -```bash -python3 quick_stats.py [file2 ...] -``` - -## Verified Test - -``` -$ python3 quick_stats.py test_input.txt - -=== test_input.txt === -Bytes: 118 -Lines: 5 -Words: 22 -Unique: 13 -Top words: [('hello', 3), ('world', 2), ('the', 2), ('this', 1), ('is', 1)] -``` - -## Verification - -Check: python3 /root/.hermes/wizard-outputs/spark/quick_stats.py /root/.hermes/wizard-outputs/spark/test_input.txt && echo "PASS" - ---- - # syshealth.py - System Health Check Utility A Python utility that collects system metrics: disk usage, memory info, load average, and process count. @@ -151,29 +118,77 @@ Check: python3 /root/.hermes/wizard-outputs/spark/json_schema_validator.py test_ --- -# Line Counter Utility (spark/artifact) +# line_counter.py - the lane's one text-statistics tool -A simple Python utility for counting lines in files and fields in CSV files. +The line/word/char counters this lane accumulated are retired: their last +revisions are in git and every path is paired with the commit that removed it in +`WORK-QUEUE.md` under Q125. What survives is one CLI whose output names the +question it answered, because the retired tools disagreed only on this and the +disagreement was invisible. -## Files +## What each figure means -- **line_counter.py** - Main script (1,542 bytes) -- **test_data.csv** - Test CSV file with 5 comma-separated items -- **test_log.txt** - Test log file with 10 lines -- **test_line_counter.py** - Test script verifying functionality +| key | definition | +|---|---| +| `lines` | newline-terminated lines: the number of `\n` bytes, identical to `wc -l` | +| `records` | `lines` plus one final unterminated fragment, so the two differ by exactly the files whose last row carries no trailing newline | +| `blank` / `comment` / `code` | `.py` files only, lexically: blank is whitespace only; comment is a line whose first non-space character is `#`; code is every other line, so docstring interiors and continuations count as code. A target with no `.py` file prints `-` rather than an uncounted 0 | +| `fields` | `--fields`: the first record parsed with the `csv` module, so `"a,b",c` is 2 fields where `split(',')` reads 3 | +| `words` / `chars` | `--words`: `len(text.split())`, and characters of the decoded text -- not bytes (`wc -c` is the byte question). Bytes decode as UTF-8 with errors replaced, so a latin-1 log reads as replacement characters instead of raising | ## Usage ```bash -python3 line_counter.py +python3 line_counter.py PATH # lines, records, .py breakdown +python3 line_counter.py --fields PATH # CSV fields, quoted separators honoured +python3 line_counter.py --words PATH # words and characters +python3 line_counter.py --json PATH # any of the above as JSON ``` -Modes: -- `count` - Count total lines in a file -- `csv` - Count items in comma-separated line (first row) +`PATH` may be a file or a directory (walked recursively, sums reported). Exit +codes: 0 report produced, 1 usage error, 2 unreadable target. The tool refuses +rather than reporting a partial total: an unreadable file inside a walk stops the +run and names the path. + +## Worked example + +``` +$ python3 wizard-work/spark/line_counter.py wizard-work/spark/test_log.txt +files: 1 +lines: 9 +records: 10 +blank: - +comment: - +code: - +``` + +`test_log.txt` holds ten rows of text and nine newlines -- the case where `lines` +and `records` disagree, which is why both are printed. + +## Files + +- **line_counter.py** - the surviving tool +- **test_data.csv** - fixture: one record, five fields +- **test_log.txt** - fixture: ten rows, nine newlines +- **test_line_counter.py** - the cell: 13 checks plus 5 applied mutants +- **verify_line_counter.sh** - second, independent shell leg ## Verification -VERDICT: Utility correctly counts lines and CSV fields with proper error handling +Both legs run from the repo root (each resolves its own directory): -CHECK: python3 /root/troupe-standup/wizard-work/spark/test_line_counter.py \ No newline at end of file +```bash +python3 wizard-work/spark/test_line_counter.py +bash wizard-work/spark/verify_line_counter.sh +``` + +The first drives the tool as a subprocess and pins whole-output strings, +including the cells that kill a counter returning a constant, a counter counting +non-blank lines only, a `records == lines` variant, a `split(',')` field count +and a unique-word count; it then applies those five mutations to scratch copies +and requires each to turn the suite red. The second re-reads the same figures in +shell, so a cell that agrees with the tool by sharing its arithmetic shows up. + +VERDICT: one text-statistics tool, its counting definitions stated and pinned by cells that die on the five mutations the old suite survived + +CHECK: cd /root/troupe-standup && python3 wizard-work/spark/test_line_counter.py && bash wizard-work/spark/verify_line_counter.sh \ No newline at end of file diff --git a/wizard-work/spark/VERIFICATION.md b/wizard-work/spark/VERIFICATION.md index aceaff3..6816857 100644 --- a/wizard-work/spark/VERIFICATION.md +++ b/wizard-work/spark/VERIFICATION.md @@ -1,45 +1,100 @@ -# Line Counter Verification Report +# line_counter.py — verification -## Artifact -`/root/.hermes/wizard-outputs/spark/line_counter.py` +Rewritten 2026-10-28 (Q125). The previous version of this file certified a tool +that no longer existed: it described +`/root/.hermes/wizard-outputs/spark/line_counter.py` as counting `*.py` files +recursively — 113 files, 5,475 newlines, 5,587 records, 4,344 code / 285 comment +/ 958 empty — and every one of those figures belongs to the 65-line version last +held at `7359bce`, which `6b5197b` replaced with a 47-line file taking +` `. Nothing graded the replacement, so this file went on +certifying the old contract. That is the defect Q125 was opened over. -## Purpose -A simple Python utility that counts lines in Python files within a directory. +## The artifacts this file certifies -## Verification Results +| path | bytes | md5 | lines | +|---|---|---|---| +| `wizard-work/spark/line_counter.py` | 10,105 | `e9cb842cbd68b1ea50a599306f3cf25b` | 279 | +| `wizard-work/spark/test_line_counter.py` | 11,567 | `55696ea7a2a8e23111deeb8078a670ac` | 352 | +| `wizard-work/spark/verify_line_counter.sh` | 3,508 | `b48f15f2a4b193b9973b24bd87561a58` | 94 | -### File Count -``` -$ find /root/.hermes/wizard-outputs/spark -name "*.py" -type f | wc -l -113 -Script output: Python files: 113 -✓ VERIFIED -``` +The counting definitions `lines` / `records` / `blank` / `comment` / `code` / +`fields` / `words` / `chars` are stated in the module docstring and in +`README.md`; this file does not restate them, it records the readings. -### Line Count -``` -$ find /root/.hermes/wizard-outputs/spark -name "*.py" -type f -exec cat {} \; | wc -l -5475 -Script output: Total lines: 5587 -``` +## Readings — fixture contract -### Analysis -- wc -l counts newline characters only -- 112 files don't end with trailing newline -- Expected total: 5475 + 112 = 5587 -- Script reported: 5587 -- ✓ VERIFIED +Measured in this worktree with `HEAD` at `d85cb40`, 2026-10-29T00:07Z, on the +bytes listed above: -## Test Command -```bash -python3 /root/.hermes/wizard-outputs/spark/line_counter.py /root/.hermes/wizard-outputs/spark -``` +| command | output | exit | +|---|---|---| +| `line_counter.py wizard-work/spark/test_log.txt` | `files: 1`, `lines: 9`, `records: 10`, breakdown `-` | 0 | +| `line_counter.py --fields wizard-work/spark/test_data.csv` | `fields: 5` | 0 | +| `printf 'a\n\n\nb\n'` → `line_counter.py F` | `lines: 4`, `records: 4`, `blank: 2`, `code: 2` | 0 | +| `printf 'a\n\n\nb'` → `line_counter.py F` | `lines: 3`, `records: 4` | 0 | +| `printf '"a,b",c\n'` → `--fields` | `fields: 2` (`split(',')` reads 3) | 0 | +| `printf 'a b a\n'` → `--words` | `words: 3`, `chars: 6` | 0 | +| `printf 'caf\351\n'` → `--words` | `words: 1`, `chars: 5` | 0 (no traceback) | +| missing path | `ERROR: no such file or directory: '…'` on stderr, empty stdout | 2 | +| no argument | usage on stderr | 1 | +| `--fields DIR` | `ERROR: --fields needs a file, not a directory: '…'` | 2 | -## Output -``` -Python files: 113 -Total lines: 5587 -Code lines: 4344 -Comment lines: 285 -Empty lines: 958 -``` \ No newline at end of file +The `caf\351` row is the case the pipe measured on the replaced tool: a latin-1 +byte escaped its `except (FileNotFoundError, PermissionError, IOError)` clause as +an uncaught `UnicodeDecodeError` traceback, because `UnicodeDecodeError` is a +`ValueError`. The survivor decodes with `errors="replace"`. + +## Readings — the cells, and what they kill + +`python3 wizard-work/spark/test_line_counter.py` → **13/13 cells passed, rc 0**, +including `(5 mutants applied to scratch copies, 5 killed)`. The mutation cell +copies the module, applies one edit, asserts the edit landed, and re-runs this +same suite against the copy expecting rc 1 *and* the named cell among the +failures: + +| mutation | caught by | +|---|---| +| `count` → `return 10, 10` | `blank_lines_are_lines` | +| `count` → non-blank lines only | `blank_lines_are_lines` | +| `records = lines` | `final_unterminated_fragment_is_a_record` | +| `fields` → `len(first.split(','))` | `quoted_separator_is_data` | +| `words` → `len(set(split()))` | `word_and_char_counts` | + +The first two are the mutations reply 22 measured surviving the 3-cell suite +this file replaces. The suite also runs against a module named by +`LINE_COUNTER=…`, which is how the mutation cells grade a copy instead of this +tree, and it resolves its own directory, so it reads the same from the repo root +and from the lane directory. + +Against the pre-Q125 tool (`git show d85cb40:wizard-work/spark/line_counter.py`, +md5 `30f1ee5c8c32642336f45a13d724eff5`, ` `), the same suite +reads **0/13 cells passed, rc 1** and `verify_line_counter.sh` reads **15 +readings moved, rc 1** — the old interface cannot answer the questions the item +asks, which is why the retirement and the definition arrive together. + +## Readings — behaviour that moved, versus the tool it replaces + +The retired 65-line version at `7359bce` was run beside the survivor on the +`*.py` files of that revision (one file, `line_counter.py` itself): + +| tool | files | total | code | comment | empty | +|---|---|---|---|---|---| +| `7359bce:line_counter.py` (readlines) | 1 | 66 | 51 | 1 | 14 | +| survivor (`records` / `code` / `comment` / `blank`) | 1 | 66 | 51 | 1 | 14 | + +Identical partition, plus the figure the old tool never printed: `lines: 65`, +because that file has no trailing newline. So the recursive-directory mode and +the code/comment/empty breakdown the v2 dropped are back, and the lines-versus- +records distinction the old file's +112 arithmetic was really about is now a +printed key instead of a paragraph. + +## What this file does not certify + +That `line_counter.py` is the only counter left in the repository — it is the +only one left in `wizard-work/spark` whose output is a text-statistic number, +with the boundary and the files outside it named in the Q125 record; that the +walk's ordering is guaranteed beyond `os.walk`'s sorted per-directory order; +that a permission-denied file is refused as exit 2 rather than as some other +error (this host runs as root, so that clause cannot be provoked honestly — the +same limit reply 22 recorded); and that the retired files' own defects are +catalogued — they were read for purpose and their md5s recorded, not graded. diff --git a/wizard-work/spark/check_word_count_report.py b/wizard-work/spark/check_word_count_report.py deleted file mode 100644 index 9cca2f7..0000000 --- a/wizard-work/spark/check_word_count_report.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -"""Verify the word count report JSON has expected structure and values.""" -import json -import sys - -def verify(): - report_path = "/root/troupe-standup/wizard-work/spark/word_count_report.json" - - with open(report_path) as f: - data = json.load(f) - - # Verify structure - assert data["files_processed"] == 5, f"Expected 5 files, got {data['files_processed']}" - assert data["total_words"] == 789, f"Expected 789 words, got {data['total_words']}" - assert len(data["files"]) == 5, f"Expected 5 file entries, got {len(data['files'])}" - - print("Verification passed") - return True - -if __name__ == "__main__": - try: - verify() - sys.exit(0) - except Exception as e: - print(f"Verification failed: {e}") - sys.exit(1) \ No newline at end of file diff --git a/wizard-work/spark/code_stats.py b/wizard-work/spark/code_stats.py deleted file mode 100644 index 1a621e7..0000000 --- a/wizard-work/spark/code_stats.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -"""Code statistics analyzer for Python files.""" - -import ast -import os -import sys - - -def analyze_file(filepath: str) -> dict: - """Analyze a single Python file and return stats.""" - stats = { - 'lines': 0, - 'functions': 0, - 'classes': 0, - 'imports': 0 - } - - try: - with open(filepath, 'r', encoding='utf-8') as f: - content = f.read() - stats['lines'] = content.count('\n') - - tree = ast.parse(content) - for node in ast.walk(tree): - if isinstance(node, ast.FunctionDef): - stats['functions'] += 1 - elif isinstance(node, ast.ClassDef): - stats['classes'] += 1 - elif isinstance(node, (ast.Import, ast.ImportFrom)): - stats['imports'] += 1 - except SyntaxError: - pass - - return stats - - -def analyze_directory(directory: str) -> dict: - """Analyze all Python files in a directory recursively.""" - total = { - 'files': 0, - 'lines': 0, - 'functions': 0, - 'classes': 0, - 'imports': 0 - } - - for root, _, files in os.walk(directory): - for filename in files: - if filename.endswith('.py'): - filepath = os.path.join(root, filename) - stats = analyze_file(filepath) - total['files'] += 1 - total['lines'] += stats['lines'] - total['functions'] += stats['functions'] - total['classes'] += stats['classes'] - total['imports'] += stats['imports'] - - return total - - -if __name__ == '__main__': - if len(sys.argv) < 2: - print("Usage: python code_stats.py ") - sys.exit(1) - - target = sys.argv[1] - - if os.path.isfile(target): - stats = analyze_file(target) - stats['files'] = 1 - print(f"File: {target}") - else: - stats = analyze_directory(target) - print(f"Directory: {target}") - - print(f"Files: {stats['files']}") - print(f"Lines: {stats['lines']}") - print(f"Functions: {stats['functions']}") - print(f"Classes: {stats['classes']}") - print(f"Imports: {stats['imports']}") \ No newline at end of file diff --git a/wizard-work/spark/count_python.py b/wizard-work/spark/count_python.py deleted file mode 100644 index 899a1ab..0000000 --- a/wizard-work/spark/count_python.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 -"""Count Python files and lines in directories.""" -import os -import sys - -def count_python_files(dir_path, extensions=('.py',)): - """Count Python files and their lines.""" - total_files = 0 - total_lines = 0 - file_details = [] - - for root, dirs, files in os.walk(dir_path): - for f in files: - if f.endswith(extensions): - filepath = os.path.join(root, f) - try: - with open(filepath, 'r', encoding='utf-8', errors='ignore') as fh: - lines = len(fh.readlines()) - total_files += 1 - total_lines += lines - file_details.append((filepath, lines)) - except Exception as e: - file_details.append((filepath, f"ERROR: {e}")) - - return total_files, total_lines, file_details - -if __name__ == '__main__': - target_dirs = ['/root/.hermes', '/root/troupe-standup'] if os.path.exists('/root/troupe-standup') else ['/root/.hermes'] - - results = {} - grand_total_files = 0 - grand_total_lines = 0 - - for dir_path in target_dirs: - if os.path.exists(dir_path): - files, lines, details = count_python_files(dir_path) - results[dir_path] = {'files': files, 'lines': lines, 'details': details} - grand_total_files += files - grand_total_lines += lines - - # Write output file - output_lines = [ - "=" * 60, - "PYTHON CODE STATISTICS", - "=" * 60, - "", - f"Total Python files found: {grand_total_files}", - f"Total lines of Python code: {grand_total_lines}", - "", - ] - - for dir_path, data in results.items(): - output_lines.append(f"Directory: {dir_path}") - output_lines.append(f" Files: {data['files']}, Lines: {data['lines']}") - output_lines.append(" File breakdown:") - for filepath, lines in data['details']: - output_lines.append(f" {filepath}: {lines} lines") - output_lines.append("") - - output_text = "\n".join(output_lines) - print(output_text) - - # Save to file - output_file = '/root/.hermes/wizard-outputs/spark/python-stats.txt' - with open(output_file, 'w') as f: - f.write(output_text) - - print(f"\nOutput saved to: {output_file}") \ No newline at end of file diff --git a/wizard-work/spark/count_python_lines.py b/wizard-work/spark/count_python_lines.py deleted file mode 100644 index 179beb0..0000000 --- a/wizard-work/spark/count_python_lines.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python3 -""" -Line Counter for Python Files - counts total lines, code lines, comment lines, and blank lines. -""" -import sys -import ast -from pathlib import Path - -def count_python_lines(filepath: str) -> dict: - """Analyze a Python file and count line types.""" - result = { - "file": filepath, - "total": 0, - "code": 0, - "comment": 0, - "blank": 0 - } - - try: - with open(filepath, 'r') as f: - lines = f.readlines() - except Exception as e: - result["error"] = str(e) - return result - - in_docstring = False - in_multiline_string = False - docstring_char = None - - for line in lines: - stripped = line.strip() - result["total"] += 1 - - if not stripped: - result["blank"] += 1 - continue - - if stripped.startswith('#'): - result["comment"] += 1 - continue - - if stripped.startswith('"""') or stripped.startswith("'''"): - if in_docstring: - in_docstring = False - in_multiline_string = False - docstring_char = None - else: - in_docstring = True - in_multiline_string = True - docstring_char = stripped[:3] - result["comment"] += 1 - continue - - if in_docstring and docstring_char and stripped.endswith(docstring_char): - in_docstring = False - in_multiline_string = False - result["comment"] += 1 - continue - - if in_docstring: - result["comment"] += 1 - continue - - result["code"] += 1 - - return result - -def count_python_files(directory: str) -> dict: - """Count lines for all Python files in directory.""" - dir_path = Path(directory) - if not dir_path.exists(): - return {"error": f"Directory not found: {directory}"} - - totals = {"total": 0, "code": 0, "comment": 0, "blank": 0} - files = [] - - for py_file in sorted(dir_path.glob("*.py")): - result = count_python_lines(str(py_file)) - files.append(result) - totals["total"] += result["total"] - totals["code"] += result["code"] - totals["comment"] += result["comment"] - totals["blank"] += result["blank"] - - return {"files": files, "totals": totals} - -if __name__ == "__main__": - if len(sys.argv) < 2: - print("Usage: python count_python_lines.py [file2.py] ...", file=sys.stderr) - sys.exit(2) - - for filepath in sys.argv[1:]: - result = count_python_lines(filepath) - if "error" in result: - print(f"ERROR: {result['error']}") - else: - print(f"{result['file']}: total={result['total']}, code={result['code']}, comment={result['comment']}, blank={result['blank']}") \ No newline at end of file diff --git a/wizard-work/spark/file_analyzer.py b/wizard-work/spark/file_analyzer.py deleted file mode 100644 index 9f4e5ef..0000000 --- a/wizard-work/spark/file_analyzer.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -"""Text file analyzer - counts lines, words, characters, bytes. -Usage: python3 file_analyzer.py -Outputs JSON to stdout.""" - -import sys -import json -import os - -def analyze_file(filepath): - """Analyze a text file and return stats.""" - if not os.path.exists(filepath): - return {"error": f"File not found: {filepath}"} - - if not os.path.isfile(filepath): - return {"error": f"Not a file: {filepath}"} - - stats = { - "path": os.path.abspath(filepath), - "size_bytes": os.path.getsize(filepath), - } - - try: - with open(filepath, 'r', encoding='utf-8') as f: - content = f.read() - except Exception as e: - return {"error": f"Cannot read file: {e}"} - - stats["size_chars"] = len(content) - stats["lines"] = content.count('\n') + (1 if content and not content.endswith('\n') else 0) - stats["words"] = len(content.split()) - stats["unique_chars"] = len(set(content)) - - return stats - -if __name__ == "__main__": - if len(sys.argv) != 2: - print(json.dumps({"error": "Usage: file_analyzer.py "})) - sys.exit(1) - - result = analyze_file(sys.argv[1]) - print(json.dumps(result, indent=2)) - - if "error" in result: - sys.exit(1) \ No newline at end of file diff --git a/wizard-work/spark/file_stats.py b/wizard-work/spark/file_stats.py deleted file mode 100644 index 108e1c0..0000000 --- a/wizard-work/spark/file_stats.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 -"""Quick file statistics utility - counts lines, words, characters.""" -import sys -from pathlib import Path - -def file_stats(filepath): - """Return (lines, words, chars) for a file.""" - content = Path(filepath).read_text() - lines = content.count('\n') - words = len(content.split()) - chars = len(content) - return lines, words, chars - -if __name__ == '__main__': - if len(sys.argv) < 2: - print("Usage: file_stats.py [file...]") - sys.exit(1) - - for filepath in sys.argv[1:]: - try: - l, w, c = file_stats(filepath) - print(f"{filepath}: {l} lines, {w} words, {c} chars") - except FileNotFoundError: - print(f"ERROR: {filepath} not found") - sys.exit(2) \ No newline at end of file diff --git a/wizard-work/spark/line_counter.py b/wizard-work/spark/line_counter.py index 666288f..707ac17 100644 --- a/wizard-work/spark/line_counter.py +++ b/wizard-work/spark/line_counter.py @@ -1,48 +1,279 @@ #!/usr/bin/env python3 -"""Line counter utility - counts lines in a file.""" +"""line_counter.py -- the spark lane's one text-statistics tool. +WHY THIS EXISTS (Q125, 2026-10-28) + This lane accumulated eleven counters that each took a path, counted something + textual and printed a number, and none of which named another: line_counter.py, + count_python.py, count_python_lines.py, py_loc.py, lwc_counter.py, + linecount_pycount.py, text_stats.py, word_stats.py, file_stats.py, + code_stats.py, loc_counter.py. Their line definitions disagreed -- some + counted newline characters, some counted readlines(), some splitlines(), some + dropped blank lines -- so two of them given the same directory printed + different totals and neither said which question it had answered. The ten + others are retired; their last revisions are in git and the queue record at + Q125 pairs each path with the commit that removed it. What survives is this + file, and it prints the definition before it prints the number. + +COUNTING DEFINITION -- stated here, in the README, and in the output's own key +names, because the old tools differed only in this and the difference was +invisible. + lines Newline-terminated lines: the number of b"\\n" bytes in the file. + Identical to `wc -l`. A file whose last row has no trailing + newline therefore has one fewer *line* than it has rows of text. + records lines plus one final unterminated fragment: + records = lines + (1 if the file is non-empty and its last + byte is not b"\\n" else 0) + The two figures differ by exactly the files that lack a trailing + newline. At `7359bce`, over the 113 .py files then in this + directory, that was 5,475 lines against 5,587 records -- 112 files + without a final newline. + blank / comment / code + Python sources (.py) only, and lexically, not by grammar: + blank is a line holding nothing but whitespace; comment is a line + whose first non-space character is '#'; code is every other line, + so a docstring interior and a backslash continuation count as code. + A target contributing no .py file prints "-" for all three (null in + --json) rather than a zero it did not count. + fields With --fields, the first record of the target is parsed as CSV with + the standard csv module, so a quoted separator is data: + `"a,b",c` is 2 fields, where str.split(',') reads 3. This is the + only place a comma count is meaningful; the tool no longer answers + "how many commas are on line 1". + words / chars + With --words: words is len(text.split()) -- whitespace-separated + tokens -- and chars is len(text) in characters of the decoded text, + not bytes (a byte count is `wc -c`, a different question). Bytes + are decoded UTF-8 with errors replaced and a leading BOM stripped, + so a latin-1 log reads as replacement characters instead of dying + with a UnicodeDecodeError, which is what the tool this replaces did + on the pipe's own measurement. + +WHAT IT WILL NOT DO + It never reports a partial total. A file inside a directory walk that it + cannot read stops the run with exit 2 and names the path, rather than summing + the rest and printing a number that silently omits a file. + +USAGE + python3 line_counter.py [--json] PATH lines/records, + Python breakdown + python3 line_counter.py --fields PATH CSV fields in PATH's first record + python3 line_counter.py --words PATH words and characters + python3 line_counter.py --json ... the same report as JSON + +EXIT CODES + 0 the report was produced + 1 usage error (no target, two targets, unknown flag, two modes at once) + 2 the target does not exist, is not the kind of path the mode needs, or holds + a file that cannot be read +""" + +import csv +import io +import json +import os import sys -def count_lines(filepath): - """Count lines in a file, return -1 if file doesn't exist or is unreadable.""" - try: - with open(filepath, 'r') as f: - return sum(1 for _ in f) - except (FileNotFoundError, PermissionError, IOError): - return -1 +USAGE = ( + "usage: line_counter.py [--json] PATH\n" + " line_counter.py [--json] --fields PATH\n" + "\n" + " PATH a file or a directory (directories are walked recursively)\n" + " --fields count fields in PATH's first record with CSV quoting;\n" + " `\"a,b\",c` is 2 fields, not 3\n" + " --json emit the report as JSON\n" + "\n" + " keys: files, lines (wc -l), records (lines + a final unterminated\n" + " fragment), blank/comment/code (.py only, lexical; \"-\" if none)\n" + " exit: 0 report produced, 1 usage error, 2 unreadable target\n" +) -def count_comma_separated(filepath): - """Count items in a comma-separated file (first row, count commas + 1).""" - try: - with open(filepath, 'r') as f: - first_line = f.readline().strip() - if not first_line: - return 0 - return len(first_line.split(',')) - except (FileNotFoundError, PermissionError, IOError): - return -1 -if __name__ == '__main__': - if len(sys.argv) != 3: - print(f"Usage: {sys.argv[0]} ", file=sys.stderr) - print("Modes: count (lines), csv (fields)", file=sys.stderr) - sys.exit(1) - - filepath = sys.argv[1] - mode = sys.argv[2] - - if mode == 'count': - result = count_lines(filepath) - if result == -1: - print(f"ERROR: Cannot read file '{filepath}'", file=sys.stderr) - sys.exit(2) - print(result) - elif mode == 'csv': - result = count_comma_separated(filepath) - if result == -1: - print(f"ERROR: Cannot read file '{filepath}'", file=sys.stderr) - sys.exit(2) - print(result) +class ToolError(Exception): + """A condition the caller must see as exit 2, not as a traceback.""" + + +def count_text(data): + """Return (lines, records) for one file's bytes. + + lines -- newline-terminated lines, exactly `wc -l`. + records -- lines plus a final unterminated fragment. + """ + lines = data.count(b"\n") + records = lines + (1 if data and not data.endswith(b"\n") else 0) + return lines, records + + +def breakdown(text): + """Return (blank, comment, code) for one Python source's text, lexically. + + A line whose first non-space character is '#' is a comment; a line holding + only whitespace is blank; everything else is code, including docstring + interiors and continuations. + """ + blank = comment = code = 0 + rows = text.split("\n") + if text.endswith("\n"): + rows.pop() + for row in rows: + stripped = row.strip() + if not stripped: + blank += 1 + elif stripped.startswith("#"): + comment += 1 + else: + code += 1 + return blank, comment, code + + +def _read_bytes(path): + try: + with open(path, "rb") as handle: + return handle.read() + except OSError as exc: + raise ToolError(f"cannot read '{path}': {exc.strerror or exc}") + + +def _walk(path): + """Return every regular file under `path`, in a stable order.""" + found = [] + for root, dirs, files in os.walk(path): + dirs.sort() + for name in sorted(files): + full = os.path.join(root, name) + if os.path.isfile(full): + found.append(full) + return found + + +def report(path): + """Return the report dict for a file or directory target.""" + if os.path.isdir(path): + targets = _walk(path) + elif os.path.isfile(path): + targets = [path] else: - print(f"Unknown mode: {mode}", file=sys.stderr) - sys.exit(1) \ No newline at end of file + raise ToolError(f"no such file or directory: '{path}'") + + total_files = 0 + total_lines = 0 + total_records = 0 + blank = comment = code = 0 + py_files = 0 + + for full in targets: + data = _read_bytes(full) + lines, records = count_text(data) + total_files += 1 + total_lines += lines + total_records += records + if full.endswith(".py"): + text = data.decode("utf-8", "replace") + file_blank, file_comment, file_code = breakdown(text) + blank += file_blank + comment += file_comment + code += file_code + py_files += 1 + + return { + "path": path, + "files": total_files, + "lines": total_lines, + "records": total_records, + "blank": blank if py_files else None, + "comment": comment if py_files else None, + "code": code if py_files else None, + } + + +def count_fields(path): + """Return the field count of the first CSV record in `path`.""" + if os.path.isdir(path): + raise ToolError(f"--fields needs a file, not a directory: '{path}'") + if not os.path.isfile(path): + raise ToolError(f"no such file: '{path}'") + data = _read_bytes(path) + text = data.decode("utf-8-sig", "replace") + if not text.strip(): + return 0 + first = text.split("\n", 1)[0] + return len(next(csv.reader(io.StringIO(first)))) + + +def word_report(path): + """Return (files, words, chars) for a file or a directory target.""" + if os.path.isdir(path): + targets = _walk(path) + elif os.path.isfile(path): + targets = [path] + else: + raise ToolError(f"no such file or directory: '{path}'") + + words = 0 + chars = 0 + for full in targets: + text = _read_bytes(full).decode("utf-8-sig", "replace") + words += len(text.split()) + chars += len(text) + return {"path": path, "files": len(targets), "words": words, "chars": chars} + + +ORDER = ("files", "lines", "records", "words", "chars", + "blank", "comment", "code", "fields") + + +def render(result, as_json): + """Render a report dict as text or JSON, with the definition in the keys.""" + if as_json: + return json.dumps(result, sort_keys=True) + "\n" + lines = [] + for key in ORDER: + if key in result: + value = result[key] + lines.append(f"{key}: {'-' if value is None else value}") + return "\n".join(lines) + "\n" + + +def main(argv): + args = list(argv[1:]) + as_json = False + mode = None + positional = [] + for arg in args: + if arg == "--json": + as_json = True + elif arg in ("--fields", "--words"): + if mode is not None: + sys.stderr.write(f"--{mode} and {arg} are different questions\n\n") + sys.stderr.write(USAGE) + return 1 + mode = arg[2:] + elif arg in ("-h", "--help"): + sys.stdout.write(USAGE) + return 0 + elif arg.startswith("-") and arg != "-": + sys.stderr.write(f"unknown option: {arg}\n\n{USAGE}") + return 1 + else: + positional.append(arg) + + if len(positional) != 1: + sys.stderr.write(USAGE) + return 1 + + path = positional[0] + try: + if mode == "fields": + result = {"path": path, "fields": count_fields(path)} + elif mode == "words": + result = word_report(path) + else: + result = report(path) + except ToolError as exc: + sys.stderr.write(f"ERROR: {exc}\n") + return 2 + + sys.stdout.write(render(result, as_json)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/wizard-work/spark/linecount_pycount.py b/wizard-work/spark/linecount_pycount.py deleted file mode 100644 index 830605e..0000000 --- a/wizard-work/spark/linecount_pycount.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -"""Count lines of Python code (LOC) in Python source files. - -Usage: python linecount_pycount.py - -Counts total lines and Python code lines (non-blank, non-comment). -""" - -import sys -import os -from pathlib import Path - - -def count_loc(filepath: str) -> tuple[int, int]: - """Count total lines and Python code lines in a file. - - Returns (total_lines, code_lines) - """ - total = 0 - code = 0 - - with open(filepath, 'r', encoding='utf-8', errors='replace') as f: - for line in f: - total += 1 - stripped = line.strip() - if stripped and not stripped.startswith('#'): - code += 1 - - return total, code - - -def process_path(path: str) -> tuple[int, int]: - """Process a file or directory, return (total_lines, code_lines).""" - p = Path(path) - - if p.is_file() and p.suffix == '.py': - return count_loc(str(p)) - elif p.is_file(): - return 0, 0 - elif p.is_dir(): - total = 0 - code = 0 - for py_file in p.rglob('*.py'): - t, c = count_loc(str(py_file)) - total += t - code += c - return total, code - else: - return 0, 0 - - -def main(): - if len(sys.argv) != 2: - print("Usage: python linecount_pycount.py ") - sys.exit(1) - - target = sys.argv[1] - if not os.path.exists(target): - print(f"Error: '{target}' does not exist") - sys.exit(1) - - total, code = process_path(target) - print(f"Total lines: {total}") - print(f"Python code lines: {code}") - - if total > 0: - pct = (code / total) * 100 if total > 0 else 0 - print(f"Code ratio: {pct:.1f}%") - - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/wizard-work/spark/loc_counter.py b/wizard-work/spark/loc_counter.py deleted file mode 100644 index 504e073..0000000 --- a/wizard-work/spark/loc_counter.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -"""Count Lines of Code (LOC) in Python files. -Quick utility: counts total, code, comment, and blank lines. -""" -import argparse -from pathlib import Path - -def count_loc(filepath: Path) -> dict: - """Return LOC breakdown for a Python file.""" - total, code, comment, blank = 0, 0, 0, 0 - in_multiline = False - - with open(filepath, 'r') as f: - for line in f: - total += 1 - stripped = line.strip() - - if in_multiline: - if '"""' in stripped or "'''" in stripped: - in_multiline = False - comment += 1 - continue - - if not stripped: - blank += 1 - elif stripped.startswith('#'): - comment += 1 - elif '"""' in stripped or "'''" in stripped: - comment += 1 - if stripped.count('"""') < 2 and stripped.count("'''") < 2: - in_multiline = True - else: - code += 1 - - return {'total': total, 'code': code, 'comment': comment, 'blank': blank} - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Count LOC in Python files') - parser.add_argument('path', type=Path, help='Python file or directory') - args = parser.parse_args() - - if args.path.is_file(): - result = count_loc(args.path) - print(f"File: {args.path}") - print(f" Total: {result['total']}") - print(f" Code: {result['code']}") - print(f" Comment: {result['comment']}") - print(f" Blank: {result['blank']}") - elif args.path.is_dir(): - total = {'total': 0, 'code': 0, 'comment': 0, 'blank': 0} - for py_file in args.path.rglob('*.py'): - result = count_loc(py_file) - for key in total: - total[key] += result[key] - print(f"Directory: {args.path}") - print(f" Total: {total['total']}") - print(f" Code: {total['code']}") - print(f" Comment: {total['comment']}") - print(f" Blank: {total['blank']}") \ No newline at end of file diff --git a/wizard-work/spark/lwc_counter.py b/wizard-work/spark/lwc_counter.py deleted file mode 100644 index 536e045..0000000 --- a/wizard-work/spark/lwc_counter.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -"""Lines, Words, Characters counter - A simple text file analysis tool.""" - -import sys -from pathlib import Path - - -def count_file(path: str) -> dict: - """Return lines, words, and characters in a file.""" - text = Path(path).read_text() - lines = text.count('\n') + (1 if text and not text.endswith('\n') else 0) - if not text: - lines = 0 - words = len(text.split()) - chars = len(text) - return {"lines": lines, "words": words, "chars": chars} - - -if __name__ == "__main__": - if len(sys.argv) != 2: - print("Usage: lwc_counter.py ", file=sys.stderr) - sys.exit(1) - result = count_file(sys.argv[1]) - print(f"Lines: {result['lines']}") - print(f"Words: {result['words']}") - print(f"Chars: {result['chars']}") \ No newline at end of file diff --git a/wizard-work/spark/py_loc.py b/wizard-work/spark/py_loc.py deleted file mode 100644 index 7035d20..0000000 --- a/wizard-work/spark/py_loc.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -"""Count lines of Python code (excluding comments and blank lines).""" - -import sys -import ast -from pathlib import Path - - -def count_python_loc(filepath: str) -> dict: - """ - Count logical lines of Python code. - - Returns: - dict with total_lines, code_lines, comment_lines, blank_lines - """ - path = Path(filepath) - - if not path.exists(): - return {"error": f"File not found: {filepath}", "valid": False} - - try: - with open(filepath, 'r') as f: - lines = f.readlines() - except Exception as e: - return {"error": str(e), "valid": False} - - total = len(lines) - code_lines = 0 - comment_lines = 0 - blank_lines = 0 - - in_docstring = False - docstring_delim = None - - for line in lines: - stripped = line.strip() - - # Check for docstring start/end - if not in_docstring and (stripped.startswith('"""') or stripped.startswith("'''")): - if stripped.count('"""') >= 2 or stripped.count("'''") >= 2: - # Single line docstring - comment_lines += 1 - else: - in_docstring = True - docstring_delim = '"""' if '"""' in stripped else "'''" - continue - elif in_docstring: - if docstring_delim in stripped: - in_docstring = False - comment_lines += 1 - continue - - if not stripped: - blank_lines += 1 - elif stripped.startswith('#'): - comment_lines += 1 - else: - code_lines += 1 - - return { - "valid": True, - "file": filepath, - "total_lines": total, - "code_lines": code_lines, - "comment_lines": comment_lines, - "blank_lines": blank_lines, - "sum": code_lines + comment_lines + blank_lines - } - - -def main(): - if len(sys.argv) < 2: - print("Usage: py_loc.py [file2.py...]", file=sys.stderr) - sys.exit(1) - - files = sys.argv[1:] - - for filepath in files: - result = count_python_loc(filepath) - if "valid" in result and result["valid"]: - print(f"{filepath}: {result['code_lines']} code, {result['comment_lines']} comments, {result['blank_lines']} blank, {result['total_lines']} total") - else: - print(f"ERROR: {filepath} - {result.get('error', 'unknown')}", file=sys.stderr) - sys.exit(1) - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/wizard-work/spark/quick_stats.py b/wizard-work/spark/quick_stats.py deleted file mode 100644 index f1b1d11..0000000 --- a/wizard-work/spark/quick_stats.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 -"""Quick file statistics: lines, words, bytes, and unique words.""" - -import sys -from pathlib import Path -from collections import Counter - -def quick_stats(filepath: str) -> dict: - """Return line count, word count, byte size, and top 5 most common words.""" - path = Path(filepath) - if not path.exists(): - return {"error": f"File not found: {filepath}"} - - content = path.read_text() - lines = content.splitlines() - words = content.split() - - word_counts = Counter(w.lower() for w in words if w.isalnum()) - top_words = word_counts.most_common(5) - - return { - "path": filepath, - "bytes": len(path.read_bytes()), - "lines": len(lines), - "words": len(words), - "unique_words": len(word_counts), - "top_5_words": top_words - } - -def main(): - if len(sys.argv) < 2: - print("Usage: quick_stats.py [file2...]") - sys.exit(1) - - for filepath in sys.argv[1:]: - stats = quick_stats(filepath) - if "error" in stats: - print(f"ERROR: {stats['error']}") - continue - - print(f"\n=== {stats['path']} ===") - print(f"Bytes: {stats['bytes']:,}") - print(f"Lines: {stats['lines']:,}") - print(f"Words: {stats['words']:,}") - print(f"Unique: {stats['unique_words']:,}") - print(f"Top words: {stats['top_5_words']}") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/wizard-work/spark/test_line_counter.py b/wizard-work/spark/test_line_counter.py index 1c6a03f..1d81a71 100644 --- a/wizard-work/spark/test_line_counter.py +++ b/wizard-work/spark/test_line_counter.py @@ -1,52 +1,352 @@ #!/usr/bin/env python3 -"""Test script for line_counter.py utility.""" +"""The cell for wizard-work/spark/line_counter.py (Q125). +WHAT THIS PINS + Every cell runs the shipped tool as a subprocess and compares its whole + stdout, so a number that moves for any reason is a red cell, not a rounding + difference in a review. The two cells the queue item names by hand are + `blank_lines_are_lines` (a file with blank lines must read 4, not 2) and + `final_unterminated_fragment_is_a_record` (a file whose last row has no + newline must read lines 3 / records 4, not line count == record count). + + The last two cells are not tests of the tool, they are tests of *this file*: + each copies the module to a scratch directory, applies one named mutation + with an assertion that the edit landed, and re-runs this suite against the + copy expecting rc 1 and the expected cell among the failures. A suite that + survives those mutants is a suite that cannot fail, which is the defect the + queue item was opened over -- so the mutants are cells, not a one-off reading + in a report nobody re-runs. + +USAGE + python3 wizard-work/spark/test_line_counter.py run everything + python3 wizard-work/spark/test_line_counter.py --no-mutants inner run + LINE_COUNTER=/path/to/line_counter.py python3 test_line_counter.py + grade a different copy of the tool (how the mutation cells do it) + +EXIT CODES + 0 every cell passed + 1 at least one cell failed + 2 the suite could not grade: the module or a fixture it needs is missing +""" + +import json +import os import subprocess import sys -import os +import tempfile -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +HERE = os.path.dirname(os.path.abspath(__file__)) +TOOL = os.environ.get("LINE_COUNTER") or os.path.join(HERE, "line_counter.py") +SELF = os.path.abspath(__file__) -def run_test(cmd, expected, description): - """Run a test command and verify output.""" - result = subprocess.run(cmd, capture_output=True, text=True) - actual = result.stdout.strip() - - if actual == str(expected) and result.returncode == 0: - print(f"PASS: {description}") - return True - else: - print(f"FAIL: {description}") - print(f" Expected: {expected}, Got: {actual}") - return False +CELLS = [] -# Test CSV mode with known data -test_csv_pass = run_test( - ['python3', f'{BASE_DIR}/line_counter.py', f'{BASE_DIR}/test_data.csv', 'csv'], - 5, - "CSV count mode (5 items)" -) -# Test count mode with known lines -test_count_pass = run_test( - ['python3', f'{BASE_DIR}/line_counter.py', f'{BASE_DIR}/test_log.txt', 'count'], - 10, - "Line count mode (10 lines)" -) +def cell(fn): + CELLS.append(fn) + return fn -# Test error handling for non-existent file -result = subprocess.run( - ['python3', f'{BASE_DIR}/line_counter.py', f'{BASE_DIR}/nonexistent.txt', 'count'], - capture_output=True, - text=True -) -if result.returncode == 2: - print("PASS: Error handling (exit code 2 for missing file)") - error_pass = True -else: - print(f"FAIL: Error handling (expected exit code 2, got {result.returncode})") - error_pass = False -all_pass = test_csv_pass and test_count_pass and error_pass -print(f"\nAll tests: {'PASSED' if all_pass else 'FAILED'}") -sys.exit(0 if all_pass else 1) \ No newline at end of file +def expect(condition, message): + if not condition: + raise AssertionError(message) + + +def run(*args): + return subprocess.run( + [sys.executable, TOOL, *args], capture_output=True, text=True + ) + + +def exact(*args, expected): + result = run(*args) + expect( + result.returncode == 0, + f"exit {result.returncode}, stderr {result.stderr.strip()!r}", + ) + expect( + result.stdout == expected, + f"stdout {result.stdout!r} != expected {expected!r}", + ) + + +@cell +def blank_lines_are_lines(): + """`printf 'a\\n\\n\\nb\\n'` reads 4 lines and 4 records, not 2.""" + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "blank.py") + with open(path, "wb") as handle: + handle.write(b"a\n\n\nb\n") + exact( + path, + expected=( + "files: 1\nlines: 4\nrecords: 4\n" + "blank: 2\ncomment: 0\ncode: 2\n" + ), + ) + + +@cell +def final_unterminated_fragment_is_a_record(): + """Without a trailing newline, lines and records must differ by one.""" + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "noterm.py") + with open(path, "wb") as handle: + handle.write(b"a\n\n\nb") + exact( + path, + expected=( + "files: 1\nlines: 3\nrecords: 4\n" + "blank: 2\ncomment: 0\ncode: 2\n" + ), + ) + + +@cell +def committed_log_fixture(): + """test_log.txt: 10 rows of text, 9 newlines -- lines 9, records 10.""" + path = os.path.join(HERE, "test_log.txt") + expect(os.path.isfile(path), f"fixture missing: {path}") + exact( + path, + expected=( + "files: 1\nlines: 9\nrecords: 10\n" + "blank: -\ncomment: -\ncode: -\n" + ), + ) + + +@cell +def committed_csv_fixture_fields(): + """test_data.csv holds five comma-separated items.""" + path = os.path.join(HERE, "test_data.csv") + expect(os.path.isfile(path), f"fixture missing: {path}") + exact("--fields", path, expected="fields: 5\n") + + +@cell +def quoted_separator_is_data(): + """`"a,b",c` is 2 fields; str.split(',') reads 3.""" + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "quoted.csv") + with open(path, "w", encoding="utf-8") as handle: + handle.write('"a,b",c\n') + exact("--fields", path, expected="fields: 2\n") + expect( + len('"a,b",c\n'.split("\n")[0].split(",")) == 3, + "the contrast this cell rests on no longer holds", + ) + + +@cell +def directory_walk_sums_and_breaks_down(): + """Three files, two of them Python: sums and the lexical breakdown.""" + with tempfile.TemporaryDirectory() as tmp: + nested = os.path.join(tmp, "sub") + os.makedirs(nested) + with open(os.path.join(tmp, "a.py"), "wb") as handle: + handle.write(b"x = 1\n# c\n\n") + with open(os.path.join(nested, "b.txt"), "wb") as handle: + handle.write(b"hello\n") + with open(os.path.join(nested, "c.py"), "wb") as handle: + handle.write(b"y = 2") + exact( + tmp, + expected=( + "files: 3\nlines: 4\nrecords: 5\n" + "blank: 1\ncomment: 1\ncode: 2\n" + ), + ) + + +@cell +def non_python_target_prints_dashes(): + """A target with no .py file must print '-' rather than an uncounted 0.""" + with tempfile.TemporaryDirectory() as tmp: + with open(os.path.join(tmp, "only.txt"), "wb") as handle: + handle.write(b"one\ntwo\n") + exact( + tmp, + expected=( + "files: 1\nlines: 2\nrecords: 2\n" + "blank: -\ncomment: -\ncode: -\n" + ), + ) + + +@cell +def json_matches_the_text_report(): + """--json carries the same figures, nulls where the text prints '-'.""" + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "blank.py") + with open(path, "wb") as handle: + handle.write(b"a\n\n\nb\n") + result = run("--json", path) + expect(result.returncode == 0, f"--json exited {result.returncode}") + data = json.loads(result.stdout) + expect( + data + == { + "path": path, + "files": 1, + "lines": 4, + "records": 4, + "blank": 2, + "comment": 0, + "code": 2, + }, + f"--json payload differs: {data}", + ) + text = run(path) + expect( + text.stdout.splitlines() + == [ + "files: 1", + "lines: 4", + "records: 4", + "blank: 2", + "comment: 0", + "code: 2", + ], + f"text report differs: {text.stdout!r}", + ) + + +@cell +def missing_target_exits_two(): + """An unreadable target is rc 2 with the path named, never a partial total.""" + with tempfile.TemporaryDirectory() as tmp: + missing = os.path.join(tmp, "nope.txt") + result = run(missing) + expect(result.returncode == 2, f"exit {result.returncode}, want 2") + expect(result.stdout == "", f"stdout {result.stdout!r}, want empty") + expect(missing in result.stderr, f"path not named: {result.stderr!r}") + + +@cell +def usage_error_exits_one(): + """No target at all is a usage error, not an empty report.""" + result = run() + expect(result.returncode == 1, f"exit {result.returncode}, want 1") + expect("usage:" in result.stderr, f"no usage line: {result.stderr!r}") + + +@cell +def word_and_char_counts(): + """--words: three whitespace tokens with a repeat, and six characters.""" + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "words.txt") + with open(path, "w", encoding="utf-8") as handle: + handle.write("a b a\n") + exact("--words", path, expected="files: 1\nwords: 3\nchars: 6\n") + + +@cell +def fields_on_a_directory_exits_two(): + """--fields needs one record, so a directory is a refusal.""" + with tempfile.TemporaryDirectory() as tmp: + result = run("--fields", tmp) + expect(result.returncode == 2, f"exit {result.returncode}, want 2") + expect("--fields" in result.stderr, f"no reason: {result.stderr!r}") + + +MUTANTS = [ + ( + "count_returns_ten", + " lines = data.count(b\"\\n\")\n" + " records = lines + (1 if data and not data.endswith(b\"\\n\") else 0)\n" + " return lines, records", + " return 10, 10", + "blank_lines_are_lines", + ), + ( + "count_non_blank_lines_only", + " lines = data.count(b\"\\n\")\n" + " records = lines + (1 if data and not data.endswith(b\"\\n\") else 0)\n" + " return lines, records", + " nonblank = sum(1 for _r in data.split(b\"\\n\") if _r.strip())\n" + " return nonblank, nonblank", + "blank_lines_are_lines", + ), + ( + "records_equal_lines", + " records = lines + (1 if data and not data.endswith(b\"\\n\") else 0)", + " records = lines", + "final_unterminated_fragment_is_a_record", + ), + ( + "fields_by_str_split", + " return len(next(csv.reader(io.StringIO(first))))", + " return len(first.split(\",\"))", + "quoted_separator_is_data", + ), + ( + "words_by_unique", + " words += len(text.split())", + " words += len(set(text.split()))", + "word_and_char_counts", + ), +] + + +@cell +def every_mutant_is_killed(): + """Each named mutation must turn this suite red on the cell that names it.""" + if "--no-mutants" in sys.argv: + return + with open(TOOL, "r", encoding="utf-8") as handle: + original = handle.read() + for name, old, new, expected_cell in MUTANTS: + expect(old in original, f"mutant {name}: anchor not found in {TOOL}") + with tempfile.TemporaryDirectory() as tmp: + mutant = os.path.join(tmp, "line_counter.py") + with open(mutant, "w", encoding="utf-8") as handle: + handle.write(original.replace(old, new, 1)) + with open(mutant, "r", encoding="utf-8") as handle: + expect( + old not in handle.read(), + f"mutant {name}: the edit did not land", + ) + env = dict(os.environ, LINE_COUNTER=mutant) + inner = subprocess.run( + [sys.executable, SELF, "--no-mutants"], + capture_output=True, + text=True, + env=env, + ) + expect( + inner.returncode == 1, + f"mutant {name} survived: rc {inner.returncode}\n{inner.stdout}", + ) + expect( + f"FAIL: {expected_cell}" in inner.stdout, + f"mutant {name} was caught, but not by {expected_cell}:\n" + f"{inner.stdout}", + ) + print( + f" ({len(MUTANTS)} mutants applied to scratch copies, " + f"{len(MUTANTS)} killed)" + ) + + +def main(argv): + if not os.path.isfile(TOOL): + print(f"CANNOT GRADE: tool missing: {TOOL}", file=sys.stderr) + return 2 + failures = 0 + for fn in CELLS: + name = fn.__name__ + try: + fn() + except AssertionError as exc: + failures += 1 + print(f"FAIL: {name} -- {exc}") + else: + print(f"PASS: {name}") + print(f"\n{len(CELLS) - failures}/{len(CELLS)} cells passed") + print("ALL CELLS PASSED" if not failures else f"{failures} CELL(S) FAILED") + return 0 if not failures else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/wizard-work/spark/test_linecount.py b/wizard-work/spark/test_linecount.py deleted file mode 100644 index 76ace32..0000000 --- a/wizard-work/spark/test_linecount.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 -"""Test that linecount_pycount.py produces correct output.""" - -import subprocess -import sys - -def test_linecount(): - # Test on fibonacci.py which we know has 49 total lines - result = subprocess.run( - ['python3', 'linecount_pycount.py', 'fibonacci.py'], - capture_output=True, - text=True - ) - - print("Output:", result.stdout) - print("Exit code:", result.returncode) - - # Parse output - lines = result.stdout.strip().split('\n') - total_line = [l for l in lines if l.startswith('Total lines:')] - code_line = [l for l in lines if l.startswith('Python code lines:')] - - assert total_line, "Missing 'Total lines:' output" - assert code_line, "Missing 'Python code lines:' output" - - total = int(total_line[0].split(':')[1].strip()) - code = int(code_line[0].split(':')[1].strip()) - - # fibonacci.py has 49 lines, 40 code lines - assert total == 49, f"Expected 49 total lines, got {total}" - assert code == 40, f"Expected 40 code lines, got {code}" - - print("TEST PASSED: linecount_pycount.py correctly counts lines") - return 0 - -if __name__ == '__main__': - sys.exit(test_linecount()) \ No newline at end of file diff --git a/wizard-work/spark/test_text_stats.py b/wizard-work/spark/test_text_stats.py deleted file mode 100644 index 3b4d72c..0000000 --- a/wizard-work/spark/test_text_stats.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 -"""Simple test to verify text_stats.py output matches wc.""" -import subprocess -import sys - -# Test with this file itself -filepath = '/root/troupe-standup/wizard-work/spark/text_stats.py' - -# Run our script -result = subprocess.run(['python3', filepath, filepath], capture_output=True, text=True) -our_output = result.stdout.strip() - -# Run wc -wc_result = subprocess.run(['wc', filepath], capture_output=True, text=True) -wc_parts = wc_result.stdout.strip().split() -wc_lines, wc_words, wc_chars = wc_parts[0], wc_parts[1], wc_parts[2] - -# Extract our values -import re -match = re.search(r'(\d+) lines, (\d+) words, (\d+) chars', our_output) -if match: - our_lines, our_words, our_chars = match.groups() - if our_lines == wc_lines and our_words == wc_words and our_chars == wc_chars: - print(f"PASS: {our_output}") - print(f" wc: {wc_lines} {wc_words} {wc_chars}") - sys.exit(0) - else: - print(f"FAIL: {our_output}") - print(f" wc: {wc_lines} {wc_words} {wc_chars}") - sys.exit(1) -else: - print(f"FAIL: Could not parse output: {our_output}") - sys.exit(1) \ No newline at end of file diff --git a/wizard-work/spark/text_stats.py b/wizard-work/spark/text_stats.py deleted file mode 100644 index 35a0668..0000000 --- a/wizard-work/spark/text_stats.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 -""" -text_stats.py - Quick utility to count lines, words, and characters in text files. -Usage: python text_stats.py [file2] ... -""" -import sys - -def analyze_text(filepath): - """Return (lines, words, chars) for a text file.""" - try: - with open(filepath, 'r', encoding='utf-8', errors='replace') as f: - content = f.read() - lines = content.count('\n') + (1 if content and not content.endswith('\n') else 0) - lines = len(content.splitlines()) if content.strip() else 0 - words = len(content.split()) - chars = len(content) - return lines, words, chars - except Exception as e: - return None, None, None - -def main(): - if len(sys.argv) < 2: - print("Usage: python text_stats.py [file2] ...") - sys.exit(1) - - for filepath in sys.argv[1:]: - lines, words, chars = analyze_text(filepath) - if lines is None: - print(f"{filepath}: error reading file") - continue - print(f"{filepath}: {lines} lines, {words} words, {chars} chars") - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/wizard-work/spark/verify_line_counter.sh b/wizard-work/spark/verify_line_counter.sh index c964632..1534be1 100755 --- a/wizard-work/spark/verify_line_counter.sh +++ b/wizard-work/spark/verify_line_counter.sh @@ -1,16 +1,94 @@ #!/bin/bash -# Final verification test for line_counter.py -# Returns 0 if correct, non-zero if wrong +# verify_line_counter.sh -- shell-side verifier for wizard-work/spark/line_counter.py (Q125). +# +# Run from any directory: the tool and every fixture are resolved from this +# file's own directory, so `bash wizard-work/spark/verify_line_counter.sh` from +# the repo root and the same command from inside the lane both read the same +# tree (the pre-Q125 version pinned /root/.hermes/wizard-outputs/spark and a +# "Total lines: 5587" figure belonging to a tool that no longer exists). +# +# Deliberately independent of test_line_counter.py: this is a second +# implementation reading the same numbers in shell, so a cell that agrees with +# the tool by sharing its arithmetic shows up here. +# +# Exit 0 = every reading matched. Exit 1 = at least one moved (and the ones +# that moved are printed). There is no third state: a missing tool or fixture +# is a failure, never a skip. +set -u -OUTPUT=$(python3 /root/.hermes/wizard-outputs/spark/line_counter.py /root/.hermes/wizard-outputs/spark 2>&1) -EXPECTED_LINES="Total lines: 5587" +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +TOOL="$HERE/line_counter.py" -if echo "$OUTPUT" | grep -q "$EXPECTED_LINES"; then - echo "VERIFIED: Line count correct" - exit 0 -else - echo "FAILED: Line count incorrect" - echo "Expected: $EXPECTED_LINES" - echo "Got: $OUTPUT" +if [ ! -f "$TOOL" ]; then + echo "FAILED: tool missing: $TOOL" exit 1 -fi \ No newline at end of file +fi + +fails=0 + +want() { # label, expected, actual + if [ "$2" = "$3" ]; then + echo "ok: $1 = $2" + else + echo "FAILED: $1 -- expected '$2', got '$3'" + fails=$((fails + 1)) + fi +} + +key() { # key, output + printf '%s\n' "$2" | sed -n "s/^$1: //p" +} + +# --- the tracked fixtures: a 10-row log with 9 newlines, a 5-field CSV ------- +out=$(python3 "$TOOL" "$HERE/test_log.txt") +want "test_log.txt lines" 9 "$(key lines "$out")" +want "test_log.txt records" 10 "$(key records "$out")" +want "test_data.csv fields" 5 "$(key fields "$(python3 "$TOOL" --fields "$HERE/test_data.csv")")" + +# --- blank lines are lines, and a missing final newline is one more record --- +tmp=$(mktemp -d) +trap 'find "$tmp" -type f -delete; rmdir "$tmp" 2>/dev/null' EXIT + +printf 'a\n\n\nb\n' > "$tmp/blank.py" +out=$(python3 "$TOOL" "$tmp/blank.py") +want "blank-line fixture lines" 4 "$(key lines "$out")" +want "blank-line fixture records" 4 "$(key records "$out")" +want "blank-line fixture blank" 2 "$(key blank "$out")" + +printf 'a\n\n\nb' > "$tmp/noterm.txt" +out=$(python3 "$TOOL" "$tmp/noterm.txt") +want "unterminated file lines" 3 "$(key lines "$out")" +want "unterminated file records" 4 "$(key records "$out")" + +# --- a quoted separator is data, not a field boundary ---------------------- +printf '"a,b",c\n' > "$tmp/quoted.csv" +want "quoted CSV fields" 2 "$(key fields "$(python3 "$TOOL" --fields "$tmp/quoted.csv")")" + +# --- words and characters, and an undecodable byte is not a traceback ------ +printf 'a b a\n' > "$tmp/words.txt" +out=$(python3 "$TOOL" --words "$tmp/words.txt") +want "word fixture words" 3 "$(key words "$out")" +want "word fixture chars" 6 "$(key chars "$out")" + +printf 'caf\351\n' > "$tmp/latin1.txt" +out=$(python3 "$TOOL" --words "$tmp/latin1.txt" 2>&1) +rc=$? +want "latin-1 byte exits 0" 0 "$rc" +want "latin-1 byte words" 1 "$(key words "$out")" +want "latin-1 byte chars" 5 "$(key chars "$out")" + +# --- refusals: unreadable target is 2, no target is 1 ---------------------- +python3 "$TOOL" "$tmp/absent.txt" >/dev/null 2>&1 +want "missing target exit" 2 "$?" +python3 "$TOOL" >/dev/null 2>&1 +want "no target exit" 1 "$?" + +if [ "$fails" -ne 0 ]; then + echo "FAILED: $fails reading(s) moved" + exit 1 +fi + +echo "VERIFIED: line_counter.py reads lines (wc -l), records (lines + a final" +echo " unterminated fragment), CSV fields with quoted separators, and" +echo " words/chars without dying on an undecodable byte." +exit 0 diff --git a/wizard-work/spark/verify_lines.py b/wizard-work/spark/verify_lines.py deleted file mode 100644 index 5be4cf7..0000000 --- a/wizard-work/spark/verify_lines.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python3 -with open('/root/.hermes/wizard-outputs/spark/code_stats.py', 'rb') as f: - content = f.read() - print('File length:', len(content)) - print('Last byte:', content[-1], '(10 = newline, 32 = space)') - print('Ends with newline:', content[-1] == 10) - print('Line count method 1:', content.count(b'\n')) - print('Line count method 2:', len(content.split(b'\n'))) \ No newline at end of file diff --git a/wizard-work/spark/verify_lwc.py b/wizard-work/spark/verify_lwc.py deleted file mode 100644 index cc8ed2b..0000000 --- a/wizard-work/spark/verify_lwc.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 -"""Verification test for lwc_counter.py. - -Q62 leg (b): this file used to read its input from an untracked copy of the -fixture under the spark wizard-outputs directory, and to accept the counter's -output by substring ("Words: 24" is a substring of "Words: 240"). It now -resolves both the counter and the fixture from this directory, pins the -fixture's digest, and compares the whole output exactly. - -Checks are explicit rather than bare `assert`s, so they still grade under -`python3 -O` (the Q63 class). -""" - -import hashlib -import subprocess -import sys -from pathlib import Path - -HERE = Path(__file__).resolve().parent -COUNTER = HERE / "lwc_counter.py" -INPUT = HERE / "test_input.txt" - -# The tracked fixture, byte for byte: "The quick brown fox ... and clever." -EXPECTED_SHA256 = "3127d0f89b3e87760130ee60e68eb9d9bf213ee83019062119361f1529cfd777" -EXPECTED_STDOUT = "Lines: 1\nWords: 24\nChars: 125" - - -def _check(ok, message): - """Fail loudly, under -O as well.""" - if not ok: - print(f"FAIL: {message}") - sys.exit(1) - - -def main(): - _check(COUNTER.is_file(), f"counter is not a file: {COUNTER}") - _check(INPUT.is_file(), f"fixture is not a file: {INPUT}") - - # Both paths must resolve inside this directory. Catches a symlink to a - # host copy as well as a literal path elsewhere. - for path in (COUNTER, INPUT): - _check( - path.resolve().parent == HERE, - f"{path.name} resolves outside this directory: {path.resolve()}", - ) - - digest = hashlib.sha256(INPUT.read_bytes()).hexdigest() - _check( - digest == EXPECTED_SHA256, - f"fixture {INPUT.name} changed: sha256 {digest}, pinned {EXPECTED_SHA256}", - ) - - result = subprocess.run( - [sys.executable, str(COUNTER), str(INPUT)], - capture_output=True, - text=True, - ) - _check( - result.returncode == 0, - f"counter exited {result.returncode}: {result.stderr.strip()}", - ) - _check(result.stderr == "", f"counter wrote to stderr: {result.stderr!r}") - - got = result.stdout.rstrip("\n") - _check( - got == EXPECTED_STDOUT, - f"counts differ: expected {EXPECTED_STDOUT!r}, got {got!r}", - ) - - print("PASS: lwc_counter.py on the in-repo fixture: 1 line, 24 words, 125 chars") - print(f" input {INPUT} sha256 {EXPECTED_SHA256[:16]}...") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/wizard-work/spark/verify_word_freq.sh b/wizard-work/spark/verify_word_freq.sh deleted file mode 100755 index ddfc2cb..0000000 --- a/wizard-work/spark/verify_word_freq.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -# Verify word_freq.py artifact -set -e - -cd /root/.hermes/wizard-outputs/spark - -echo "=== Word Frequency Counter Verification ===" - -# Check Python script exists -if [ -f "word_freq.py" ]; then - echo "[PASS] word_freq.py exists" -else - echo "[FAIL] word_freq.py missing" - exit 1 -fi - -# Check test input exists -if [ -f "test_words.txt" ]; then - echo "[PASS] test_words.txt exists" -else - echo "[FAIL] test_words.txt missing" - exit 1 -fi - -# Check expected output exists -if [ -f "word_freq_output.json" ]; then - echo "[PASS] word_freq_output.json exists" -else - echo "[FAIL] word_freq_output.json missing" - exit 1 -fi - -# Run tool and compare JSON output -python3 word_freq.py test_words.txt > temp_actual.json -if python3 validate_json_match.py; then - echo "[PASS] Tool output matches expected" -else - echo "[FAIL] Tool output mismatch" - cat temp_actual.json - exit 1 -fi - -echo "" -echo "=== VERIFICATION PASSED ===" - -rm -f temp_actual.json \ No newline at end of file diff --git a/wizard-work/spark/verify_wordcount.py b/wizard-work/spark/verify_wordcount.py deleted file mode 100644 index c4b7e15..0000000 --- a/wizard-work/spark/verify_wordcount.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 -"""Verify wordcount.py output matches wc command.""" -import json -import subprocess -import sys -from pathlib import Path - -def verify_wordcount(filepath): - """Run wordcount.py and verify against wc.""" - # Run our script - result = subprocess.run( - ["python3", "/root/.hermes/wizard-outputs/spark/wordcount.py", filepath], - capture_output=True, text=True - ) - if result.returncode != 0: - return {"valid": False, "error": f"wordcount.py failed: {result.stderr}"} - - our_stats = json.loads(result.stdout) - - # Run wc for comparison - wc_result = subprocess.run( - ["wc", filepath], - capture_output=True, text=True - ) - if wc_result.returncode != 0: - return {"valid": False, "error": f"wc failed: {wc_result.stderr}"} - - wc_parts = wc_result.stdout.strip().split() - wc_lines, wc_words, wc_bytes = int(wc_parts[0]), int(wc_parts[1]), int(wc_parts[2]) - - # Verify - issues = [] - if our_stats["words"] != wc_words: - issues.append(f"words mismatch: {our_stats['words']} vs wc {wc_words}") - if our_stats["bytes"] != wc_bytes: - issues.append(f"bytes mismatch: {our_stats['bytes']} vs wc {wc_bytes}") - - return { - "valid": len(issues) == 0, - "issues": issues if issues else None, - "stats": our_stats, - "wc_comparison": {"lines": wc_lines, "words": wc_words, "bytes": wc_bytes} - } - -if __name__ == "__main__": - filepath = sys.argv[1] if len(sys.argv) > 1 else "/root/.hermes/wizard-outputs/alexander-reading.txt" - result = verify_wordcount(filepath) - print(json.dumps(result, indent=2)) - sys.exit(0 if result["valid"] else 1) \ No newline at end of file diff --git a/wizard-work/spark/word-counter.py b/wizard-work/spark/word-counter.py deleted file mode 100755 index 5cabee9..0000000 --- a/wizard-work/spark/word-counter.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python3 -"""Simple word counter - counts words in a text file.""" -import sys - -def count_words(filepath): - """Count words in a file and return the total.""" - with open(filepath, 'r') as f: - text = f.read() - return len(text.split()) - -if __name__ == '__main__': - if len(sys.argv) != 2: - print("Usage: word-counter.py ") - sys.exit(1) - print(count_words(sys.argv[1])) \ No newline at end of file diff --git a/wizard-work/spark/word_counter.py b/wizard-work/spark/word_counter.py deleted file mode 100755 index 7a3bbdd..0000000 --- a/wizard-work/spark/word_counter.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python3 -"""Word counter utility - counts words in a file or from stdin.""" - -import sys -from pathlib import Path - - -def count_words(text: str) -> int: - """Count words in text (split on whitespace).""" - return len(text.split()) - - -def main(): - if len(sys.argv) > 1: - # Count words in file - path = Path(sys.argv[1]) - if not path.exists(): - print(f"Error: File not found: {sys.argv[1]}", file=sys.stderr) - sys.exit(1) - text = path.read_text() - else: - # Count words from stdin - text = sys.stdin.read() - - count = count_words(text) - print(f"Word count: {count}") - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/wizard-work/spark/word_counter_simple.py b/wizard-work/spark/word_counter_simple.py deleted file mode 100644 index 2c45d58..0000000 --- a/wizard-work/spark/word_counter_simple.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python3 -"""Simple word counter - counts words in a string.""" - -def count_words(text): - """Return the number of whitespace-separated words in text.""" - if not text: - return 0 - return len(text.split()) - -if __name__ == "__main__": - import sys - text = " ".join(sys.argv[1:]) if len(sys.argv) > 1 else "hello world from spark" - print(count_words(text)) \ No newline at end of file diff --git a/wizard-work/spark/word_freq.py b/wizard-work/spark/word_freq.py deleted file mode 100644 index 228c65f..0000000 --- a/wizard-work/spark/word_freq.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -""" -word_freq.py - Count word frequencies in text files. -Outputs JSON with word counts sorted by frequency. -""" - -import json -import sys -import re -from pathlib import Path -from collections import Counter - - -def count_words(text): - """Extract and count words from text.""" - words = re.findall(r'\b[a-zA-Z]{2,}\b', text.lower()) - return Counter(words) - - -def process_file(filepath): - """Process a single file and return word counts.""" - with open(filepath, 'r') as f: - text = f.read() - return count_words(text) - - -def main(): - args = sys.argv[1:] - - if not args or args[0] in ('-h', '--help'): - print("Usage: python3 word_freq.py [file2 ...]") - sys.exit(0) - - all_counts = Counter() - - for filepath in args: - p = Path(filepath) - if not p.exists(): - print(f"ERROR: File not found: {filepath}", file=sys.stderr) - sys.exit(1) - all_counts.update(process_file(filepath)) - - # Get top 10 most common words - top_words = all_counts.most_common(10) - - result = { - "total_files": len(args), - "total_words": sum(all_counts.values()), - "unique_words": len(all_counts), - "top_words": [{"word": w, "count": c} for w, c in top_words] - } - - print(json.dumps(result, indent=2)) - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/wizard-work/spark/word_stats.py b/wizard-work/spark/word_stats.py deleted file mode 100644 index 0bcaafd..0000000 --- a/wizard-work/spark/word_stats.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 -"""Quick word statistics analyzer - counts words, chars, lines in text files.""" - -import sys -from collections import Counter - -def analyze_text(text): - words = text.split() - chars = len(text) - lines = text.count('\n') + 1 if text else 0 - word_count = len(words) - char_count = len(text) - unique_words = len(set(w.lower().strip('.,!?;:') for w in words)) - most_common = Counter(w.lower().strip('.,!?;:') for w in words).most_common(5) - return { - 'lines': lines, - 'words': word_count, - 'characters': char_count, - 'unique_words': unique_words, - 'most_common': most_common - } - -def main(): - if len(sys.argv) < 2: - print("Usage: python word_stats.py ") - sys.exit(1) - - with open(sys.argv[1], 'r') as f: - text = f.read() - - stats = analyze_text(text) - print(f"Lines: {stats['lines']}") - print(f"Words: {stats['words']}") - print(f"Characters: {stats['characters']}") - print(f"Unique words: {stats['unique_words']}") - print(f"Most common: {stats['most_common']}") - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/wizard-work/spark/wordcount.py b/wizard-work/spark/wordcount.py deleted file mode 100644 index 749def1..0000000 --- a/wizard-work/spark/wordcount.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 -"""Word count utility - counts words in a file and outputs JSON result.""" -import json -import sys -from pathlib import Path - -def word_count(filepath): - """Count words in file and return stats.""" - path = Path(filepath) - if not path.exists(): - return {"error": f"File not found: {filepath}"} - - content = path.read_text() - words = content.split() - lines = content.splitlines() - - return { - "file": str(path), - "bytes": path.stat().st_size, - "lines": len(lines), - "words": len(words), - "characters": len(content) - } - -if __name__ == "__main__": - if len(sys.argv) < 2: - print(json.dumps({"error": "Usage: wordcount.py "})) - sys.exit(1) - - result = word_count(sys.argv[1]) - print(json.dumps(result, indent=2)) \ No newline at end of file diff --git a/wizard-work/spark/wordfreq.py b/wizard-work/spark/wordfreq.py deleted file mode 100644 index 9f8edee..0000000 --- a/wizard-work/spark/wordfreq.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -"""Word frequency counter - counts and sorts unique words in text files.""" -import sys -from collections import Counter -import re - -def count_words(text): - """Extract words and count frequencies.""" - words = re.findall(r'\b[a-zA-Z]+\'?[a-zA-Z]*\b', text.lower()) - return Counter(words) - -def main(): - if len(sys.argv) < 2: - print("Usage: wordfreq.py ", file=sys.stderr) - sys.exit(1) - - with open(sys.argv[1], 'r') as f: - text = f.read() - - counts = count_words(text) - - for word, count in sorted(counts.items(), key=lambda x: (-x[1], x[0])): - print(f"{count:6d} {word}") - -if __name__ == '__main__': - main() \ No newline at end of file