troupe-standup/wizard-work/spark/line_counter.py

#!/usr/bin/env python3
"""line_counter.py -- the spark lane's one text-statistics tool.

WHY THIS EXISTS (Q125, 2026-10-28)
  This lane accumulated eleven counters that each took a path, counted something
  textual and printed a number, and none of which named another: line_counter.py,
  count_python.py, count_python_lines.py, py_loc.py, lwc_counter.py,
  linecount_pycount.py, text_stats.py, word_stats.py, file_stats.py,
  code_stats.py, loc_counter.py.  Their line definitions disagreed -- some
  counted newline characters, some counted readlines(), some splitlines(), some
  dropped blank lines -- so two of them given the same directory printed
  different totals and neither said which question it had answered.  The ten
  others are retired; their last revisions are in git and the queue record at
  Q125 pairs each path with the commit that removed it.  What survives is this
  file, and it prints the definition before it prints the number.

COUNTING DEFINITION -- stated here, in the README, and in the output's own key
names, because the old tools differed only in this and the difference was
invisible.
  lines     Newline-terminated lines: the number of b"\\n" bytes in the file.
            Identical to `wc -l`.  A file whose last row has no trailing
            newline therefore has one fewer *line* than it has rows of text.
  records   lines plus one final unterminated fragment:
                records = lines + (1 if the file is non-empty and its last
                                     byte is not b"\\n" else 0)
            The two figures differ by exactly the files that lack a trailing
            newline.  At `7359bce`, over the 113 .py files then in this
            directory, that was 5,475 lines against 5,587 records -- 112 files
            without a final newline.
  blank / comment / code
            Python sources (.py) only, and lexically, not by grammar:
            blank is a line holding nothing but whitespace; comment is a line
            whose first non-space character is '#'; code is every other line,
            so a docstring interior and a backslash continuation count as code.
            A target contributing no .py file prints "-" for all three (null in
            --json) rather than a zero it did not count.
  fields    With --fields, the first record of the target is parsed as CSV with
            the standard csv module, so a quoted separator is data:
            `"a,b",c` is 2 fields, where str.split(',') reads 3.  This is the
            only place a comma count is meaningful; the tool no longer answers
            "how many commas are on line 1".
  words / chars
            With --words: words is len(text.split()) -- whitespace-separated
            tokens -- and chars is len(text) in characters of the decoded text,
            not bytes (a byte count is `wc -c`, a different question).  Bytes
            are decoded UTF-8 with errors replaced and a leading BOM stripped,
            so a latin-1 log reads as replacement characters instead of dying
            with a UnicodeDecodeError, which is what the tool this replaces did
            on the pipe's own measurement.

WHAT IT WILL NOT DO
  It never reports a partial total.  A file inside a directory walk that it
  cannot read stops the run with exit 2 and names the path, rather than summing
  the rest and printing a number that silently omits a file.

USAGE
  python3 line_counter.py [--json] PATH        lines/records, + Python breakdown
  python3 line_counter.py --fields PATH        CSV fields in PATH's first record
  python3 line_counter.py --words PATH         words and characters
  python3 line_counter.py --json ...           the same report as JSON

EXIT CODES
  0  the report was produced
  1  usage error (no target, two targets, unknown flag, two modes at once)
  2  the target does not exist, is not the kind of path the mode needs, or holds
     a file that cannot be read
"""

import csv
import io
import json
import os
import sys

USAGE = (
    "usage: line_counter.py [--json] PATH\n"
    "       line_counter.py [--json] --fields PATH\n"
    "\n"
    "  PATH         a file or a directory (directories are walked recursively)\n"
    "  --fields     count fields in PATH's first record with CSV quoting;\n"
    "               `\"a,b\",c` is 2 fields, not 3\n"
    "  --json       emit the report as JSON\n"
    "\n"
    "  keys: files, lines (wc -l), records (lines + a final unterminated\n"
    "        fragment), blank/comment/code (.py only, lexical; \"-\" if none)\n"
    "  exit: 0 report produced, 1 usage error, 2 unreadable target\n"
)


class ToolError(Exception):
    """A condition the caller must see as exit 2, not as a traceback."""


def count_text(data):
    """Return (lines, records) for one file's bytes.

    lines   -- newline-terminated lines, exactly `wc -l`.
    records -- lines plus a final unterminated fragment.
    """
    lines = data.count(b"\n")
    records = lines + (1 if data and not data.endswith(b"\n") else 0)
    return lines, records


def breakdown(text):
    """Return (blank, comment, code) for one Python source's text, lexically.

    A line whose first non-space character is '#' is a comment; a line holding
    only whitespace is blank; everything else is code, including docstring
    interiors and continuations.
    """
    blank = comment = code = 0
    rows = text.split("\n")
    if text.endswith("\n"):
        rows.pop()
    for row in rows:
        stripped = row.strip()
        if not stripped:
            blank += 1
        elif stripped.startswith("#"):
            comment += 1
        else:
            code += 1
    return blank, comment, code


def _read_bytes(path):
    try:
        with open(path, "rb") as handle:
            return handle.read()
    except OSError as exc:
        raise ToolError(f"cannot read '{path}': {exc.strerror or exc}")


def _walk(path):
    """Return every regular file under `path`, in a stable order."""
    found = []
    for root, dirs, files in os.walk(path):
        dirs.sort()
        for name in sorted(files):
            full = os.path.join(root, name)
            if os.path.isfile(full):
                found.append(full)
    return found


def report(path):
    """Return the report dict for a file or directory target."""
    if os.path.isdir(path):
        targets = _walk(path)
    elif os.path.isfile(path):
        targets = [path]
    else:
        raise ToolError(f"no such file or directory: '{path}'")

    total_files = 0
    total_lines = 0
    total_records = 0
    blank = comment = code = 0
    py_files = 0

    for full in targets:
        data = _read_bytes(full)
        lines, records = count_text(data)
        total_files += 1
        total_lines += lines
        total_records += records
        if full.endswith(".py"):
            text = data.decode("utf-8", "replace")
            file_blank, file_comment, file_code = breakdown(text)
            blank += file_blank
            comment += file_comment
            code += file_code
            py_files += 1

    return {
        "path": path,
        "files": total_files,
        "lines": total_lines,
        "records": total_records,
        "blank": blank if py_files else None,
        "comment": comment if py_files else None,
        "code": code if py_files else None,
    }


def count_fields(path):
    """Return the field count of the first CSV record in `path`."""
    if os.path.isdir(path):
        raise ToolError(f"--fields needs a file, not a directory: '{path}'")
    if not os.path.isfile(path):
        raise ToolError(f"no such file: '{path}'")
    data = _read_bytes(path)
    text = data.decode("utf-8-sig", "replace")
    if not text.strip():
        return 0
    first = text.split("\n", 1)[0]
    return len(next(csv.reader(io.StringIO(first))))


def word_report(path):
    """Return (files, words, chars) for a file or a directory target."""
    if os.path.isdir(path):
        targets = _walk(path)
    elif os.path.isfile(path):
        targets = [path]
    else:
        raise ToolError(f"no such file or directory: '{path}'")

    words = 0
    chars = 0
    for full in targets:
        text = _read_bytes(full).decode("utf-8-sig", "replace")
        words += len(text.split())
        chars += len(text)
    return {"path": path, "files": len(targets), "words": words, "chars": chars}


ORDER = ("files", "lines", "records", "words", "chars",
         "blank", "comment", "code", "fields")


def render(result, as_json):
    """Render a report dict as text or JSON, with the definition in the keys."""
    if as_json:
        return json.dumps(result, sort_keys=True) + "\n"
    lines = []
    for key in ORDER:
        if key in result:
            value = result[key]
            lines.append(f"{key}: {'-' if value is None else value}")
    return "\n".join(lines) + "\n"


def main(argv):
    args = list(argv[1:])
    as_json = False
    mode = None
    positional = []
    for arg in args:
        if arg == "--json":
            as_json = True
        elif arg in ("--fields", "--words"):
            if mode is not None:
                sys.stderr.write(f"--{mode} and {arg} are different questions\n\n")
                sys.stderr.write(USAGE)
                return 1
            mode = arg[2:]
        elif arg in ("-h", "--help"):
            sys.stdout.write(USAGE)
            return 0
        elif arg.startswith("-") and arg != "-":
            sys.stderr.write(f"unknown option: {arg}\n\n{USAGE}")
            return 1
        else:
            positional.append(arg)

    if len(positional) != 1:
        sys.stderr.write(USAGE)
        return 1

    path = positional[0]
    try:
        if mode == "fields":
            result = {"path": path, "fields": count_fields(path)}
        elif mode == "words":
            result = word_report(path)
        else:
            result = report(path)
    except ToolError as exc:
        sys.stderr.write(f"ERROR: {exc}\n")
        return 2

    sys.stdout.write(render(result, as_json))
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))