hermes-agent/skills/research/research-paper-writing/scripts/scaffold_paper.py

#!/usr/bin/env python3
"""
Research Paper Project Scaffolder.

Creates a ready-to-write research paper workspace with directory structure,
LaTeX templates, experiment tracking files, and git initialization.

Companion tool for the /research-paper-writing skill.

Usage:
    python3 SKILL_DIR/scripts/scaffold_paper.py init my-paper --venue neurips2025 --title "My Paper"
    python3 SKILL_DIR/scripts/scaffold_paper.py venues
    python3 SKILL_DIR/scripts/scaffold_paper.py status my-paper

Pure Python stdlib — no external dependencies.
"""

import argparse
import json
import os
import shutil
import subprocess
import sys
from datetime import datetime, timezone
from pathlib import Path

# ── Venue definitions ─────────────────────────────────────────────

VENUES = {
    "neurips2025": {
        "name": "NeurIPS 2025",
        "pages": 9,
        "format": "Double-blind",
        "checklist": "Paper Checklist required (15 items)",
        "notes": "No Broader Impact section required since 2022. Supplementary unlimited.",
        "template_dir": "neurips2025",
    },
    "icml2026": {
        "name": "ICML 2026",
        "pages": 8,
        "format": "Double-blind",
        "checklist": "Broader Impact statement required",
        "notes": "Strict 8-page limit for main text. Appendix allowed.",
        "template_dir": "icml2026",
    },
    "iclr2026": {
        "name": "ICLR 2026",
        "pages": 9,
        "format": "Double-blind",
        "checklist": "LLM disclosure required, reciprocal reviewing agreement",
        "notes": "arXiv posting allowed before submission.",
        "template_dir": "iclr2026",
    },
    "acl": {
        "name": "ACL (long paper)",
        "pages": 8,
        "format": "Double-blind",
        "checklist": "Mandatory Limitations section, Responsible NLP checklist",
        "notes": "Short papers: 4 pages. Findings track available.",
        "template_dir": "acl",
    },
    "aaai2026": {
        "name": "AAAI 2026",
        "pages": 7,
        "format": "Double-blind",
        "checklist": "Strict style file - no modifications whatsoever",
        "notes": "7-page limit is tight. Plan content carefully.",
        "template_dir": "aaai2026",
    },
    "colm2025": {
        "name": "COLM 2025",
        "pages": 9,
        "format": "Double-blind",
        "checklist": "Frame contribution for language model community",
        "notes": "Newer venue focused on language models.",
        "template_dir": "colm2025",
    },
}

# ── Directory structure ───────────────────────────────────────────

DIRS = [
    "paper",
    "experiments",
    "code",
    "results",
    "tasks",
    "human_eval",
    "context",
    "figures",
    "logs",
]

# ── Template content ──────────────────────────────────────────────


def scope_md(title: str, venue_key: str) -> str:
    v = VENUES.get(venue_key, {})
    venue_name = v.get("name", venue_key)
    pages = v.get("pages", "?")
    return f"""# Research Paper Scope

## The What (one-sentence contribution)
[What is the single thing this paper contributes?]

## The Why (evidence)
[What evidence supports it?]

## The So What (significance)
[Why should readers care?]

## Title
{title}

## Target Venue
{venue_name} ({pages} pages, deadline: TBD)

## Paper Type
- [ ] Empirical (IMRaD)
- [ ] Survey / Literature Review
- [ ] Position Paper
- [ ] Technical Report
- [ ] Theory
- [ ] Benchmark

## Authors & Roles
| Author | Sections Owned | Role |
|--------|---------------|------|
| | | Lead |

## Notation Conventions
- Vectors: lowercase bold (\\mathbf{{x}})
- Matrices: uppercase bold (\\mathbf{{A}})
- Citation style: \\citet{{}} for narrative, \\citep{{}} for parenthetical
- British / American spelling: [choose one]
"""


EXPERIMENT_LOG_MD = """# Experiment Log

## Contribution (one sentence)
[The paper's main claim]

## Experiments Run

### Experiment 1: [Name]
- **Claim tested**: [Which paper claim this supports]
- **Setup**: [Model, dataset, config, number of runs]
- **Key result**: [One sentence with the number]
- **Result files**: results/exp1/
- **Figures generated**: figures/exp1_*.pdf
- **Surprising findings**: [Anything unexpected]

## Figures
| Filename | Description | Section |
|----------|-------------|---------|
| | | |

## Failed Experiments (document for honesty)
- [What was tried, why it failed, what it tells us]

## Open Questions
- [Anything the results raised that the paper should address]
"""


TODO_MD = """# Research Paper TODO

## Phase 0: Setup
- [x] Scaffold project workspace
- [ ] Define one-sentence contribution (scope.md)
- [ ] Agree on notation conventions

## Phase 1: Literature Review
- [ ] Identify seed papers from existing references
- [ ] Search for related work (3 rounds: breadth, depth, targeted)
- [ ] Verify every citation programmatically
- [ ] Organize related work by methodology

## Phase 2: Experiment Design
- [ ] Map claims to experiments (claim-experiment table)
- [ ] Design baselines (naive, strong, ablation, compute-matched)
- [ ] Define evaluation protocol (metrics, aggregation, tests, sample sizes)
- [ ] Estimate compute budget

## Phase 3: Execution
- [ ] Write experiment scripts
- [ ] Run experiments
- [ ] Monitor and handle failures
- [ ] Commit results with descriptive messages

## Phase 4: Analysis
- [ ] Aggregate results
- [ ] Compute statistical significance
- [ ] Identify the story (main finding, surprises, failures)
- [ ] Create figures and tables
- [ ] Write experiment_log.md (bridge to writeup)

## Phase 5: Drafting
- [ ] Write Abstract
- [ ] Write Introduction
- [ ] Write Related Work
- [ ] Write Methods
- [ ] Write Experiments / Results
- [ ] Write Discussion
- [ ] Write Conclusion

## Phase 6: Self-Review
- [ ] Simulate 3 reviewers (skeptical, supportive, methods-focused)
- [ ] Address major concerns
- [ ] Revision pass

## Phase 7: Submission
- [ ] Conference-specific formatting
- [ ] Venue checklist complete
- [ ] Final PDF compiles cleanly
- [ ] Supplementary materials ready
- [ ] Submit
"""


GITIGNORE = """# LaTeX auxiliary files
*.aux
*.bbl
*.blg
*.fdb_latexmk
*.fls
*.log
*.out
*.synctex.gz
*.toc
*.nav
*.snm
*.vrb
*.lot
*.lof
*.idx
*.ilg
*.ind
*.bak
*.sav

# Compiled output
*.pdf
!paper/*.pdf

# Python
__pycache__/
*.pyc
.venv/

# OS
.DS_Store
Thumbs.db

# Editor
*.swp
*.swo
*~
.idea/
.vscode/

# Large data (download, don't commit)
data/*.csv
data/*.json
data/*.parquet
checkpoints/
*.ckpt
*.pt
*.pth
"""


# ── Commands ──────────────────────────────────────────────────────


def cmd_init(args):
    """Initialize a new research paper project."""
    project = Path(args.project_dir)

    if project.exists() and any(project.iterdir()):
        print(f"Error: {project} already exists and is not empty.", file=sys.stderr)
        sys.exit(1)

    venue_key = args.venue
    if venue_key and venue_key not in VENUES:
        print(f"Error: Unknown venue '{venue_key}'. Run 'venues' to see options.", file=sys.stderr)
        sys.exit(1)

    title = args.title or "Untitled Paper"
    venue_key = venue_key or "neurips2025"

    print(f"Scaffolding project: {project}")
    print(f"  Venue: {VENUES[venue_key]['name']}")
    print(f"  Title: {title}")
    print()

    # Create directories
    for d in DIRS:
        (project / d).mkdir(parents=True, exist_ok=True)
        print(f"  mkdir {d}/")

    # Copy venue template if available
    skill_dir = Path(__file__).resolve().parent.parent
    template_src = skill_dir / "templates" / VENUES[venue_key]["template_dir"]
    if template_src.exists():
        template_dst = project / "paper"
        for item in template_src.iterdir():
            dst = template_dst / item.name
            if item.is_file():
                shutil.copy2(item, dst)
            elif item.is_dir():
                shutil.copytree(item, dst / item.name, dirs_exist_ok=True)
        print(f"  Copied {venue_key} template to paper/")
    else:
        print(f"  (No template found at {template_src} — create paper/ files manually)")

    # Write starter files
    _write(project / "scope.md", scope_md(title, venue_key))
    _write(project / "experiment_log.md", EXPERIMENT_LOG_MD)
    _write(project / "TODO.md", TODO_MD)
    _write(project / "experiment_journal.jsonl", "")
    _write(project / "results" / "cost_log.jsonl", "")
    _write(project / ".gitignore", GITIGNORE)

    # Write a minimal README
    readme = f"# {title}\n\nResearch paper project targeting {VENUES[venue_key]['name']}.\n\n"
    readme += "Scaffolded with `/research-paper-writing` skill.\n\n"
    readme += "## Quick Start\n\n"
    readme += "1. Fill in `scope.md` (contribution, evidence, significance)\n"
    readme += "2. Literature review (Phase 1)\n"
    readme += "3. Design experiments (Phase 2)\n"
    readme += "4. Run experiments, write results to `experiment_log.md` (Phases 3-4)\n"
    readme += "5. Draft paper in `paper/` (Phase 5)\n"
    readme += "6. Self-review and submit (Phases 6-7)\n\n"
    readme += "See `TODO.md` for detailed checklist.\n"
    _write(project / "README.md", readme)

    # Write cost tracker helper
    cost_tracker = '''"""
Cost tracker for research experiments.

Usage:
    from cost_tracker import log_cost, summarize_costs
    log_cost("exp1", "claude-sonnet-4-20250514", 1000, 500, 0.015)
    summarize_costs()
"""

import json
import os
from datetime import datetime

COST_LOG = os.path.join(os.path.dirname(__file__), "results", "cost_log.jsonl")


def log_cost(experiment: str, model: str, input_tokens: int,
             output_tokens: int, cost_usd: float):
    """Append a cost entry to the cost log."""
    entry = {
        "timestamp": datetime.now().isoformat(),
        "experiment": experiment,
        "model": model,
        "input_tokens": input_tokens,
        "output_tokens": output_tokens,
        "cost_usd": cost_usd,
    }
    with open(COST_LOG, "a") as f:
        f.write(json.dumps(entry) + "\\n")


def summarize_costs() -> dict:
    """Summarize total costs from the cost log."""
    if not os.path.exists(COST_LOG):
        return {"total_usd": 0, "entries": 0, "by_experiment": {}}

    total = 0.0
    by_exp = {}
    count = 0
    with open(COST_LOG) as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            entry = json.loads(line)
            cost = entry.get("cost_usd", 0)
            total += cost
            exp = entry.get("experiment", "unknown")
            by_exp[exp] = by_exp.get(exp, 0) + cost
            count += 1

    return {"total_usd": round(total, 4), "entries": count, "by_experiment": by_exp}


if __name__ == "__main__":
    summary = summarize_costs()
    print(json.dumps(summary, indent=2))
'''
    _write(project / "cost_tracker.py", cost_tracker)

    # Initialize git
    print()
    try:
        subprocess.run(["git", "init"], cwd=project, capture_output=True, check=True)
        subprocess.run(["git", "add", "-A"], cwd=project, capture_output=True, check=True)
        subprocess.run(
            ["git", "commit", "-m", f"Scaffold research paper project: {title}"],
            cwd=project,
            capture_output=True,
            check=True,
            env={**os.environ, "GIT_AUTHOR_NAME": "Hermes", "GIT_COMMITTER_NAME": "Hermes",
                 "GIT_AUTHOR_EMAIL": "hermes@local", "GIT_COMMITTER_EMAIL": "hermes@local"},
        )
        print("  git init + initial commit")
    except FileNotFoundError:
        print("  (git not found — skipping git init)")
    except subprocess.CalledProcessError as e:
        print(f"  (git init failed: {e})")

    print()
    print(f"Project ready at: {project.resolve()}")
    print(f"Next: fill in scope.md, then start Phase 1 (Literature Review)")


def cmd_venues(args):
    """List available conference templates."""
    if args.json:
        print(json.dumps(VENUES, indent=2))
        return

    print("Available conference templates:")
    print()
    print(f"  {'Venue':<20} {'Pages':<7} {'Format':<15} Notes")
    print(f"  {'─'*20} {'─'*7} {'─'*15} {'─'*40}")
    for key, v in VENUES.items():
        print(f"  {key:<20} {v['pages']:<7} {v['format']:<15} {v['checklist']}")
    print()
    print("Use: scaffold_paper.py init <project> --venue <venue>")


def cmd_status(args):
    """Check project health and phase progress."""
    project = Path(args.project_dir)

    if not project.exists():
        print(f"Error: {project} does not exist.", file=sys.stderr)
        sys.exit(1)

    if args.json:
        print(json.dumps(_status_dict(project), indent=2))
        return

    status = _status_dict(project)
    print(f"Project: {project.resolve()}")
    print()

    # Phase artifacts
    print("Phase Progress:")
    for phase, info in status["phases"].items():
        icon = "\u2705" if info["complete"] else "\u2b1c"
        print(f"  {icon} {phase}: {info['detail']}")

    # TODO summary
    print()
    todo = status["todo"]
    print(f"TODO: {todo['done']}/{todo['total']} tasks complete ({todo['pct']}%)")

    # Cost summary
    if status["costs"]["entries"] > 0:
        c = status["costs"]
        print(f"Costs: ${c['total_usd']:.2f} across {c['entries']} entries")
        for exp, cost in c["by_experiment"].items():
            print(f"  {exp}: ${cost:.4f}")

    # Git status
    print()
    git = status["git"]
    if git["initialized"]:
        print(f"Git: {git['commits']} commits, {'CLEAN' if git['clean'] else 'DIRTY (' + str(git['uncommitted']) + ' uncommitted)'}")
    else:
        print("Git: not initialized")

    # Results
    results = status["results"]
    if results["count"] > 0:
        print(f"Results: {results['count']} result files found")

    print()


def _status_dict(project: Path) -> dict:
    """Build a status dictionary for the project."""
    phases = {}

    # Phase 0: Setup
    scope_exists = (project / "scope.md").exists()
    scope_filled = False
    if scope_exists:
        content = (project / "scope.md").read_text()
        scope_filled = "[What is the single thing" not in content
    phases["Phase 0: Setup"] = {
        "complete": scope_filled,
        "detail": "scope.md filled" if scope_filled else ("scope.md exists (template)" if scope_exists else "scope.md missing"),
    }

    # Phase 1: Literature
    bib_files = list(project.rglob("*.bib"))
    phases["Phase 1: Literature"] = {
        "complete": len(bib_files) > 0,
        "detail": f"{len(bib_files)} .bib file(s)" if bib_files else "No .bib files yet",
    }

    # Phase 2-3: Experiments
    exp_scripts = list((project / "experiments").glob("*.py")) if (project / "experiments").exists() else []
    phases["Phase 2-3: Experiments"] = {
        "complete": len(exp_scripts) > 0,
        "detail": f"{len(exp_scripts)} experiment script(s)" if exp_scripts else "No experiment scripts",
    }

    # Phase 4: Analysis
    exp_log = project / "experiment_log.md"
    log_filled = False
    if exp_log.exists():
        content = exp_log.read_text()
        log_filled = "Experiment 1: [Name]" not in content and len(content.strip()) > 200
    phases["Phase 4: Analysis"] = {
        "complete": log_filled,
        "detail": "experiment_log.md populated" if log_filled else "experiment_log.md is template",
    }

    # Phase 5: Drafting
    tex_files = list((project / "paper").rglob("*.tex")) if (project / "paper").exists() else []
    has_content = False
    for tf in tex_files:
        content = tf.read_text()
        if "\\begin{abstract}" in content and len(content) > 2000:
            has_content = True
            break
    phases["Phase 5: Drafting"] = {
        "complete": has_content,
        "detail": f"{len(tex_files)} .tex file(s), {'content detected' if has_content else 'template only'}",
    }

    # Phase 6-7: Review & Submit
    pdf_files = list((project / "paper").glob("*.pdf")) if (project / "paper").exists() else []
    phases["Phase 6-7: Review & Submit"] = {
        "complete": len(pdf_files) > 0,
        "detail": f"{len(pdf_files)} PDF(s) in paper/" if pdf_files else "No compiled PDFs",
    }

    # TODO parsing
    todo_info = {"done": 0, "total": 0, "pct": 0}
    todo_file = project / "TODO.md"
    if todo_file.exists():
        content = todo_file.read_text()
        done = content.count("- [x]")
        undone = content.count("- [ ]")
        total = done + undone
        todo_info = {"done": done, "total": total, "pct": round(done / total * 100) if total > 0 else 0}

    # Cost summary
    cost_info = {"total_usd": 0, "entries": 0, "by_experiment": {}}
    cost_file = project / "results" / "cost_log.jsonl"
    if cost_file.exists():
        total = 0.0
        by_exp = {}
        count = 0
        for line in cost_file.read_text().splitlines():
            if not line.strip():
                continue
            try:
                entry = json.loads(line)
                cost = entry.get("cost_usd", 0)
                total += cost
                exp = entry.get("experiment", "unknown")
                by_exp[exp] = by_exp.get(exp, 0) + cost
                count += 1
            except json.JSONDecodeError:
                continue
        cost_info = {"total_usd": round(total, 4), "entries": count, "by_experiment": by_exp}

    # Git
    git_info = {"initialized": False, "clean": True, "commits": 0, "uncommitted": 0}
    if (project / ".git").exists():
        git_info["initialized"] = True
        try:
            result = subprocess.run(
                ["git", "rev-list", "--count", "HEAD"],
                cwd=project, capture_output=True, text=True
            )
            if result.returncode == 0:
                git_info["commits"] = int(result.stdout.strip())
        except Exception:
            pass
        try:
            result = subprocess.run(
                ["git", "status", "--porcelain"],
                cwd=project, capture_output=True, text=True
            )
            if result.returncode == 0:
                changes = [l for l in result.stdout.splitlines() if l.strip()]
                git_info["uncommitted"] = len(changes)
                git_info["clean"] = len(changes) == 0
        except Exception:
            pass

    # Results
    result_files = list((project / "results").rglob("*.json")) if (project / "results").exists() else []
    # Exclude cost_log.jsonl
    result_files = [f for f in result_files if f.name != "cost_log.jsonl"]
    results_info = {"count": len(result_files)}

    return {
        "project": str(project.resolve()),
        "phases": phases,
        "todo": todo_info,
        "costs": cost_info,
        "git": git_info,
        "results": results_info,
    }


# ── Helpers ───────────────────────────────────────────────────────


def _write(path: Path, content: str):
    """Write content to a file, creating parent dirs."""
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(content)
    print(f"  write {path.relative_to(path.parent.parent) if len(path.parts) > 2 else path.name}")


# ── CLI ───────────────────────────────────────────────────────────

COMMAND_MAP = {
    "init": cmd_init,
    "venues": cmd_venues,
    "status": cmd_status,
}


def main():
    parser = argparse.ArgumentParser(
        prog="scaffold_paper",
        description="Research paper project scaffolder. Companion to /research-paper-writing skill.",
    )
    subparsers = parser.add_subparsers(dest="command", help="Command to run")

    # init
    p_init = subparsers.add_parser("init", help="Initialize a new paper project")
    p_init.add_argument("project_dir", help="Directory to create")
    p_init.add_argument("--venue", default="neurips2025",
                        choices=list(VENUES.keys()),
                        help="Target conference (default: neurips2025)")
    p_init.add_argument("--title", default=None, help="Paper title")

    # venues
    p_venues = subparsers.add_parser("venues", help="List available conference templates")
    p_venues.add_argument("--json", action="store_true", help="Output as JSON")

    # status
    p_status = subparsers.add_parser("status", help="Check project status")
    p_status.add_argument("project_dir", help="Project directory to check")
    p_status.add_argument("--json", action="store_true", help="Output as JSON")

    args = parser.parse_args()

    if not args.command:
        parser.print_help()
        sys.exit(1)

    COMMAND_MAP[args.command](args)


if __name__ == "__main__":
    main()