Compare commits
2 Commits
fix/679
...
fix/536-be
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4fd78ace44 | ||
|
|
b8b8bb65fd |
296
GENOME.md
296
GENOME.md
@@ -1,209 +1,141 @@
|
||||
# GENOME.md — the-nexus
|
||||
# GENOME.md — Timmy_Foundation/timmy-home
|
||||
|
||||
Generated by `pipelines/codebase_genome.py`.
|
||||
|
||||
## Project Overview
|
||||
|
||||
`the-nexus` is a hybrid repo that combines three layers in one codebase:
|
||||
Timmy Foundation's home repository for development operations and configurations.
|
||||
|
||||
1. A browser-facing world shell rooted in `index.html`, `boot.js`, `bootstrap.mjs`, `app.js`, `style.css`, `portals.json`, `vision.json`, `manifest.json`, and `gofai_worker.js`
|
||||
2. A Python realtime bridge centered on `server.py` plus harness code under `nexus/`
|
||||
3. A memory / fleet / operator layer spanning `mempalace/`, `mcp_servers/`, `multi_user_bridge.py`, and supporting scripts
|
||||
- Text files indexed: 3004
|
||||
- Source and script files: 186
|
||||
- Test files: 28
|
||||
- Documentation files: 701
|
||||
|
||||
The repo is not a clean single-purpose frontend and not just a backend harness. It is a mixed world/runtime/ops repository where browser rendering, WebSocket telemetry, MCP-driven game harnesses, and fleet memory tooling coexist.
|
||||
|
||||
Grounded repo facts from this checkout:
|
||||
- Browser shell files exist at repo root: `index.html`, `app.js`, `style.css`, `manifest.json`, `gofai_worker.js`
|
||||
- Data/config files also live at repo root: `portals.json`, `vision.json`
|
||||
- Realtime bridge exists in `server.py`
|
||||
- Game harnesses exist in `nexus/morrowind_harness.py` and `nexus/bannerlord_harness.py`
|
||||
- Memory/fleet sync exists in `mempalace/tunnel_sync.py`
|
||||
- Desktop/game automation MCP servers exist in `mcp_servers/desktop_control_server.py` and `mcp_servers/steam_info_server.py`
|
||||
- Validation exists in `tests/test_browser_smoke.py`, `tests/test_portals_json.py`, `tests/test_index_html_integrity.py`, and `tests/test_repo_truth.py`
|
||||
|
||||
The current architecture is best understood as a sovereign world shell plus operator/game harness backend, with accumulated documentation drift from multiple restoration and migration efforts.
|
||||
|
||||
## Architecture Diagram
|
||||
## Architecture
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
browser[Index HTML Shell\nindex.html -> boot.js -> bootstrap.mjs -> app.js]
|
||||
assets[Root Assets\nstyle.css\nmanifest.json\ngofai_worker.js]
|
||||
data[World Data\nportals.json\nvision.json]
|
||||
ws[Realtime Bridge\nserver.py\nWebSocket broadcast hub]
|
||||
gofai[In-browser GOFAI\nSymbolicEngine\nNeuroSymbolicBridge\nsetupGOFAI/updateGOFAI]
|
||||
harnesses[Python Harnesses\nnexus/morrowind_harness.py\nnexus/bannerlord_harness.py]
|
||||
mcp[MCP Adapters\nmcp_servers/desktop_control_server.py\nmcp_servers/steam_info_server.py]
|
||||
memory[Memory + Fleet\nmempalace/tunnel_sync.py\nmempalace.js]
|
||||
bridge[Operator / MUD Bridge\nmulti_user_bridge.py\ncommands/timmy_commands.py]
|
||||
tests[Verification\ntests/test_browser_smoke.py\ntests/test_portals_json.py\ntests/test_repo_truth.py]
|
||||
docs[Contracts + Drift Docs\nBROWSER_CONTRACT.md\nREADME.md\nCLAUDE.md\nINVESTIGATION_ISSUE_1145.md]
|
||||
|
||||
browser --> assets
|
||||
browser --> data
|
||||
browser --> gofai
|
||||
browser --> ws
|
||||
harnesses --> mcp
|
||||
harnesses --> ws
|
||||
bridge --> ws
|
||||
memory --> ws
|
||||
tests --> browser
|
||||
tests --> data
|
||||
tests --> docs
|
||||
docs --> browser
|
||||
repo_root["repo"]
|
||||
angband["angband"]
|
||||
briefings["briefings"]
|
||||
config["config"]
|
||||
conftest["conftest"]
|
||||
evennia["evennia"]
|
||||
evennia_tools["evennia_tools"]
|
||||
evolution["evolution"]
|
||||
gemini_fallback_setup["gemini-fallback-setup"]
|
||||
heartbeat["heartbeat"]
|
||||
infrastructure["infrastructure"]
|
||||
repo_root --> angband
|
||||
repo_root --> briefings
|
||||
repo_root --> config
|
||||
repo_root --> conftest
|
||||
repo_root --> evennia
|
||||
repo_root --> evennia_tools
|
||||
```
|
||||
|
||||
## Entry Points and Data Flow
|
||||
## Entry Points
|
||||
|
||||
### Primary entry points
|
||||
- `gemini-fallback-setup.sh` — operational script (`bash gemini-fallback-setup.sh`)
|
||||
- `morrowind/hud.sh` — operational script (`bash morrowind/hud.sh`)
|
||||
- `pipelines/codebase_genome.py` — python main guard (`python3 pipelines/codebase_genome.py`)
|
||||
- `scripts/auto_restart_agent.sh` — operational script (`bash scripts/auto_restart_agent.sh`)
|
||||
- `scripts/backup_pipeline.sh` — operational script (`bash scripts/backup_pipeline.sh`)
|
||||
- `scripts/big_brain_manager.py` — operational script (`python3 scripts/big_brain_manager.py`)
|
||||
- `scripts/big_brain_repo_audit.py` — operational script (`python3 scripts/big_brain_repo_audit.py`)
|
||||
- `scripts/codebase_genome_nightly.py` — operational script (`python3 scripts/codebase_genome_nightly.py`)
|
||||
- `scripts/detect_secrets.py` — operational script (`python3 scripts/detect_secrets.py`)
|
||||
- `scripts/dynamic_dispatch_optimizer.py` — operational script (`python3 scripts/dynamic_dispatch_optimizer.py`)
|
||||
- `scripts/emacs-fleet-bridge.py` — operational script (`python3 scripts/emacs-fleet-bridge.py`)
|
||||
- `scripts/emacs-fleet-poll.sh` — operational script (`bash scripts/emacs-fleet-poll.sh`)
|
||||
|
||||
- `index.html` — root browser entry point
|
||||
- `boot.js` — startup selector; `tests/boot.test.js` shows it chooses file-mode vs HTTP/module-mode and injects `bootstrap.mjs` when served over HTTP
|
||||
- `bootstrap.mjs` — module bootstrap for the browser shell
|
||||
- `app.js` — main browser runtime; owns world state, GOFAI wiring, metrics polling, and portal/UI logic
|
||||
- `server.py` — WebSocket broadcast bridge on `ws://0.0.0.0:8765`
|
||||
- `nexus/morrowind_harness.py` — GamePortal/MCP harness for OpenMW Morrowind
|
||||
- `nexus/bannerlord_harness.py` — GamePortal/MCP harness for Bannerlord
|
||||
- `mempalace/tunnel_sync.py` — pulls remote fleet closets into the local palace over HTTP
|
||||
- `multi_user_bridge.py` — HTTP bridge for multi-user chat/session integration
|
||||
- `mcp_servers/desktop_control_server.py` — stdio MCP server exposing screenshots/mouse/keyboard control
|
||||
## Data Flow
|
||||
|
||||
### Data flow
|
||||
|
||||
1. Browser startup begins at `index.html`
|
||||
2. `boot.js` decides whether the page is being served correctly; in HTTP mode it injects `bootstrap.mjs`
|
||||
3. `bootstrap.mjs` hands off to `app.js`
|
||||
4. `app.js` loads world configuration from `portals.json` and `vision.json`
|
||||
5. `app.js` constructs the Three.js scene and in-browser reasoning components, including `SymbolicEngine`, `NeuroSymbolicBridge`, `setupGOFAI()`, and `updateGOFAI()`
|
||||
6. Browser state and external runtimes connect through `server.py`, which broadcasts messages between connected clients
|
||||
7. Python harnesses (`nexus/morrowind_harness.py`, `nexus/bannerlord_harness.py`) spawn MCP subprocesses for desktop control / Steam metadata, capture state, execute actions, and feed telemetry into the Nexus bridge
|
||||
8. Memory/fleet tools like `mempalace/tunnel_sync.py` import remote palace data into local closets, extending what the operator/runtime layers can inspect
|
||||
9. Tests validate both the static browser contract and the higher-level repo-truth/memory contracts
|
||||
|
||||
### Important repo-specific runtime facts
|
||||
|
||||
- `portals.json` is a JSON array of portal/world/operator entries; examples in this checkout include `morrowind`, `bannerlord`, `workshop`, `archive`, `chapel`, and `courtyard`
|
||||
- `server.py` is a plain broadcast hub: clients send messages, the server forwards them to other connected clients
|
||||
- `nexus/morrowind_harness.py` and `nexus/bannerlord_harness.py` both implement a GamePortal pattern with MCP subprocess clients over stdio and WebSocket telemetry uplink
|
||||
- `mempalace/tunnel_sync.py` is not speculative; it is a real client that discovers remote wings, searches remote rooms, and writes `.closet.json` payloads locally
|
||||
1. Operators enter through `gemini-fallback-setup.sh`, `morrowind/hud.sh`, `pipelines/codebase_genome.py`.
|
||||
2. Core logic fans into top-level components: `angband`, `briefings`, `config`, `conftest`, `evennia`, `evennia_tools`.
|
||||
3. Validation is incomplete around `wizards/allegro/home/skills/red-teaming/godmode/scripts/auto_jailbreak.py`, `timmy-local/cache/agent_cache.py`, `wizards/allegro/home/skills/red-teaming/godmode/scripts/parseltongue.py`, so changes there carry regression risk.
|
||||
4. Final artifacts land as repository files, docs, or runtime side effects depending on the selected entry point.
|
||||
|
||||
## Key Abstractions
|
||||
|
||||
### Browser runtime
|
||||
|
||||
- `app.js`
|
||||
- Defines in-browser reasoning/state machinery, including `class SymbolicEngine`, `class NeuroSymbolicBridge`, `setupGOFAI()`, and `updateGOFAI()`
|
||||
- Couples rendering, local symbolic reasoning, metrics polling, and portal/UI logic in one very large root module
|
||||
- `BROWSER_CONTRACT.md`
|
||||
- Acts like an executable architecture contract for the browser surface
|
||||
- Declares required files, DOM IDs, Three.js expectations, provenance rules, and WebSocket expectations
|
||||
|
||||
### Realtime bridge
|
||||
|
||||
- `server.py`
|
||||
- Single hub abstraction: a WebSocket broadcast server maintaining a `clients` set and forwarding messages from one client to the others
|
||||
- This is the seam between browser shell, harnesses, and external telemetry producers
|
||||
|
||||
### GamePortal harness layer
|
||||
|
||||
- `nexus/morrowind_harness.py`
|
||||
- `nexus/bannerlord_harness.py`
|
||||
- Both define MCP client wrappers, `GameState` / `ActionResult`-style data classes, and an Observe-Decide-Act telemetry loop
|
||||
- The harnesses are symmetric enough to be understood as reusable portal adapters with game-specific context injected on top
|
||||
|
||||
### Memory / fleet layer
|
||||
|
||||
- `mempalace/tunnel_sync.py`
|
||||
- Encodes the fleet-memory sync client contract: discover wings, pull broad room queries, write closet files, support dry-run
|
||||
- `mempalace.js`
|
||||
- Minimal browser/Electron bridge to MemPalace commands via `window.electronAPI.execPython(...)`
|
||||
- Important because it shows a second memory integration surface distinct from the Python fleet sync path
|
||||
|
||||
### Operator / interaction bridge
|
||||
|
||||
- `multi_user_bridge.py`
|
||||
- `commands/timmy_commands.py`
|
||||
- These bridge user-facing conversations or MUD/Evennia interactions back into Timmy/Nexus services
|
||||
- `evennia/timmy_world/game.py` — classes `World`:91, `ActionSystem`:421, `TimmyAI`:539, `NPCAI`:550; functions `get_narrative_phase()`:55, `get_phase_transition_event()`:65
|
||||
- `evennia/timmy_world/world/game.py` — classes `World`:19, `ActionSystem`:326, `TimmyAI`:444, `NPCAI`:455; functions none detected
|
||||
- `timmy-world/game.py` — classes `World`:19, `ActionSystem`:349, `TimmyAI`:467, `NPCAI`:478; functions none detected
|
||||
- `wizards/allegro/home/skills/red-teaming/godmode/scripts/auto_jailbreak.py` — classes none detected; functions none detected
|
||||
- `uniwizard/self_grader.py` — classes `SessionGrade`:23, `WeeklyReport`:55, `SelfGrader`:74; functions `main()`:713
|
||||
- `uni-wizard/v3/intelligence_engine.py` — classes `ExecutionPattern`:27, `ModelPerformance`:44, `AdaptationEvent`:58, `PatternDatabase`:69; functions none detected
|
||||
- `scripts/know_thy_father/crossref_audit.py` — classes `ThemeCategory`:30, `Principle`:160, `MeaningKernel`:169, `CrossRefFinding`:178; functions `extract_themes_from_text()`:192, `parse_soul_md()`:206, `parse_kernels()`:264, `cross_reference()`:296, `generate_report()`:440, `main()`:561
|
||||
- `timmy-local/cache/agent_cache.py` — classes `CacheStats`:28, `LRUCache`:52, `ResponseCache`:94, `ToolCache`:205; functions none detected
|
||||
|
||||
## API Surface
|
||||
|
||||
### Browser / static surface
|
||||
- CLI: `bash gemini-fallback-setup.sh` — operational script (`gemini-fallback-setup.sh`)
|
||||
- CLI: `bash morrowind/hud.sh` — operational script (`morrowind/hud.sh`)
|
||||
- CLI: `python3 pipelines/codebase_genome.py` — python main guard (`pipelines/codebase_genome.py`)
|
||||
- CLI: `bash scripts/auto_restart_agent.sh` — operational script (`scripts/auto_restart_agent.sh`)
|
||||
- CLI: `bash scripts/backup_pipeline.sh` — operational script (`scripts/backup_pipeline.sh`)
|
||||
- CLI: `python3 scripts/big_brain_manager.py` — operational script (`scripts/big_brain_manager.py`)
|
||||
- CLI: `python3 scripts/big_brain_repo_audit.py` — operational script (`scripts/big_brain_repo_audit.py`)
|
||||
- CLI: `python3 scripts/codebase_genome_nightly.py` — operational script (`scripts/codebase_genome_nightly.py`)
|
||||
- Python: `get_narrative_phase()` from `evennia/timmy_world/game.py:55`
|
||||
- Python: `get_phase_transition_event()` from `evennia/timmy_world/game.py:65`
|
||||
- Python: `main()` from `uniwizard/self_grader.py:713`
|
||||
|
||||
- `index.html` served over HTTP
|
||||
- `boot.js` exports `bootPage()`; verified by `node --test tests/boot.test.js`
|
||||
- Data APIs are file-based inside the repo: `portals.json`, `vision.json`, `manifest.json`
|
||||
## Test Coverage Report
|
||||
|
||||
### Network/runtime surface
|
||||
- Source and script files inspected: 186
|
||||
- Test files inspected: 28
|
||||
- Coverage gaps:
|
||||
- `wizards/allegro/home/skills/red-teaming/godmode/scripts/auto_jailbreak.py` — no matching test reference detected
|
||||
- `timmy-local/cache/agent_cache.py` — no matching test reference detected
|
||||
- `wizards/allegro/home/skills/red-teaming/godmode/scripts/parseltongue.py` — no matching test reference detected
|
||||
- `twitter-archive/multimodal_pipeline.py` — no matching test reference detected
|
||||
- `wizards/allegro/home/skills/red-teaming/godmode/scripts/godmode_race.py` — no matching test reference detected
|
||||
- `skills/productivity/google-workspace/scripts/google_api.py` — no matching test reference detected
|
||||
- `wizards/allegro/home/skills/productivity/google-workspace/scripts/google_api.py` — no matching test reference detected
|
||||
- `morrowind/pilot.py` — no matching test reference detected
|
||||
- `morrowind/mcp_server.py` — no matching test reference detected
|
||||
- `skills/research/domain-intel/scripts/domain_intel.py` — no matching test reference detected
|
||||
- `wizards/allegro/home/skills/research/domain-intel/scripts/domain_intel.py` — no matching test reference detected
|
||||
- `timmy-local/scripts/ingest.py` — no matching test reference detected
|
||||
|
||||
- `python3 server.py`
|
||||
- Starts the WebSocket bridge on port `8765`
|
||||
- `python3 l402_server.py`
|
||||
- Local HTTP microservice for cost-estimate style responses
|
||||
- `python3 multi_user_bridge.py`
|
||||
- Multi-user HTTP/chat bridge
|
||||
## Security Audit Findings
|
||||
|
||||
### Harness / operator CLI surfaces
|
||||
- [medium] `briefings/briefing_20260325.json:37` — hardcoded http endpoint: plaintext or fixed HTTP endpoints can drift or leak across environments. Evidence: `"gitea_error": "Gitea 404: {\"errors\":null,\"message\":\"not found\",\"url\":\"http://143.198.27.163:3000/api/swagger\"}\n [http://143.198.27.163:3000/api/v1/repos/Timmy_Foundation/sovereign-orchestration/issues?state=open&type=issues&sort=created&direction=desc&limit=1&page=1]",`
|
||||
- [medium] `briefings/briefing_20260328.json:11` — hardcoded http endpoint: plaintext or fixed HTTP endpoints can drift or leak across environments. Evidence: `"provider_base_url": "http://localhost:8081/v1",`
|
||||
- [medium] `briefings/briefing_20260329.json:11` — hardcoded http endpoint: plaintext or fixed HTTP endpoints can drift or leak across environments. Evidence: `"provider_base_url": "http://localhost:8081/v1",`
|
||||
- [medium] `config.yaml:37` — hardcoded http endpoint: plaintext or fixed HTTP endpoints can drift or leak across environments. Evidence: `summary_base_url: http://localhost:11434/v1`
|
||||
- [medium] `config.yaml:47` — hardcoded http endpoint: plaintext or fixed HTTP endpoints can drift or leak across environments. Evidence: `base_url: 'http://localhost:11434/v1'`
|
||||
- [medium] `config.yaml:52` — hardcoded http endpoint: plaintext or fixed HTTP endpoints can drift or leak across environments. Evidence: `base_url: 'http://localhost:11434/v1'`
|
||||
- [medium] `config.yaml:57` — hardcoded http endpoint: plaintext or fixed HTTP endpoints can drift or leak across environments. Evidence: `base_url: 'http://localhost:11434/v1'`
|
||||
- [medium] `config.yaml:62` — hardcoded http endpoint: plaintext or fixed HTTP endpoints can drift or leak across environments. Evidence: `base_url: 'http://localhost:11434/v1'`
|
||||
- [medium] `config.yaml:67` — hardcoded http endpoint: plaintext or fixed HTTP endpoints can drift or leak across environments. Evidence: `base_url: 'http://localhost:11434/v1'`
|
||||
- [medium] `config.yaml:77` — hardcoded http endpoint: plaintext or fixed HTTP endpoints can drift or leak across environments. Evidence: `base_url: 'http://localhost:11434/v1'`
|
||||
- [medium] `config.yaml:82` — hardcoded http endpoint: plaintext or fixed HTTP endpoints can drift or leak across environments. Evidence: `base_url: 'http://localhost:11434/v1'`
|
||||
- [medium] `config.yaml:174` — hardcoded http endpoint: plaintext or fixed HTTP endpoints can drift or leak across environments. Evidence: `base_url: http://localhost:11434/v1`
|
||||
|
||||
- `python3 nexus/morrowind_harness.py`
|
||||
- `python3 nexus/bannerlord_harness.py`
|
||||
- `python3 mempalace/tunnel_sync.py --peer <url> [--dry-run] [--n N]`
|
||||
- `python3 mcp_servers/desktop_control_server.py`
|
||||
- `python3 mcp_servers/steam_info_server.py`
|
||||
## Dead Code Candidates
|
||||
|
||||
### Validation surface
|
||||
- `wizards/allegro/home/skills/red-teaming/godmode/scripts/auto_jailbreak.py` — not imported by indexed Python modules and not referenced by tests
|
||||
- `timmy-local/cache/agent_cache.py` — not imported by indexed Python modules and not referenced by tests
|
||||
- `wizards/allegro/home/skills/red-teaming/godmode/scripts/parseltongue.py` — not imported by indexed Python modules and not referenced by tests
|
||||
- `twitter-archive/multimodal_pipeline.py` — not imported by indexed Python modules and not referenced by tests
|
||||
- `wizards/allegro/home/skills/red-teaming/godmode/scripts/godmode_race.py` — not imported by indexed Python modules and not referenced by tests
|
||||
- `skills/productivity/google-workspace/scripts/google_api.py` — not imported by indexed Python modules and not referenced by tests
|
||||
- `wizards/allegro/home/skills/productivity/google-workspace/scripts/google_api.py` — not imported by indexed Python modules and not referenced by tests
|
||||
- `morrowind/pilot.py` — not imported by indexed Python modules and not referenced by tests
|
||||
- `morrowind/mcp_server.py` — not imported by indexed Python modules and not referenced by tests
|
||||
- `skills/research/domain-intel/scripts/domain_intel.py` — not imported by indexed Python modules and not referenced by tests
|
||||
|
||||
- `python3 -m pytest tests/test_portals_json.py tests/test_index_html_integrity.py tests/test_repo_truth.py -q`
|
||||
- `node --test tests/boot.test.js`
|
||||
- `python3 -m py_compile server.py nexus/morrowind_harness.py nexus/bannerlord_harness.py mempalace/tunnel_sync.py mcp_servers/desktop_control_server.py`
|
||||
- `tests/test_browser_smoke.py` defines the higher-cost Playwright smoke contract for the world shell
|
||||
## Performance Bottleneck Analysis
|
||||
|
||||
## Test Coverage Gaps
|
||||
|
||||
Strongly covered in this checkout:
|
||||
- `tests/test_portals_json.py` validates `portals.json`
|
||||
- `tests/test_index_html_integrity.py` checks merge-marker/DOM-integrity regressions in `index.html`
|
||||
- `tests/boot.test.js` verifies `boot.js` startup behavior
|
||||
- `tests/test_repo_truth.py` validates the repo-truth documents
|
||||
- Multiple `tests/test_mempalace_*.py` files cover the palace layer
|
||||
- `tests/test_bannerlord_harness.py` exists for the Bannerlord harness
|
||||
|
||||
Notable gaps or weak seams:
|
||||
- `nexus/morrowind_harness.py` is large and operationally critical, but the generated baseline still flags it as a gap relative to its size/complexity
|
||||
- `mcp_servers/desktop_control_server.py` exposes high-power automation but has no obvious dedicated test file in the root `tests/` suite
|
||||
- `app.js` is the dominant browser runtime file and mixes rendering, GOFAI, metrics, and integration logic in one place; browser smoke exists, but there is limited unit-level decomposition around those subsystems
|
||||
- `mempalace.js` appears minimally bridged and stale relative to the richer Python MemPalace layer
|
||||
- `multi_user_bridge.py` is a large integration surface and should be treated as high regression risk even though it is central to operator/chat flow
|
||||
|
||||
## Security Considerations
|
||||
|
||||
- `server.py` binds `HOST = "0.0.0.0"`, exposing the broadcast bridge beyond localhost unless network controls limit it
|
||||
- The WebSocket bridge is a broadcast hub without visible authentication in `server.py`; connected clients are trusted to send messages into the bus
|
||||
- `mcp_servers/desktop_control_server.py` exposes mouse/keyboard/screenshot control through a stdio MCP server. In any non-local or poorly isolated runtime, this is a privileged automation surface
|
||||
- `app.js` contains hardcoded local/network endpoints such as `http://localhost:${L402_PORT}/api/cost-estimate` and `http://localhost:8082/metrics`; these are convenient for local development but create environment drift and deployment assumptions
|
||||
- `app.js` also embeds explicit endpoint/status references like `ws://143.198.27.163:8765`, which is operationally brittle and the kind of hardcoded location data that drifts across environments
|
||||
- `mempalace.js` shells out through `window.electronAPI.execPython(...)`; this is powerful and useful, but it is a clear trust boundary between UI and host execution
|
||||
- `INVESTIGATION_ISSUE_1145.md` documents an earlier integrity hazard: agents writing to `public/nexus/` instead of canonical root paths. That path confusion is both an operational and security concern because it makes provenance harder to reason about
|
||||
|
||||
## Runtime Truth and Docs Drift
|
||||
|
||||
The most important architecture finding in this repo is not a class or subsystem. It is a truth mismatch.
|
||||
|
||||
- README.md says current `main` does not ship a browser 3D world
|
||||
- CLAUDE.md declares root `app.js` and `index.html` as canonical frontend paths
|
||||
- tests and browser contract now assume the root frontend exists
|
||||
|
||||
All three statements are simultaneously present in this checkout.
|
||||
|
||||
Grounded evidence:
|
||||
- `README.md` still says the repo does not contain an active root frontend such as `index.html`, `app.js`, or `style.css`
|
||||
- the current checkout does contain `index.html`, `app.js`, `style.css`, `manifest.json`, and `gofai_worker.js`
|
||||
- `BROWSER_CONTRACT.md` explicitly treats those root files as required browser assets
|
||||
- `tests/test_browser_smoke.py` serves those exact files and validates DOM/WebGL contracts against them
|
||||
- `tests/test_index_html_integrity.py` assumes `index.html` is canonical and production-relevant
|
||||
- `CLAUDE.md` says frontend code lives at repo root and explicitly warns against `public/nexus/`
|
||||
- `INVESTIGATION_ISSUE_1145.md` explains why `public/nexus/` is a bad/corrupt duplicate path and confirms the real classical AI code lives in root `app.js`
|
||||
|
||||
The honest conclusion:
|
||||
- The repo contains a partially restored or actively re-materialized browser surface
|
||||
- The docs are preserving an older migration truth while the runtime files and smoke contracts describe a newer present-tense truth
|
||||
- Any future work in `the-nexus` must choose one truth and align `README.md`, `CLAUDE.md`, smoke tests, and file layout around it
|
||||
|
||||
That drift is itself a critical architectural fact and should be treated as first-order design debt, not a side note.
|
||||
- `angband/mcp_server.py` — large module (353 lines) likely hides multiple responsibilities
|
||||
- `evennia/timmy_world/game.py` — large module (1541 lines) likely hides multiple responsibilities
|
||||
- `evennia/timmy_world/world/game.py` — large module (1345 lines) likely hides multiple responsibilities
|
||||
- `morrowind/mcp_server.py` — large module (451 lines) likely hides multiple responsibilities
|
||||
- `morrowind/pilot.py` — large module (459 lines) likely hides multiple responsibilities
|
||||
- `pipelines/codebase_genome.py` — large module (557 lines) likely hides multiple responsibilities
|
||||
- `scripts/know_thy_father/crossref_audit.py` — large module (657 lines) likely hides multiple responsibilities
|
||||
- `scripts/know_thy_father/index_media.py` — large module (405 lines) likely hides multiple responsibilities
|
||||
- `scripts/know_thy_father/synthesize_kernels.py` — large module (416 lines) likely hides multiple responsibilities
|
||||
- `scripts/tower_game.py` — large module (395 lines) likely hides multiple responsibilities
|
||||
|
||||
110
evennia_tools/batch_cmds_bezalel.ev
Normal file
110
evennia_tools/batch_cmds_bezalel.ev
Normal file
@@ -0,0 +1,110 @@
|
||||
#
|
||||
# Bezalel World Builder — Evennia batch commands
|
||||
# Creates the Bezalel Evennia world from evennia_tools/bezalel_layout.py specs.
|
||||
#
|
||||
# Load with: @batchcommand bezalel_world
|
||||
#
|
||||
# Part of #536
|
||||
|
||||
# Create rooms
|
||||
@create/drop Limbo:evennia.objects.objects.DefaultRoom
|
||||
@desc here = The void between worlds. The air carries the pulse of three houses: Mac, VPS, and this one. Everything begins here before it is given form.
|
||||
|
||||
@create/drop Gatehouse:evennia.objects.objects.DefaultRoom
|
||||
@desc here = A stone guard tower at the edge of Bezalel world. The walls are carved with runes of travel, proof, and return. Every arrival is weighed before it is trusted.
|
||||
|
||||
@create/drop Great Hall:evennia.objects.objects.DefaultRoom
|
||||
@desc here = A vast hall with a long working table. Maps of the three houses hang beside sketches, benchmarks, and deployment notes. This is where the forge reports back to the house.
|
||||
|
||||
@create/drop The Library of Bezalel:evennia.objects.objects.DefaultRoom
|
||||
@desc here = Shelves of technical manuals, Evennia code, test logs, and bridge schematics rise to the ceiling. This room holds plans waiting to be made real.
|
||||
|
||||
@create/drop The Observatory:evennia.objects.objects.DefaultRoom
|
||||
@desc here = A high chamber with telescopes pointing toward the Mac, the VPS, and the wider net. Screens glow with status lights, latency traces, and long-range signals.
|
||||
|
||||
@create/drop The Workshop:evennia.objects.objects.DefaultRoom
|
||||
@desc here = A forge and workbench share the same heat. Scattered here are half-finished bridges, patched harnesses, and tools laid out for proof before pride.
|
||||
|
||||
@create/drop The Server Room:evennia.objects.objects.DefaultRoom
|
||||
@desc here = Racks of humming servers line the walls. Fans push warm air through the chamber while status LEDs beat like a mechanical heart. This is the pulse of Bezalel house.
|
||||
|
||||
@create/drop The Garden of Code:evennia.objects.objects.DefaultRoom
|
||||
@desc here = A quiet garden where ideas are left long enough to grow roots. Code-shaped leaves flutter in patterned wind, and a stone path invites patient thought.
|
||||
|
||||
@create/drop The Portal Room:evennia.objects.objects.DefaultRoom
|
||||
@desc here = Three shimmering doorways stand in a ring: one marked for the Mac house, one for the VPS, and one for the wider net. The room hums like a bridge waiting for traffic.
|
||||
|
||||
# Create exits
|
||||
@open gatehouse:gate,tower = Gatehouse
|
||||
@open limbo:void,back = Limbo
|
||||
@open greathall:hall,great hall = Great Hall
|
||||
@open gatehouse:gate,tower = Gatehouse
|
||||
@open library:books,study = The Library of Bezalel
|
||||
@open hall:great hall,back = Great Hall
|
||||
@open observatory:telescope,tower top = The Observatory
|
||||
@open hall:great hall,back = Great Hall
|
||||
@open workshop:forge,bench = The Workshop
|
||||
@open hall:great hall,back = Great Hall
|
||||
@open serverroom:servers,server room = The Server Room
|
||||
@open workshop:forge,bench = The Workshop
|
||||
@open garden:garden of code,grove = The Garden of Code
|
||||
@open workshop:forge,bench = The Workshop
|
||||
@open portalroom:portal,portals = The Portal Room
|
||||
@open gatehouse:gate,back = Gatehouse
|
||||
|
||||
# Create objects
|
||||
@create Threshold Ledger
|
||||
@desc Threshold Ledger = A heavy ledger where arrivals, departures, and field notes are recorded before the work begins.
|
||||
@tel Threshold Ledger = Gatehouse
|
||||
|
||||
@create Three-House Map
|
||||
@desc Three-House Map = A long map showing Mac, VPS, and remote edges in one continuous line of work.
|
||||
@tel Three-House Map = Great Hall
|
||||
|
||||
@create Bridge Schematics
|
||||
@desc Bridge Schematics = Rolled plans describing world bridges, Evennia layouts, and deployment paths.
|
||||
@tel Bridge Schematics = The Library of Bezalel
|
||||
|
||||
@create Compiler Manuals
|
||||
@desc Compiler Manuals = Manuals annotated in the margins with warnings against cleverness without proof.
|
||||
@tel Compiler Manuals = The Library of Bezalel
|
||||
|
||||
@create Tri-Axis Telescope
|
||||
@desc Tri-Axis Telescope = A brass telescope assembly that can be turned toward the Mac, the VPS, or the open net.
|
||||
@tel Tri-Axis Telescope = The Observatory
|
||||
|
||||
@create Forge Anvil
|
||||
@desc Forge Anvil = Scarred metal used for turning rough plans into testable form.
|
||||
@tel Forge Anvil = The Workshop
|
||||
|
||||
@create Bridge Workbench
|
||||
@desc Bridge Workbench = A wide bench covered in harness patches, relay notes, and half-soldered bridge parts.
|
||||
@tel Bridge Workbench = The Workshop
|
||||
|
||||
@create Heartbeat Console
|
||||
@desc Heartbeat Console = A monitoring console showing service health, latency, and the steady hum of the house.
|
||||
@tel Heartbeat Console = The Server Room
|
||||
|
||||
@create Server Racks
|
||||
@desc Server Racks = Stacked machines that keep the world awake even when no one is watching.
|
||||
@tel Server Racks = The Server Room
|
||||
|
||||
@create Code Orchard
|
||||
@desc Code Orchard = Trees with code-shaped leaves. Some branches bear elegant abstractions; others hold broken prototypes.
|
||||
@tel Code Orchard = The Garden of Code
|
||||
|
||||
@create Stone Bench
|
||||
@desc Stone Bench = A place to sit long enough for a hard implementation problem to become clear.
|
||||
@tel Stone Bench = The Garden of Code
|
||||
|
||||
@create Mac Portal:mac arch
|
||||
@desc Mac Portal = A silver doorway whose frame vibrates with the local sovereign house.
|
||||
@tel Mac Portal = The Portal Room
|
||||
|
||||
@create VPS Portal:vps arch
|
||||
@desc VPS Portal = A cobalt doorway tuned toward the testbed VPS house.
|
||||
@tel VPS Portal = The Portal Room
|
||||
|
||||
@create Net Portal:net arch,network arch
|
||||
@desc Net Portal = A pale doorway pointed toward the wider net and every uncertain edge beyond it.
|
||||
@tel Net Portal = The Portal Room
|
||||
85
evennia_tools/build_bezalel_world.py
Normal file
85
evennia_tools/build_bezalel_world.py
Normal file
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
""
|
||||
build_bezalel_world.py — Build Bezalel Evennia world from layout specs.
|
||||
|
||||
Programmatically creates rooms, exits, objects, and characters in a running
|
||||
Evennia instance using the specs from evennia_tools/bezalel_layout.py.
|
||||
|
||||
Usage (in Evennia game shell):
|
||||
from evennia_tools.build_bezalel_world import build_world
|
||||
build_world()
|
||||
|
||||
Or via batch command:
|
||||
@batchcommand evennia_tools/batch_cmds_bezalel.ev
|
||||
|
||||
Part of #536
|
||||
""
|
||||
|
||||
from evennia_tools.bezalel_layout import (
|
||||
ROOMS, EXITS, OBJECTS, CHARACTERS, PORTAL_COMMANDS,
|
||||
room_keys, reachable_rooms_from
|
||||
)
|
||||
|
||||
|
||||
def build_world():
|
||||
"""Build the Bezalel Evennia world from layout specs."""
|
||||
from evennia.objects.models import ObjectDB
|
||||
from evennia.utils.create import create_object, create_exit, create_message
|
||||
|
||||
print("Building Bezalel world...")
|
||||
|
||||
# Create rooms
|
||||
rooms = {}
|
||||
for spec in ROOMS:
|
||||
room = create_object(
|
||||
"evennia.objects.objects.DefaultRoom",
|
||||
key=spec.key,
|
||||
attributes=(("desc", spec.desc),),
|
||||
)
|
||||
rooms[spec.key] = room
|
||||
print(f" Room: {spec.key}")
|
||||
|
||||
# Create exits
|
||||
for spec in EXITS:
|
||||
source = rooms.get(spec.source)
|
||||
dest = rooms.get(spec.destination)
|
||||
if not source or not dest:
|
||||
print(f" WARNING: Exit {spec.key} — missing room")
|
||||
continue
|
||||
exit_obj = create_exit(
|
||||
key=spec.key,
|
||||
location=source,
|
||||
destination=dest,
|
||||
aliases=list(spec.aliases),
|
||||
)
|
||||
print(f" Exit: {spec.source} -> {spec.destination} ({spec.key})")
|
||||
|
||||
# Create objects
|
||||
for spec in OBJECTS:
|
||||
location = rooms.get(spec.location)
|
||||
if not location:
|
||||
print(f" WARNING: Object {spec.key} — missing room {spec.location}")
|
||||
continue
|
||||
obj = create_object(
|
||||
"evennia.objects.objects.DefaultObject",
|
||||
key=spec.key,
|
||||
location=location,
|
||||
attributes=(("desc", spec.desc),),
|
||||
aliases=list(spec.aliases),
|
||||
)
|
||||
print(f" Object: {spec.key} in {spec.location}")
|
||||
|
||||
# Verify reachability
|
||||
all_rooms = set(room_keys())
|
||||
reachable = reachable_rooms_from("Limbo")
|
||||
unreachable = all_rooms - reachable
|
||||
if unreachable:
|
||||
print(f" WARNING: Unreachable rooms: {unreachable}")
|
||||
else:
|
||||
print(f" All {len(all_rooms)} rooms reachable from Limbo")
|
||||
|
||||
print("Bezalel world built.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
build_world()
|
||||
@@ -1,101 +0,0 @@
|
||||
# GENOME.md — Burn Fleet (Timmy_Foundation/burn-fleet)
|
||||
|
||||
> Codebase Genome v1.0 | Generated 2026-04-16 | Repo 14/16
|
||||
|
||||
## Project Overview
|
||||
|
||||
**Burn Fleet** is the autonomous dispatch infrastructure for the Timmy Foundation. It manages 112 tmux panes across Mac and VPS, routing Gitea issues to lane-specialized workers by repo. Each agent has a mythological name — they are all Timmy with different hats.
|
||||
|
||||
**Core principle:** Dispatch ALL panes. Never scan for idle. Stale work beats idle workers.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Mac (M3 Max, 14 cores, 36GB) Allegro (VPS, 2 cores, 8GB)
|
||||
┌─────────────────────────────┐ ┌─────────────────────────────┐
|
||||
│ CRUCIBLE 14 panes (bugs) │ │ FORGE 14 panes (bugs) │
|
||||
│ GNOMES 12 panes (cron) │ │ ANVIL 14 panes (nexus) │
|
||||
│ LOOM 12 panes (home) │ │ CRUCIBLE-2 10 panes (home) │
|
||||
│ FOUNDRY 10 panes (nexus) │ │ SENTINEL 6 panes (council)│
|
||||
│ WARD 12 panes (fleet) │ └─────────────────────────────┘
|
||||
│ COUNCIL 8 panes (sages) │ 44 panes (36 workers)
|
||||
└─────────────────────────────┘
|
||||
68 panes (60 workers)
|
||||
```
|
||||
|
||||
**Total: 112 panes, 96 workers + 12 council members + 4 sentinel advisors**
|
||||
|
||||
## Key Files
|
||||
|
||||
| File | LOC | Purpose |
|
||||
|------|-----|---------|
|
||||
| `fleet-spec.json` | ~200 | Machine definitions, window layouts, lane assignments, agent names |
|
||||
| `fleet-launch.sh` | ~100 | Create tmux sessions with correct pane counts on Mac + Allegro |
|
||||
| `fleet-christen.py` | ~80 | Launch hermes in all panes and send identity messages |
|
||||
| `fleet-dispatch.py` | ~250 | Pull Gitea issues and route to correct panes by lane |
|
||||
| `fleet-status.py` | ~100 | Health check across all machines |
|
||||
| `allegro/docker-compose.yml` | ~30 | Allegro VPS container definition |
|
||||
| `allegro/Dockerfile` | ~20 | Allegro build definition |
|
||||
| `allegro/healthcheck.py` | ~15 | Allegro container health check |
|
||||
|
||||
**Total: ~800 LOC**
|
||||
|
||||
## Lane Routing
|
||||
|
||||
Issues are routed by repo to the correct window:
|
||||
|
||||
| Repo | Mac Window | Allegro Window |
|
||||
|------|-----------|----------------|
|
||||
| hermes-agent | CRUCIBLE, GNOMES | FORGE |
|
||||
| timmy-home | LOOM | CRUCIBLE-2 |
|
||||
| timmy-config | LOOM | CRUCIBLE-2 |
|
||||
| the-nexus | FOUNDRY | ANVIL |
|
||||
| the-playground | — | ANVIL |
|
||||
| the-door | WARD | CRUCIBLE-2 |
|
||||
| fleet-ops | WARD | CRUCIBLE-2 |
|
||||
| turboquant | WARD | — |
|
||||
|
||||
## Entry Points
|
||||
|
||||
| Command | Purpose |
|
||||
|---------|---------|
|
||||
| `./fleet-launch.sh both` | Create tmux layout on Mac + Allegro |
|
||||
| `python3 fleet-christen.py both` | Wake all agents with identity messages |
|
||||
| `python3 fleet-dispatch.py --cycles 1` | Single dispatch cycle |
|
||||
| `python3 fleet-dispatch.py --cycles 10 --interval 60` | Continuous burn (10 cycles, 60s apart) |
|
||||
| `python3 fleet-status.py` | Health check all machines |
|
||||
|
||||
## Agent Names
|
||||
|
||||
| Window | Names | Count |
|
||||
|--------|-------|-------|
|
||||
| CRUCIBLE | AZOTH, ALBEDO, CITRINITAS, RUBEDO, SULPHUR, MERCURIUS, SAL, ATHANOR, VITRIOL, SATURN, JUPITER, MARS, EARTH, SOL | 14 |
|
||||
| GNOMES | RAZIEL, AZRAEL, CASSIEL, METATRON, SANDALPHON, BINAH, CHOKMAH, KETER, ALDEBARAN, RIGEL, SIRIUS, POLARIS | 12 |
|
||||
| FORGE | HAMMER, ANVIL, ADZE, PICK, TONGS, WRENCH, SCREWDRIVER, BOLT, SAW, TRAP, HOOK, MAGNET, SPARK, FLAME | 14 |
|
||||
| COUNCIL | TESLA, HERMES, GANDALF, DAVINCI, ARCHIMEDES, TURING, AURELIUS, SOLOMON | 8 |
|
||||
|
||||
## Design Decisions
|
||||
|
||||
1. **Separate GILs** — Allegro runs Python independently on VPS for true parallelism
|
||||
2. **Queue, not send-keys** — Workers process at their own pace, no interruption
|
||||
3. **Lane enforcement** — Panes stay in one repo to build deep context
|
||||
4. **Dispatch ALL panes** — Never scan for idle; stale work beats idle workers
|
||||
5. **Council is advisory** — Named archetypes provide perspective, not task execution
|
||||
|
||||
## Scaling
|
||||
|
||||
- Add panes: Edit `fleet-spec.json` → `fleet-launch.sh` → `fleet-christen.py`
|
||||
- Add machines: Edit `fleet-spec.json` → Add routing in `fleet-dispatch.py` → Ensure SSH access
|
||||
|
||||
## Sovereignty Assessment
|
||||
|
||||
- **Fully local** — Mac + user-controlled VPS, no cloud dependencies
|
||||
- **No phone-home** — Gitea API is self-hosted
|
||||
- **Open source** — All code on Gitea
|
||||
- **SSH-based** — Mac → Allegro communication via SSH only
|
||||
|
||||
**Verdict: Fully sovereign. Autonomous fleet dispatch with no external dependencies.**
|
||||
|
||||
---
|
||||
|
||||
*"Dispatch ALL panes. Never scan for idle — stale work beats idle workers."*
|
||||
@@ -1,200 +1,138 @@
|
||||
# GENOME.md — TurboQuant (Timmy_Foundation/turboquant)
|
||||
|
||||
> Codebase Genome v1.1 | Refreshed 2026-04-21 for timmy-home #679
|
||||
> Codebase Genome v1.0 | Generated 2026-04-15 | Repo 12/16
|
||||
|
||||
## Project Overview
|
||||
|
||||
TurboQuant is a local-first KV-cache compression repo for long-context inference on Apple Silicon. Its center of gravity is not a web service or product UI; it is a compact implementation and verification surface around PolarQuant, QJL, and TurboQuant compression modes for llama.cpp-style runtimes.
|
||||
**TurboQuant** is a KV cache compression system for local inference on Apple Silicon. Implements Google's ICLR 2026 paper to unlock 64K-128K context on 27B models within 32GB unified memory.
|
||||
|
||||
The repo mixes three lanes:
|
||||
1. a small native core (`llama-turbo.cpp`, `llama-turbo.h`, `ggml-metal-turbo.metal`)
|
||||
2. benchmark and evaluation scripts (`benchmarks/`)
|
||||
3. operational selection/profile helpers (`evolution/`, `profiles/`)
|
||||
**Three-stage compression:**
|
||||
1. **PolarQuant** — WHT rotation + polar coordinates + Lloyd-Max codebook (~4.2x compression)
|
||||
2. **QJL** — 1-bit quantized Johnson-Lindenstrauss residual correction
|
||||
3. **TurboQuant** — PolarQuant + QJL = ~3.5 bits/channel, zero accuracy loss
|
||||
|
||||
Current `main` already builds a standalone CMake target and roundtrip C++ test locally, but the checked-in `.gitea/workflows/smoke.yml` still performs only parse + secret scanning. That mismatch matters: the codebase can be build-healthy while CI remains blind to the most important executable path.
|
||||
**Key result:** 73% KV memory savings with 1% prompt processing overhead, 11% generation overhead.
|
||||
|
||||
## Architecture
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph Compression_Core
|
||||
H[llama-turbo.h] --> CPP[llama-turbo.cpp]
|
||||
CPP --> PQ[PolarQuant encode/decode]
|
||||
CPP --> QJL[QJL residual helpers]
|
||||
subgraph "Compression Pipeline"
|
||||
KV[Raw KV Cache fp16] --> WHT[WHT Rotation]
|
||||
WHT --> POLAR[PolarQuant 4-bit]
|
||||
POLAR --> QJL[QJL Residual]
|
||||
QJL --> PACKED[Packed KV ~3.5bit]
|
||||
end
|
||||
|
||||
subgraph Metal_Runtime
|
||||
MTL[ggml-metal-turbo.metal]
|
||||
PQ --> MTL
|
||||
QJL --> MTL
|
||||
MTL --> ATTN[KV dequant + attention path]
|
||||
subgraph "Metal Shaders"
|
||||
PACKED --> DECODE[Polar Decode Kernel]
|
||||
DECODE --> ATTEN[Flash Attention]
|
||||
ATTEN --> OUTPUT[Model Output]
|
||||
end
|
||||
|
||||
subgraph Verification
|
||||
CMAKE[CMakeLists.txt] --> ROUNDTRIP[tests/roundtrip_test.cpp]
|
||||
ROUNDTRIP --> CTEST[ctest --test-dir build --output-on-failure]
|
||||
BENCH[benchmarks/run_benchmarks.py] --> PPL[benchmarks/run_perplexity.py]
|
||||
PYTEST[tests/test_quant_selector.py]
|
||||
PYTEST2[tests/test_tool_call_integration.py]
|
||||
subgraph "Build System"
|
||||
CMAKE[CMakeLists.txt] --> LIB[turboquant.a]
|
||||
LIB --> TEST[turboquant_roundtrip_test]
|
||||
LIB --> LLAMA[llama.cpp fork integration]
|
||||
end
|
||||
|
||||
subgraph Ops_Surface
|
||||
QS[evolution/quant_selector.py]
|
||||
HO[evolution/hardware_optimizer.py]
|
||||
PROFILE[profiles/hermes-profile-gemma4-turboquant.yaml]
|
||||
end
|
||||
|
||||
ATTN --> PROFILE
|
||||
BENCH --> QS
|
||||
QS --> PROFILE
|
||||
```
|
||||
|
||||
## Entry Points
|
||||
|
||||
| Entry point | File | Purpose |
|
||||
| --- | --- | --- |
|
||||
| Native encode/decode API | `llama-turbo.h` / `llama-turbo.cpp` | Public compression functions and implementation |
|
||||
| Metal kernels | `ggml-metal-turbo.metal` | Apple GPU execution path for dequant / rotation helpers |
|
||||
| Standalone build | `CMakeLists.txt` | Builds static library plus `turboquant_roundtrip_test` |
|
||||
| Benchmark suite | `benchmarks/run_benchmarks.py` | Speed / quality benchmark runner |
|
||||
| Perplexity harness | `benchmarks/run_perplexity.py` | Prompt-quality regression measurement |
|
||||
| Long-session harness | `benchmarks/run_long_session.py` | Extended runtime exercise |
|
||||
| Quant chooser | `evolution/quant_selector.py` | Memory-aware preset selection |
|
||||
| Hardware optimizer stub | `evolution/hardware_optimizer.py` | Hardware tuning placeholder/stub |
|
||||
| Hermes profile | `profiles/hermes-profile-gemma4-turboquant.yaml` | Deployment config for compressed long-context runs |
|
||||
| Repo CI floor | `.gitea/workflows/smoke.yml` | Parse + secret scan only on current `main` |
|
||||
|
||||
## Data Flow
|
||||
|
||||
1. Input KV vectors enter the native compression path in `llama-turbo.cpp`.
|
||||
2. The encoder performs WHT rotation, transforms into polar/codebook space, and writes packed KV outputs plus norms.
|
||||
3. `ggml-metal-turbo.metal` supplies the Apple Silicon GPU-side decode and attention helpers for runtime execution.
|
||||
4. `tests/roundtrip_test.cpp` validates encode→decode fidelity via the standalone CMake/CTest path.
|
||||
5. Python benchmark scripts consume the native/built surface to measure speed and quality.
|
||||
6. `evolution/quant_selector.py` translates machine/resource constraints into quantization choices that feed the deployment profile in `profiles/hermes-profile-gemma4-turboquant.yaml`.
|
||||
| Entry Point | File | Purpose |
|
||||
|-------------|------|---------|
|
||||
| `polar_quant_encode_turbo4()` | llama-turbo.cpp | Encode float KV → 4-bit packed |
|
||||
| `polar_quant_decode_turbo4()` | llama-turbo.cpp | Decode 4-bit packed → float KV |
|
||||
| `cmake build` | CMakeLists.txt | Build static library + tests |
|
||||
| `run_benchmarks.py` | benchmarks/ | Run perplexity benchmarks |
|
||||
|
||||
## Key Abstractions
|
||||
|
||||
| Abstraction | File | Role |
|
||||
| --- | --- | --- |
|
||||
| `polar_quant_encode_turbo4` / `polar_quant_decode_turbo4` | `llama-turbo.cpp` | Core public encode/decode path for TurboQuant |
|
||||
| `turboquant_roundtrip_test` | `tests/roundtrip_test.cpp` | Executable spec for standalone correctness |
|
||||
| `QuantLevel` / `QUANT_LEVELS` | `evolution/quant_selector.py` | Encodes quality/compression presets and fallback ordering |
|
||||
| `select_quant_for_memory(...)`-style selection logic | `evolution/quant_selector.py` | Converts hardware budgets into quant choices |
|
||||
| benchmark runners | `benchmarks/run_benchmarks.py`, `benchmarks/run_perplexity.py` | Measure regression envelope for performance/quality |
|
||||
| deployment profile | `profiles/hermes-profile-gemma4-turboquant.yaml` | Operationalizes the repo into a Hermes runtime |
|
||||
| Symbol | File | Purpose |
|
||||
|--------|------|---------|
|
||||
| `polar_quant_encode_turbo4()` | llama-turbo.h/.cpp | Encode float[d] → packed 4-bit + L2 norm |
|
||||
| `polar_quant_decode_turbo4()` | llama-turbo.h/.cpp | Decode packed 4-bit + norm → float[d] |
|
||||
| `turbo_dequantize_k()` | ggml-metal-turbo.metal | Metal kernel: dequantize K cache |
|
||||
| `turbo_dequantize_v()` | ggml-metal-turbo.metal | Metal kernel: dequantize V cache |
|
||||
| `turbo_fwht_128()` | ggml-metal-turbo.metal | Fast Walsh-Hadamard Transform |
|
||||
| `run_perplexity.py` | benchmarks/ | Measure perplexity impact |
|
||||
| `run_benchmarks.py` | benchmarks/ | Full benchmark suite (speed + quality) |
|
||||
|
||||
## API Surface
|
||||
## Data Flow
|
||||
|
||||
### Native API
|
||||
The repo exposes a narrow C/C++ surface through `llama-turbo.h`:
|
||||
- encode float KV blocks into packed TurboQuant form
|
||||
- decode packed blocks back into float KV
|
||||
|
||||
### Verification / CLI surface
|
||||
Operationally, the repo is exercised via command entry points rather than a service API:
|
||||
- `cmake -S . -B build -DTURBOQUANT_BUILD_TESTS=ON`
|
||||
- `cmake --build build`
|
||||
- `ctest --test-dir build --output-on-failure`
|
||||
- `python3 benchmarks/run_benchmarks.py`
|
||||
- `python3 benchmarks/run_perplexity.py`
|
||||
|
||||
### Python operational surface
|
||||
The repo also exposes Python analysis / selection code:
|
||||
- `evolution/quant_selector.py`
|
||||
- `evolution/hardware_optimizer.py`
|
||||
- `tests/test_quant_selector.py`
|
||||
- `tests/test_tool_call_integration.py`
|
||||
|
||||
## Repository Surface Snapshot
|
||||
|
||||
Current high-signal files on `main`:
|
||||
- `README.md`
|
||||
- `docs/PROJECT_STATUS.md`
|
||||
- `CMakeLists.txt`
|
||||
- `llama-turbo.cpp`
|
||||
- `llama-turbo.h`
|
||||
- `ggml-metal-turbo.metal`
|
||||
- `tests/roundtrip_test.cpp`
|
||||
- `tests/test_quant_selector.py`
|
||||
- `tests/test_tool_call_integration.py`
|
||||
- `evolution/quant_selector.py`
|
||||
- `evolution/hardware_optimizer.py`
|
||||
- `profiles/hermes-profile-gemma4-turboquant.yaml`
|
||||
- `.gitea/workflows/smoke.yml`
|
||||
|
||||
Using `pygount --format=summary`, the current repo footprint is roughly:
|
||||
- 29 tracked source/doc/config files in the main tree (excluding build output)
|
||||
- 1535 lines of code
|
||||
- Python is the largest language block, followed by YAML and C++
|
||||
|
||||
## Runtime Verification (2026-04-21)
|
||||
|
||||
Verified directly against a fresh clone of `Timmy_Foundation/turboquant`:
|
||||
|
||||
```bash
|
||||
cmake -S . -B build -DTURBOQUANT_BUILD_TESTS=ON
|
||||
cmake --build build
|
||||
ctest --test-dir build --output-on-failure
|
||||
python3 -m pytest tests/test_quant_selector.py -q
|
||||
```
|
||||
Input: float KV vectors [d=128 per head]
|
||||
↓
|
||||
1. WHT rotation (in-place, O(d log d))
|
||||
↓
|
||||
2. Convert to polar coords (radius, angles)
|
||||
↓
|
||||
3. Lloyd-Max quantize angles → 4-bit indices
|
||||
↓
|
||||
4. Store: packed indices [d/2 bytes] + float norm [4 bytes]
|
||||
↓
|
||||
Decode: indices → codebook lookup → polar → cartesian → inverse WHT
|
||||
↓
|
||||
Output: reconstructed float KV [d=128]
|
||||
```
|
||||
|
||||
Observed results:
|
||||
- standalone CMake configure/build passes
|
||||
- `ctest --test-dir build --output-on-failure` passes on current `main`
|
||||
- `tests/test_quant_selector.py` is not fully green on current `main`
|
||||
- the failing lane is already tracked as `turboquant #139`
|
||||
## File Index
|
||||
|
||||
| File | LOC | Purpose |
|
||||
|------|-----|---------|
|
||||
| `llama-turbo.h` | 24 | C API: encode/decode function declarations |
|
||||
| `llama-turbo.cpp` | 78 | Implementation: PolarQuant encode/decode |
|
||||
| `ggml-metal-turbo.metal` | 76 | Metal shaders: dequantize + flash attention |
|
||||
| `CMakeLists.txt` | 44 | Build system: static lib + tests |
|
||||
| `tests/roundtrip_test.cpp` | 104 | Roundtrip encode→decode validation |
|
||||
| `benchmarks/run_benchmarks.py` | 227 | Benchmark suite |
|
||||
| `benchmarks/run_perplexity.py` | ~100 | Perplexity measurement |
|
||||
| `evolution/hardware_optimizer.py` | 5 | Hardware detection stub |
|
||||
|
||||
**Total: ~660 LOC | C++ core: 206 LOC | Python benchmarks: 232 LOC**
|
||||
|
||||
## Dependencies
|
||||
|
||||
| Dependency | Purpose |
|
||||
|------------|---------|
|
||||
| CMake 3.16+ | Build system |
|
||||
| C++17 compiler | Core implementation |
|
||||
| Metal (macOS) | GPU shader execution |
|
||||
| Python 3.11+ | Benchmarks |
|
||||
| llama.cpp fork | Integration target |
|
||||
|
||||
## Source Repos (Upstream)
|
||||
|
||||
| Repo | Role |
|
||||
|------|------|
|
||||
| TheTom/llama-cpp-turboquant | llama.cpp fork with Metal shaders |
|
||||
| TheTom/turboquant_plus | Reference impl, 511+ tests |
|
||||
| amirzandieh/QJL | Author QJL code (CUDA) |
|
||||
| rachittshah/mlx-turboquant | MLX fallback |
|
||||
|
||||
## Test Coverage
|
||||
|
||||
Present coverage on the target repo includes:
|
||||
- `tests/roundtrip_test.cpp` — core encode/decode roundtrip executable validation
|
||||
- `tests/test_quant_selector.py` — Python quant preset / selection behavior
|
||||
- `tests/test_tool_call_integration.py` — integration-oriented Python coverage
|
||||
- benchmark scripts for performance and perplexity measurement
|
||||
|
||||
## Test Coverage Gaps
|
||||
|
||||
1. `.gitea/workflows/smoke.yml` on current `main` still omits the standalone build/test path.
|
||||
- The workflow checks parse/secret hygiene only.
|
||||
- It does not execute `cmake --build build` or `ctest --test-dir build --output-on-failure`.
|
||||
- This gap is already being addressed separately in turboquant issue/PR work around `#50` / PR `#137`.
|
||||
|
||||
2. `tests/test_quant_selector.py` currently has a real failing assertion on current `main`.
|
||||
- The failure is tracked in `turboquant #139`.
|
||||
- The mismatch is around ordering expectations between `turbo2` and `q4_0` presets.
|
||||
|
||||
3. `evolution/hardware_optimizer.py` remains a very thin stub relative to the repo's ambition.
|
||||
- The genome should treat it as a placeholder, not a mature optimizer layer.
|
||||
|
||||
4. There is still no broad end-to-end CI lane that combines:
|
||||
- native build
|
||||
- Python tests
|
||||
- benchmark smoke
|
||||
- profile validation
|
||||
| Test | File | Validates |
|
||||
|------|------|-----------|
|
||||
| `turboquant_roundtrip` | tests/roundtrip_test.cpp | Encode→decode roundtrip fidelity |
|
||||
| Perplexity benchmarks | benchmarks/run_perplexity.py | Quality preservation across prompts |
|
||||
| Speed benchmarks | benchmarks/run_benchmarks.py | Compression overhead measurement |
|
||||
|
||||
## Security Considerations
|
||||
|
||||
- Strong privacy posture: the repo is fundamentally local-first and computational, not network-oriented.
|
||||
- No obvious telemetry surface in the core implementation.
|
||||
- Native code and Metal shaders increase the blast radius of correctness bugs; correctness verification matters more than in pure Python tooling.
|
||||
- Build artifacts are not sandboxed by default; consumers inherit the trust boundary of local execution.
|
||||
- Deployment profiles can route this code into larger local inference systems, so stale configuration guidance is an operational risk even when the core compression code is sound.
|
||||
1. **No network calls** — Pure local computation, no telemetry
|
||||
2. **Memory safety** — C++ code uses raw pointers; roundtrip tests validate correctness
|
||||
3. **Build isolation** — CMake builds static library; no dynamic linking
|
||||
|
||||
## Current Drift and Operational Findings
|
||||
## Sovereignty Assessment
|
||||
|
||||
- The refreshed genome here is more current than the older generated snapshot because it explicitly incorporates the present repo surface (`tests/test_quant_selector.py`, `tests/test_tool_call_integration.py`, `evolution/quant_selector.py`, and `profiles/hermes-profile-gemma4-turboquant.yaml`).
|
||||
- Current `main` still has a CI/runtime visibility gap: `.gitea/workflows/smoke.yml` does not yet run the native build/test path even though local runtime verification passes.
|
||||
- `turboquant #139` captures the currently failing `tests/test_quant_selector.py` ordering assertion and should remain referenced as live drift until fixed upstream.
|
||||
- **Fully local** — No cloud dependencies, no API calls
|
||||
- **Open source** — All code on Gitea, upstream repos public
|
||||
- **No telemetry** — Pure computation
|
||||
- **Hardware-specific** — Metal shaders target Apple Silicon; CUDA upstream for other GPUs
|
||||
|
||||
## Operational Risk Summary
|
||||
**Verdict: Fully sovereign. No corporate lock-in. Pure local inference enhancement.**
|
||||
|
||||
- No network requirement for core compression logic.
|
||||
- Benchmarks and profiles are local-file driven.
|
||||
- The main security/quality risks are correctness drift, stale CI, and native-code trust boundaries rather than data exfiltration.
|
||||
---
|
||||
|
||||
## Conclusion
|
||||
|
||||
TurboQuant is a compact but real codebase: a native compression core, a Metal runtime surface, benchmark harnesses, quant-selection logic, and Hermes deployment configuration. The core standalone path is healthy under manual verification, but the repo still depends on human discipline because CI does not yet enforce the same native build/test contract and the quant-selector lane remains partially red on `main`.
|
||||
|
||||
That makes the repo strong on local sovereignty and promising on runtime execution, but still operationally immature in continuous verification. The refreshed genome should be read as both an architecture map and a warning: the compression core is ahead of its automated guardrails.
|
||||
*"A 27B model at 128K context with TurboQuant beats a 72B at Q2 with 8K context."*
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
# MemPalace v3.0.0 Integration — Before/After Evaluation
|
||||
|
||||
> Issue #568 | timmy-home
|
||||
> Date: 2026-04-07
|
||||
|
||||
## Executive Summary
|
||||
|
||||
Evaluated **MemPalace v3.0.0** as a memory layer for the Timmy/Hermes agent stack.
|
||||
|
||||
**Installed:** ✅ `mempalace 3.0.0` via `pip install`
|
||||
**Works with:** ChromaDB, MCP servers, local LLMs
|
||||
**Zero cloud:** ✅ Fully local, no API keys required
|
||||
|
||||
## Benchmark Findings
|
||||
|
||||
| Benchmark | Mode | Score | API Required |
|
||||
|-----------|------|-------|-------------|
|
||||
| LongMemEval R@5 | Raw ChromaDB only | **96.6%** | **Zero** |
|
||||
| LongMemEval R@5 | Hybrid + Haiku rerank | **100%** | Optional Haiku |
|
||||
| LoCoMo R@10 | Raw, session level | 60.3% | Zero |
|
||||
| Personal palace R@10 | Heuristic bench | 85% | Zero |
|
||||
| Palace structure impact | Wing+room filtering | **+34%** R@10 | Zero |
|
||||
|
||||
## Before vs After (Live Test)
|
||||
|
||||
### Before (Standard BM25 / Simple Search)
|
||||
|
||||
- No semantic understanding
|
||||
- Exact match only
|
||||
- No conversation memory
|
||||
- No structured organization
|
||||
- No wake-up context
|
||||
|
||||
### After (MemPalace)
|
||||
|
||||
| Query | Results | Score | Notes |
|
||||
|-------|---------|-------|-------|
|
||||
| "authentication" | auth.md, main.py | -0.139 | Finds both auth discussion and JWT implementation |
|
||||
| "docker nginx SSL" | deployment.md, auth.md | 0.447 | Exact match on deployment, related JWT context |
|
||||
| "keycloak OAuth" | auth.md, main.py | -0.029 | Finds OAuth discussion and JWT usage |
|
||||
| "postgresql database" | README.md, main.py | 0.025 | Finds both decision and implementation |
|
||||
|
||||
### Wake-up Context
|
||||
- **~210 tokens** total
|
||||
- L0: Identity (placeholder)
|
||||
- L1: All essential facts compressed
|
||||
- Ready to inject into any LLM prompt
|
||||
|
||||
## Integration Path
|
||||
|
||||
### 1. Memory Mining
|
||||
```bash
|
||||
mempalace mine ~/.hermes/sessions/ --mode convos
|
||||
mempalace mine ~/.hermes/hermes-agent/
|
||||
mempalace mine ~/.hermes/
|
||||
```
|
||||
|
||||
### 2. Wake-up Protocol
|
||||
```bash
|
||||
mempalace wake-up > /tmp/timmy-context.txt
|
||||
```
|
||||
|
||||
### 3. MCP Integration
|
||||
```bash
|
||||
hermes mcp add mempalace -- python -m mempalace.mcp_server
|
||||
```
|
||||
|
||||
### 4. Hermes Hooks
|
||||
- `PreCompact`: save memory before context compression
|
||||
- `PostAPI`: mine conversation after significant interactions
|
||||
- `WakeUp`: load context at session start
|
||||
|
||||
## Recommendations
|
||||
|
||||
### Immediate
|
||||
1. Add `mempalace` to Hermes venv requirements
|
||||
2. Create mine script for ~/.hermes/ and ~/.timmy/
|
||||
3. Add wake-up hook to Hermes session start
|
||||
4. Test with real conversation exports
|
||||
|
||||
### Short-term
|
||||
1. Mine last 30 days of Timmy sessions
|
||||
2. Build wake-up context for all agents
|
||||
3. Add MemPalace MCP tools to Hermes toolset
|
||||
4. Test retrieval quality on real queries
|
||||
|
||||
### Medium-term
|
||||
1. Replace homebrew memory system with MemPalace
|
||||
2. Build palace structure: wings for projects, halls for topics
|
||||
3. Compress with AAAK for 30x storage efficiency
|
||||
4. Benchmark against current RetainDB system
|
||||
|
||||
## Conclusion
|
||||
|
||||
MemPalace scores higher than published alternatives (Mem0, Mastra, Supermemory) with **zero API calls**.
|
||||
|
||||
Key advantages:
|
||||
1. **Verbatim retrieval** — never loses the "why" context
|
||||
2. **Palace structure** — +34% boost from organization
|
||||
3. **Local-only** — aligns with sovereignty mandate
|
||||
4. **MCP compatible** — drops into existing tool chain
|
||||
5. **AAAK compression** — 30x storage reduction coming
|
||||
|
||||
---
|
||||
|
||||
*Evaluated by Timmy | Issue #568*
|
||||
171
scripts/genome_analyzer.py
Executable file
171
scripts/genome_analyzer.py
Executable file
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
genome_analyzer.py — Generate a GENOME.md from a codebase.
|
||||
|
||||
Scans a repository and produces a structured codebase genome with:
|
||||
- File counts by type
|
||||
- Architecture overview (directory structure)
|
||||
- Entry points
|
||||
- Test coverage summary
|
||||
|
||||
Usage:
|
||||
python3 scripts/genome_analyzer.py /path/to/repo
|
||||
python3 scripts/genome_analyzer.py /path/to/repo --output GENOME.md
|
||||
python3 scripts/genome_analyzer.py /path/to/repo --dry-run
|
||||
|
||||
Part of #666: GENOME.md Template + Single-Repo Analyzer.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
SKIP_DIRS = {".git", "__pycache__", ".venv", "venv", "node_modules", ".tox", ".pytest_cache", ".DS_Store"}
|
||||
|
||||
|
||||
def count_files(repo_path: Path) -> Dict[str, int]:
|
||||
counts = defaultdict(int)
|
||||
for f in repo_path.rglob("*"):
|
||||
if any(part in SKIP_DIRS for part in f.parts):
|
||||
continue
|
||||
if f.is_file():
|
||||
ext = f.suffix or "(no ext)"
|
||||
counts[ext] += 1
|
||||
return dict(sorted(counts.items(), key=lambda x: -x[1]))
|
||||
|
||||
|
||||
def find_entry_points(repo_path: Path) -> List[str]:
|
||||
entry_points = []
|
||||
candidates = [
|
||||
"main.py", "app.py", "server.py", "cli.py", "manage.py",
|
||||
"index.html", "index.js", "index.ts",
|
||||
"Makefile", "Dockerfile", "docker-compose.yml",
|
||||
"README.md", "deploy.sh", "setup.py", "pyproject.toml",
|
||||
]
|
||||
for name in candidates:
|
||||
if (repo_path / name).exists():
|
||||
entry_points.append(name)
|
||||
scripts_dir = repo_path / "scripts"
|
||||
if scripts_dir.is_dir():
|
||||
for f in sorted(scripts_dir.iterdir()):
|
||||
if f.suffix in (".py", ".sh") and not f.name.startswith("test_"):
|
||||
entry_points.append(f"scripts/{f.name}")
|
||||
return entry_points[:15]
|
||||
|
||||
|
||||
def find_tests(repo_path: Path) -> Tuple[List[str], int]:
|
||||
test_files = []
|
||||
for f in repo_path.rglob("*"):
|
||||
if any(part in SKIP_DIRS for part in f.parts):
|
||||
continue
|
||||
if f.is_file() and (f.name.startswith("test_") or f.name.endswith("_test.py") or f.name.endswith("_test.js")):
|
||||
test_files.append(str(f.relative_to(repo_path)))
|
||||
return sorted(test_files), len(test_files)
|
||||
|
||||
|
||||
def find_directories(repo_path: Path, max_depth: int = 2) -> List[str]:
|
||||
dirs = []
|
||||
for d in sorted(repo_path.rglob("*")):
|
||||
if d.is_dir() and len(d.relative_to(repo_path).parts) <= max_depth:
|
||||
if not any(part in SKIP_DIRS for part in d.parts):
|
||||
rel = str(d.relative_to(repo_path))
|
||||
if rel != ".":
|
||||
dirs.append(rel)
|
||||
return dirs[:30]
|
||||
|
||||
|
||||
def read_readme(repo_path: Path) -> str:
|
||||
for name in ["README.md", "README.rst", "README.txt", "README"]:
|
||||
readme = repo_path / name
|
||||
if readme.exists():
|
||||
lines = readme.read_text(encoding="utf-8", errors="replace").split("\n")
|
||||
para = []
|
||||
started = False
|
||||
for line in lines:
|
||||
if line.startswith("#") and not started:
|
||||
continue
|
||||
if line.strip():
|
||||
started = True
|
||||
para.append(line.strip())
|
||||
elif started:
|
||||
break
|
||||
return " ".join(para[:5])
|
||||
return "(no README found)"
|
||||
|
||||
|
||||
def generate_genome(repo_path: Path, repo_name: str = "") -> str:
|
||||
if not repo_name:
|
||||
repo_name = repo_path.name
|
||||
date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
readme_desc = read_readme(repo_path)
|
||||
file_counts = count_files(repo_path)
|
||||
total_files = sum(file_counts.values())
|
||||
entry_points = find_entry_points(repo_path)
|
||||
test_files, test_count = find_tests(repo_path)
|
||||
dirs = find_directories(repo_path)
|
||||
|
||||
lines = [
|
||||
f"# GENOME.md — {repo_name}", "",
|
||||
f"> Codebase analysis generated {date}. {readme_desc[:100]}.", "",
|
||||
"## Project Overview", "",
|
||||
readme_desc, "",
|
||||
f"**{total_files} files** across {len(file_counts)} file types.", "",
|
||||
"## Architecture", "",
|
||||
"```",
|
||||
]
|
||||
for d in dirs[:20]:
|
||||
lines.append(f" {d}/")
|
||||
lines.append("```")
|
||||
lines += ["", "### File Types", "", "| Type | Count |", "|------|-------|"]
|
||||
for ext, count in list(file_counts.items())[:15]:
|
||||
lines.append(f"| {ext} | {count} |")
|
||||
lines += ["", "## Entry Points", ""]
|
||||
for ep in entry_points:
|
||||
lines.append(f"- `{ep}`")
|
||||
lines += ["", "## Test Coverage", "", f"**{test_count} test files** found.", ""]
|
||||
if test_files:
|
||||
for tf in test_files[:10]:
|
||||
lines.append(f"- `{tf}`")
|
||||
if len(test_files) > 10:
|
||||
lines.append(f"- ... and {len(test_files) - 10} more")
|
||||
else:
|
||||
lines.append("No test files found.")
|
||||
lines += ["", "## Security Considerations", "", "(To be filled during analysis)", ""]
|
||||
lines += ["## Design Decisions", "", "(To be filled during analysis)", ""]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Generate GENOME.md from a codebase")
|
||||
parser.add_argument("repo_path", help="Path to repository")
|
||||
parser.add_argument("--output", default="", help="Output file (default: stdout)")
|
||||
parser.add_argument("--name", default="", help="Repository name")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Print stats only")
|
||||
args = parser.parse_args()
|
||||
repo_path = Path(args.repo_path).resolve()
|
||||
if not repo_path.is_dir():
|
||||
print(f"ERROR: {repo_path} is not a directory", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
repo_name = args.name or repo_path.name
|
||||
if args.dry_run:
|
||||
counts = count_files(repo_path)
|
||||
_, test_count = find_tests(repo_path)
|
||||
print(f"Repo: {repo_name}")
|
||||
print(f"Total files: {sum(counts.values())}")
|
||||
print(f"Test files: {test_count}")
|
||||
print(f"Top types: {', '.join(f'{k}={v}' for k,v in list(counts.items())[:5])}")
|
||||
sys.exit(0)
|
||||
genome = generate_genome(repo_path, repo_name)
|
||||
if args.output:
|
||||
with open(args.output, "w") as f:
|
||||
f.write(genome)
|
||||
print(f"Written: {args.output}")
|
||||
else:
|
||||
print(genome)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
46
templates/GENOME-template.md
Normal file
46
templates/GENOME-template.md
Normal file
@@ -0,0 +1,46 @@
|
||||
# GENOME.md — {{REPO_NAME}}
|
||||
|
||||
> Codebase analysis generated {{DATE}}. {{SHORT_DESCRIPTION}}.
|
||||
|
||||
## Project Overview
|
||||
|
||||
{{OVERVIEW}}
|
||||
|
||||
## Architecture
|
||||
|
||||
{{ARCHITECTURE_DIAGRAM}}
|
||||
|
||||
## Entry Points
|
||||
|
||||
{{ENTRY_POINTS}}
|
||||
|
||||
## Data Flow
|
||||
|
||||
{{DATA_FLOW}}
|
||||
|
||||
## Key Abstractions
|
||||
|
||||
{{ABSTRACTIONS}}
|
||||
|
||||
## API Surface
|
||||
|
||||
{{API_SURFACE}}
|
||||
|
||||
## Test Coverage
|
||||
|
||||
### Existing Tests
|
||||
{{EXISTING_TESTS}}
|
||||
|
||||
### Coverage Gaps
|
||||
{{COVERAGE_GAPS}}
|
||||
|
||||
### Critical paths that need tests:
|
||||
{{CRITICAL_PATHS}}
|
||||
|
||||
## Security Considerations
|
||||
|
||||
{{SECURITY}}
|
||||
|
||||
## Design Decisions
|
||||
|
||||
{{DESIGN_DECISIONS}}
|
||||
@@ -1,56 +0,0 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
GENOME = Path("GENOME.md")
|
||||
|
||||
|
||||
def read_genome() -> str:
|
||||
assert GENOME.exists(), "GENOME.md must exist at repo root"
|
||||
return GENOME.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_the_nexus_genome_has_required_sections() -> None:
|
||||
text = read_genome()
|
||||
required = [
|
||||
"# GENOME.md — the-nexus",
|
||||
"## Project Overview",
|
||||
"## Architecture Diagram",
|
||||
"```mermaid",
|
||||
"## Entry Points and Data Flow",
|
||||
"## Key Abstractions",
|
||||
"## API Surface",
|
||||
"## Test Coverage Gaps",
|
||||
"## Security Considerations",
|
||||
"## Runtime Truth and Docs Drift",
|
||||
]
|
||||
missing = [item for item in required if item not in text]
|
||||
assert not missing, missing
|
||||
|
||||
|
||||
def test_the_nexus_genome_captures_current_runtime_contract() -> None:
|
||||
text = read_genome()
|
||||
required = [
|
||||
"server.py",
|
||||
"app.js",
|
||||
"index.html",
|
||||
"portals.json",
|
||||
"vision.json",
|
||||
"BROWSER_CONTRACT.md",
|
||||
"tests/test_browser_smoke.py",
|
||||
"tests/test_repo_truth.py",
|
||||
"nexus/morrowind_harness.py",
|
||||
"nexus/bannerlord_harness.py",
|
||||
"mempalace/tunnel_sync.py",
|
||||
"mcp_servers/desktop_control_server.py",
|
||||
"public/nexus/",
|
||||
]
|
||||
missing = [item for item in required if item not in text]
|
||||
assert not missing, missing
|
||||
|
||||
|
||||
def test_the_nexus_genome_explains_docs_runtime_drift() -> None:
|
||||
text = read_genome()
|
||||
assert "README.md says current `main` does not ship a browser 3D world" in text
|
||||
assert "CLAUDE.md declares root `app.js` and `index.html` as canonical frontend paths" in text
|
||||
assert "tests and browser contract now assume the root frontend exists" in text
|
||||
assert len(text) >= 5000
|
||||
@@ -1,44 +0,0 @@
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
GENOME_PATH = ROOT / "genomes" / "turboquant" / "GENOME.md"
|
||||
|
||||
|
||||
class TestTurboquantGenome(unittest.TestCase):
|
||||
def test_genome_exists_with_required_sections(self):
|
||||
self.assertTrue(GENOME_PATH.exists(), "missing genomes/turboquant/GENOME.md")
|
||||
text = GENOME_PATH.read_text(encoding="utf-8")
|
||||
required_sections = [
|
||||
"# GENOME.md — TurboQuant (Timmy_Foundation/turboquant)",
|
||||
"## Project Overview",
|
||||
"## Architecture",
|
||||
"```mermaid",
|
||||
"## Entry Points",
|
||||
"## Data Flow",
|
||||
"## Key Abstractions",
|
||||
"## Test Coverage",
|
||||
"## Security Considerations",
|
||||
]
|
||||
for section in required_sections:
|
||||
self.assertIn(section, text)
|
||||
|
||||
def test_genome_mentions_current_runtime_and_ci_findings(self):
|
||||
text = GENOME_PATH.read_text(encoding="utf-8")
|
||||
required_snippets = [
|
||||
".gitea/workflows/smoke.yml",
|
||||
"ctest --test-dir build --output-on-failure",
|
||||
"tests/test_quant_selector.py",
|
||||
"tests/test_tool_call_integration.py",
|
||||
"evolution/quant_selector.py",
|
||||
"profiles/hermes-profile-gemma4-turboquant.yaml",
|
||||
"pygount",
|
||||
"turboquant #139",
|
||||
]
|
||||
for snippet in required_snippets:
|
||||
self.assertIn(snippet, text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user