From 38bcaa1e86dfd0c03c0aba1735823297af25dffe Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Mon, 6 Apr 2026 11:08:55 +0530 Subject: [PATCH] chore: remove langfuse doc, smoketest script, and installed-plugin test Made-with: Cursor --- docs/langfuse-tracing-local-setup.md | 262 ------------------ scripts/langfuse_smoketest.py | 215 -------------- .../test_langfuse_tracing_plugin_installed.py | 102 ------- 3 files changed, 579 deletions(-) delete mode 100644 docs/langfuse-tracing-local-setup.md delete mode 100644 scripts/langfuse_smoketest.py delete mode 100644 tests/test_langfuse_tracing_plugin_installed.py diff --git a/docs/langfuse-tracing-local-setup.md b/docs/langfuse-tracing-local-setup.md deleted file mode 100644 index 6e1fbab48..000000000 --- a/docs/langfuse-tracing-local-setup.md +++ /dev/null @@ -1,262 +0,0 @@ -# Langfuse Tracing for Hermes - -Opt-in tracing plugin that sends LLM calls, tool calls, and per-turn spans to -Langfuse. The plugin lives **outside** the hermes-agent repo so pulling -upstream updates never causes conflicts. - ---- - -## Quick start (copy-paste recipe) - -This gets you from zero to working traces. Every command is meant to be run -in order in a single terminal session. - -```bash -# ── 1. Prerequisites ────────────────────────────────────────────────── -cd /path/to/hermes-agent -source .venv/bin/activate -pip install langfuse # into the repo venv, not global - -# ── 2. Fetch the plugin source ──────────────────────────────────────── -# The plugin lives on the fork branch feat/langfuse_tracing. -# Pick ONE of the two fetch commands depending on your remote setup: - -# (a) Your origin IS the fork (kshitijk4poor/hermes-agent): -git fetch origin feat/langfuse_tracing -PLUGIN_REF="origin/feat/langfuse_tracing" - -# (b) Your origin is upstream (NousResearch/hermes-agent): -git fetch git@github.com:kshitijk4poor/hermes-agent.git \ - feat/langfuse_tracing:refs/remotes/fork/feat/langfuse_tracing -PLUGIN_REF="fork/feat/langfuse_tracing" - -# ── 3. Determine your plugin directory ──────────────────────────────── -# Hermes loads user plugins from $HERMES_HOME/plugins/. -# HERMES_HOME defaults to ~/.hermes for the default profile. -# If you use `hermes -p `, it becomes ~/.hermes/profiles//. -# The CLI sets HERMES_HOME internally — it may not be in your shell env. - -# Default profile: -PLUGIN_DIR="$HOME/.hermes/plugins/langfuse_tracing" - -# Named profile (uncomment and edit): -# PLUGIN_DIR="$HOME/.hermes/profiles//plugins/langfuse_tracing" - -# ── 4. Install the plugin ──────────────────────────────────────────── -mkdir -p "$PLUGIN_DIR" -git show "$PLUGIN_REF:.hermes/plugins/langfuse_tracing/__init__.py" \ - > "$PLUGIN_DIR/__init__.py" -git show "$PLUGIN_REF:.hermes/plugins/langfuse_tracing/plugin.yaml" \ - > "$PLUGIN_DIR/plugin.yaml" - -# ── 5. Set credentials ─────────────────────────────────────────────── -# Add these to your shell profile (~/.zshrc, ~/.bashrc, etc.) or .env. -# Tracing is completely dormant without them — no errors, no network calls. -export HERMES_LANGFUSE_ENABLED=true -export HERMES_LANGFUSE_PUBLIC_KEY=pk-lf-... -export HERMES_LANGFUSE_SECRET_KEY=sk-lf-... - -# ── 6. Verify ───────────────────────────────────────────────────────── -# Start a NEW terminal / hermes process (plugins load at startup only). -hermes plugins list # should show langfuse_tracing: enabled -HERMES_LANGFUSE_DEBUG=true hermes chat -q "hello" -# Look for: "Langfuse tracing: started trace ..." in stderr -``` - -That's it. The plugin is outside the repo tree, so `git pull upstream main` -will never touch it. - ---- - -## Updating hermes without breaking tracing - -The plugin hooks into hermes via the standard plugin system and uses `**_` in -every hook signature to absorb new kwargs. Per-API-call tracing uses -`pre_api_request` / `post_api_request` (not `pre_llm_call` / `post_llm_call`, which -are once per user turn). Those hooks receive **summary fields only** (message -counts, tool counts, token usage dict, etc.) — not full `messages`, `tools`, or -raw provider `response` objects — so keep span metadata small and the contract -stable. - -This means: - -```bash -# Just pull upstream as usual -git fetch upstream -git merge upstream/main -# or: git pull upstream main -``` - -Nothing else is needed. The plugin at `$PLUGIN_DIR` is not inside the repo, -so there are no merge conflicts. - -### Updating the plugin itself - -When the plugin code on `feat/langfuse_tracing` is updated: - -```bash -git fetch origin feat/langfuse_tracing # or the fork fetch from step 2b -git show "$PLUGIN_REF:.hermes/plugins/langfuse_tracing/__init__.py" \ - > "$PLUGIN_DIR/__init__.py" -git show "$PLUGIN_REF:.hermes/plugins/langfuse_tracing/plugin.yaml" \ - > "$PLUGIN_DIR/plugin.yaml" -# Restart hermes to pick up changes -``` - ---- - -## Alternative: symlink for plugin development - -If you're actively editing the plugin and want it version-controlled separately: - -```bash -# Create a standalone plugin repo -mkdir -p ~/Projects/hermes-langfuse-plugin/langfuse_tracing -git show "$PLUGIN_REF:.hermes/plugins/langfuse_tracing/__init__.py" \ - > ~/Projects/hermes-langfuse-plugin/langfuse_tracing/__init__.py -git show "$PLUGIN_REF:.hermes/plugins/langfuse_tracing/plugin.yaml" \ - > ~/Projects/hermes-langfuse-plugin/langfuse_tracing/plugin.yaml -cd ~/Projects/hermes-langfuse-plugin && git init && git add -A && git commit -m "init" - -# Symlink into hermes plugin dir (remove existing dir/link first) -rm -rf "$PLUGIN_DIR" -ln -s ~/Projects/hermes-langfuse-plugin/langfuse_tracing "$PLUGIN_DIR" -``` - -Edits to `~/Projects/hermes-langfuse-plugin/langfuse_tracing/` take effect on -next hermes restart. Upstream hermes updates are still conflict-free. - ---- - -## Environment variables reference - -All variables are optional. Tracing does nothing unless `ENABLED` + both keys are set. - -| Variable | Required | Default | Notes | -|----------|----------|---------|-------| -| `HERMES_LANGFUSE_ENABLED` | yes | `false` | Must be `true`/`1`/`yes`/`on` | -| `HERMES_LANGFUSE_PUBLIC_KEY` | yes | — | Langfuse project public key | -| `HERMES_LANGFUSE_SECRET_KEY` | yes | — | Langfuse project secret key | -| `HERMES_LANGFUSE_BASE_URL` | no | `https://cloud.langfuse.com` | Self-hosted Langfuse URL | -| `HERMES_LANGFUSE_ENV` | no | — | Environment tag (e.g. `development`) | -| `HERMES_LANGFUSE_RELEASE` | no | — | Release tag | -| `HERMES_LANGFUSE_SAMPLE_RATE` | no | `1.0` | Float 0.0-1.0 | -| `HERMES_LANGFUSE_MAX_CHARS` | no | `12000` | Max chars per traced value | -| `HERMES_LANGFUSE_DEBUG` | no | `false` | Verbose logging to stderr | - -Each variable also accepts `CC_LANGFUSE_*` and bare `LANGFUSE_*` prefixes as -fallbacks (checked in order: `HERMES_` > `CC_` > bare). - ---- - -## Troubleshooting - -| Symptom | Cause | Fix | -|---------|-------|-----| -| `hermes plugins list` doesn't show `langfuse_tracing` | Plugin files not in the right dir | Check `$PLUGIN_DIR` matches your profile. Must contain both `__init__.py` and `plugin.yaml`. | -| Listed as `disabled` | In `plugins.disabled` in config.yaml | Run `hermes plugins enable langfuse_tracing` | -| No trace output with `HERMES_LANGFUSE_DEBUG=true` | Plugin loaded but dormant | Verify all 3 required env vars are set and exported | -| `"Could not initialize Langfuse client: ..."` | Bad credentials or unreachable server | Check public/secret keys; check base URL if self-hosted | -| Traces appear but background reviews aren't tagged | `feat/turn-type-hooks` not merged upstream | Plugin still works — `turn_type` defaults to `"user"`. Background reviews just won't be filterable until the upstream PR lands. | -| Plugin works in `hermes` but not `hermes -p coder` | Profile-scoped plugin dirs | Install plugin into `~/.hermes/profiles/coder/plugins/langfuse_tracing/` | - ---- - -## Disabling tracing - -Three options, from least to most permanent: - -1. **Unset env vars** — unset `HERMES_LANGFUSE_ENABLED`. Plugin loads but does nothing. -2. **CLI toggle** — `hermes plugins disable langfuse_tracing`. Plugin is skipped at startup. -3. **Remove files** — `rm -rf "$PLUGIN_DIR"`. - ---- - -## What gets traced - -Each user turn becomes a root trace with nested child observations: - -``` -Hermes turn (or "Hermes background review") - |-- LLM call 0 (generation — with usage/cost) - |-- Tool: search_files (tool — with parsed JSON output) - |-- Tool: read_file (tool — head/tail preview, not raw content) - |-- LLM call 1 (generation) - \-- ... -``` - -Root trace metadata: `source`, `task_id`, `session_id`, `platform`, `provider`, -`model`, `api_mode`, `turn_type`. - -Tags: `hermes`, `langfuse`, plus `background_review` for auto-generated passes. - -Data normalization applied: -- Tool result JSON strings parsed into dicts -- Trailing `[Hint: ...]` extracted into `_hint` key -- `read_file` content replaced with head/tail line preview -- `base64_content` omitted (replaced with length) -- Usage/cost extracted when `agent.usage_pricing` is available - ---- - -## Running tests - -Tests live on the fork branch only — not on upstream or `main`. - -```bash -git checkout feat/langfuse_tracing -source .venv/bin/activate -python -m pytest tests/test_langfuse_tracing_plugin.py -q -``` - -12 tests covering payload parsing, observation nesting, tool call aggregation, -and `turn_type` propagation. No credentials or network access needed. - ---- - -## Project history - -### Branches - -| Branch | Remote | Purpose | -|--------|--------|---------| -| `feat/turn-type-hooks` | `origin` (fork) | Upstream PR: `turn_type` hook plumbing in `run_agent.py` + `model_tools.py` | -| `feat/langfuse_tracing` | `origin` (fork) | Plugin code, tests, optional skill, skills hub changes | - -Fork remote: `git@github.com:kshitijk4poor/hermes-agent.git` -Upstream remote: `https://github.com/NousResearch/hermes-agent.git` - -### Commit log (chronological) - -| Date | Commit | Description | -|------|--------|-------------| -| 2026-03-28 | `b0a64856` | Initial plugin + hook emission patches + langfuse dependency | -| 2026-03-28 | `e691abda` | Parse JSON tool payloads into structured data | -| 2026-03-28 | `00dbff19` | Handle trailing `[Hint: ...]` after JSON in tool outputs | -| 2026-03-28 | `fd54a008` | Fix child observation nesting (use parent span API) | -| 2026-03-28 | `8752aed1` | Format read_file traces as head/tail previews | -| 2026-03-28 | `93f9c338` | Aggregate tool calls onto root trace output | -| 2026-03-29 | `dd714b2a` | Optional skill installer + skills hub enhancements | -| 2026-03-29 | `4b2f865e` | Distinguish background review traces via `turn_type` | -| 2026-03-29 | `aef4b44d` | Upstream-clean `turn_type` hook plumbing (2 files only) | - -### File inventory - -**Plugin** (`$HERMES_HOME/plugins/langfuse_tracing/`): -`__init__.py` (hook handlers + `register()`), `plugin.yaml` (manifest) - -**Upstream PR** (`feat/turn-type-hooks`): -`run_agent.py` (+`_turn_type` attr, hook propagation), `model_tools.py` (+`turn_type` param) - -**Fork branch** (`feat/langfuse_tracing`): -`.hermes/plugins/langfuse_tracing/` (plugin source), -`optional-skills/observability/` (installer skill), -`tools/skills_hub.py` + `hermes_cli/skills_hub.py` (hub enhancements), -`tests/test_langfuse_tracing_plugin.py` + `tests/tools/test_skills_hub.py` (tests) - -### Known limitations - -1. `pre_llm_call`/`post_llm_call` fire once per user turn. Hermes (this branch) adds `pre_api_request`/`post_api_request` per actual LLM HTTP request; the Langfuse plugin on `feat/langfuse_tracing` should register those names and read the summary kwargs documented above. -2. No session-level parent trace — turns are independent, linked by `session_id` in metadata. -3. Background review filtering requires the `feat/turn-type-hooks` upstream PR. -4. Plugin is profile-scoped — must be installed per Hermes profile. diff --git a/scripts/langfuse_smoketest.py b/scripts/langfuse_smoketest.py deleted file mode 100644 index c298a3a02..000000000 --- a/scripts/langfuse_smoketest.py +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/env python3 -"""Verify Langfuse credentials and that the user plugin can emit a trace. - -Loads ``~/.hermes/.env`` (and optional repo ``.env``) like Hermes. Run from repo: - - uv run python scripts/langfuse_smoketest.py - -Exit codes: 0 ok, 1 connectivity/plugin failure, 2 missing keys/plugin files. -""" - -from __future__ import annotations - -import argparse -import base64 -import importlib.util -import json -import os -import sys -import uuid -from pathlib import Path -from urllib.error import HTTPError, URLError -from urllib.request import Request, urlopen - - -def _repo_root() -> Path: - return Path(__file__).resolve().parents[1] - - -def _pick(*keys: str) -> str: - for k in keys: - v = os.getenv(k, "").strip() - if v: - return v - return "" - - -def _load_hermes_env() -> None: - repo = _repo_root() - sys.path.insert(0, str(repo)) - from hermes_cli.env_loader import load_hermes_dotenv - from hermes_constants import get_hermes_home - - load_hermes_dotenv(hermes_home=get_hermes_home(), project_env=repo / ".env") - - -def _sdk_smoke() -> str: - from langfuse import Langfuse - - pk = _pick("HERMES_LANGFUSE_PUBLIC_KEY", "LANGFUSE_PUBLIC_KEY", "CC_LANGFUSE_PUBLIC_KEY") - sk = _pick("HERMES_LANGFUSE_SECRET_KEY", "LANGFUSE_SECRET_KEY", "CC_LANGFUSE_SECRET_KEY") - base = _pick("HERMES_LANGFUSE_BASE_URL", "LANGFUSE_BASE_URL", "CC_LANGFUSE_BASE_URL") - if not base: - base = "https://cloud.langfuse.com" - if not pk or not sk: - print("ERROR: set HERMES_LANGFUSE_PUBLIC_KEY and HERMES_LANGFUSE_SECRET_KEY (or LANGFUSE_* aliases).") - sys.exit(2) - - lf = Langfuse(public_key=pk, secret_key=sk, base_url=base) - if not lf.auth_check(): - print("ERROR: Langfuse auth_check() returned False.") - sys.exit(1) - - trace_id = lf.create_trace_id(seed="hermes-langfuse-smoketest") - root = lf.start_observation( - trace_context={"trace_id": trace_id}, - name="Hermes langfuse_smoketest (SDK)", - as_type="chain", - input={"check": "sdk"}, - metadata={"source": "scripts/langfuse_smoketest.py"}, - ) - child = root.start_observation( - name="sub-span", - as_type="generation", - input={"ping": True}, - model="smoke/test", - ) - child.update(output={"pong": True}) - child.end() - root.end() - lf.flush() - try: - url = lf.get_trace_url(trace_id=trace_id) - except Exception: - url = f"{base.rstrip('/')}/traces/{trace_id}" - print("SDK smoke: OK") - print(" trace_id:", trace_id) - print(" url:", url) - return trace_id - - -def _plugin_smoke() -> None: - plugin_path = Path.home() / ".hermes" / "plugins" / "langfuse_tracing" / "__init__.py" - if not plugin_path.is_file(): - print("SKIP plugin smoke: no file at", plugin_path) - return - - spec = importlib.util.spec_from_file_location("langfuse_tracing_smoke", plugin_path) - if spec is None or spec.loader is None: - print("ERROR: cannot load plugin module spec") - sys.exit(1) - mod = importlib.util.module_from_spec(spec) - sys.modules["langfuse_tracing_smoke"] = mod - spec.loader.exec_module(mod) - - mod._TRACE_STATE.clear() - mod._LANGFUSE_CLIENT = None - - session_id = f"smoke_sess_{uuid.uuid4().hex[:8]}" - effective_task_id = str(uuid.uuid4()) - user_msg = "Langfuse plugin smoketest message." - - mod.on_pre_llm_call( - session_id=session_id, - user_message=user_msg, - conversation_history=[], - model="smoke/model", - platform="cli", - ) - mod.on_pre_api_request( - task_id=effective_task_id, - session_id=session_id, - platform="cli", - model="smoke/model", - provider="test", - base_url="http://localhost", - api_mode="chat_completions", - api_call_count=1, - message_count=1, - tool_count=0, - approx_input_tokens=10, - request_char_count=40, - max_tokens=256, - ) - mod.on_post_api_request( - task_id=effective_task_id, - session_id=session_id, - provider="test", - base_url="http://localhost", - api_mode="chat_completions", - model="smoke/model", - api_call_count=1, - api_duration=0.01, - finish_reason="stop", - usage={ - "input_tokens": 5, - "output_tokens": 5, - "total_tokens": 10, - "reasoning_tokens": 0, - "cache_read_tokens": 0, - "cache_write_tokens": 0, - }, - assistant_content_chars=4, - assistant_tool_call_count=0, - response_model="smoke/model", - ) - mod.on_post_llm_call( - session_id=session_id, - user_message=user_msg, - assistant_response="pong", - conversation_history=[], - model="smoke/model", - platform="cli", - ) - - client = mod._get_langfuse() - if client is None: - print("SKIP plugin smoke: Langfuse disabled or keys missing (_get_langfuse is None).") - return - client.flush() - print("Plugin hook chain: OK (flushed)") - print(" session_id:", session_id) - - -def _api_list_traces(limit: int = 2) -> None: - pk = _pick("HERMES_LANGFUSE_PUBLIC_KEY", "LANGFUSE_PUBLIC_KEY", "CC_LANGFUSE_PUBLIC_KEY") - sk = _pick("HERMES_LANGFUSE_SECRET_KEY", "LANGFUSE_SECRET_KEY", "CC_LANGFUSE_SECRET_KEY") - base = _pick("HERMES_LANGFUSE_BASE_URL", "LANGFUSE_BASE_URL", "CC_LANGFUSE_BASE_URL") - if not base or not pk or not sk: - return - base = base.rstrip("/") - auth = base64.b64encode(f"{pk}:{sk}".encode()).decode() - req = Request( - f"{base}/api/public/traces?limit={limit}", - headers={"Authorization": f"Basic {auth}"}, - ) - try: - with urlopen(req, timeout=15) as resp: - payload = json.loads(resp.read().decode()) - except (HTTPError, URLError, TimeoutError, json.JSONDecodeError) as exc: - print("REST list traces: failed:", exc) - return - rows = payload.get("data") or [] - print(f"REST /api/public/traces?limit={limit}: {len(rows)} row(s)") - for row in rows: - name = row.get("name") - tid = row.get("id") - ts = row.get("timestamp") - print(f" - {ts} {name!r} id={tid}") - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--no-plugin", action="store_true", help="Only run SDK smoke + REST list") - args = parser.parse_args() - - _load_hermes_env() - _sdk_smoke() - if not args.no_plugin: - _plugin_smoke() - _api_list_traces(limit=3) - print("Done.") - - -if __name__ == "__main__": - main() diff --git a/tests/test_langfuse_tracing_plugin_installed.py b/tests/test_langfuse_tracing_plugin_installed.py deleted file mode 100644 index d85d83a5c..000000000 --- a/tests/test_langfuse_tracing_plugin_installed.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Smoke tests for the user-installed Langfuse plugin (when present). - -The canonical plugin lives under ``~/.hermes/plugins/langfuse_tracing/``. -These tests are skipped in CI unless that directory exists locally. -""" - -from __future__ import annotations - -import importlib.util -import sys -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest - -PLUGIN_INIT = Path.home() / ".hermes" / "plugins" / "langfuse_tracing" / "__init__.py" - -needs_user_plugin = pytest.mark.skipif( - not PLUGIN_INIT.is_file(), - reason="langfuse_tracing plugin not installed at ~/.hermes/plugins/langfuse_tracing/", -) - - -def _load_user_plugin(): - name = "langfuse_tracing_user_plugin" - if name in sys.modules: - return sys.modules[name] - spec = importlib.util.spec_from_file_location(name, PLUGIN_INIT) - if spec is None or spec.loader is None: - raise RuntimeError("cannot load langfuse plugin") - mod = importlib.util.module_from_spec(spec) - sys.modules[name] = mod - spec.loader.exec_module(mod) - return mod - - -@needs_user_plugin -def test_langfuse_plugin_registers_api_request_hooks(): - mod = _load_user_plugin() - ctx = MagicMock() - ctx.manifest.name = "langfuse_tracing" - mod.register(ctx) - registered = [c[0][0] for c in ctx.register_hook.call_args_list] - assert "pre_api_request" in registered - assert "post_api_request" in registered - assert "pre_llm_call" in registered - - -@needs_user_plugin -def test_pre_post_api_request_smoke_with_mock_langfuse(): - mod = _load_user_plugin() - mod._TRACE_STATE.clear() - - gen_obs = MagicMock() - root_obs = MagicMock() - root_obs.start_observation.return_value = gen_obs - - client = MagicMock() - client.create_trace_id.return_value = "trace-smoke-test" - client.start_observation.return_value = root_obs - - with patch.object(mod, "_get_langfuse", return_value=client): - mod.on_pre_api_request( - task_id="t1", - session_id="s1", - platform="cli", - model="test/model", - provider="openrouter", - base_url="https://openrouter.ai/api/v1", - api_mode="chat_completions", - api_call_count=1, - message_count=3, - tool_count=5, - approx_input_tokens=100, - request_char_count=400, - max_tokens=4096, - ) - mod.on_post_api_request( - task_id="t1", - session_id="s1", - provider="openrouter", - base_url="https://openrouter.ai/api/v1", - api_mode="chat_completions", - model="test/model", - api_call_count=1, - api_duration=0.05, - finish_reason="stop", - usage={ - "input_tokens": 10, - "output_tokens": 20, - "total_tokens": 30, - "reasoning_tokens": 0, - "cache_read_tokens": 0, - "cache_write_tokens": 0, - }, - assistant_content_chars=42, - assistant_tool_call_count=0, - response_model="test/model", - ) - - gen_obs.update.assert_called() - gen_obs.end.assert_called()