55d53c513c
Wire arXiv relevance filter output directly into DPO pair generation,
closing the loop between research synthesis and overnight training data.
New module: dpo_generator.py
- DPOPairGenerator class with 3 pair strategies:
* summarize: paper → fleet-grounded analysis (chosen) vs generic (rejected)
* relevance: 'what matters to Hermes?' → scored context vs vague
* implication: 'what should we do?' → actionable insight vs platitude
- Extracts synthesis excerpts matched to each ranked item
- Outputs to ~/.timmy/training-data/dpo-pairs/deepdive_{timestamp}.jsonl
- Format: {prompt, chosen, rejected, task_type, evidence_ids,
source_session, safety_flags, metadata}
Pipeline changes (pipeline.py):
- Import DPOPairGenerator with graceful degradation
- Initialize from config deepdive.training.dpo section
- Execute as Phase 3.5 between synthesis and audio
- DPO results included in pipeline return dict
- Wrapped in try/except — DPO failure never blocks delivery
Config changes (config.yaml):
- New deepdive.training.dpo section with:
enabled, output_dir, min_score, max_pairs_per_run, pair_types
Integration tested: 2 mock items × 3 pair types = 6 valid JSONL pairs.
Chosen responses consistently richer than rejected (assert-verified).
124 lines
3.1 KiB
YAML
124 lines
3.1 KiB
YAML
# Deep Dive Configuration
|
|
# Copy to config.yaml and customize
|
|
|
|
deepdive:
|
|
# Schedule
|
|
schedule:
|
|
daily_time: "06:00"
|
|
timezone: "America/New_York"
|
|
|
|
# Phase 1: Aggregation
|
|
sources:
|
|
- name: "arxiv_cs_ai"
|
|
url: "http://export.arxiv.org/rss/cs.AI"
|
|
type: "rss"
|
|
fetch_window_hours: 24
|
|
max_items: 50
|
|
|
|
- name: "arxiv_cs_cl"
|
|
url: "http://export.arxiv.org/rss/cs.CL"
|
|
type: "rss"
|
|
fetch_window_hours: 24
|
|
max_items: 50
|
|
|
|
- name: "arxiv_cs_lg"
|
|
url: "http://export.arxiv.org/rss/cs.LG"
|
|
type: "rss"
|
|
fetch_window_hours: 24
|
|
max_items: 50
|
|
|
|
- name: "openai_blog"
|
|
url: "https://openai.com/blog/rss.xml"
|
|
type: "rss"
|
|
fetch_window_hours: 48
|
|
max_items: 5
|
|
|
|
- name: "anthropic_blog"
|
|
url: "https://www.anthropic.com/blog/rss.xml"
|
|
type: "rss"
|
|
fetch_window_hours: 48
|
|
max_items: 5
|
|
|
|
- name: "deepmind_blog"
|
|
url: "https://deepmind.google/blog/rss.xml"
|
|
type: "rss"
|
|
fetch_window_hours: 48
|
|
max_items: 5
|
|
|
|
# Phase 2: Relevance
|
|
relevance:
|
|
model: "all-MiniLM-L6-v2" # ~80MB embeddings model
|
|
top_n: 10 # Items selected for briefing
|
|
min_score: 0.25 # Hard cutoff
|
|
keywords:
|
|
- "LLM agent"
|
|
- "agent architecture"
|
|
- "tool use"
|
|
- "function calling"
|
|
- "chain of thought"
|
|
- "reasoning"
|
|
- "reinforcement learning"
|
|
- "RLHF"
|
|
- "GRPO"
|
|
- "PPO"
|
|
- "fine-tuning"
|
|
- "transformer"
|
|
- "attention mechanism"
|
|
- "inference optimization"
|
|
- "quantization"
|
|
- "local LLM"
|
|
- "llama.cpp"
|
|
- "ollama"
|
|
- "vLLM"
|
|
- "Hermes"
|
|
- "open source AI"
|
|
|
|
# Phase 3: Synthesis
|
|
synthesis:
|
|
llm_endpoint: "http://localhost:4000/v1" # Local llama-server
|
|
llm_model: "gemma4:12b"
|
|
max_summary_length: 800
|
|
temperature: 0.7
|
|
|
|
# Phase 4: Audio
|
|
tts:
|
|
engine: "piper"
|
|
model_path: "~/.local/share/piper/models"
|
|
voice: "en_US-amy-medium"
|
|
speed: 1.0
|
|
output_format: "mp3" # piper outputs WAV, convert for Telegram
|
|
|
|
# Phase 3.5: DPO Training Pair Generation
|
|
training:
|
|
dpo:
|
|
enabled: true
|
|
output_dir: "~/.timmy/training-data/dpo-pairs"
|
|
min_score: 0.5 # Only generate pairs from items above this relevance score
|
|
max_pairs_per_run: 30 # Cap pairs per pipeline execution
|
|
pair_types: # Which pair strategies to use
|
|
- "summarize" # Paper summary → fleet-grounded analysis
|
|
- "relevance" # Relevance analysis → scored fleet context
|
|
- "implication" # Implications → actionable insight
|
|
|
|
# Phase 0: Fleet Context Grounding
|
|
fleet_context:
|
|
enabled: true
|
|
gitea_url: "https://forge.alexanderwhitestone.com"
|
|
token: "${GITEA_TOKEN}" # From environment
|
|
owner: "Timmy_Foundation"
|
|
repos:
|
|
- "timmy-config"
|
|
- "the-nexus"
|
|
- "timmy-home"
|
|
- "hermes-agent"
|
|
|
|
# Phase 5: Delivery
|
|
delivery:
|
|
method: "telegram"
|
|
bot_token: "${TELEGRAM_BOT_TOKEN}" # From env
|
|
channel_id: "-1003664764329"
|
|
send_text_summary: true
|
|
|
|
output_dir: "~/briefings"
|
|
log_level: "INFO"
|