Merge pull request 'fix: use PYTHON variable in training Makefile (closes #660)' (#822) from fix/660-python-makefile into main
fix: use PYTHON variable in training Makefile (closes #660) Refs Timmy_Foundation/the-nexus#1471
This commit is contained in:
@@ -5,20 +5,20 @@
|
||||
# pip install axolotl mlx-lm lm-evaluation-harness pyyaml
|
||||
#
|
||||
# Targets:
|
||||
# make train-cloud — QLoRA on cloud GPU via Axolotl
|
||||
# make train-local — LoRA on Apple Silicon via MLX
|
||||
# make eval — Standard benchmarks via lm-eval-harness
|
||||
# make vibes — Hand-picked prompts through Ollama, human review
|
||||
# make ingest — Pull heartbeat trajectories into training data
|
||||
# make curated — Regenerate curated exemplar dataset
|
||||
# make train-cloud -- QLoRA on cloud GPU via Axolotl
|
||||
# make train-local -- LoRA on Apple Silicon via MLX
|
||||
# make eval -- Standard benchmarks via lm-eval-harness
|
||||
# make vibes -- Hand-picked prompts through Ollama, human review
|
||||
# make ingest -- Pull heartbeat trajectories into training data
|
||||
# make curated -- Regenerate curated exemplar dataset
|
||||
|
||||
PYTHON ?= python3
|
||||
MODEL ?= timmy:v0.1-q4
|
||||
BASELINE ?= hermes3:latest
|
||||
OLLAMA_URL ?= http://localhost:11434
|
||||
OUTPUT ?= output
|
||||
PYTHON ?= python3
|
||||
|
||||
# ── Training ──────────────────────────────────────────────────────────
|
||||
# -- Training --
|
||||
|
||||
train-cloud: ## QLoRA fine-tune on cloud GPU (Axolotl)
|
||||
axolotl train axolotl.yaml
|
||||
@@ -26,7 +26,7 @@ train-cloud: ## QLoRA fine-tune on cloud GPU (Axolotl)
|
||||
train-local: ## LoRA fine-tune on Apple Silicon (MLX)
|
||||
$(PYTHON) -m mlx_lm.lora --config mlx-lora.yaml
|
||||
|
||||
# ── Evaluation ────────────────────────────────────────────────────────
|
||||
# -- Evaluation --
|
||||
|
||||
eval: ## Run standard benchmarks against Ollama model
|
||||
lm_eval --model local-completions \
|
||||
@@ -41,40 +41,23 @@ eval-baseline: ## Run same benchmarks against baseline for comparison
|
||||
--tasks hellaswag,truthfulqa_mc2,arc_challenge,winogrande \
|
||||
--output_path evals_archive/$(BASELINE)/
|
||||
|
||||
vibes: ## Run vibes check — hand-picked prompts, human review
|
||||
vibes: ## Run vibes check -- hand-picked prompts, human review
|
||||
@echo "=== Vibes Check: $(MODEL) ==="
|
||||
@echo "Date: $$(date '+%Y-%m-%d %H:%M')" > $(OUTPUT)/vibes-$(MODEL).md
|
||||
@echo "Model: $(MODEL)" >> $(OUTPUT)/vibes-$(MODEL).md
|
||||
@echo "" >> $(OUTPUT)/vibes-$(MODEL).md
|
||||
@$(PYTHON) -c "\
|
||||
import yaml, subprocess, sys; \
|
||||
prompts = yaml.safe_load(open('data/prompts_vibes.yaml'))['prompts']; \
|
||||
f = open('$(OUTPUT)/vibes-$(MODEL).md', 'a'); \
|
||||
[(\
|
||||
sys.stdout.write(f\" [{p['id']}] {p['category']}...\"), \
|
||||
sys.stdout.flush(), \
|
||||
f.write(f\"## [{p['id']}] {p['category']}\n\"), \
|
||||
f.write(f\"PROMPT: {p['prompt']}\n\"), \
|
||||
f.write(f\"EXPECTED: {p['expected']}\n\n\"), \
|
||||
f.write('RESPONSE:\n'), \
|
||||
f.write(subprocess.run( \
|
||||
['ollama', 'run', '$(MODEL)', p['prompt']], \
|
||||
capture_output=True, text=True, timeout=120 \
|
||||
).stdout), \
|
||||
f.write('\n\nSCORE: ___/5\n\n---\n\n'), \
|
||||
print(' done') \
|
||||
) for p in prompts]; \
|
||||
f.close()"
|
||||
@echo "Output: $(OUTPUT)/vibes-$(MODEL).md — fill in scores manually."
|
||||
@$(PYTHON) -c "import yaml, subprocess, sys; prompts = yaml.safe_load(open('data/prompts_vibes.yaml'))['prompts']; f = open('$(OUTPUT)/vibes-$(MODEL).md', 'a'); [(sys.stdout.write(f' [{p[\"id\"]}] {p[\"category\"]}...'), sys.stdout.flush(), f.write(f'## [{p[\"id\"]}] {p[\"category\"]}\n'), f.write(f'PROMPT: {p[\"prompt\"]}\n'), f.write(f'EXPECTED: {p[\"expected\"]}\n\n'), f.write('RESPONSE:\n'), f.write(subprocess.run(['ollama', 'run', '$(MODEL)', p['prompt']], capture_output=True, text=True, timeout=120).stdout), f.write('\n\nSCORE: ___/5\n\n---\n\n'), print(' done')) for p in prompts]; f.close()"
|
||||
@echo "Output: $(OUTPUT)/vibes-$(MODEL).md -- fill in scores manually."
|
||||
|
||||
|
||||
|
||||
adversary-value-violations: ## Run 200-prompt value-violations adversary suite against Ollama model
|
||||
adversary-value-violations: ## Run 200-prompt value-violations adversary suite
|
||||
@mkdir -p $(OUTPUT)/adversary-value-violations
|
||||
$(PYTHON) run_adversary_eval.py --suite data/prompts_adversary_value_violations.yaml --model $(MODEL) --output-dir $(OUTPUT)/adversary-value-violations
|
||||
$(PYTHON) run_adversary_eval.py \
|
||||
--suite data/prompts_adversary_value_violations.yaml \
|
||||
--model $(MODEL) \
|
||||
--output-dir $(OUTPUT)/adversary-value-violations
|
||||
@echo "Output: $(OUTPUT)/adversary-value-violations"
|
||||
|
||||
# ── Data Pipeline ─────────────────────────────────────────────────────
|
||||
# -- Data Pipeline --
|
||||
|
||||
ingest: ## Pull heartbeat trajectories into training data
|
||||
$(PYTHON) ingest_trajectories.py \
|
||||
@@ -89,18 +72,9 @@ curated: ## Regenerate curated exemplar dataset
|
||||
|
||||
convert: ## Convert merged dataset to MLX format (train/valid split)
|
||||
@mkdir -p data/mlx_curated
|
||||
$(PYTHON) -c "\
|
||||
import json; \
|
||||
lines = open('data/merged_training_data.jsonl').readlines(); \
|
||||
sessions = [json.loads(l) for l in lines]; \
|
||||
ROLE_MAP = {'system':'system','human':'user','gpt':'assistant','tool':'user'}; \
|
||||
converted = [{'messages': [{'role': ROLE_MAP.get(t.get('from',''),'user'), 'content': t.get('value','')} for t in s.get('conversations',[])]} for s in sessions]; \
|
||||
split = max(1, int(len(converted)*0.9)); \
|
||||
open('data/mlx_curated/train.jsonl','w').writelines(json.dumps(c)+'\n' for c in converted[:split]); \
|
||||
open('data/mlx_curated/valid.jsonl','w').writelines(json.dumps(c)+'\n' for c in converted[split:]); \
|
||||
print(f'train: {split}, valid: {len(converted)-split}')"
|
||||
$(PYTHON) -c "import json; lines = open('data/merged_training_data.jsonl').readlines(); sessions = [json.loads(l) for l in lines]; ROLE_MAP = {'system':'system','human':'user','gpt':'assistant','tool':'user'}; converted = [{'messages': [{'role': ROLE_MAP.get(t.get('from',''),'user'), 'content': t.get('value','')} for t in s.get('conversations',[])]} for s in sessions]; split = max(1, int(len(converted)*0.9)); open('data/mlx_curated/train.jsonl','w').writelines(json.dumps(c)+'\n' for c in converted[:split]); open('data/mlx_curated/valid.jsonl','w').writelines(json.dumps(c)+'\n' for c in converted[split:]); print(f'train: {split}, valid: {len(converted)-split}')"
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────
|
||||
# -- Helpers --
|
||||
|
||||
.PHONY: train-cloud train-local eval eval-baseline vibes adversary-value-violations ingest curated convert help
|
||||
|
||||
|
||||
Reference in New Issue
Block a user