33 lines
838 B
Bash
33 lines
838 B
Bash
|
#!/bin/bash
|
||
|
|
|
||
|
|
# Terminal-Bench 2.0 Evaluation
|
||
|
|
#
|
||
|
|
# Run from repo root:
|
||
|
|
# bash environments/benchmarks/terminalbench_2/run_eval.sh
|
||
|
|
#
|
||
|
|
# Override model:
|
||
|
|
# bash environments/benchmarks/terminalbench_2/run_eval.sh \
|
||
|
|
# --openai.model_name anthropic/claude-sonnet-4
|
||
|
|
#
|
||
|
|
# Run a subset:
|
||
|
|
# bash environments/benchmarks/terminalbench_2/run_eval.sh \
|
||
|
|
# --env.task_filter fix-git,git-multibranch
|
||
|
|
|
||
|
|
mkdir -p logs evals/terminal-bench-2
|
||
|
|
LOG_FILE="logs/terminalbench2_$(date +%Y%m%d_%H%M%S).log"
|
||
|
|
|
||
|
|
echo "Terminal-Bench 2.0 Evaluation"
|
||
|
|
echo "Log: $LOG_FILE"
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
export TERMINAL_ENV=modal
|
||
|
|
export TERMINAL_TIMEOUT=300
|
||
|
|
|
||
|
|
python environments/benchmarks/terminalbench_2/terminalbench2_env.py evaluate \
|
||
|
|
--config environments/benchmarks/terminalbench_2/default.yaml \
|
||
|
|
"$@" \
|
||
|
|
2>&1 | tee "$LOG_FILE"
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo "Log saved to: $LOG_FILE"
|