#!/bin/sh
# Runs a definitions test: Haiku and Sonnet in the battery's sandbox, then Qwen five
# times and Gemma twice in Ollama with no tools. Usage: run.sh N -> answers in N/
D=$(dirname "$0")
B=$D/..
mkdir -p "$D/$1"
for m in haiku sonnet; do
  mkdir -p "$D/$1/claude-$m"
  python3 "$B/box.py" claude $m "$D/prompt-$1.txt" "$D/$1/claude-$m" &
done
for spec in "qwen2.5-coder:7b qwen 5" "gemma4:31b gemma 2"; do
  set -- $1 $spec
  n=1
  while [ "$n" -le "$4" ]; do
    python3 - "$2" "$D/prompt-$1.txt" "$D/$1/$3-$n.md" <<'PY'
import json, sys, urllib.request
body = json.dumps({"model": sys.argv[1], "stream": False, "options": {"num_ctx": 16384, "temperature": 0.2, "num_predict": 8000},
                   "messages": [{"role": "user", "content": open(sys.argv[2]).read()}]}).encode()
req = urllib.request.Request("http://127.0.0.1:11434/api/chat", data=body, headers={"Content-Type": "application/json"})
open(sys.argv[3], "w").write(json.load(urllib.request.urlopen(req, timeout=1800))["message"]["content"])
PY
    n=$((n + 1))
  done
  curl -s -m 60 http://127.0.0.1:11434/api/generate -d "{\"model\":\"$2\",\"keep_alive\":0}" > /dev/null
done
wait