#!/usr/bin/env bash
# Fleet status: services, models, disk, bench state, running benches.
set -uo pipefail
for h in rain@miche.local melon@byte.local; do
  echo "== $h"
  ssh -o ConnectTimeout=6 -o BatchMode=yes "$h" '
    printf "  service: "; systemctl --user is-active llama-swap
    curl -sf http://127.0.0.1:9292/v1/models | python3 -c "
import json,sys
d=json.load(sys.stdin)[\"data\"]
print(\"  models: %d\" % len(d))
for m in sorted(d, key=lambda m: m[\"id\"]): print(\"    -\", m[\"id\"])
" || echo "  models: API DOWN"
    printf "  disk:    "; df -h /home | tail -1 | awk "{print \$3\" / \"\$2\" (\"\$5\" used)\"}"
  ' || echo "  SSH FAILED"
done
echo "== miche bench"
ssh -o BatchMode=yes rain@miche.local '
  cd ~/AI/strix-halo-fleet-bench
  python3 -c "
import json
s = json.load(open(\"results/state.json\"))
done = {}
for e in s[\"completed\"]:
    done.setdefault(e[\"model\"], []).append(e[\"tier\"] if not e[\"error\"] else str(e[\"tier\"])+\"!\")
print(\"  completed tiers:\")
for k in sorted(done): print(\"   \", k, sorted(int(str(t).rstrip(\"!\")) for t in done[k]))
"
  printf "  running benches: "; pgrep -fa "[b]ench_framework.py" || echo none
'