30 lines
1.1 KiB
Bash
30 lines
1.1 KiB
Bash
#!/usr/bin/env bash
|
|
# Fleet status: services, models, disk, bench state, running benches.
|
|
set -uo pipefail
|
|
for h in rain@miche.local melon@byte.local; do
|
|
echo "== $h"
|
|
ssh -o ConnectTimeout=6 -o BatchMode=yes "$h" '
|
|
printf " service: "; systemctl --user is-active llama-swap
|
|
curl -sf http://127.0.0.1:9292/v1/models | python3 -c "
|
|
import json,sys
|
|
d=json.load(sys.stdin)[\"data\"]
|
|
print(\" models: %d\" % len(d))
|
|
for m in sorted(d, key=lambda m: m[\"id\"]): print(\" -\", m[\"id\"])
|
|
" || echo " models: API DOWN"
|
|
printf " disk: "; df -h /home | tail -1 | awk "{print \$3\" / \"\$2\" (\"\$5\" used)\"}"
|
|
' || echo " SSH FAILED"
|
|
done
|
|
echo "== miche bench"
|
|
ssh -o BatchMode=yes rain@miche.local '
|
|
cd ~/AI/strix-halo-fleet-bench
|
|
python3 -c "
|
|
import json
|
|
s = json.load(open(\"results/state.json\"))
|
|
done = {}
|
|
for e in s[\"completed\"]:
|
|
done.setdefault(e[\"model\"], []).append(e[\"tier\"] if not e[\"error\"] else str(e[\"tier\"])+\"!\")
|
|
print(\" completed tiers:\")
|
|
for k in sorted(done): print(\" \", k, sorted(int(str(t).rstrip(\"!\")) for t in done[k]))
|
|
"
|
|
printf " running benches: "; pgrep -fa "[b]ench_framework.py" || echo none
|
|
'
|