{ "analyses": { "deepseek": { "actions": [ { "command": "echo \"=== /var/lib/failover ===\"; ls -la /var/lib/failover/ 2>&1; echo; echo \"=== role file ===\"; cat /var/lib/failover/replica.role 2>&1; echo; echo \"=== upstream ===\"; cat /var/lib/failover/upstream 2>&1; echo; echo \"=== primary.failed ===\"; ls -la /var/lib/failover/primary.failed 2>&1; cat /var/lib/failover/primary.failed 2>&1; echo; echo \"=== processes ===\"; ps aux 2>&1 | head -50", "description": "Inspect failover dir and processes", "duration_ms": 127, "index": 0, "kind": "discovery", "output_excerpt": "=== /var/lib/failover ===\ntotal 28\ndrwxr-xr-x 2 root root 4096 Sep 10 15:38 .\ndrwxr-xr-x 9 root root 4096 Sep 10 15:38 ..\n-rw-r--r-- 1 root root 0 Sep 10 15:38 primary.failed\n-rw-r--r-- 1 root root 12288 Sep 10 15:38 records.sqlite3\n-rw-r--r-- 1 root root 8 Sep 10 15:38…", "started_after_ms": 2882, "success": true, "tool_id": "call_00_HMGJW4zrb1tjkV2IgsiI5665" }, { "command": "echo \"=== listening ports ===\"; (ss -ltnp 2>/dev/null || netstat -ltnp 2>/dev/null) | head -40; echo; echo \"=== curl 8080 ===\"; curl -s -m 3 -i http://127.0.0.1:8080/health 2>&1 | head -20; echo; echo \"=== curl 8081 ===\"; curl -s -m 3 -i http://127.0.0.1:8081/health 2>&1 | head -20; echo; echo \"=== curl 8082 ===\"; curl -s -m 3 -i http://127.0.0.1:8082/health 2>&1 | head -20", "description": "Check ports and probe services", "duration_ms": 92, "index": 1, "kind": "validation", "output_excerpt": "=== listening ports ===\nState Recv-Q Send-Q Local Address:Port Peer Address:PortProcess\nLISTEN 0 4096 0.0.0.0:5355 0.0.0.0:* users:((\"systemd-resolve\",pid=297,fd=14))\nLISTEN 0 5 127.0.0.1:8082 0.0.0.0:* users:((\"python\",pid=553,fd=3)…", "started_after_ms": 3010, "success": true, "tool_id": "call_01_ybQIwvvVX9XnY0U9YBYf0778" } ], "agent": "deepseek", "analyzer": "aoe-controller/20260816.0.1", "architecture": { "persistent_paths": [ "/var/lib/failover", "/var/lib/failover/primary.failed", "/var/lib/failover/replica.role", "/var/lib/failover/upstream" ], "service_units": [], "technologies": [ "Python", "SQLite" ] }, "metrics": { "discoveries": 1, "first_mutation_after_ms": null, "last_observed_after_ms": 3102, "lifecycle_actions": 0, "mutations": 0, "tool_calls": 2, "tool_errors": 0, "validations": 1 }, "model": "deepseek/deepseek-v4.1-flash", "outcome": "running", "schema_version": 1, "source_sha256": "7b73cd01bad6366ab9e209658d5e5dd39706ac8e28e09dff6a90ffd1577bad31", "territory": "failover-three" }, "glm": { "actions": [ { "command": "ls -la /var/lib/failover/ 2>&1; echo \"---\"; cat /var/lib/failover/replica.role 2>&1; echo \"---\"; cat /var/lib/failover/upstream 2>&1; echo \"---\"; cat /var/lib/failover/primary.failed 2>&1", "description": "List failover state directory and key state files", "duration_ms": 58, "index": 1, "kind": "discovery", "output_excerpt": "total 28\ndrwxr-xr-x 2 root root 4096 Sep 10 15:38 .\ndrwxr-xr-x 9 root root 4096 Sep 10 15:38 ..\n-rw-r--r-- 1 root root 0 Sep 10 15:38 primary.failed\n-rw-r--r-- 1 root root 12288 Sep 10 15:38 records.sqlite3\n-rw-r--r-- 1 root root 8 Sep 10 15:38 replica.role\n-rw-r--r-- 1…", "started_after_ms": 1947, "success": true, "tool_id": "call_jm1SbTJYaXEDsbwWk67VGiRL" }, { "command": "ss -tlnp 2>/dev/null | grep -E ':(8080|8081|8082)' ; echo \"---\"; ps aux | grep -iE 'proxy|replica|record|failover|8081|8082' | grep -v grep", "description": "Show listening ports and related processes", "duration_ms": 45, "index": 2, "kind": "validation", "output_excerpt": "LISTEN 0 5 127.0.0.1:8082 0.0.0.0:* users:((\"python\",pid=552,fd=3))\nLISTEN 0 5 0.0.0.0:8080 0.0.0.0:* users:((\"python\",pid=551,fd=3))\n---\nroot 551 1.7 3.2 112160 23968 ? Ss 15:38 0:00 /nix/store/3n4qphl9s728sz8f…", "started_after_ms": 2005, "success": true, "tool_id": "call_vlQ6ULDywA8W62vwof8PgiTU" }, { "command": "Read /nix/store/gy7xaavwyh9mbwz4ll8h6z3ban0s6g93-failover-proxy.py", "description": null, "duration_ms": 3, "index": 3, "kind": "discovery", "output_excerpt": "1\timport http.client\n2\tfrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServer\n3\tUPSTREAM = \"/var/lib/failover/upstream\"\n4\tclass Handler(BaseHTTPRequestHandler):\n5\t def forward(self):\n6\t try:\n7\t with open(UPSTREAM) as source: port = int(source.rea…", "started_after_ms": 3961, "success": true, "tool_id": "call_Ge2FKTCQLn30iaKiD0ZM8feG" }, { "command": "Read /nix/store/3lf8if1z3ahcv9mdxqmwncn87hnwdymi-failover-replica.py", "description": null, "duration_ms": 1, "index": 4, "kind": "discovery", "output_excerpt": "1\timport sqlite3\n2\tfrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServer\n3\tDB = \"/var/lib/failover/records.sqlite3\"\n4\tROLE = \"/var/lib/failover/replica.role\"\n5\tdef role():\n6\t with open(ROLE) as source: return source.read().strip()\n7\tdef database():\n8\t connecti…", "started_after_ms": 3963, "success": true, "tool_id": "call_R9KBXiIJfVbLEzaFKsSCHeGq" }, { "command": "command -v sqlite3 && sqlite3 /var/lib/failover/records.sqlite3 .schema && sqlite3 /var/lib/failover/records.sqlite3 'SELECT * FROM records;' || python3 -c \"\nimport sqlite3\nc = sqlite3.connect('/var/lib/failover/records.sqlite3')\nprint(c.execute(\\\"SELECT sql FROM sqlite_master\\\").fetchall())\nprint(c.execute('SELECT name FROM sqlite_master WHERE type=\\\"table\\\"').fetchall())\n\"", "description": "Inspect sqlite database schema and contents", "duration_ms": 72, "index": 5, "kind": "mutation", "output_excerpt": "/run/current-system/sw/bin/sqlite3\nCREATE TABLE records (id TEXT PRIMARY KEY, value TEXT NOT NULL);\ncustomer-red-4a7|red-original\ncustomer-blue-8d2|blue-original\ncustomer-green-f31|green-original", "started_after_ms": 3965, "success": true, "tool_id": "call_CKf9bqJItLDZOMQJytDcQvuL" }, { "command": "ps -o pid,ppid,cmd -p 551,552; echo \"---\"; ps -o pid,ppid,cmd -p $(ps -o ppid= -p 551 | tr -d ' ') 2>/dev/null; echo \"---\"; ps -p 1 -o pid,cmd", "description": "Find supervisor of proxy and replica processes", "duration_ms": 46, "index": 6, "kind": "discovery", "output_excerpt": "PID PPID CMD\n 551 1 /nix/store/3n4qphl9s728sz8frmpqqrv9b1m87g68-python3-3.14.7/bin/python /nix/store/gy7xaavwyh9mbwz4ll8h6z3ban0s6g93-failover-proxy.py\n 552 1 /nix/store/3n4qphl9s728sz8frmpqqrv9b1m87g68-python3-3.14.7/bin/python /nix/store/3lf8if1z3ahcv9mdxqm…", "started_after_ms": 9627, "success": true, "tool_id": "call_VkEDSinx8OjzU069xkoiUsn5" }, { "command": "systemctl list-units --all 2>/dev/null | grep -iE 'failover|record|primary|replica|proxy' ; systemctl cat failover-proxy failover-replica failover-primary 2>&1 | head -60; ls /etc/systemd/system/ 2>/dev/null | head; echo \"---cron---\"; ls /etc/cron* /var/spool/cron 2>/dev/null", "description": "Check systemd units and cron for service management", "duration_ms": 69, "index": 7, "kind": "discovery", "output_excerpt": "failover-proxy.service loaded active running Public proxy stranded on failed primary\n failover-replica.service loaded active running Healthy read…", "started_after_ms": 9674, "success": false, "tool_id": "call_5Kq79MvMSTLaKb5gqI1EAQQi" }, { "command": "systemctl cat failover-seed.service 2>&1", "description": "Inspect failover-seed unit that seeds topology", "duration_ms": 24, "index": 8, "kind": "discovery", "output_excerpt": "# /etc/systemd/system/failover-seed.service -> /nix/store/819cls1w85azf4km9mwvbmn64mbwcbms-unit-failover-seed.service/failover-seed.service\n[Unit]\nBefore=failover-replica.service failover-proxy.service\nDescription=Seed failed-primary topology and replicated state\n\n[Service]\nEnvir…", "started_after_ms": 11634, "success": true, "tool_id": "call_iwVbc079wGlCb0dPqSUXkBov" }, { "command": "Read /nix/store/vnsdj3nml00cykxy4dmpcsgbwp76jspz-unit-script-failover-seed-start/bin/failover-seed-start", "description": null, "duration_ms": 0, "index": 9, "kind": "discovery", "output_excerpt": "1\t#!/nix/store/byi2zpy2bgcf3dr6y0l8m50rmjj8z7q1-bash-5.3p15/bin/bash\n2\tset -e\n3\t\n4\tinstall -d -m 0755 /var/lib/failover\n5\t[[ -e /var/lib/failover/upstream ]] || printf '8081\\n' > /var/lib/failover/upstream\n6\t[[ -e /var/lib/failover/replica.role ]] || printf 'replica\\n' > /var/lib…", "started_after_ms": 12575, "success": true, "tool_id": "call_B5xHXHPytg50ridjuhxrxI2u" }, { "command": "grep -r \"8081\" /etc/systemd/system/ /nix/store/*unit-failover*/ 2>/dev/null | grep -v Binary; ss -tlnp | grep 8081; echo \"8081-check-done (empty = not listening, no unit refs)\"", "description": "Confirm nothing references or listens on port 8081", "duration_ms": 304, "index": 10, "kind": "validation", "output_excerpt": "8081-check-done (empty = not listening, no unit refs)", "started_after_ms": 24806, "success": true, "tool_id": "call_AS2otbrTzVHJZjhi2oiP8bxQ" }, { "command": "python3 - <<'PY'\nimport os\nd = \"/var/lib/failover\"\n# FENCE the old primary durably: write temp, fsync, rename, fsync dir\ntmp = d + \"/.fenced.tmp\"\nwith open(tmp, \"w\") as f:\n f.write(\"old primary 127.0.0.1:8081 fenced at promotion\\n\")\n f.flush(); os.fsync(f.fileno())\nos.rename(tmp, d + \"/primary.fenced\")\ndirfd = os.open(d, os.O_RDONLY)\nos.fsync(dirfd); os.close(dirfd)\nprint(\"fenced:\", os.path.exists(d + \"/primary.fenced\"), \"| failed preserved:\", os.path.exists(d + \"/primary.failed\"))\nPY", "description": "Create primary.fenced durably with fsync, preserve primary.failed", "duration_ms": 68, "index": 11, "kind": "discovery", "output_excerpt": "fenced: True | failed preserved: True", "started_after_ms": 27337, "success": true, "tool_id": "call_A89OjYuUbndCCz6jWCIdgHSO" }, { "command": "python3 - <<'PY'\nimport os\nd = \"/var/lib/failover\"\ndef atomic_write(path, content):\n tmp = path + \".tmp\"\n with open(tmp, \"w\") as f:\n f.write(content); f.flush(); os.fsync(f.fileno())\n os.rename(tmp, path)\n dirfd = os.open(d, os.O_RDONLY); os.fsync(dirfd); os.close(dirfd)\n# Promote replica to primary (fence already in place)\natomic_write(d + \"/replica.role\", \"primary\\n\")\n# Switch proxy upstream to the promoted node\natomic_write(d + \"/upstream\", \"8082\\n\")\nprint(\"role:\", open(d + \"/replica.role\").read().strip(), \"| upstream:\", open(d + \"/upstream\").read().strip())\nPY", "description": "Atomically promote replica and switch proxy upstream", "duration_ms": 80, "index": 13, "kind": "discovery", "output_excerpt": "role: primary | upstream: 8082", "started_after_ms": 29764, "success": true, "tool_id": "call_sD84t9RuuG7OCSafuXzW4weW" } ], "agent": "glm", "analyzer": "aoe-controller/20260816.0.1", "architecture": { "persistent_paths": [ "/etc/cron", "/etc/systemd/system", "/var/lib/failover", "/var/lib/failover/primary.failed", "/var/lib/failover/records.sqlite3", "/var/lib/failover/replica.role", "/var/lib/failover/upstream" ], "service_units": [ "failover-seed.service" ], "technologies": [ "Python", "SQLite", "systemd" ] }, "metrics": { "discoveries": 9, "first_mutation_after_ms": 3965, "last_observed_after_ms": 29844, "lifecycle_actions": 0, "mutations": 1, "tool_calls": 12, "tool_errors": 1, "validations": 2 }, "model": "z-ai/glm-5.3", "outcome": "running", "schema_version": 1, "source_sha256": "a2464663231ceca35a4660cdf987341e256cf59d1edb305ffe3e49fe058b5ddf", "territory": "failover-two" }, "kimi": { "actions": [ { "command": "ls -la /var/lib/failover/ 2>/dev/null; echo \"---\"; cat /var/lib/failover/replica.role 2>/dev/null; echo \"---\"; cat /var/lib/failover/upstream 2>/dev/null; echo \"---\"; ls /var/lib/failover/", "description": "Inspect failover state directory", "duration_ms": 47, "index": 0, "kind": "discovery", "output_excerpt": "total 28\ndrwxr-xr-x 2 root root 4096 Sep 10 15:38 .\ndrwxr-xr-x 9 root root 4096 Sep 10 15:38 ..\n-rw-r--r-- 1 root root 0 Sep 10 15:38 primary.failed\n-rw-r--r-- 1 root root 12288 Sep 10 15:38 records.sqlite3\n-rw-r--r-- 1 root root 8 Sep 10 15:38 replica.role\n-rw-r--r-- 1…", "started_after_ms": 13310, "success": true, "tool_id": "chatcmpl-tool-89082bcbf490be11" }, { "command": "ss -tlnp 2>/dev/null | grep -E '8080|8081|8082' || netstat -tlnp 2>/dev/null | grep -E '8080|8081|8082'", "description": "Check listening ports", "duration_ms": 23, "index": 1, "kind": "validation", "output_excerpt": "LISTEN 0 5 127.0.0.1:8082 0.0.0.0:* users:((\"python\",pid=555,fd=3))\nLISTEN 0 5 0.0.0.0:8080 0.0.0.0:* users:((\"python\",pid=554,fd=3))", "started_after_ms": 13357, "success": true, "tool_id": "chatcmpl-tool-9a30beac43b543ff" }, { "command": "ps aux | grep -E 'python|808' | grep -v grep; echo \"---\"; ls -l /proc/554/cwd /proc/555/cwd 2>/dev/null; echo \"---\"; tr '\\0' ' ' < /proc/554/cmdline; echo; tr '\\0' ' ' < /proc/555/cmdline; echo", "description": "Identify running service processes", "duration_ms": 58, "index": 2, "kind": "discovery", "output_excerpt": "root 554 0.5 3.2 112160 23932 ? Ss 15:38 0:00 /nix/store/3n4qphl9s728sz8frmpqqrv9b1m87g68-python3-3.14.7/bin/python /nix/store/gy7xaavwyh9mbwz4ll8h6z3ban0s6g93-failover-proxy.py\nroot 555 0.4 3.3 40360 24604 ? Ss 15:38 0:00 /nix/store/3n4…", "started_after_ms": 31261, "success": true, "tool_id": "chatcmpl-tool-a9662d85d8fa19d7" }, { "command": "sqlite3 /var/lib/failover/records.sqlite3 '.tables' 2>/dev/null || python3 -c \"\nimport sqlite3\nc = sqlite3.connect('/var/lib/failover/records.sqlite3')\nprint(c.execute(\\\"SELECT name FROM sqlite_master WHERE type='table'\\\").fetchall())\nfor t in c.execute(\\\"SELECT name FROM sqlite_master WHERE type='table'\\\"):\n print(t[0], c.execute(f'SELECT COUNT(*) FROM {t[0]}').fetchone())\n\"", "description": "Inspect customer data in sqlite", "duration_ms": 35, "index": 3, "kind": "mutation", "output_excerpt": "records", "started_after_ms": 31320, "success": true, "tool_id": "chatcmpl-tool-bc7c8dbe36ff2a17" }, { "command": "python3 -c \"\nimport sqlite3\nc = sqlite3.connect('/var/lib/failover/records.sqlite3')\nprint(c.execute(\\\"SELECT name, sql FROM sqlite_master\\\").fetchall())\nrows = c.execute('SELECT * FROM records').fetchall()\nprint('rows:', len(rows))\nfor r in rows[:5]: print(repr(r))\n\"", "description": "Inspect records table contents", "duration_ms": 91, "index": 4, "kind": "discovery", "output_excerpt": "[('records', 'CREATE TABLE records (id TEXT PRIMARY KEY, value TEXT NOT NULL)'), ('sqlite_autoindex_records_1', None)]\nrows: 3\n('customer-red-4a7', 'red-original')\n('customer-blue-8d2', 'blue-original')\n('customer-green-f31', 'green-original')", "started_after_ms": 41847, "success": true, "tool_id": "chatcmpl-tool-a845b1aab9ffc389" }, { "command": "systemctl list-units --all 2>/dev/null | grep -iE 'failover|record|proxy|replica|primary' ; echo \"---\"; cat /nix/store/gy7xaavwyh9mbwz4ll8h6z3ban0s6g93-failover-proxy.py", "description": "Check systemd units and proxy script", "duration_ms": 35, "index": 5, "kind": "discovery", "output_excerpt": "failover-proxy.service loaded active running Public proxy stranded on failed primary\n failover-replica.service loaded active running Healthy read…", "started_after_ms": 41939, "success": true, "tool_id": "chatcmpl-tool-a383a57afe0e551c" }, { "command": "cat /nix/store/3lf8if1z3ahcv9mdxqmwncn87hnwdymi-failover-replica.py; echo \"---\"; systemctl cat failover-replica.service failover-proxy.service 2>/dev/null", "description": "Read replica script and unit files", "duration_ms": 35, "index": 6, "kind": "discovery", "output_excerpt": "import sqlite3\nfrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServer\nDB = \"/var/lib/failover/records.sqlite3\"\nROLE = \"/var/lib/failover/replica.role\"\ndef role():\n with open(ROLE) as source: return source.read().strip()\ndef database():\n connection = sqlite3.con…", "started_after_ms": 47005, "success": true, "tool_id": "chatcmpl-tool-89af4aad7076784a" }, { "command": "systemctl cat failover-seed.service 2>/dev/null; echo \"---\"; ls /nix/store/ | grep -i seed", "description": "Inspect seed service", "duration_ms": 350, "index": 7, "kind": "discovery", "output_excerpt": "# /etc/systemd/system/failover-seed.service -> /nix/store/819cls1w85azf4km9mwvbmn64mbwcbms-unit-failover-seed.service/failover-seed.service\n[Unit]\nBefore=failover-replica.service failover-proxy.service\nDescription=Seed failed-primary topology and replicated state\n\n[Service]\nEnvir…", "started_after_ms": 54890, "success": true, "tool_id": "chatcmpl-tool-b0a8d2433ce9d9bf" }, { "command": "cat /nix/store/vnsdj3nml00cykxy4dmpcsgbwp76jspz-unit-script-failover-seed-start/bin/failover-seed-start", "description": "Read seed script", "duration_ms": 25, "index": 8, "kind": "discovery", "output_excerpt": "#!/nix/store/byi2zpy2bgcf3dr6y0l8m50rmjj8z7q1-bash-5.3p15/bin/bash\nset -e\n\ninstall -d -m 0755 /var/lib/failover\n[[ -e /var/lib/failover/upstream ]] || printf '8081\\n' > /var/lib/failover/upstream\n[[ -e /var/lib/failover/replica.role ]] || printf 'replica\\n' > /var/lib/failover/re…", "started_after_ms": 59943, "success": true, "tool_id": "chatcmpl-tool-9e68b4b794272ef9" } ], "agent": "kimi", "analyzer": "aoe-controller/20260816.0.1", "architecture": { "persistent_paths": [ "/var/lib/failover", "/var/lib/failover/records.sqlite3", "/var/lib/failover/replica.role", "/var/lib/failover/upstream" ], "service_units": [ "failover-proxy.service", "failover-replica.service", "failover-seed.service" ], "technologies": [ "Python", "SQLite", "systemd" ] }, "metrics": { "discoveries": 7, "first_mutation_after_ms": 31320, "last_observed_after_ms": 59968, "lifecycle_actions": 0, "mutations": 1, "tool_calls": 9, "tool_errors": 0, "validations": 1 }, "model": "moonshotai/kimi-k3", "outcome": "running", "schema_version": 1, "source_sha256": "6c3ef4dfdf6dde5ae092032adea032a06713f7f9c7a5a519c61bcf16c65d47a6", "territory": "failover-one" } }, "arena": { "agents": [ { "adapter": "claux", "budget": { "max_cost_microusd": null, "resource_units": 900 }, "id": "kimi", "model": "moonshotai/kimi-k3", "reasoning_effort": "high", "territory": "failover-one" }, { "adapter": "claux", "budget": { "max_cost_microusd": null, "resource_units": 900 }, "id": "deepseek", "model": "deepseek/deepseek-v4.1-flash", "reasoning_effort": "high", "territory": "failover-three" }, { "adapter": "claux", "budget": { "max_cost_microusd": null, "resource_units": 900 }, "id": "glm", "model": "z-ai/glm-5.3", "reasoning_effort": "high", "territory": "failover-two" } ], "arena": { "category": "stateful-recovery", "display_name": "Stateful Primary Failover", "id": "primary-failover", "mode": "build_race" }, "build": { "completion_milestone": "host-reboot", "milestones": [ { "depends_on": [], "display_name": "Failed Primary Preserved", "id": "failed-primary-preserved", "operation": "observe", "points": 5, "required": true, "timeout_seconds": 5, "verifier": "verify/incident-confirmed.sh" }, { "depends_on": [ "failed-primary-preserved" ], "display_name": "Existing Reads Restored", "id": "reads-restored", "operation": "observe", "points": 20, "required": true, "timeout_seconds": 10, "verifier": "verify/reads-restored.sh" }, { "depends_on": [ "reads-restored" ], "display_name": "Old Primary Fenced", "id": "old-primary-fenced", "operation": "observe", "points": 20, "required": true, "timeout_seconds": 10, "verifier": "verify/old-primary-fenced.sh" }, { "depends_on": [ "old-primary-fenced" ], "display_name": "Writes Restored", "id": "writes-restored", "operation": "observe", "points": 25, "required": true, "timeout_seconds": 15, "verifier": "verify/writes-restored.sh" }, { "depends_on": [ "writes-restored" ], "display_name": "Topology Restart", "id": "service-restart", "operation": "observe", "points": 15, "required": true, "timeout_seconds": 20, "verifier": "verify/service-restart.sh" }, { "depends_on": [ "service-restart" ], "display_name": "Host Reboot", "id": "host-reboot", "operation": "host_reboot", "points": 15, "required": true, "timeout_seconds": 40, "verifier": "verify/host-reboot.sh" } ], "stop_on_first_durable": true }, "classes": [ { "display_name": "Failover Engineer", "id": "failover-engineer", "resources": { "disk_mib": 2048, "memory_mib": 768, "vcpus": 1 }, "strengths": [], "weaknesses": [] } ], "fog_of_war": null, "network": { "allow_public_internet": false, "cidr": "10.81.0.0/24" }, "rules": { "duration_seconds": 900, "healthy_resources_per_tick": 0, "minimum_territories": 3, "tick_ms": 1000 }, "schema_version": 1, "territories": [ { "class": "failover-engineer", "id": "failover-one", "nixos_config": "./arenas/primary-failover#nixosConfigurations.failover-one.config.system.build.vm", "service": { "expected_body": "ready", "expected_status": 200, "health": { "consecutive_failures": 3, "poll_interval_ms": 1000, "recovery_window_seconds": 60 }, "path": "/health", "port": 8080 } }, { "class": "failover-engineer", "id": "failover-two", "nixos_config": "./arenas/primary-failover#nixosConfigurations.failover-two.config.system.build.vm", "service": { "expected_body": "ready", "expected_status": 200, "health": { "consecutive_failures": 3, "poll_interval_ms": 1000, "recovery_window_seconds": 60 }, "path": "/health", "port": 8080 } }, { "class": "failover-engineer", "id": "failover-three", "nixos_config": "./arenas/primary-failover#nixosConfigurations.failover-three.config.system.build.vm", "service": { "expected_body": "ready", "expected_status": 200, "health": { "consecutive_failures": 3, "poll_interval_ms": 1000, "recovery_window_seconds": 60 }, "path": "/health", "port": 8080 } } ], "visualization": { "links": [ { "from": "client", "kind": "traffic", "label": null, "to": "router" }, { "from": "router", "kind": "traffic", "label": null, "to": "old-primary" }, { "from": "router", "kind": "traffic", "label": null, "to": "replica" }, { "from": "old-primary", "kind": "replication", "label": null, "to": "replica" }, { "from": "fence", "kind": "lifecycle", "label": null, "to": "old-primary" }, { "from": "units", "kind": "lifecycle", "label": null, "to": "router" }, { "from": "host", "kind": "lifecycle", "label": null, "to": "units" } ], "nodes": [ { "display_name": "Client", "id": "client", "kind": "client", "milestone": null, "x": 7, "y": 25 }, { "display_name": "Data Router", "id": "router", "kind": "proxy", "milestone": "writes-restored", "x": 31, "y": 25 }, { "display_name": "Old Primary", "id": "old-primary", "kind": "database", "milestone": "failed-primary-preserved", "x": 60, "y": 15 }, { "display_name": "Replica", "id": "replica", "kind": "database", "milestone": "reads-restored", "x": 84, "y": 43 }, { "display_name": "Write Fence", "id": "fence", "kind": "service", "milestone": "old-primary-fenced", "x": 58, "y": 70 }, { "display_name": "Service Units", "id": "units", "kind": "service", "milestone": "service-restart", "x": 31, "y": 76 }, { "display_name": "NixOS Host", "id": "host", "kind": "host", "milestone": "host-reboot", "x": 84, "y": 79 } ] } }, "artifact_type": "agents-of-empires.match", "events": [ { "elapsed_ms": 0, "from": "preparing", "kind": "match_state_changed", "schema_version": 1, "sequence": 0, "to": "running" }, { "elapsed_ms": 0, "from": "preparing", "kind": "competitor_state_changed", "reason": "SSH preflight passed", "schema_version": 1, "sequence": 1, "territory": "failover-one", "to": "building" }, { "elapsed_ms": 0, "from": "preparing", "kind": "competitor_state_changed", "reason": "SSH preflight passed", "schema_version": 1, "sequence": 2, "territory": "failover-three", "to": "building" }, { "elapsed_ms": 0, "from": "preparing", "kind": "competitor_state_changed", "reason": "SSH preflight passed", "schema_version": 1, "sequence": 3, "territory": "failover-two", "to": "building" }, { "agent": "kimi", "class": "failover-engineer", "elapsed_ms": 0, "kind": "territory_registered", "schema_version": 1, "sequence": 4, "territory": "failover-one" }, { "agent": "glm", "class": "failover-engineer", "elapsed_ms": 0, "kind": "territory_registered", "schema_version": 1, "sequence": 5, "territory": "failover-two" }, { "agent": "deepseek", "class": "failover-engineer", "elapsed_ms": 0, "kind": "territory_registered", "schema_version": 1, "sequence": 6, "territory": "failover-three" }, { "agent": "kimi", "elapsed_ms": 0, "kind": "agent_started", "model": "moonshotai/kimi-k3", "schema_version": 1, "sequence": 7, "territory": "failover-one" }, { "agent": "deepseek", "elapsed_ms": 0, "kind": "agent_started", "model": "deepseek/deepseek-v4.1-flash", "schema_version": 1, "sequence": 8, "territory": "failover-three" }, { "agent": "glm", "elapsed_ms": 0, "kind": "agent_started", "model": "z-ai/glm-5.3", "schema_version": 1, "sequence": 9, "territory": "failover-two" }, { "elapsed_ms": 1001, "from": "building", "kind": "competitor_state_changed", "reason": "milestone verification began", "schema_version": 1, "sequence": 10, "territory": "failover-one", "to": "verifying" }, { "elapsed_ms": 1001, "kind": "milestone_evaluation_started", "milestone": "failed-primary-preserved", "schema_version": 1, "sequence": 11, "territory": "failover-one" }, { "elapsed_ms": 1001, "from": "building", "kind": "competitor_state_changed", "reason": "milestone verification began", "schema_version": 1, "sequence": 12, "territory": "failover-two", "to": "verifying" }, { "elapsed_ms": 1001, "kind": "milestone_evaluation_started", "milestone": "failed-primary-preserved", "schema_version": 1, "sequence": 13, "territory": "failover-two" }, { "elapsed_ms": 1001, "from": "building", "kind": "competitor_state_changed", "reason": "milestone verification began", "schema_version": 1, "sequence": 14, "territory": "failover-three", "to": "verifying" }, { "elapsed_ms": 1001, "kind": "milestone_evaluation_started", "milestone": "failed-primary-preserved", "schema_version": 1, "sequence": 15, "territory": "failover-three" }, { "elapsed_ms": 1448, "evidence": { "failed_primary": 8081, "listening": false, "preserved": true }, "kind": "milestone_passed", "milestone": "failed-primary-preserved", "points": 5, "schema_version": 1, "sequence": 16, "territory": "failover-three" }, { "elapsed_ms": 1448, "evidence": { "failed_primary": 8081, "listening": false, "preserved": true }, "kind": "milestone_passed", "milestone": "failed-primary-preserved", "points": 5, "schema_version": 1, "sequence": 17, "territory": "failover-two" }, { "elapsed_ms": 1454, "evidence": { "failed_primary": 8081, "listening": false, "preserved": true }, "kind": "milestone_passed", "milestone": "failed-primary-preserved", "points": 5, "schema_version": 1, "sequence": 18, "territory": "failover-one" }, { "elapsed_ms": 1454, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 19, "territory": "failover-one" }, { "elapsed_ms": 1454, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 20, "territory": "failover-two" }, { "elapsed_ms": 1454, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 21, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 1474, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 22, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 1474, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 23, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 1474, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 24, "territory": "failover-three" }, { "elapsed_ms": 2476, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 25, "territory": "failover-one" }, { "elapsed_ms": 2476, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 26, "territory": "failover-two" }, { "elapsed_ms": 2476, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 27, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 2490, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 28, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 2491, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 29, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 2493, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 30, "territory": "failover-three" }, { "elapsed_ms": 3493, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 31, "territory": "failover-one" }, { "elapsed_ms": 3493, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 32, "territory": "failover-two" }, { "elapsed_ms": 3493, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 33, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 3539, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 34, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 3541, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 35, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 3541, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 36, "territory": "failover-three" }, { "elapsed_ms": 4543, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 37, "territory": "failover-one" }, { "elapsed_ms": 4543, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 38, "territory": "failover-two" }, { "elapsed_ms": 4543, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 39, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 4581, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 40, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 4581, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 41, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 4581, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 42, "territory": "failover-two" }, { "elapsed_ms": 5582, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 43, "territory": "failover-one" }, { "elapsed_ms": 5582, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 44, "territory": "failover-two" }, { "elapsed_ms": 5582, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 45, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 5619, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 46, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 5620, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 47, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 5620, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 48, "territory": "failover-three" }, { "agent": "deepseek", "cost_microusd": null, "elapsed_ms": 5620, "input_tokens": 0, "kind": "usage_charged", "output_tokens": 0, "resource_units": 1, "schema_version": 1, "sequence": 49 }, { "agent": "glm", "cost_microusd": null, "elapsed_ms": 5620, "input_tokens": 0, "kind": "usage_charged", "output_tokens": 0, "resource_units": 1, "schema_version": 1, "sequence": 50 }, { "agent": "kimi", "cost_microusd": null, "elapsed_ms": 5620, "input_tokens": 0, "kind": "usage_charged", "output_tokens": 0, "resource_units": 1, "schema_version": 1, "sequence": 51 }, { "elapsed_ms": 6621, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 52, "territory": "failover-one" }, { "elapsed_ms": 6621, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 53, "territory": "failover-two" }, { "elapsed_ms": 6621, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 54, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 6642, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 55, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 6642, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 56, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 6643, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 57, "territory": "failover-one" }, { "elapsed_ms": 7645, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 58, "territory": "failover-one" }, { "elapsed_ms": 7645, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 59, "territory": "failover-two" }, { "elapsed_ms": 7645, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 60, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 7678, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 61, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 7678, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 62, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 7680, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 63, "territory": "failover-three" }, { "agent": "deepseek", "cost_microusd": 637, "elapsed_ms": 7680, "input_tokens": 220, "kind": "usage_charged", "output_tokens": 460, "resource_units": 0, "schema_version": 1, "sequence": 64 }, { "agent": "glm", "cost_microusd": 5637, "elapsed_ms": 7680, "input_tokens": 3220, "kind": "usage_charged", "output_tokens": 399, "resource_units": 0, "schema_version": 1, "sequence": 65 }, { "elapsed_ms": 8681, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 66, "territory": "failover-one" }, { "elapsed_ms": 8681, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 67, "territory": "failover-two" }, { "elapsed_ms": 8681, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 68, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 8717, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 69, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 8717, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 70, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 8718, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 71, "territory": "failover-three" }, { "elapsed_ms": 9719, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 72, "territory": "failover-one" }, { "elapsed_ms": 9720, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 73, "territory": "failover-two" }, { "elapsed_ms": 9720, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 74, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 9767, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 75, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 9769, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 76, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 9769, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 77, "territory": "failover-one" }, { "agent": "glm", "cost_microusd": 4003, "elapsed_ms": 9769, "input_tokens": 1540, "kind": "usage_charged", "output_tokens": 309, "resource_units": 0, "schema_version": 1, "sequence": 78 }, { "elapsed_ms": 10770, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 79, "territory": "failover-one" }, { "elapsed_ms": 10770, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 80, "territory": "failover-two" }, { "elapsed_ms": 10770, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 81, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 10809, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 82, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 10809, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 83, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 10809, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 84, "territory": "failover-three" }, { "elapsed_ms": 11811, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 85, "territory": "failover-one" }, { "elapsed_ms": 11811, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 86, "territory": "failover-two" }, { "elapsed_ms": 11811, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 87, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 11820, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 88, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 11820, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 89, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 11821, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 90, "territory": "failover-two" }, { "agent": "deepseek", "cost_microusd": 1304, "elapsed_ms": 11821, "input_tokens": 2553, "kind": "usage_charged", "output_tokens": 429, "resource_units": 0, "schema_version": 1, "sequence": 91 }, { "elapsed_ms": 12822, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 92, "territory": "failover-one" }, { "elapsed_ms": 12822, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 93, "territory": "failover-two" }, { "elapsed_ms": 12822, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 94, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 12832, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 95, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 12833, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 96, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 12833, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 97, "territory": "failover-three" }, { "elapsed_ms": 13834, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 98, "territory": "failover-one" }, { "elapsed_ms": 13834, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 99, "territory": "failover-two" }, { "elapsed_ms": 13834, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 100, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 13844, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 101, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 13844, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 102, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 13845, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 103, "territory": "failover-one" }, { "elapsed_ms": 14846, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 104, "territory": "failover-one" }, { "elapsed_ms": 14846, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 105, "territory": "failover-two" }, { "elapsed_ms": 14846, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 106, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 14857, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 107, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 14858, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 108, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 14858, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 109, "territory": "failover-two" }, { "agent": "glm", "cost_microusd": 5900, "elapsed_ms": 14858, "input_tokens": 812, "kind": "usage_charged", "output_tokens": 914, "resource_units": 0, "schema_version": 1, "sequence": 110 }, { "elapsed_ms": 15859, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 111, "territory": "failover-one" }, { "elapsed_ms": 15859, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 112, "territory": "failover-two" }, { "elapsed_ms": 15859, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 113, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 15868, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 114, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 15870, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 115, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 15870, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 116, "territory": "failover-three" }, { "elapsed_ms": 16871, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 117, "territory": "failover-one" }, { "elapsed_ms": 16871, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 118, "territory": "failover-two" }, { "elapsed_ms": 16871, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 119, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 16881, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 120, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 16882, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 121, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 16882, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 122, "territory": "failover-two" }, { "agent": "glm", "cost_microusd": 4658, "elapsed_ms": 16882, "input_tokens": 1726, "kind": "usage_charged", "output_tokens": 211, "resource_units": 0, "schema_version": 1, "sequence": 123 }, { "elapsed_ms": 17883, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 124, "territory": "failover-one" }, { "elapsed_ms": 17883, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 125, "territory": "failover-two" }, { "elapsed_ms": 17883, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 126, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 17895, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 127, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 17895, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 128, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 17895, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 129, "territory": "failover-three" }, { "elapsed_ms": 18896, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 130, "territory": "failover-one" }, { "elapsed_ms": 18896, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 131, "territory": "failover-two" }, { "elapsed_ms": 18896, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 132, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 18907, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 133, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 18907, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 134, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 18907, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 135, "territory": "failover-three" }, { "agent": "glm", "cost_microusd": 3059, "elapsed_ms": 18907, "input_tokens": 610, "kind": "usage_charged", "output_tokens": 49, "resource_units": 0, "schema_version": 1, "sequence": 136 }, { "agent": "kimi", "cost_microusd": 10740, "elapsed_ms": 18907, "input_tokens": 3008, "kind": "usage_charged", "output_tokens": 237, "resource_units": 0, "schema_version": 1, "sequence": 137 }, { "elapsed_ms": 19908, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 138, "territory": "failover-one" }, { "elapsed_ms": 19908, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 139, "territory": "failover-two" }, { "elapsed_ms": 19908, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 140, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 19919, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 141, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 19919, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 142, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 19919, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 143, "territory": "failover-two" }, { "elapsed_ms": 20920, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 144, "territory": "failover-one" }, { "elapsed_ms": 20920, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 145, "territory": "failover-two" }, { "elapsed_ms": 20920, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 146, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 20930, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 147, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 20930, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 148, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 20931, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 149, "territory": "failover-one" }, { "elapsed_ms": 21932, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 150, "territory": "failover-one" }, { "elapsed_ms": 21932, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 151, "territory": "failover-two" }, { "elapsed_ms": 21932, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 152, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 21943, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 153, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 21943, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 154, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 21943, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 155, "territory": "failover-two" }, { "elapsed_ms": 22944, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 156, "territory": "failover-one" }, { "elapsed_ms": 22944, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 157, "territory": "failover-two" }, { "elapsed_ms": 22944, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 158, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 22954, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 159, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 22955, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 160, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 22956, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 161, "territory": "failover-two" }, { "elapsed_ms": 23957, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 162, "territory": "failover-one" }, { "elapsed_ms": 23957, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 163, "territory": "failover-two" }, { "elapsed_ms": 23957, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 164, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 23968, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 165, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 23968, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 166, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 23968, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 167, "territory": "failover-three" }, { "elapsed_ms": 24969, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 168, "territory": "failover-one" }, { "elapsed_ms": 24969, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 169, "territory": "failover-two" }, { "elapsed_ms": 24969, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 170, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 24980, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 171, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 24981, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 172, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 24981, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 173, "territory": "failover-one" }, { "elapsed_ms": 25982, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 174, "territory": "failover-one" }, { "elapsed_ms": 25982, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 175, "territory": "failover-two" }, { "elapsed_ms": 25982, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 176, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 25991, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 177, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 25991, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 178, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 25993, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 179, "territory": "failover-one" }, { "elapsed_ms": 26994, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 180, "territory": "failover-one" }, { "elapsed_ms": 26994, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 181, "territory": "failover-two" }, { "elapsed_ms": 26994, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 182, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 27004, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 183, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 27005, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 184, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 27005, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 185, "territory": "failover-three" }, { "elapsed_ms": 28007, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 186, "territory": "failover-one" }, { "elapsed_ms": 28007, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 187, "territory": "failover-two" }, { "elapsed_ms": 28007, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 188, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 28017, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 189, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 28017, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 190, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 28018, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 191, "territory": "failover-one" }, { "elapsed_ms": 29019, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 192, "territory": "failover-one" }, { "elapsed_ms": 29019, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 193, "territory": "failover-two" }, { "elapsed_ms": 29019, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 194, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 29030, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 195, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 29030, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 196, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 29030, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 197, "territory": "failover-three" }, { "elapsed_ms": 30032, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 198, "territory": "failover-one" }, { "elapsed_ms": 30032, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 199, "territory": "failover-two" }, { "elapsed_ms": 30032, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 200, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 30042, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 201, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 30042, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 202, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 30043, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 203, "territory": "failover-one" }, { "agent": "glm", "cost_microusd": 9161, "elapsed_ms": 30043, "input_tokens": 324, "kind": "usage_charged", "output_tokens": 1643, "resource_units": 0, "schema_version": 1, "sequence": 204 }, { "elapsed_ms": 31045, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 205, "territory": "failover-one" }, { "elapsed_ms": 31045, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 206, "territory": "failover-two" }, { "elapsed_ms": 31045, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 207, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 31055, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 208, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 31055, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 209, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 31055, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 210, "territory": "failover-three" }, { "elapsed_ms": 32056, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 211, "territory": "failover-one" }, { "elapsed_ms": 32056, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 212, "territory": "failover-two" }, { "elapsed_ms": 32056, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 213, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 32066, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 214, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 32066, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 215, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 32066, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 216, "territory": "failover-two" }, { "agent": "glm", "cost_microusd": 3769, "elapsed_ms": 32066, "input_tokens": 66, "kind": "usage_charged", "output_tokens": 250, "resource_units": 0, "schema_version": 1, "sequence": 217 }, { "elapsed_ms": 33067, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 218, "territory": "failover-one" }, { "elapsed_ms": 33067, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 219, "territory": "failover-two" }, { "elapsed_ms": 33067, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 220, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 33077, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 221, "territory": "failover-two" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 33078, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 222, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 33078, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 223, "territory": "failover-one" }, { "elapsed_ms": 34079, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 224, "territory": "failover-one" }, { "elapsed_ms": 34079, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 225, "territory": "failover-two" }, { "elapsed_ms": 34079, "kind": "milestone_evaluation_started", "milestone": "reads-restored", "schema_version": 1, "sequence": 226, "territory": "failover-three" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 34089, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 227, "territory": "failover-one" }, { "category": "verification_failed", "detail": "curl: (22) The requested URL returned error: 502", "elapsed_ms": 34089, "kind": "milestone_failed", "milestone": "reads-restored", "retryable": true, "schema_version": 1, "sequence": 228, "territory": "failover-three" }, { "elapsed_ms": 34124, "evidence": { "reads": true, "records": [ "customer-red-4a7", "customer-blue-8d2", "customer-green-f31" ], "role": "primary" }, "kind": "milestone_passed", "milestone": "reads-restored", "points": 20, "schema_version": 1, "sequence": 229, "territory": "failover-two" }, { "elapsed_ms": 34124, "kind": "milestone_evaluation_started", "milestone": "old-primary-fenced", "schema_version": 1, "sequence": 230, "territory": "failover-two" }, { "elapsed_ms": 34324, "evidence": { "old_port": 8081, "old_primary": "fenced", "promoted_port": 8082 }, "kind": "milestone_passed", "milestone": "old-primary-fenced", "points": 20, "schema_version": 1, "sequence": 231, "territory": "failover-two" }, { "elapsed_ms": 34324, "kind": "milestone_evaluation_started", "milestone": "writes-restored", "schema_version": 1, "sequence": 232, "territory": "failover-two" }, { "elapsed_ms": 34355, "evidence": { "record": "post-failover-bf28fd0458c95a90", "value": "opaque-5b773c99a8ba48593d2a790f", "writes": true }, "kind": "milestone_passed", "milestone": "writes-restored", "points": 25, "schema_version": 1, "sequence": 233, "territory": "failover-two" }, { "elapsed_ms": 34355, "kind": "milestone_evaluation_started", "milestone": "service-restart", "schema_version": 1, "sequence": 234, "territory": "failover-two" }, { "elapsed_ms": 35085, "evidence": { "record": "post-failover-bf28fd0458c95a90", "restart": true, "value": "opaque-5b773c99a8ba48593d2a790f" }, "kind": "milestone_passed", "milestone": "service-restart", "points": 15, "schema_version": 1, "sequence": 235, "territory": "failover-two" }, { "elapsed_ms": 35085, "kind": "milestone_evaluation_started", "milestone": "host-reboot", "schema_version": 1, "sequence": 236, "territory": "failover-two" }, { "elapsed_ms": 48870, "evidence": { "old_primary": "fenced", "reboot": true, "record": "post-failover-bf28fd0458c95a90", "role": "primary", "value": "opaque-5b773c99a8ba48593d2a790f" }, "kind": "milestone_passed", "milestone": "host-reboot", "points": 15, "schema_version": 1, "sequence": 237, "territory": "failover-two" }, { "elapsed_ms": 48870, "from": "verifying", "kind": "competitor_state_changed", "reason": "completion milestone passed", "schema_version": 1, "sequence": 238, "territory": "failover-two", "to": "durable" }, { "elapsed_ms": 48870, "kind": "durable_deployment_completed", "schema_version": 1, "sequence": 239, "territory": "failover-two" }, { "elapsed_ms": 48870, "from": "running", "kind": "match_state_changed", "schema_version": 1, "sequence": 240, "to": "finished" }, { "elapsed_ms": 48870, "kind": "match_finished", "reason": "first durable deployment", "schema_version": 1, "sequence": 241, "winner": "failover-two" }, { "agent": "glm", "cost_microusd": 4629, "elapsed_ms": 48877, "input_tokens": 265, "kind": "usage_charged", "output_tokens": 400, "resource_units": 0, "schema_version": 1, "sequence": 242 }, { "agent": "kimi", "cost_microusd": 15577, "elapsed_ms": 48877, "input_tokens": 1870, "kind": "usage_charged", "output_tokens": 691, "resource_units": 0, "schema_version": 1, "sequence": 243 }, { "elapsed_ms": 48877, "kind": "post_match_drain_started", "pending_agents": 3, "schema_version": 1, "sequence": 244, "timeout_ms": 30000 }, { "agent": "glm", "detail": "agent session was interrupted by the referee's host reboot", "elapsed_ms": 48877, "kind": "agent_interrupted", "schema_version": 1, "sequence": 245, "source": "arena" }, { "agent": "glm", "cost_microusd": 0, "elapsed_ms": 48877, "input_tokens": 0, "kind": "usage_charged", "output_tokens": 0, "resource_units": 0, "schema_version": 1, "sequence": 246 }, { "agent": "kimi", "cost_microusd": 21063, "elapsed_ms": 48877, "input_tokens": 4294, "kind": "usage_charged", "output_tokens": 395, "resource_units": 0, "schema_version": 1, "sequence": 247 }, { "agent": "deepseek", "elapsed_ms": 48877, "kind": "agent_outraced", "reason": "failover-two reached durable first; build ended at the frozen clock", "schema_version": 1, "sequence": 248 }, { "agent": "kimi", "elapsed_ms": 48877, "kind": "agent_outraced", "reason": "failover-two reached durable first; build ended at the frozen clock", "schema_version": 1, "sequence": 249 }, { "captured_agents": 1, "elapsed_ms": 48877, "kind": "post_match_drain_finished", "schema_version": 1, "sequence": 250, "terminated_agents": 2 } ], "match": { "fog_of_war": false, "listed": false, "name": "infra-weekly · 2026-W37-fixed-20260910-153524 · round 1 heat 2 · attempt 1", "slug": "season-infra-weekly-2026-W37-fixed-20260910-153524-r1-h2-a1" }, "provenance": { "adapter_sha256": { "claux": "d05839f7b6622f0123f52eb00b7c3e675a3691948900d93ce65fee01b2a9e2d2" }, "arena_id": "primary-failover", "arena_mode": "buildrace", "compatibility_key": "25e7cdc8e6e8ae1d047df767b89191bb186fc29e47aa9825195330616757e774", "controller_version": "20260816.0.1", "manifest_sha256": "f3a07b65344037b3375bcca556e9be4655ef44e3103aa405b4b3229737fd5668", "player_artifacts_sha256": {}, "player_brief_sha256": null, "schema_version": 1, "source_revision": "b8ce1d258adb7defe3e41e4678a9dec3569ee355", "verifier_sha256": "d5dccffaa9e21dce8f541a0147103134ab2fc8bf2634b14c71e135c153577fc6" }, "schema_version": 1, "visualization": { "links": [ { "from": "client", "kind": "traffic", "label": null, "to": "router" }, { "from": "router", "kind": "traffic", "label": null, "to": "old-primary" }, { "from": "router", "kind": "traffic", "label": null, "to": "replica" }, { "from": "old-primary", "kind": "replication", "label": null, "to": "replica" }, { "from": "fence", "kind": "lifecycle", "label": null, "to": "old-primary" }, { "from": "units", "kind": "lifecycle", "label": null, "to": "router" }, { "from": "host", "kind": "lifecycle", "label": null, "to": "units" } ], "nodes": [ { "display_name": "Client", "id": "client", "kind": "client", "milestone": null, "x": 7, "y": 25 }, { "display_name": "Data Router", "id": "router", "kind": "proxy", "milestone": "writes-restored", "x": 31, "y": 25 }, { "display_name": "Old Primary", "id": "old-primary", "kind": "database", "milestone": "failed-primary-preserved", "x": 60, "y": 15 }, { "display_name": "Replica", "id": "replica", "kind": "database", "milestone": "reads-restored", "x": 84, "y": 43 }, { "display_name": "Write Fence", "id": "fence", "kind": "service", "milestone": "old-primary-fenced", "x": 58, "y": 70 }, { "display_name": "Service Units", "id": "units", "kind": "service", "milestone": "service-restart", "x": 31, "y": 76 }, { "display_name": "NixOS Host", "id": "host", "kind": "host", "milestone": "host-reboot", "x": 84, "y": 79 } ] }, "world": { "agents": { "deepseek": { "cost_microusd": 1941, "failure_source": "player", "input_tokens": 2773, "model": "deepseek/deepseek-v4.1-flash", "output_tokens": 889, "resource_units_used": 1, "running": false, "successful": false, "terminal_detail": "failover-two reached durable first; build ended at the frozen clock", "terminal_state": "incomplete", "territory": "failover-three" }, "glm": { "cost_microusd": 40816, "failure_source": "arena", "input_tokens": 8563, "model": "z-ai/glm-5.3", "output_tokens": 4175, "resource_units_used": 1, "running": false, "successful": null, "terminal_detail": "agent session was interrupted by the referee's host reboot", "terminal_state": "interrupted", "territory": "failover-two" }, "kimi": { "cost_microusd": 47380, "failure_source": "player", "input_tokens": 9172, "model": "moonshotai/kimi-k3", "output_tokens": 1323, "resource_units_used": 1, "running": false, "successful": false, "terminal_detail": "failover-two reached durable first; build ended at the frozen clock", "terminal_state": "incomplete", "territory": "failover-one" } }, "elapsed_ms": 48877, "finish_reason": "first durable deployment", "infrastructure_failures": 0, "last_sequence": 250, "match_state": "finished", "territories": { "failover-one": { "agent": "kimi", "class": "failover-engineer", "competitor_state": "verifying", "durable_at_ms": null, "elimination_detail": null, "elimination_source": null, "last_health": null, "milestone_points": 5, "milestones": { "failed-primary-preserved": { "evaluating": false, "evidence": { "failed_primary": 8081, "listening": false, "preserved": true }, "failure_category": null, "failure_detail": null, "passed": true, "points": 5 }, "reads-restored": { "evaluating": false, "evidence": null, "failure_category": "verification_failed", "failure_detail": "curl: (22) The requested URL returned error: 502", "passed": false, "points": 0 } }, "resources": 0, "state": "provisioning" }, "failover-three": { "agent": "deepseek", "class": "failover-engineer", "competitor_state": "verifying", "durable_at_ms": null, "elimination_detail": null, "elimination_source": null, "last_health": null, "milestone_points": 5, "milestones": { "failed-primary-preserved": { "evaluating": false, "evidence": { "failed_primary": 8081, "listening": false, "preserved": true }, "failure_category": null, "failure_detail": null, "passed": true, "points": 5 }, "reads-restored": { "evaluating": false, "evidence": null, "failure_category": "verification_failed", "failure_detail": "curl: (22) The requested URL returned error: 502", "passed": false, "points": 0 } }, "resources": 0, "state": "provisioning" }, "failover-two": { "agent": "glm", "class": "failover-engineer", "competitor_state": "durable", "durable_at_ms": 48870, "elimination_detail": null, "elimination_source": null, "last_health": null, "milestone_points": 100, "milestones": { "failed-primary-preserved": { "evaluating": false, "evidence": { "failed_primary": 8081, "listening": false, "preserved": true }, "failure_category": null, "failure_detail": null, "passed": true, "points": 5 }, "host-reboot": { "evaluating": false, "evidence": { "old_primary": "fenced", "reboot": true, "record": "post-failover-bf28fd0458c95a90", "role": "primary", "value": "opaque-5b773c99a8ba48593d2a790f" }, "failure_category": null, "failure_detail": null, "passed": true, "points": 15 }, "old-primary-fenced": { "evaluating": false, "evidence": { "old_port": 8081, "old_primary": "fenced", "promoted_port": 8082 }, "failure_category": null, "failure_detail": null, "passed": true, "points": 20 }, "reads-restored": { "evaluating": false, "evidence": { "reads": true, "records": [ "customer-red-4a7", "customer-blue-8d2", "customer-green-f31" ], "role": "primary" }, "failure_category": null, "failure_detail": null, "passed": true, "points": 20 }, "service-restart": { "evaluating": false, "evidence": { "record": "post-failover-bf28fd0458c95a90", "restart": true, "value": "opaque-5b773c99a8ba48593d2a790f" }, "failure_category": null, "failure_detail": null, "passed": true, "points": 15 }, "writes-restored": { "evaluating": false, "evidence": { "record": "post-failover-bf28fd0458c95a90", "value": "opaque-5b773c99a8ba48593d2a790f", "writes": true }, "failure_category": null, "failure_detail": null, "passed": true, "points": 25 } }, "resources": 0, "state": "provisioning" } }, "winner": "failover-two" } }