Add self-check via control-api (self_check.methods)

control-api is hosted outside the cloud and validators reach it directly,
so it sees the floating IP as the connection's source address. New open
route GET /api/v1/agents/{id}/observed-ip returns that address (taken only
from the TCP peer; forwarding headers are ignored so a validator cannot
forge it).

The agent gets self_check.methods, a priority-ordered list of ip_echo
(unchanged) and control_api; the default stays [ip_echo]. The self-check
passes when any method confirms the address; the next method is tried on
no answer and on a mismatch. Each method has its own timeout so a hung
first method cannot starve the fallback, and control_api uses a new TCP
connection per call (a connection opened before the floating IP was
attached would keep reporting the old address).

Also: docker agent template/env, example config, docs, plan in
docs/changes, e2e script switch E2E_SELF_CHECK_METHODS, rebuilt
bin/control-api and bin/validator-agent.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

deploy/docker/docker-compose.yml

@@ -100,6 +100,7 @@ services:
       VALIDATOR_AGENT_CONTROL_API_URL: "${VALIDATOR_AGENT_CONTROL_API_URL:-http://control-api:8080}"
       VALIDATOR_AGENT_POLL_INTERVAL_SECONDS: "${VALIDATOR_AGENT_POLL_INTERVAL_SECONDS:-5}"
       VALIDATOR_AGENT_SELF_CHECK_TIMEOUT_SECONDS: "${VALIDATOR_AGENT_SELF_CHECK_TIMEOUT_SECONDS:-10}"
+      VALIDATOR_AGENT_SELF_CHECK_METHODS: "${VALIDATOR_AGENT_SELF_CHECK_METHODS:-[ip_echo]}"
       VALIDATOR_AGENT_HTTPS_TIMEOUT_SECONDS: "${VALIDATOR_AGENT_HTTPS_TIMEOUT_SECONDS:-10}"
       VALIDATOR_AGENT_ICMP_TIMEOUT_SECONDS: "${VALIDATOR_AGENT_ICMP_TIMEOUT_SECONDS:-5}"
       VALIDATOR_AGENT_ICMP_COUNT: "${VALIDATOR_AGENT_ICMP_COUNT:-3}"

deploy/docker/validator-agent/docker-entrypoint.sh

@@ -6,6 +6,7 @@ set -eu
 
 export VALIDATOR_AGENT_POLL_INTERVAL_SECONDS="${VALIDATOR_AGENT_POLL_INTERVAL_SECONDS:-5}"
 export VALIDATOR_AGENT_SELF_CHECK_TIMEOUT_SECONDS="${VALIDATOR_AGENT_SELF_CHECK_TIMEOUT_SECONDS:-10}"
+export VALIDATOR_AGENT_SELF_CHECK_METHODS="${VALIDATOR_AGENT_SELF_CHECK_METHODS:-[ip_echo]}"
 export VALIDATOR_AGENT_HTTPS_TIMEOUT_SECONDS="${VALIDATOR_AGENT_HTTPS_TIMEOUT_SECONDS:-10}"
 export VALIDATOR_AGENT_ICMP_TIMEOUT_SECONDS="${VALIDATOR_AGENT_ICMP_TIMEOUT_SECONDS:-5}"
 export VALIDATOR_AGENT_ICMP_COUNT="${VALIDATOR_AGENT_ICMP_COUNT:-3}"
@@ -16,7 +17,7 @@ export VALIDATOR_AGENT_SSH_TIMEOUT_SECONDS="${VALIDATOR_AGENT_SSH_TIMEOUT_SECOND
 # NOT templated into the YAML: the binary reads them straight from this
 # container's environment (names default in the config loader), so secrets
 # never land in a file inside the container.
-envsubst '${VALIDATOR_AGENT_VALIDATOR_ID} ${VALIDATOR_AGENT_CONTROL_API_URL} ${VALIDATOR_AGENT_POLL_INTERVAL_SECONDS} ${VALIDATOR_AGENT_SELF_CHECK_TIMEOUT_SECONDS} ${VALIDATOR_AGENT_HTTPS_TIMEOUT_SECONDS} ${VALIDATOR_AGENT_ICMP_TIMEOUT_SECONDS} ${VALIDATOR_AGENT_ICMP_COUNT} ${VALIDATOR_AGENT_SSH_ENABLED} ${VALIDATOR_AGENT_SSH_TIMEOUT_SECONDS}' \
+envsubst '${VALIDATOR_AGENT_VALIDATOR_ID} ${VALIDATOR_AGENT_CONTROL_API_URL} ${VALIDATOR_AGENT_POLL_INTERVAL_SECONDS} ${VALIDATOR_AGENT_SELF_CHECK_TIMEOUT_SECONDS} ${VALIDATOR_AGENT_SELF_CHECK_METHODS} ${VALIDATOR_AGENT_HTTPS_TIMEOUT_SECONDS} ${VALIDATOR_AGENT_ICMP_TIMEOUT_SECONDS} ${VALIDATOR_AGENT_ICMP_COUNT} ${VALIDATOR_AGENT_SSH_ENABLED} ${VALIDATOR_AGENT_SSH_TIMEOUT_SECONDS}' \
     < /etc/validator-agent/validator-agent.yaml.tmpl > /etc/validator-agent/validator-agent.yaml
 
 exec /usr/local/bin/validator-agent -config /etc/validator-agent/validator-agent.yaml

deploy/docker/validator-agent/validator-agent.yaml.tmpl

@@ -4,6 +4,7 @@ poll_interval_seconds: ${VALIDATOR_AGENT_POLL_INTERVAL_SECONDS}
 
 self_check:
   timeout_seconds: ${VALIDATOR_AGENT_SELF_CHECK_TIMEOUT_SECONDS}
+  methods: ${VALIDATOR_AGENT_SELF_CHECK_METHODS}
 
 checks:
   https_timeout_seconds: ${VALIDATOR_AGENT_HTTPS_TIMEOUT_SECONDS}