cloud-ip-validator/scripts/run-local-e2e.sh

#!/usr/bin/env bash
# Offline end-to-end smoke test for the Cloud IP Validator (see
# docs/LOCAL_E2E.md). Runs control-api + 1 validator-agent + 3 probers as
# local processes against a mock OpenStack backend, with loopback stand-ins
# for both the "floating IP" under test and the outbound targets — no real
# cloud or internet access required.
#
# Usage: scripts/run-local-e2e.sh
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$REPO_ROOT"

GO="${GO:-go}"
if ! command -v "$GO" >/dev/null 2>&1 && [ -x /usr/local/go/bin/go ]; then
	GO=/usr/local/go/bin/go
fi

WORK_DIR="$(mktemp -d /tmp/cloud-ip-validator-e2e.XXXXXX)"
BIN_DIR="$WORK_DIR/bin"
LOG_DIR="$WORK_DIR/logs"
mkdir -p "$BIN_DIR" "$LOG_DIR"

PIDS=()
cleanup() {
	echo "--- cleaning up (workdir: $WORK_DIR) ---"
	for pid in "${PIDS[@]:-}"; do
		kill "$pid" >/dev/null 2>&1 || true
	done
	wait >/dev/null 2>&1 || true
}
trap cleanup EXIT

echo "--- building binaries ---"
"$GO" build -o "$BIN_DIR/control-api" ./cmd/control-api
"$GO" build -o "$BIN_DIR/validator-agent" ./cmd/validator-agent
"$GO" build -o "$BIN_DIR/prober" ./cmd/prober
"$GO" build -o "$BIN_DIR/httpstub" ./scripts/httpstub

echo "--- starting stub outbound targets (stand-ins for hub.docker.com / github.com / packages.ubuntu.com) ---"
"$BIN_DIR/httpstub" -addr 127.0.0.1:29091 >"$LOG_DIR/stub-1.log" 2>&1 & PIDS+=($!)
"$BIN_DIR/httpstub" -addr 127.0.0.1:29092 >"$LOG_DIR/stub-2.log" 2>&1 & PIDS+=($!)
"$BIN_DIR/httpstub" -addr 127.0.0.1:29093 >"$LOG_DIR/stub-3.log" 2>&1 & PIDS+=($!)
sleep 0.5

cat >"$WORK_DIR/control-api.yaml" <<EOF
server:
  listen_addr: "127.0.0.1:28080"
database:
  path: "$WORK_DIR/control-api.db"
openstack:
  mode: "mock"
orchestrator:
  poll_interval_seconds: 1
  self_check_timeout_seconds: 5
  max_self_check_retries: 3
  checking_window_seconds: 15
  max_retries: 3
  lease_ttl_seconds: 8
  heartbeat_timeout_seconds: 10
aggregation:
  missing_counts_as_fail: true
validators:
  - validator_id: "validator_01"
    os_port_id: "mock-port-1"
sites:
  - site_id: "site-1"
    index: 1
  - site_id: "site-2"
    index: 2
  - site_id: "site-3"
    index: 3
check_types:
  - name: "https"
    enabled: true
    targets: ["stub-targets"]
  - name: "icmp"
    enabled: true
    targets: ["stub-targets"]
targets:
  stub-targets:
    - "http://127.0.0.1:29091"
    - "http://127.0.0.1:29092"
    - "http://127.0.0.1:29093"
inbound_checks:
  ports: [22022, 28081, 28443, 28888]
  icmp: true
# Only 127.0.0.1 is usable here: the self-check mechanism compares the
# TCP source address the validator-agent's own outbound connection to
# control-api arrives with (see httpapi.remoteIP) against the assigned
# address. In a real deployment Neutron actually SNATs traffic through the
# newly attached FIP, so any configured address works; this offline
# harness has no real network-level SNAT, so only the literal loopback
# address the agent's traffic really originates from can ever pass.
ip_addresses:
  - "127.0.0.1"
EOF

cat >"$WORK_DIR/validator-agent.yaml" <<EOF
validator_id: "validator_01"
control_api_url: "http://127.0.0.1:28080"
poll_interval_seconds: 1
self_check:
  timeout_seconds: 5
checks:
  https_timeout_seconds: 5
  icmp_timeout_seconds: 3
  icmp_count: 2
  ssh:
    enabled: false
    timeout_seconds: 3
EOF

for site in site-1 site-2 site-3; do
	cat >"$WORK_DIR/prober-$site.yaml" <<EOF
site_id: "$site"
control_api_url: "http://127.0.0.1:28080"
poll_interval_seconds: 1
checks:
  tcp_timeout_seconds: 3
  icmp_timeout_seconds: 3
  icmp_count: 2
EOF
done

echo "--- starting control-api ---"
"$BIN_DIR/control-api" -config "$WORK_DIR/control-api.yaml" >"$LOG_DIR/control-api.log" 2>&1 & CONTROL_PID=$!
PIDS+=("$CONTROL_PID")

echo -n "waiting for control-api /healthz "
for i in $(seq 1 30); do
	if curl -fs "http://127.0.0.1:28080/healthz" >/dev/null 2>&1; then
		echo "ok"
		break
	fi
	echo -n "."
	sleep 0.5
	if [ "$i" -eq 30 ]; then
		echo "FAILED"
		cat "$LOG_DIR/control-api.log"
		exit 1
	fi
done

start_validator_agent() {
	"$BIN_DIR/validator-agent" -config "$WORK_DIR/validator-agent.yaml" -stub-ports "22022,28081,28443,28888" \
		>>"$LOG_DIR/validator-agent.log" 2>&1 & VALIDATOR_PID=$!
	PIDS+=("$VALIDATOR_PID")
	echo "validator-agent started (pid $VALIDATOR_PID)"
}

status() { curl -fs "http://127.0.0.1:28080/api/v1/admin/status"; }
ips() { curl -fs "http://127.0.0.1:28080/api/v1/admin/ips"; }
ip_state() { ips | python3 -c "import json,sys; d=json.load(sys.stdin); print(d[0]['State'] if d else 'none')" 2>/dev/null || echo "?"; }

echo "--- starting validator-agent ---"
start_validator_agent

# Poll tightly (every 0.1s) for the IP to leave 'queued' — i.e. control-api
# has claimed it and associated the mock FIP — then kill the agent right
# there, before it has had a chance to self-check or run any checks. This
# proves the lease sweep reclaims genuinely in-flight, not-yet-reported
# work, rather than just racing against how fast the local checks happen
# to run.
echo "--- waiting for control-api to claim+associate the ip, to kill the agent mid-flight ---"
CAUGHT_MIDFLIGHT=0
for i in $(seq 1 50); do
	st="$(ip_state)"
	if [ "$st" != "queued" ] && [ "$st" != "none" ] && [ "$st" != "?" ]; then
		CAUGHT_MIDFLIGHT=1
		break
	fi
	sleep 0.1
done

if [ "$CAUGHT_MIDFLIGHT" -eq 1 ]; then
	echo ">>> caught ip in state '$st'; killing validator-agent mid-run to demonstrate lease reclaim <<<"
	kill "$VALIDATOR_PID" >/dev/null 2>&1 || true
	wait "$VALIDATOR_PID" 2>/dev/null || true
	sleep 9 # let the 8s lease_ttl_seconds expire and get reclaimed
	echo ">>> restarting validator-agent <<<"
	start_validator_agent
else
	echo ">>> never observed the ip leave 'queued' in time; skipping the reclaim demonstration <<<"
fi

echo "--- starting 3 probers ---"
for site in site-1 site-2 site-3; do
	"$BIN_DIR/prober" -config "$WORK_DIR/prober-$site.yaml" >"$LOG_DIR/prober-$site.log" 2>&1 & PIDS+=($!)
done

echo "--- waiting for the queue to drain (up to 60s) ---"
for i in $(seq 1 120); do
	sleep 0.5
	remaining=$(status | python3 -c "import json,sys; d=json.load(sys.stdin); s=d['ips_by_state']; print(sum(v for k,v in s.items() if k not in ('done','failed')))" 2>/dev/null || echo "?")
	if [ "$remaining" = "0" ]; then
		echo "queue drained after ~$((i / 2))s"
		break
	fi
done

echo "--- final status ---"
status | python3 -m json.tool
echo "--- final ip results ---"
ips | python3 -m json.tool

echo "--- logs are in $WORK_DIR (kept for inspection; workdir NOT auto-deleted) ---"
trap - EXIT
cleanup