repo init
This commit is contained in:
ayurishchev committed
commit
7e44db87b2
48 files changed
+5346
No files matched your search
Executable
+209
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env bash
|
||||
# Offline end-to-end smoke test for the Cloud IP Validator (see
|
||||
# docs/LOCAL_E2E.md). Runs control-api + 1 validator-agent + 3 probers as
|
||||
# local processes against a mock OpenStack backend, with loopback stand-ins
|
||||
# for both the "floating IP" under test and the outbound targets — no real
|
||||
# cloud or internet access required.
|
||||
#
|
||||
# Usage: scripts/run-local-e2e.sh
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
GO="${GO:-go}"
|
||||
if ! command -v "$GO" >/dev/null 2>&1 && [ -x /usr/local/go/bin/go ]; then
|
||||
GO=/usr/local/go/bin/go
|
||||
fi
|
||||
|
||||
WORK_DIR="$(mktemp -d /tmp/cloud-ip-validator-e2e.XXXXXX)"
|
||||
BIN_DIR="$WORK_DIR/bin"
|
||||
LOG_DIR="$WORK_DIR/logs"
|
||||
mkdir -p "$BIN_DIR" "$LOG_DIR"
|
||||
|
||||
PIDS=()
|
||||
cleanup() {
|
||||
echo "--- cleaning up (workdir: $WORK_DIR) ---"
|
||||
for pid in "${PIDS[@]:-}"; do
|
||||
kill "$pid" >/dev/null 2>&1 || true
|
||||
done
|
||||
wait >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "--- building binaries ---"
|
||||
"$GO" build -o "$BIN_DIR/control-api" ./cmd/control-api
|
||||
"$GO" build -o "$BIN_DIR/validator-agent" ./cmd/validator-agent
|
||||
"$GO" build -o "$BIN_DIR/prober" ./cmd/prober
|
||||
"$GO" build -o "$BIN_DIR/httpstub" ./scripts/httpstub
|
||||
|
||||
echo "--- starting stub outbound targets (stand-ins for hub.docker.com / github.com / packages.ubuntu.com) ---"
|
||||
"$BIN_DIR/httpstub" -addr 127.0.0.1:29091 >"$LOG_DIR/stub-1.log" 2>&1 & PIDS+=($!)
|
||||
"$BIN_DIR/httpstub" -addr 127.0.0.1:29092 >"$LOG_DIR/stub-2.log" 2>&1 & PIDS+=($!)
|
||||
"$BIN_DIR/httpstub" -addr 127.0.0.1:29093 >"$LOG_DIR/stub-3.log" 2>&1 & PIDS+=($!)
|
||||
sleep 0.5
|
||||
|
||||
cat >"$WORK_DIR/control-api.yaml" <<EOF
|
||||
server:
|
||||
listen_addr: "127.0.0.1:28080"
|
||||
database:
|
||||
path: "$WORK_DIR/control-api.db"
|
||||
openstack:
|
||||
mode: "mock"
|
||||
orchestrator:
|
||||
poll_interval_seconds: 1
|
||||
self_check_timeout_seconds: 5
|
||||
max_self_check_retries: 3
|
||||
checking_window_seconds: 15
|
||||
max_retries: 3
|
||||
lease_ttl_seconds: 8
|
||||
heartbeat_timeout_seconds: 10
|
||||
aggregation:
|
||||
missing_counts_as_fail: true
|
||||
validators:
|
||||
- validator_id: "validator_01"
|
||||
os_port_id: "mock-port-1"
|
||||
sites:
|
||||
- site_id: "site-1"
|
||||
index: 1
|
||||
- site_id: "site-2"
|
||||
index: 2
|
||||
- site_id: "site-3"
|
||||
index: 3
|
||||
check_types:
|
||||
- name: "https"
|
||||
enabled: true
|
||||
targets: ["stub-targets"]
|
||||
- name: "icmp"
|
||||
enabled: true
|
||||
targets: ["stub-targets"]
|
||||
targets:
|
||||
stub-targets:
|
||||
- "http://127.0.0.1:29091"
|
||||
- "http://127.0.0.1:29092"
|
||||
- "http://127.0.0.1:29093"
|
||||
inbound_checks:
|
||||
ports: [22022, 28081, 28443, 28888]
|
||||
icmp: true
|
||||
# Only 127.0.0.1 is usable here: the self-check mechanism compares the
|
||||
# TCP source address the validator-agent's own outbound connection to
|
||||
# control-api arrives with (see httpapi.remoteIP) against the assigned
|
||||
# address. In a real deployment Neutron actually SNATs traffic through the
|
||||
# newly attached FIP, so any configured address works; this offline
|
||||
# harness has no real network-level SNAT, so only the literal loopback
|
||||
# address the agent's traffic really originates from can ever pass.
|
||||
ip_addresses:
|
||||
- "127.0.0.1"
|
||||
EOF
|
||||
|
||||
cat >"$WORK_DIR/validator-agent.yaml" <<EOF
|
||||
validator_id: "validator_01"
|
||||
control_api_url: "http://127.0.0.1:28080"
|
||||
poll_interval_seconds: 1
|
||||
self_check:
|
||||
timeout_seconds: 5
|
||||
checks:
|
||||
https_timeout_seconds: 5
|
||||
icmp_timeout_seconds: 3
|
||||
icmp_count: 2
|
||||
ssh:
|
||||
enabled: false
|
||||
timeout_seconds: 3
|
||||
EOF
|
||||
|
||||
for site in site-1 site-2 site-3; do
|
||||
cat >"$WORK_DIR/prober-$site.yaml" <<EOF
|
||||
site_id: "$site"
|
||||
control_api_url: "http://127.0.0.1:28080"
|
||||
poll_interval_seconds: 1
|
||||
checks:
|
||||
tcp_timeout_seconds: 3
|
||||
icmp_timeout_seconds: 3
|
||||
icmp_count: 2
|
||||
EOF
|
||||
done
|
||||
|
||||
echo "--- starting control-api ---"
|
||||
"$BIN_DIR/control-api" -config "$WORK_DIR/control-api.yaml" >"$LOG_DIR/control-api.log" 2>&1 & CONTROL_PID=$!
|
||||
PIDS+=("$CONTROL_PID")
|
||||
|
||||
echo -n "waiting for control-api /healthz "
|
||||
for i in $(seq 1 30); do
|
||||
if curl -fs "http://127.0.0.1:28080/healthz" >/dev/null 2>&1; then
|
||||
echo "ok"
|
||||
break
|
||||
fi
|
||||
echo -n "."
|
||||
sleep 0.5
|
||||
if [ "$i" -eq 30 ]; then
|
||||
echo "FAILED"
|
||||
cat "$LOG_DIR/control-api.log"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
start_validator_agent() {
|
||||
"$BIN_DIR/validator-agent" -config "$WORK_DIR/validator-agent.yaml" -stub-ports "22022,28081,28443,28888" \
|
||||
>>"$LOG_DIR/validator-agent.log" 2>&1 & VALIDATOR_PID=$!
|
||||
PIDS+=("$VALIDATOR_PID")
|
||||
echo "validator-agent started (pid $VALIDATOR_PID)"
|
||||
}
|
||||
|
||||
status() { curl -fs "http://127.0.0.1:28080/api/v1/admin/status"; }
|
||||
ips() { curl -fs "http://127.0.0.1:28080/api/v1/admin/ips"; }
|
||||
ip_state() { ips | python3 -c "import json,sys; d=json.load(sys.stdin); print(d[0]['State'] if d else 'none')" 2>/dev/null || echo "?"; }
|
||||
|
||||
echo "--- starting validator-agent ---"
|
||||
start_validator_agent
|
||||
|
||||
# Poll tightly (every 0.1s) for the IP to leave 'queued' — i.e. control-api
|
||||
# has claimed it and associated the mock FIP — then kill the agent right
|
||||
# there, before it has had a chance to self-check or run any checks. This
|
||||
# proves the lease sweep reclaims genuinely in-flight, not-yet-reported
|
||||
# work, rather than just racing against how fast the local checks happen
|
||||
# to run.
|
||||
echo "--- waiting for control-api to claim+associate the ip, to kill the agent mid-flight ---"
|
||||
CAUGHT_MIDFLIGHT=0
|
||||
for i in $(seq 1 50); do
|
||||
st="$(ip_state)"
|
||||
if [ "$st" != "queued" ] && [ "$st" != "none" ] && [ "$st" != "?" ]; then
|
||||
CAUGHT_MIDFLIGHT=1
|
||||
break
|
||||
fi
|
||||
sleep 0.1
|
||||
done
|
||||
|
||||
if [ "$CAUGHT_MIDFLIGHT" -eq 1 ]; then
|
||||
echo ">>> caught ip in state '$st'; killing validator-agent mid-run to demonstrate lease reclaim <<<"
|
||||
kill "$VALIDATOR_PID" >/dev/null 2>&1 || true
|
||||
wait "$VALIDATOR_PID" 2>/dev/null || true
|
||||
sleep 9 # let the 8s lease_ttl_seconds expire and get reclaimed
|
||||
echo ">>> restarting validator-agent <<<"
|
||||
start_validator_agent
|
||||
else
|
||||
echo ">>> never observed the ip leave 'queued' in time; skipping the reclaim demonstration <<<"
|
||||
fi
|
||||
|
||||
echo "--- starting 3 probers ---"
|
||||
for site in site-1 site-2 site-3; do
|
||||
"$BIN_DIR/prober" -config "$WORK_DIR/prober-$site.yaml" >"$LOG_DIR/prober-$site.log" 2>&1 & PIDS+=($!)
|
||||
done
|
||||
|
||||
echo "--- waiting for the queue to drain (up to 60s) ---"
|
||||
for i in $(seq 1 120); do
|
||||
sleep 0.5
|
||||
remaining=$(status | python3 -c "import json,sys; d=json.load(sys.stdin); s=d['ips_by_state']; print(sum(v for k,v in s.items() if k not in ('done','failed')))" 2>/dev/null || echo "?")
|
||||
if [ "$remaining" = "0" ]; then
|
||||
echo "queue drained after ~$((i / 2))s"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
echo "--- final status ---"
|
||||
status | python3 -m json.tool
|
||||
echo "--- final ip results ---"
|
||||
ips | python3 -m json.tool
|
||||
|
||||
echo "--- logs are in $WORK_DIR (kept for inspection; workdir NOT auto-deleted) ---"
|
||||
trap - EXIT
|
||||
cleanup
|
||||
Reference in new issue
Block a user