cloud-ip-validator/internal/httpapi/handlers_config_test.go

package httpapi

import (
	"context"
	"encoding/json"
	"log/slog"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"testing"
	"time"

	"cloudipvalidator/internal/config"
	"cloudipvalidator/internal/db"
	"cloudipvalidator/internal/openstack"
	"cloudipvalidator/internal/orchestrator"
)

// newConfigTestHarness sets up a control-api stack with an *empty*
// control-api.yaml (no validators/sites/targets/check_types) — everything
// in this test is created purely through the admin API, to prove the
// dynamic-config path works with no YAML at all.
func newConfigTestHarness(t *testing.T) (*fakeClient, *db.DB, *orchestrator.Orchestrator, *openstack.MockClient) {
	t.Helper()
	ctx := context.Background()
	d, err := db.Open(ctx, filepath.Join(t.TempDir(), "test.db"))
	if err != nil {
		t.Fatalf("open db: %v", err)
	}
	t.Cleanup(func() { d.Close() })

	mock := openstack.NewMockClient()

	cfg := &config.ControlAPI{
		Orchestrator: config.OrchestratorConfig{
			PollIntervalSeconds: 1, SelfCheckTimeoutSeconds: 10, MaxSelfCheckRetries: 3,
			CheckingWindowSeconds: 120, MaxRetries: 3, LeaseTTLSeconds: 180, HeartbeatTimeoutSeconds: 30,
		},
		Aggregation: config.AggregationConfig{MissingCountsAsFail: true},
		Inbound:     config.InboundConfig{Ports: []int{22, 80}, ICMP: true},
	}
	if err := d.BootstrapFromConfig(ctx, cfg); err != nil {
		t.Fatalf("bootstrap from (empty) config: %v", err)
	}

	log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))
	orch := orchestrator.New(d, mock, cfg, log)
	srv := New(d, orch, log)
	ts := httptest.NewServer(srv.Handler())
	t.Cleanup(ts.Close)

	return &fakeClient{t: t, base: ts.URL, client: ts.Client()}, d, orch, mock
}

// TestConfigManagedEntirelyViaAPI proves an operator can stand up a working
// validator/site/target-group/check-type configuration using only the
// admin API — no YAML at all — and that an IP submitted afterwards passes
// through the full checking cycle to `done`.
func TestConfigManagedEntirelyViaAPI(t *testing.T) {
	fc, d, orch, mock := newConfigTestHarness(t)
	ctx := context.Background()

	mock.Seed("fip-1", "9.9.9.9", "svc-project")

	resp, body := fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{
		ValidatorID: "validator-1", OSPortID: "port-1",
	})
	if resp.StatusCode != http.StatusCreated {
		t.Fatalf("create validator: status=%d body=%s", resp.StatusCode, body)
	}

	resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/targets/web", putTargetGroupRequest{
		Targets: []string{"https://example.test"},
	})
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("put target group: status=%d body=%s", resp.StatusCode, body)
	}

	resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/check-types/https", putCheckTypeRequest{
		Enabled: true, Targets: []string{"web"},
	})
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("put check type: status=%d body=%s", resp.StatusCode, body)
	}

	// No sites configured -> inbound checks stay opt-out; egress alone
	// should be enough to reach `done`.

	resp, body = fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}})
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("submit ips: status=%d body=%s", resp.StatusCode, body)
	}
	var submitResp submitIPsResponse
	if err := json.Unmarshal(body, &submitResp); err != nil {
		t.Fatalf("unmarshal submit response: %v", err)
	}
	if len(submitResp.Added) != 1 || submitResp.Added[0] != "9.9.9.9" {
		t.Fatalf("expected 9.9.9.9 added, got %+v", submitResp)
	}

	resp, body = fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"})
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("register agent: status=%d body=%s", resp.StatusCode, body)
	}

	orch.Tick(ctx)

	resp, body = fc.do(http.MethodGet, "/api/v1/agents/validator-1/assignment", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("assignment: status=%d body=%s", resp.StatusCode, body)
	}
	var assignment assignmentResponse
	if err := json.Unmarshal(body, &assignment); err != nil {
		t.Fatalf("unmarshal assignment: %v", err)
	}
	if len(assignment.CheckConfig) != 1 || assignment.CheckConfig[0].Type != "https" {
		t.Fatalf("expected the API-created https check type in the assignment, got %+v", assignment.CheckConfig)
	}

	resp, body = fc.do(http.MethodPost, "/api/v1/agents/validator-1/self-check", selfCheckRequest{
		IPID: assignment.IPID, DetectedEgress: "9.9.9.9", Success: true,
	})
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("self-check: status=%d body=%s", resp.StatusCode, body)
	}
	resp, body = fc.do(http.MethodPost, "/api/v1/agents/validator-1/results", agentResultsRequest{
		Results: []checkResultDTO{{
			IPID: assignment.IPID, CheckType: "https", Target: "https://example.test",
			Success: true, CheckedAt: time.Now().Format(time.RFC3339Nano),
		}},
	})
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("results: status=%d body=%s", resp.StatusCode, body)
	}
	resp, body = fc.do(http.MethodPost, "/api/v1/agents/validator-1/complete", agentCompleteRequest{IPID: assignment.IPID})
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("complete: status=%d body=%s", resp.StatusCode, body)
	}

	orch.Tick(ctx)

	item, err := d.GetIPByAddress(ctx, "9.9.9.9")
	if err != nil {
		t.Fatalf("get ip: %v", err)
	}
	if item.State != db.IPDone || item.OverallResult != db.ResultPass {
		t.Fatalf("expected done/pass, got state=%s result=%s", item.State, item.OverallResult)
	}
}

// TestSubmitIPsForcesRecheckOfFinishedAddress proves that resubmitting an
// address that already reached `done` starts a brand-new checking cycle
// rather than being ignored.
func TestSubmitIPsForcesRecheckOfFinishedAddress(t *testing.T) {
	fc, d, orch, mock := newConfigTestHarness(t)
	ctx := context.Background()
	mock.Seed("fip-1", "9.9.9.9", "svc-project")

	fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ValidatorID: "validator-1", OSPortID: "port-1"})
	fc.do(http.MethodPut, "/api/v1/admin/config/targets/web", putTargetGroupRequest{Targets: []string{"https://example.test"}})
	fc.do(http.MethodPut, "/api/v1/admin/config/check-types/https", putCheckTypeRequest{Enabled: true, Targets: []string{"web"}})

	runOneCycle := func() {
		orch.Tick(ctx)
		_, body := fc.do(http.MethodGet, "/api/v1/agents/validator-1/assignment", nil)
		var assignment assignmentResponse
		if err := json.Unmarshal(body, &assignment); err != nil {
			t.Fatalf("unmarshal assignment: %v", err)
		}
		fc.do(http.MethodPost, "/api/v1/agents/validator-1/self-check", selfCheckRequest{
			IPID: assignment.IPID, DetectedEgress: "9.9.9.9", Success: true,
		})
		fc.do(http.MethodPost, "/api/v1/agents/validator-1/results", agentResultsRequest{
			Results: []checkResultDTO{{
				IPID: assignment.IPID, CheckType: "https", Target: "https://example.test",
				Success: true, CheckedAt: time.Now().Format(time.RFC3339Nano),
			}},
		})
		fc.do(http.MethodPost, "/api/v1/agents/validator-1/complete", agentCompleteRequest{IPID: assignment.IPID})
		orch.Tick(ctx)
	}

	fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"})
	fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}})
	runOneCycle()

	item, err := d.GetIPByAddress(ctx, "9.9.9.9")
	if err != nil {
		t.Fatalf("get ip: %v", err)
	}
	if item.State != db.IPDone || item.AttemptNumber != 1 {
		t.Fatalf("expected done after first cycle with attempt_number=1, got state=%s attempt=%d", item.State, item.AttemptNumber)
	}

	// Force a recheck of the same, already-finished address. The mock FIP
	// was disassociated at the end of the first cycle; associateFIP will
	// simply re-associate it during the second cycle.
	resp, body := fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}})
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("submit ips (recheck): status=%d body=%s", resp.StatusCode, body)
	}
	var submitResp submitIPsResponse
	if err := json.Unmarshal(body, &submitResp); err != nil {
		t.Fatalf("unmarshal submit response: %v", err)
	}
	if len(submitResp.Requeued) != 1 || submitResp.Requeued[0] != "9.9.9.9" {
		t.Fatalf("expected 9.9.9.9 to be requeued, got %+v", submitResp)
	}

	item, err = d.GetIPByAddress(ctx, "9.9.9.9")
	if err != nil {
		t.Fatalf("get ip after resubmit: %v", err)
	}
	if item.State != db.IPQueued || item.AttemptNumber != 2 || item.OverallResult != "" {
		t.Fatalf("expected freshly queued with attempt_number=2, got %+v", item)
	}

	runOneCycle()
	item, err = d.GetIPByAddress(ctx, "9.9.9.9")
	if err != nil {
		t.Fatalf("get ip after second cycle: %v", err)
	}
	if item.State != db.IPDone || item.OverallResult != db.ResultPass || item.AttemptNumber != 2 {
		t.Fatalf("expected done/pass on second attempt, got %+v", item)
	}
}

// TestForceCancelMidCheck proves POST /admin/ips/{ip}/cancel stops an
// in-progress check, disassociates its floating IP, and frees the
// validator, without waiting for the checking window to elapse.
func TestForceCancelMidCheck(t *testing.T) {
	fc, d, orch, mock := newConfigTestHarness(t)
	ctx := context.Background()
	mock.Seed("fip-1", "9.9.9.9", "svc-project")

	fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ValidatorID: "validator-1", OSPortID: "port-1"})
	fc.do(http.MethodPut, "/api/v1/admin/config/targets/web", putTargetGroupRequest{Targets: []string{"https://example.test"}})
	fc.do(http.MethodPut, "/api/v1/admin/config/check-types/https", putCheckTypeRequest{Enabled: true, Targets: []string{"web"}})
	fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"})
	fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}})

	orch.Tick(ctx) // claim + associate FIP -> awaiting_self_check

	item, err := d.GetIPByAddress(ctx, "9.9.9.9")
	if err != nil {
		t.Fatalf("get ip: %v", err)
	}
	if item.State != db.IPAwaitingSelfCheck {
		t.Fatalf("expected awaiting_self_check before cancel, got %s", item.State)
	}
	if fip, _ := mock.GetFloatingIPByAddress(ctx, "9.9.9.9"); fip.PortID == "" {
		t.Fatalf("expected fip associated before cancel")
	}

	resp, body := fc.do(http.MethodPost, "/api/v1/admin/ips/9.9.9.9/cancel", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("cancel: status=%d body=%s", resp.StatusCode, body)
	}

	item, err = d.GetIPByAddress(ctx, "9.9.9.9")
	if err != nil {
		t.Fatalf("get ip after cancel: %v", err)
	}
	if item.State != db.IPFailed || item.OverallResult != db.ResultCancelled {
		t.Fatalf("expected failed/cancelled, got state=%s result=%s", item.State, item.OverallResult)
	}
	if fip, _ := mock.GetFloatingIPByAddress(ctx, "9.9.9.9"); fip.PortID != "" {
		t.Fatalf("expected fip disassociated after cancel, still on port %q", fip.PortID)
	}

	v, err := d.GetValidator(ctx, "validator-1")
	if err != nil {
		t.Fatalf("get validator: %v", err)
	}
	if v.State != db.ValidatorIdle || v.CurrentIPID != nil {
		t.Fatalf("expected validator freed, got state=%s current_ip=%v", v.State, v.CurrentIPID)
	}

	// Cancelling again is rejected — nothing left to cancel.
	resp, body = fc.do(http.MethodPost, "/api/v1/admin/ips/9.9.9.9/cancel", nil)
	if resp.StatusCode != http.StatusConflict {
		t.Fatalf("expected 409 cancelling an already-finished ip, status=%d body=%s", resp.StatusCode, body)
	}

	// Cancelling an unknown address is a 404.
	resp, body = fc.do(http.MethodPost, "/api/v1/admin/ips/1.1.1.1/cancel", nil)
	if resp.StatusCode != http.StatusNotFound {
		t.Fatalf("expected 404 cancelling unknown ip, status=%d body=%s", resp.StatusCode, body)
	}
}

// TestDeleteIPMidCheck proves DELETE /admin/ips/{ip} disassociates the
// floating IP and permanently removes the address — unlike cancel, a
// subsequent GET on the same address is a 404, not a cancelled record.
func TestDeleteIPMidCheck(t *testing.T) {
	fc, d, orch, mock := newConfigTestHarness(t)
	ctx := context.Background()
	mock.Seed("fip-1", "9.9.9.9", "svc-project")

	fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ValidatorID: "validator-1", OSPortID: "port-1"})
	fc.do(http.MethodPut, "/api/v1/admin/config/targets/web", putTargetGroupRequest{Targets: []string{"https://example.test"}})
	fc.do(http.MethodPut, "/api/v1/admin/config/check-types/https", putCheckTypeRequest{Enabled: true, Targets: []string{"web"}})
	fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"})
	fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}})

	orch.Tick(ctx) // claim + associate FIP -> awaiting_self_check

	if fip, _ := mock.GetFloatingIPByAddress(ctx, "9.9.9.9"); fip.PortID == "" {
		t.Fatalf("expected fip associated before delete")
	}

	resp, body := fc.do(http.MethodDelete, "/api/v1/admin/ips/9.9.9.9", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("delete: status=%d body=%s", resp.StatusCode, body)
	}

	if fip, _ := mock.GetFloatingIPByAddress(ctx, "9.9.9.9"); fip.PortID != "" {
		t.Fatalf("expected fip disassociated after delete, still on port %q", fip.PortID)
	}
	if _, err := d.GetIPByAddress(ctx, "9.9.9.9"); err == nil {
		t.Fatalf("expected ip row gone from db after delete")
	}
	v, err := d.GetValidator(ctx, "validator-1")
	if err != nil {
		t.Fatalf("get validator: %v", err)
	}
	if v.State != db.ValidatorIdle || v.CurrentIPID != nil {
		t.Fatalf("expected validator freed, got state=%s current_ip=%v", v.State, v.CurrentIPID)
	}

	resp, body = fc.do(http.MethodGet, "/api/v1/admin/ips/9.9.9.9", nil)
	if resp.StatusCode != http.StatusNotFound {
		t.Fatalf("expected 404 for deleted ip, status=%d body=%s", resp.StatusCode, body)
	}

	resp, body = fc.do(http.MethodDelete, "/api/v1/admin/ips/9.9.9.9", nil)
	if resp.StatusCode != http.StatusNotFound {
		t.Fatalf("expected 404 deleting already-gone ip, status=%d body=%s", resp.StatusCode, body)
	}
}

// TestDeleteIPsAndClearQueue exercises POST /admin/ips/delete against a
// mixed known/unknown list, then POST /admin/ips/clear against whatever
// remains in the queue (including an actively checking address).
func TestDeleteIPsAndClearQueue(t *testing.T) {
	fc, d, orch, mock := newConfigTestHarness(t)
	ctx := context.Background()
	mock.Seed("fip-1", "1.1.1.1", "svc-project")

	fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ValidatorID: "validator-1", OSPortID: "port-1"})
	fc.do(http.MethodPut, "/api/v1/admin/config/targets/web", putTargetGroupRequest{Targets: []string{"https://example.test"}})
	fc.do(http.MethodPut, "/api/v1/admin/config/check-types/https", putCheckTypeRequest{Enabled: true, Targets: []string{"web"}})
	fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"})
	fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"1.1.1.1", "2.2.2.2", "3.3.3.3"}})

	orch.Tick(ctx) // claims 1.1.1.1 for validator-1, associates its fip

	// POST /admin/ips/delete with an empty list is a 400.
	resp, body := fc.do(http.MethodPost, "/api/v1/admin/ips/delete", deleteIPsRequest{})
	if resp.StatusCode != http.StatusBadRequest {
		t.Fatalf("expected 400 for empty delete list, status=%d body=%s", resp.StatusCode, body)
	}

	// Delete a mix of a queued address and an unknown one.
	resp, body = fc.do(http.MethodPost, "/api/v1/admin/ips/delete", deleteIPsRequest{Addresses: []string{"2.2.2.2", "no-such-ip"}})
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("delete ips: status=%d body=%s", resp.StatusCode, body)
	}
	var delResp deleteIPsResponse
	if err := json.Unmarshal(body, &delResp); err != nil {
		t.Fatalf("unmarshal delete response: %v", err)
	}
	if len(delResp.Deleted) != 1 || delResp.Deleted[0] != "2.2.2.2" {
		t.Fatalf("expected 2.2.2.2 deleted, got %+v", delResp)
	}
	if len(delResp.NotFound) != 1 || delResp.NotFound[0] != "no-such-ip" {
		t.Fatalf("expected no-such-ip in not_found, got %+v", delResp)
	}

	// Clear whatever's left — 1.1.1.1 (mid-check, fip attached) and 3.3.3.3
	// (still queued).
	resp, body = fc.do(http.MethodPost, "/api/v1/admin/ips/clear", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("clear queue: status=%d body=%s", resp.StatusCode, body)
	}
	var clearResp clearQueueResponse
	if err := json.Unmarshal(body, &clearResp); err != nil {
		t.Fatalf("unmarshal clear response: %v", err)
	}
	if len(clearResp.Deleted) != 2 {
		t.Fatalf("expected both remaining addresses deleted, got %+v", clearResp)
	}

	if fip, _ := mock.GetFloatingIPByAddress(ctx, "1.1.1.1"); fip.PortID != "" {
		t.Fatalf("expected fip disassociated on clear, still on port %q", fip.PortID)
	}
	ips, err := d.ListIPs(ctx)
	if err != nil {
		t.Fatalf("list ips: %v", err)
	}
	if len(ips) != 0 {
		t.Fatalf("expected empty queue after clear, got %+v", ips)
	}
	v, err := d.GetValidator(ctx, "validator-1")
	if err != nil {
		t.Fatalf("get validator: %v", err)
	}
	if v.State != db.ValidatorIdle || v.CurrentIPID != nil {
		t.Fatalf("expected validator freed after clear, got state=%s current_ip=%v", v.State, v.CurrentIPID)
	}
}

// TestOrchestratorSettingsGetPut proves the settle-delay setting round-trips
// through GET/PUT /api/v1/admin/config/orchestrator.
func TestOrchestratorSettingsGetPut(t *testing.T) {
	fc, _, _, _ := newConfigTestHarness(t)

	resp, body := fc.do(http.MethodGet, "/api/v1/admin/config/orchestrator", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("get settings: status=%d body=%s", resp.StatusCode, body)
	}
	var got orchestratorSettingsDTO
	if err := json.Unmarshal(body, &got); err != nil {
		t.Fatalf("unmarshal get response: %v", err)
	}
	if got.FIPSettleSeconds != 0 {
		t.Fatalf("expected default 0, got %+v", got)
	}

	resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", orchestratorSettingsDTO{FIPSettleSeconds: 20})
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("put settings: status=%d body=%s", resp.StatusCode, body)
	}
	if err := json.Unmarshal(body, &got); err != nil {
		t.Fatalf("unmarshal put response: %v", err)
	}
	if got.FIPSettleSeconds != 20 {
		t.Fatalf("expected 20, got %+v", got)
	}

	resp, body = fc.do(http.MethodGet, "/api/v1/admin/config/orchestrator", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("get settings after put: status=%d body=%s", resp.StatusCode, body)
	}
	if err := json.Unmarshal(body, &got); err != nil {
		t.Fatalf("unmarshal get-after-put response: %v", err)
	}
	if got.FIPSettleSeconds != 20 {
		t.Fatalf("expected 20 to persist, got %+v", got)
	}
}

// TestOrchestratorSettingsPutValidation proves a value that would leave no
// room for self-check inside the claim lease is rejected with 400.
func TestOrchestratorSettingsPutValidation(t *testing.T) {
	fc, _, _, _ := newConfigTestHarness(t)
	// newConfigTestHarness: LeaseTTLSeconds=180, SelfCheckTimeoutSeconds=10.

	resp, body := fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", orchestratorSettingsDTO{FIPSettleSeconds: 175})
	if resp.StatusCode != http.StatusBadRequest {
		t.Fatalf("expected 400 for settle seconds too close to lease ttl, status=%d body=%s", resp.StatusCode, body)
	}

	resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", orchestratorSettingsDTO{FIPSettleSeconds: -1})
	if resp.StatusCode != http.StatusBadRequest {
		t.Fatalf("expected 400 for negative settle seconds, status=%d body=%s", resp.StatusCode, body)
	}
}

// TestFIPSettleDelayGatesAssignmentEndpoint proves GET
// /api/v1/agents/{id}/assignment returns 204 while the settle window is
// open and 200 once it has elapsed — the end-to-end proof of the whole
// feature over the real HTTP wire contract.
func TestFIPSettleDelayGatesAssignmentEndpoint(t *testing.T) {
	fc, _, orch, mock := newConfigTestHarness(t)
	ctx := context.Background()
	mock.Seed("fip-1", "9.9.9.9", "svc-project")

	resp, body := fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", orchestratorSettingsDTO{FIPSettleSeconds: 1})
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("put settings: status=%d body=%s", resp.StatusCode, body)
	}

	fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ValidatorID: "validator-1", OSPortID: "port-1"})
	fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"})
	fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}})

	orch.Tick(ctx) // claim + associate -> awaiting_self_check, fip attached

	resp, body = fc.do(http.MethodGet, "/api/v1/agents/validator-1/assignment", nil)
	if resp.StatusCode != http.StatusNoContent {
		t.Fatalf("expected 204 during settle window, status=%d body=%s", resp.StatusCode, body)
	}

	time.Sleep(1100 * time.Millisecond)

	resp, body = fc.do(http.MethodGet, "/api/v1/agents/validator-1/assignment", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("expected 200 after settle window elapsed, status=%d body=%s", resp.StatusCode, body)
	}
}

// TestInboundChecksGetPut proves the prober check config round-trips
// through GET/PUT /api/v1/admin/config/inbound-checks.
func TestInboundChecksGetPut(t *testing.T) {
	fc, _, _, _ := newConfigTestHarness(t)

	resp, body := fc.do(http.MethodGet, "/api/v1/admin/config/inbound-checks", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("get inbound checks: status=%d body=%s", resp.StatusCode, body)
	}
	var got inboundChecksDTO
	if err := json.Unmarshal(body, &got); err != nil {
		t.Fatalf("unmarshal get response: %v", err)
	}
	// newConfigTestHarness seeds Ports:[22,80], ICMP:true.
	if len(got.Ports) != 2 || !got.ICMP {
		t.Fatalf("expected seeded {[22 80] true}, got %+v", got)
	}

	resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/inbound-checks", inboundChecksDTO{Ports: []int{443, 8080}, ICMP: false})
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("put inbound checks: status=%d body=%s", resp.StatusCode, body)
	}
	if err := json.Unmarshal(body, &got); err != nil {
		t.Fatalf("unmarshal put response: %v", err)
	}
	if len(got.Ports) != 2 || got.ICMP {
		t.Fatalf("expected {[443 8080] false}, got %+v", got)
	}

	resp, body = fc.do(http.MethodGet, "/api/v1/admin/config/inbound-checks", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("get inbound checks after put: status=%d body=%s", resp.StatusCode, body)
	}
	if err := json.Unmarshal(body, &got); err != nil {
		t.Fatalf("unmarshal get-after-put response: %v", err)
	}
	if got.Ports[0] != 443 || got.Ports[1] != 8080 || got.ICMP {
		t.Fatalf("expected {[443 8080] false} to persist, got %+v", got)
	}
}

// TestInboundChecksPutValidation proves out-of-range and duplicate ports
// are rejected with 400.
func TestInboundChecksPutValidation(t *testing.T) {
	fc, _, _, _ := newConfigTestHarness(t)

	resp, body := fc.do(http.MethodPut, "/api/v1/admin/config/inbound-checks", inboundChecksDTO{Ports: []int{0}, ICMP: false})
	if resp.StatusCode != http.StatusBadRequest {
		t.Fatalf("expected 400 for port 0, status=%d body=%s", resp.StatusCode, body)
	}

	resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/inbound-checks", inboundChecksDTO{Ports: []int{70000}, ICMP: false})
	if resp.StatusCode != http.StatusBadRequest {
		t.Fatalf("expected 400 for port 70000, status=%d body=%s", resp.StatusCode, body)
	}

	resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/inbound-checks", inboundChecksDTO{Ports: []int{22, 22}, ICMP: false})
	if resp.StatusCode != http.StatusBadRequest {
		t.Fatalf("expected 400 for duplicate port, status=%d body=%s", resp.StatusCode, body)
	}
}

// TestInboundChecksReflectedInProberAssignmentsWithoutRestart proves the
// fix to the formerly-static handlers_prober.go read: a PUT to
// /api/v1/admin/config/inbound-checks changes what GET
// /api/v1/probers/{site_id}/assignments hands back to an already-registered
// prober, for an IP already in `checking`, with no control-api restart.
func TestInboundChecksReflectedInProberAssignmentsWithoutRestart(t *testing.T) {
	fc, _, orch, mock := newConfigTestHarness(t)
	ctx := context.Background()
	mock.Seed("fip-1", "9.9.9.9", "svc-project")

	fc.do(http.MethodPut, "/api/v1/admin/config/sites/1", putSiteRequest{SiteID: "site-1"})
	fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ValidatorID: "validator-1", OSPortID: "port-1"})
	fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"})
	fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}})

	orch.Tick(ctx) // claim + associate -> awaiting_self_check

	resp, body := fc.do(http.MethodGet, "/api/v1/agents/validator-1/assignment", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("assignment: status=%d body=%s", resp.StatusCode, body)
	}
	var assignment assignmentResponse
	if err := json.Unmarshal(body, &assignment); err != nil {
		t.Fatalf("unmarshal assignment: %v", err)
	}
	fc.do(http.MethodPost, "/api/v1/agents/validator-1/self-check", selfCheckRequest{
		IPID: assignment.IPID, DetectedEgress: "9.9.9.9", Success: true, Detail: "matched",
	})
	// Now item.State == "checking" — a prober assignment target.

	fc.do(http.MethodPost, "/api/v1/probers/register", registerProberRequest{SiteID: "site-1"})

	resp, body = fc.do(http.MethodGet, "/api/v1/probers/site-1/assignments", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("prober assignments: status=%d body=%s", resp.StatusCode, body)
	}
	var assignments []proberAssignment
	if err := json.Unmarshal(body, &assignments); err != nil {
		t.Fatalf("unmarshal assignments: %v", err)
	}
	if len(assignments) != 1 || len(assignments[0].Ports) != 2 || !assignments[0].ICMP {
		t.Fatalf("expected seeded {[22 80] true} before PUT, got %+v", assignments)
	}

	resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/inbound-checks", inboundChecksDTO{Ports: []int{8080}, ICMP: false})
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("put inbound checks: status=%d body=%s", resp.StatusCode, body)
	}

	resp, body = fc.do(http.MethodGet, "/api/v1/probers/site-1/assignments", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("prober assignments after put: status=%d body=%s", resp.StatusCode, body)
	}
	if err := json.Unmarshal(body, &assignments); err != nil {
		t.Fatalf("unmarshal assignments after put: %v", err)
	}
	if len(assignments) != 1 || len(assignments[0].Ports) != 1 || assignments[0].Ports[0] != 8080 || assignments[0].ICMP {
		t.Fatalf("expected updated {[8080] false} without restart, got %+v", assignments)
	}
}

// TestProberRegisterSetsHostnameAndIdleState proves POST
// /api/v1/probers/register persists the calling prober's hostname and
// flips the site's state to idle, visible via GET
// /api/v1/admin/config/sites — the prober-availability analog of
// validator registration.
func TestProberRegisterSetsHostnameAndIdleState(t *testing.T) {
	fc, _, _, _ := newConfigTestHarness(t)

	fc.do(http.MethodPut, "/api/v1/admin/config/sites/1", putSiteRequest{SiteID: "site-1"})

	resp, body := fc.do(http.MethodGet, "/api/v1/admin/config/sites", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("get sites: status=%d body=%s", resp.StatusCode, body)
	}
	var sites []siteDTO
	if err := json.Unmarshal(body, &sites); err != nil {
		t.Fatalf("unmarshal sites: %v", err)
	}
	if len(sites) != 1 || sites[0].State != "unregistered" {
		t.Fatalf("expected freshly-created slot unregistered, got %+v", sites)
	}

	resp, body = fc.do(http.MethodPost, "/api/v1/probers/register", registerProberRequest{SiteID: "site-1", Hostname: "probe-host-1"})
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("register prober: status=%d body=%s", resp.StatusCode, body)
	}

	resp, body = fc.do(http.MethodGet, "/api/v1/admin/config/sites", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("get sites after register: status=%d body=%s", resp.StatusCode, body)
	}
	if err := json.Unmarshal(body, &sites); err != nil {
		t.Fatalf("unmarshal sites: %v", err)
	}
	if sites[0].State != "idle" || sites[0].Hostname != "probe-host-1" {
		t.Fatalf("expected {state:idle, hostname:probe-host-1}, got %+v", sites[0])
	}
	if sites[0].LastHeartbeatAt == nil {
		t.Fatalf("expected last_heartbeat_at stamped")
	}
}

// TestProberHeartbeat404UnknownSite proves the heartbeat endpoint rejects
// an unconfigured site_id, mirroring the validator heartbeat's 404.
func TestProberHeartbeat404UnknownSite(t *testing.T) {
	fc, _, _, _ := newConfigTestHarness(t)
	resp, body := fc.do(http.MethodPost, "/api/v1/probers/unknown-site/heartbeat", nil)
	if resp.StatusCode != http.StatusNotFound {
		t.Fatalf("expected 404 for unknown site_id, status=%d body=%s", resp.StatusCode, body)
	}
}

// TestSweepStaleSiteHeartbeatsMarksUnreachable is the end-to-end proof that
// a prober that stops heartbeating gets marked unreachable by the sweep,
// visible via the admin API — the prober-side analog of
// TestFIPSettleDelayGatesAssignmentEndpoint's wire-level style.
func TestSweepStaleSiteHeartbeatsMarksUnreachable(t *testing.T) {
	fc, d, orch, _ := newConfigTestHarness(t)
	ctx := context.Background()

	fc.do(http.MethodPut, "/api/v1/admin/config/sites/1", putSiteRequest{SiteID: "site-1"})
	resp, body := fc.do(http.MethodPost, "/api/v1/probers/register", registerProberRequest{SiteID: "site-1", Hostname: "probe-host-1"})
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("register prober: status=%d body=%s", resp.StatusCode, body)
	}

	// Backdate last_heartbeat_at past HeartbeatTimeoutSeconds (30s, per
	// newConfigTestHarness) directly in the DB, rather than sleeping 30+
	// real seconds in the test.
	old := db.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano)
	if _, err := d.ExecContext(ctx, `UPDATE sites SET last_heartbeat_at=? WHERE site_id=?`, old, "site-1"); err != nil {
		t.Fatalf("backdate heartbeat: %v", err)
	}

	if err := orch.SweepStaleSiteHeartbeats(ctx); err != nil {
		t.Fatalf("sweep stale site heartbeats: %v", err)
	}

	resp, body = fc.do(http.MethodGet, "/api/v1/admin/config/sites", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("get sites: status=%d body=%s", resp.StatusCode, body)
	}
	var sites []siteDTO
	if err := json.Unmarshal(body, &sites); err != nil {
		t.Fatalf("unmarshal sites: %v", err)
	}
	if len(sites) != 1 || sites[0].State != "unreachable" {
		t.Fatalf("expected site-1 marked unreachable after sweep, got %+v", sites)
	}

	// A fresh heartbeat brings it back to idle.
	resp, body = fc.do(http.MethodPost, "/api/v1/probers/site-1/heartbeat", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("heartbeat: status=%d body=%s", resp.StatusCode, body)
	}
	resp, body = fc.do(http.MethodGet, "/api/v1/admin/config/sites", nil)
	if resp.StatusCode != http.StatusOK {
		t.Fatalf("get sites after heartbeat: status=%d body=%s", resp.StatusCode, body)
	}
	if err := json.Unmarshal(body, &sites); err != nil {
		t.Fatalf("unmarshal sites: %v", err)
	}
	if sites[0].State != "idle" {
		t.Fatalf("expected site-1 back to idle after heartbeat, got %+v", sites)
	}
}