package httpapi import ( "context" "encoding/json" "log/slog" "net/http" "net/http/httptest" "os" "path/filepath" "testing" "time" "cloudipvalidator/internal/config" "cloudipvalidator/internal/db" "cloudipvalidator/internal/openstack" "cloudipvalidator/internal/orchestrator" ) // newConfigTestHarness sets up a control-api stack with an *empty* // control-api.yaml (no validators/sites/targets/check_types) — everything // in this test is created purely through the admin API, to prove the // dynamic-config path works with no YAML at all. func newConfigTestHarness(t *testing.T) (*fakeClient, *db.DB, *orchestrator.Orchestrator, *openstack.MockClient) { t.Helper() ctx := context.Background() d, err := db.Open(ctx, filepath.Join(t.TempDir(), "test.db")) if err != nil { t.Fatalf("open db: %v", err) } t.Cleanup(func() { d.Close() }) mock := openstack.NewMockClient() cfg := &config.ControlAPI{ Orchestrator: config.OrchestratorConfig{ PollIntervalSeconds: 1, SelfCheckTimeoutSeconds: 10, MaxSelfCheckRetries: 3, CheckingWindowSeconds: 120, MaxRetries: 3, LeaseTTLSeconds: 180, HeartbeatTimeoutSeconds: 30, }, Aggregation: config.AggregationConfig{MissingCountsAsFail: true}, Inbound: config.InboundConfig{Ports: []int{22, 80}, ICMP: true}, } if err := d.BootstrapFromConfig(ctx, cfg); err != nil { t.Fatalf("bootstrap from (empty) config: %v", err) } log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})) orch := orchestrator.New(d, mock, cfg, log) srv := New(d, orch, log) ts := httptest.NewServer(srv.Handler()) t.Cleanup(ts.Close) return &fakeClient{t: t, base: ts.URL, client: ts.Client()}, d, orch, mock } // TestConfigManagedEntirelyViaAPI proves an operator can stand up a working // validator/site/target-group/check-type configuration using only the // admin API — no YAML at all — and that an IP submitted afterwards passes // through the full checking cycle to `done`. func TestConfigManagedEntirelyViaAPI(t *testing.T) { fc, d, orch, mock := newConfigTestHarness(t) ctx := context.Background() mock.Seed("fip-1", "9.9.9.9", "svc-project") resp, body := fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ ValidatorID: "validator-1", OSPortID: "port-1", }) if resp.StatusCode != http.StatusCreated { t.Fatalf("create validator: status=%d body=%s", resp.StatusCode, body) } resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/targets/web", putTargetGroupRequest{ Targets: []string{"https://example.test"}, }) if resp.StatusCode != http.StatusOK { t.Fatalf("put target group: status=%d body=%s", resp.StatusCode, body) } resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/check-types/https", putCheckTypeRequest{ Enabled: true, Targets: []string{"web"}, }) if resp.StatusCode != http.StatusOK { t.Fatalf("put check type: status=%d body=%s", resp.StatusCode, body) } // No sites configured -> inbound checks stay opt-out; egress alone // should be enough to reach `done`. resp, body = fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}}) if resp.StatusCode != http.StatusOK { t.Fatalf("submit ips: status=%d body=%s", resp.StatusCode, body) } var submitResp submitIPsResponse if err := json.Unmarshal(body, &submitResp); err != nil { t.Fatalf("unmarshal submit response: %v", err) } if len(submitResp.Added) != 1 || submitResp.Added[0] != "9.9.9.9" { t.Fatalf("expected 9.9.9.9 added, got %+v", submitResp) } resp, body = fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"}) if resp.StatusCode != http.StatusOK { t.Fatalf("register agent: status=%d body=%s", resp.StatusCode, body) } orch.Tick(ctx) resp, body = fc.do(http.MethodGet, "/api/v1/agents/validator-1/assignment", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("assignment: status=%d body=%s", resp.StatusCode, body) } var assignment assignmentResponse if err := json.Unmarshal(body, &assignment); err != nil { t.Fatalf("unmarshal assignment: %v", err) } if len(assignment.CheckConfig) != 1 || assignment.CheckConfig[0].Type != "https" { t.Fatalf("expected the API-created https check type in the assignment, got %+v", assignment.CheckConfig) } resp, body = fc.do(http.MethodPost, "/api/v1/agents/validator-1/self-check", selfCheckRequest{ IPID: assignment.IPID, DetectedEgress: "9.9.9.9", Success: true, }) if resp.StatusCode != http.StatusOK { t.Fatalf("self-check: status=%d body=%s", resp.StatusCode, body) } resp, body = fc.do(http.MethodPost, "/api/v1/agents/validator-1/results", agentResultsRequest{ Results: []checkResultDTO{{ IPID: assignment.IPID, CheckType: "https", Target: "https://example.test", Success: true, CheckedAt: time.Now().Format(time.RFC3339Nano), }}, }) if resp.StatusCode != http.StatusOK { t.Fatalf("results: status=%d body=%s", resp.StatusCode, body) } resp, body = fc.do(http.MethodPost, "/api/v1/agents/validator-1/complete", agentCompleteRequest{IPID: assignment.IPID}) if resp.StatusCode != http.StatusOK { t.Fatalf("complete: status=%d body=%s", resp.StatusCode, body) } orch.Tick(ctx) item, err := d.GetIPByAddress(ctx, "9.9.9.9") if err != nil { t.Fatalf("get ip: %v", err) } if item.State != db.IPDone || item.OverallResult != db.ResultPass { t.Fatalf("expected done/pass, got state=%s result=%s", item.State, item.OverallResult) } } // TestSubmitIPsForcesRecheckOfFinishedAddress proves that resubmitting an // address that already reached `done` starts a brand-new checking cycle // rather than being ignored. func TestSubmitIPsForcesRecheckOfFinishedAddress(t *testing.T) { fc, d, orch, mock := newConfigTestHarness(t) ctx := context.Background() mock.Seed("fip-1", "9.9.9.9", "svc-project") fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ValidatorID: "validator-1", OSPortID: "port-1"}) fc.do(http.MethodPut, "/api/v1/admin/config/targets/web", putTargetGroupRequest{Targets: []string{"https://example.test"}}) fc.do(http.MethodPut, "/api/v1/admin/config/check-types/https", putCheckTypeRequest{Enabled: true, Targets: []string{"web"}}) runOneCycle := func() { orch.Tick(ctx) _, body := fc.do(http.MethodGet, "/api/v1/agents/validator-1/assignment", nil) var assignment assignmentResponse if err := json.Unmarshal(body, &assignment); err != nil { t.Fatalf("unmarshal assignment: %v", err) } fc.do(http.MethodPost, "/api/v1/agents/validator-1/self-check", selfCheckRequest{ IPID: assignment.IPID, DetectedEgress: "9.9.9.9", Success: true, }) fc.do(http.MethodPost, "/api/v1/agents/validator-1/results", agentResultsRequest{ Results: []checkResultDTO{{ IPID: assignment.IPID, CheckType: "https", Target: "https://example.test", Success: true, CheckedAt: time.Now().Format(time.RFC3339Nano), }}, }) fc.do(http.MethodPost, "/api/v1/agents/validator-1/complete", agentCompleteRequest{IPID: assignment.IPID}) orch.Tick(ctx) } fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"}) fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}}) runOneCycle() item, err := d.GetIPByAddress(ctx, "9.9.9.9") if err != nil { t.Fatalf("get ip: %v", err) } if item.State != db.IPDone || item.AttemptNumber != 1 { t.Fatalf("expected done after first cycle with attempt_number=1, got state=%s attempt=%d", item.State, item.AttemptNumber) } // Force a recheck of the same, already-finished address. The mock FIP // was disassociated at the end of the first cycle; associateFIP will // simply re-associate it during the second cycle. resp, body := fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}}) if resp.StatusCode != http.StatusOK { t.Fatalf("submit ips (recheck): status=%d body=%s", resp.StatusCode, body) } var submitResp submitIPsResponse if err := json.Unmarshal(body, &submitResp); err != nil { t.Fatalf("unmarshal submit response: %v", err) } if len(submitResp.Requeued) != 1 || submitResp.Requeued[0] != "9.9.9.9" { t.Fatalf("expected 9.9.9.9 to be requeued, got %+v", submitResp) } item, err = d.GetIPByAddress(ctx, "9.9.9.9") if err != nil { t.Fatalf("get ip after resubmit: %v", err) } if item.State != db.IPQueued || item.AttemptNumber != 2 || item.OverallResult != "" { t.Fatalf("expected freshly queued with attempt_number=2, got %+v", item) } runOneCycle() item, err = d.GetIPByAddress(ctx, "9.9.9.9") if err != nil { t.Fatalf("get ip after second cycle: %v", err) } if item.State != db.IPDone || item.OverallResult != db.ResultPass || item.AttemptNumber != 2 { t.Fatalf("expected done/pass on second attempt, got %+v", item) } } // TestForceCancelMidCheck proves POST /admin/ips/{ip}/cancel stops an // in-progress check, disassociates its floating IP, and frees the // validator, without waiting for the checking window to elapse. func TestForceCancelMidCheck(t *testing.T) { fc, d, orch, mock := newConfigTestHarness(t) ctx := context.Background() mock.Seed("fip-1", "9.9.9.9", "svc-project") fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ValidatorID: "validator-1", OSPortID: "port-1"}) fc.do(http.MethodPut, "/api/v1/admin/config/targets/web", putTargetGroupRequest{Targets: []string{"https://example.test"}}) fc.do(http.MethodPut, "/api/v1/admin/config/check-types/https", putCheckTypeRequest{Enabled: true, Targets: []string{"web"}}) fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"}) fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}}) orch.Tick(ctx) // claim + associate FIP -> awaiting_self_check item, err := d.GetIPByAddress(ctx, "9.9.9.9") if err != nil { t.Fatalf("get ip: %v", err) } if item.State != db.IPAwaitingSelfCheck { t.Fatalf("expected awaiting_self_check before cancel, got %s", item.State) } if fip, _ := mock.GetFloatingIPByAddress(ctx, "9.9.9.9"); fip.PortID == "" { t.Fatalf("expected fip associated before cancel") } resp, body := fc.do(http.MethodPost, "/api/v1/admin/ips/9.9.9.9/cancel", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("cancel: status=%d body=%s", resp.StatusCode, body) } item, err = d.GetIPByAddress(ctx, "9.9.9.9") if err != nil { t.Fatalf("get ip after cancel: %v", err) } if item.State != db.IPFailed || item.OverallResult != db.ResultCancelled { t.Fatalf("expected failed/cancelled, got state=%s result=%s", item.State, item.OverallResult) } if fip, _ := mock.GetFloatingIPByAddress(ctx, "9.9.9.9"); fip.PortID != "" { t.Fatalf("expected fip disassociated after cancel, still on port %q", fip.PortID) } v, err := d.GetValidator(ctx, "validator-1") if err != nil { t.Fatalf("get validator: %v", err) } if v.State != db.ValidatorIdle || v.CurrentIPID != nil { t.Fatalf("expected validator freed, got state=%s current_ip=%v", v.State, v.CurrentIPID) } // Cancelling again is rejected — nothing left to cancel. resp, body = fc.do(http.MethodPost, "/api/v1/admin/ips/9.9.9.9/cancel", nil) if resp.StatusCode != http.StatusConflict { t.Fatalf("expected 409 cancelling an already-finished ip, status=%d body=%s", resp.StatusCode, body) } // Cancelling an unknown address is a 404. resp, body = fc.do(http.MethodPost, "/api/v1/admin/ips/1.1.1.1/cancel", nil) if resp.StatusCode != http.StatusNotFound { t.Fatalf("expected 404 cancelling unknown ip, status=%d body=%s", resp.StatusCode, body) } } // TestDeleteIPMidCheck proves DELETE /admin/ips/{ip} disassociates the // floating IP and permanently removes the address — unlike cancel, a // subsequent GET on the same address is a 404, not a cancelled record. func TestDeleteIPMidCheck(t *testing.T) { fc, d, orch, mock := newConfigTestHarness(t) ctx := context.Background() mock.Seed("fip-1", "9.9.9.9", "svc-project") fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ValidatorID: "validator-1", OSPortID: "port-1"}) fc.do(http.MethodPut, "/api/v1/admin/config/targets/web", putTargetGroupRequest{Targets: []string{"https://example.test"}}) fc.do(http.MethodPut, "/api/v1/admin/config/check-types/https", putCheckTypeRequest{Enabled: true, Targets: []string{"web"}}) fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"}) fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}}) orch.Tick(ctx) // claim + associate FIP -> awaiting_self_check if fip, _ := mock.GetFloatingIPByAddress(ctx, "9.9.9.9"); fip.PortID == "" { t.Fatalf("expected fip associated before delete") } resp, body := fc.do(http.MethodDelete, "/api/v1/admin/ips/9.9.9.9", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("delete: status=%d body=%s", resp.StatusCode, body) } if fip, _ := mock.GetFloatingIPByAddress(ctx, "9.9.9.9"); fip.PortID != "" { t.Fatalf("expected fip disassociated after delete, still on port %q", fip.PortID) } if _, err := d.GetIPByAddress(ctx, "9.9.9.9"); err == nil { t.Fatalf("expected ip row gone from db after delete") } v, err := d.GetValidator(ctx, "validator-1") if err != nil { t.Fatalf("get validator: %v", err) } if v.State != db.ValidatorIdle || v.CurrentIPID != nil { t.Fatalf("expected validator freed, got state=%s current_ip=%v", v.State, v.CurrentIPID) } resp, body = fc.do(http.MethodGet, "/api/v1/admin/ips/9.9.9.9", nil) if resp.StatusCode != http.StatusNotFound { t.Fatalf("expected 404 for deleted ip, status=%d body=%s", resp.StatusCode, body) } resp, body = fc.do(http.MethodDelete, "/api/v1/admin/ips/9.9.9.9", nil) if resp.StatusCode != http.StatusNotFound { t.Fatalf("expected 404 deleting already-gone ip, status=%d body=%s", resp.StatusCode, body) } } // TestDeleteIPsAndClearQueue exercises POST /admin/ips/delete against a // mixed known/unknown list, then POST /admin/ips/clear against whatever // remains in the queue (including an actively checking address). func TestDeleteIPsAndClearQueue(t *testing.T) { fc, d, orch, mock := newConfigTestHarness(t) ctx := context.Background() mock.Seed("fip-1", "1.1.1.1", "svc-project") fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ValidatorID: "validator-1", OSPortID: "port-1"}) fc.do(http.MethodPut, "/api/v1/admin/config/targets/web", putTargetGroupRequest{Targets: []string{"https://example.test"}}) fc.do(http.MethodPut, "/api/v1/admin/config/check-types/https", putCheckTypeRequest{Enabled: true, Targets: []string{"web"}}) fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"}) fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"1.1.1.1", "2.2.2.2", "3.3.3.3"}}) orch.Tick(ctx) // claims 1.1.1.1 for validator-1, associates its fip // POST /admin/ips/delete with an empty list is a 400. resp, body := fc.do(http.MethodPost, "/api/v1/admin/ips/delete", deleteIPsRequest{}) if resp.StatusCode != http.StatusBadRequest { t.Fatalf("expected 400 for empty delete list, status=%d body=%s", resp.StatusCode, body) } // Delete a mix of a queued address and an unknown one. resp, body = fc.do(http.MethodPost, "/api/v1/admin/ips/delete", deleteIPsRequest{Addresses: []string{"2.2.2.2", "no-such-ip"}}) if resp.StatusCode != http.StatusOK { t.Fatalf("delete ips: status=%d body=%s", resp.StatusCode, body) } var delResp deleteIPsResponse if err := json.Unmarshal(body, &delResp); err != nil { t.Fatalf("unmarshal delete response: %v", err) } if len(delResp.Deleted) != 1 || delResp.Deleted[0] != "2.2.2.2" { t.Fatalf("expected 2.2.2.2 deleted, got %+v", delResp) } if len(delResp.NotFound) != 1 || delResp.NotFound[0] != "no-such-ip" { t.Fatalf("expected no-such-ip in not_found, got %+v", delResp) } // Clear whatever's left — 1.1.1.1 (mid-check, fip attached) and 3.3.3.3 // (still queued). resp, body = fc.do(http.MethodPost, "/api/v1/admin/ips/clear", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("clear queue: status=%d body=%s", resp.StatusCode, body) } var clearResp clearQueueResponse if err := json.Unmarshal(body, &clearResp); err != nil { t.Fatalf("unmarshal clear response: %v", err) } if len(clearResp.Deleted) != 2 { t.Fatalf("expected both remaining addresses deleted, got %+v", clearResp) } if fip, _ := mock.GetFloatingIPByAddress(ctx, "1.1.1.1"); fip.PortID != "" { t.Fatalf("expected fip disassociated on clear, still on port %q", fip.PortID) } ips, err := d.ListIPs(ctx) if err != nil { t.Fatalf("list ips: %v", err) } if len(ips) != 0 { t.Fatalf("expected empty queue after clear, got %+v", ips) } v, err := d.GetValidator(ctx, "validator-1") if err != nil { t.Fatalf("get validator: %v", err) } if v.State != db.ValidatorIdle || v.CurrentIPID != nil { t.Fatalf("expected validator freed after clear, got state=%s current_ip=%v", v.State, v.CurrentIPID) } } // TestOrchestratorSettingsGetPut proves the settle-delay setting round-trips // through GET/PUT /api/v1/admin/config/orchestrator. func TestOrchestratorSettingsGetPut(t *testing.T) { fc, _, _, _ := newConfigTestHarness(t) resp, body := fc.do(http.MethodGet, "/api/v1/admin/config/orchestrator", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("get settings: status=%d body=%s", resp.StatusCode, body) } var got orchestratorSettingsDTO if err := json.Unmarshal(body, &got); err != nil { t.Fatalf("unmarshal get response: %v", err) } if got.FIPSettleSeconds != 0 { t.Fatalf("expected default 0, got %+v", got) } resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", orchestratorSettingsDTO{FIPSettleSeconds: 20}) if resp.StatusCode != http.StatusOK { t.Fatalf("put settings: status=%d body=%s", resp.StatusCode, body) } if err := json.Unmarshal(body, &got); err != nil { t.Fatalf("unmarshal put response: %v", err) } if got.FIPSettleSeconds != 20 { t.Fatalf("expected 20, got %+v", got) } resp, body = fc.do(http.MethodGet, "/api/v1/admin/config/orchestrator", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("get settings after put: status=%d body=%s", resp.StatusCode, body) } if err := json.Unmarshal(body, &got); err != nil { t.Fatalf("unmarshal get-after-put response: %v", err) } if got.FIPSettleSeconds != 20 { t.Fatalf("expected 20 to persist, got %+v", got) } } // TestOrchestratorSettingsPutValidation proves a value that would leave no // room for self-check inside the claim lease is rejected with 400. func TestOrchestratorSettingsPutValidation(t *testing.T) { fc, _, _, _ := newConfigTestHarness(t) // newConfigTestHarness: LeaseTTLSeconds=180, SelfCheckTimeoutSeconds=10. resp, body := fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", orchestratorSettingsDTO{FIPSettleSeconds: 175}) if resp.StatusCode != http.StatusBadRequest { t.Fatalf("expected 400 for settle seconds too close to lease ttl, status=%d body=%s", resp.StatusCode, body) } resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", orchestratorSettingsDTO{FIPSettleSeconds: -1}) if resp.StatusCode != http.StatusBadRequest { t.Fatalf("expected 400 for negative settle seconds, status=%d body=%s", resp.StatusCode, body) } } // TestFIPSettleDelayGatesAssignmentEndpoint proves GET // /api/v1/agents/{id}/assignment returns 204 while the settle window is // open and 200 once it has elapsed — the end-to-end proof of the whole // feature over the real HTTP wire contract. func TestFIPSettleDelayGatesAssignmentEndpoint(t *testing.T) { fc, _, orch, mock := newConfigTestHarness(t) ctx := context.Background() mock.Seed("fip-1", "9.9.9.9", "svc-project") resp, body := fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", orchestratorSettingsDTO{FIPSettleSeconds: 1}) if resp.StatusCode != http.StatusOK { t.Fatalf("put settings: status=%d body=%s", resp.StatusCode, body) } fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ValidatorID: "validator-1", OSPortID: "port-1"}) fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"}) fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}}) orch.Tick(ctx) // claim + associate -> awaiting_self_check, fip attached resp, body = fc.do(http.MethodGet, "/api/v1/agents/validator-1/assignment", nil) if resp.StatusCode != http.StatusNoContent { t.Fatalf("expected 204 during settle window, status=%d body=%s", resp.StatusCode, body) } time.Sleep(1100 * time.Millisecond) resp, body = fc.do(http.MethodGet, "/api/v1/agents/validator-1/assignment", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("expected 200 after settle window elapsed, status=%d body=%s", resp.StatusCode, body) } } // TestInboundChecksGetPut proves the prober check config round-trips // through GET/PUT /api/v1/admin/config/inbound-checks. func TestInboundChecksGetPut(t *testing.T) { fc, _, _, _ := newConfigTestHarness(t) resp, body := fc.do(http.MethodGet, "/api/v1/admin/config/inbound-checks", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("get inbound checks: status=%d body=%s", resp.StatusCode, body) } var got inboundChecksDTO if err := json.Unmarshal(body, &got); err != nil { t.Fatalf("unmarshal get response: %v", err) } // newConfigTestHarness seeds Ports:[22,80], ICMP:true. if len(got.Ports) != 2 || !got.ICMP { t.Fatalf("expected seeded {[22 80] true}, got %+v", got) } resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/inbound-checks", inboundChecksDTO{Ports: []int{443, 8080}, ICMP: false}) if resp.StatusCode != http.StatusOK { t.Fatalf("put inbound checks: status=%d body=%s", resp.StatusCode, body) } if err := json.Unmarshal(body, &got); err != nil { t.Fatalf("unmarshal put response: %v", err) } if len(got.Ports) != 2 || got.ICMP { t.Fatalf("expected {[443 8080] false}, got %+v", got) } resp, body = fc.do(http.MethodGet, "/api/v1/admin/config/inbound-checks", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("get inbound checks after put: status=%d body=%s", resp.StatusCode, body) } if err := json.Unmarshal(body, &got); err != nil { t.Fatalf("unmarshal get-after-put response: %v", err) } if got.Ports[0] != 443 || got.Ports[1] != 8080 || got.ICMP { t.Fatalf("expected {[443 8080] false} to persist, got %+v", got) } } // TestInboundChecksPutValidation proves out-of-range and duplicate ports // are rejected with 400. func TestInboundChecksPutValidation(t *testing.T) { fc, _, _, _ := newConfigTestHarness(t) resp, body := fc.do(http.MethodPut, "/api/v1/admin/config/inbound-checks", inboundChecksDTO{Ports: []int{0}, ICMP: false}) if resp.StatusCode != http.StatusBadRequest { t.Fatalf("expected 400 for port 0, status=%d body=%s", resp.StatusCode, body) } resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/inbound-checks", inboundChecksDTO{Ports: []int{70000}, ICMP: false}) if resp.StatusCode != http.StatusBadRequest { t.Fatalf("expected 400 for port 70000, status=%d body=%s", resp.StatusCode, body) } resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/inbound-checks", inboundChecksDTO{Ports: []int{22, 22}, ICMP: false}) if resp.StatusCode != http.StatusBadRequest { t.Fatalf("expected 400 for duplicate port, status=%d body=%s", resp.StatusCode, body) } } // TestInboundChecksReflectedInProberAssignmentsWithoutRestart proves the // fix to the formerly-static handlers_prober.go read: a PUT to // /api/v1/admin/config/inbound-checks changes what GET // /api/v1/probers/{site_id}/assignments hands back to an already-registered // prober, for an IP already in `checking`, with no control-api restart. func TestInboundChecksReflectedInProberAssignmentsWithoutRestart(t *testing.T) { fc, _, orch, mock := newConfigTestHarness(t) ctx := context.Background() mock.Seed("fip-1", "9.9.9.9", "svc-project") fc.do(http.MethodPut, "/api/v1/admin/config/sites/1", putSiteRequest{SiteID: "site-1"}) fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ValidatorID: "validator-1", OSPortID: "port-1"}) fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"}) fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}}) orch.Tick(ctx) // claim + associate -> awaiting_self_check resp, body := fc.do(http.MethodGet, "/api/v1/agents/validator-1/assignment", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("assignment: status=%d body=%s", resp.StatusCode, body) } var assignment assignmentResponse if err := json.Unmarshal(body, &assignment); err != nil { t.Fatalf("unmarshal assignment: %v", err) } fc.do(http.MethodPost, "/api/v1/agents/validator-1/self-check", selfCheckRequest{ IPID: assignment.IPID, DetectedEgress: "9.9.9.9", Success: true, Detail: "matched", }) // Now item.State == "checking" — a prober assignment target. fc.do(http.MethodPost, "/api/v1/probers/register", registerProberRequest{SiteID: "site-1"}) resp, body = fc.do(http.MethodGet, "/api/v1/probers/site-1/assignments", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("prober assignments: status=%d body=%s", resp.StatusCode, body) } var assignments []proberAssignment if err := json.Unmarshal(body, &assignments); err != nil { t.Fatalf("unmarshal assignments: %v", err) } if len(assignments) != 1 || len(assignments[0].Ports) != 2 || !assignments[0].ICMP { t.Fatalf("expected seeded {[22 80] true} before PUT, got %+v", assignments) } resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/inbound-checks", inboundChecksDTO{Ports: []int{8080}, ICMP: false}) if resp.StatusCode != http.StatusOK { t.Fatalf("put inbound checks: status=%d body=%s", resp.StatusCode, body) } resp, body = fc.do(http.MethodGet, "/api/v1/probers/site-1/assignments", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("prober assignments after put: status=%d body=%s", resp.StatusCode, body) } if err := json.Unmarshal(body, &assignments); err != nil { t.Fatalf("unmarshal assignments after put: %v", err) } if len(assignments) != 1 || len(assignments[0].Ports) != 1 || assignments[0].Ports[0] != 8080 || assignments[0].ICMP { t.Fatalf("expected updated {[8080] false} without restart, got %+v", assignments) } } // TestProberRegisterSetsHostnameAndIdleState proves POST // /api/v1/probers/register persists the calling prober's hostname and // flips the site's state to idle, visible via GET // /api/v1/admin/config/sites — the prober-availability analog of // validator registration. func TestProberRegisterSetsHostnameAndIdleState(t *testing.T) { fc, _, _, _ := newConfigTestHarness(t) fc.do(http.MethodPut, "/api/v1/admin/config/sites/1", putSiteRequest{SiteID: "site-1"}) resp, body := fc.do(http.MethodGet, "/api/v1/admin/config/sites", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("get sites: status=%d body=%s", resp.StatusCode, body) } var sites []siteDTO if err := json.Unmarshal(body, &sites); err != nil { t.Fatalf("unmarshal sites: %v", err) } if len(sites) != 1 || sites[0].State != "unregistered" { t.Fatalf("expected freshly-created slot unregistered, got %+v", sites) } resp, body = fc.do(http.MethodPost, "/api/v1/probers/register", registerProberRequest{SiteID: "site-1", Hostname: "probe-host-1"}) if resp.StatusCode != http.StatusOK { t.Fatalf("register prober: status=%d body=%s", resp.StatusCode, body) } resp, body = fc.do(http.MethodGet, "/api/v1/admin/config/sites", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("get sites after register: status=%d body=%s", resp.StatusCode, body) } if err := json.Unmarshal(body, &sites); err != nil { t.Fatalf("unmarshal sites: %v", err) } if sites[0].State != "idle" || sites[0].Hostname != "probe-host-1" { t.Fatalf("expected {state:idle, hostname:probe-host-1}, got %+v", sites[0]) } if sites[0].LastHeartbeatAt == nil { t.Fatalf("expected last_heartbeat_at stamped") } } // TestProberHeartbeat404UnknownSite proves the heartbeat endpoint rejects // an unconfigured site_id, mirroring the validator heartbeat's 404. func TestProberHeartbeat404UnknownSite(t *testing.T) { fc, _, _, _ := newConfigTestHarness(t) resp, body := fc.do(http.MethodPost, "/api/v1/probers/unknown-site/heartbeat", nil) if resp.StatusCode != http.StatusNotFound { t.Fatalf("expected 404 for unknown site_id, status=%d body=%s", resp.StatusCode, body) } } // TestSweepStaleSiteHeartbeatsMarksUnreachable is the end-to-end proof that // a prober that stops heartbeating gets marked unreachable by the sweep, // visible via the admin API — the prober-side analog of // TestFIPSettleDelayGatesAssignmentEndpoint's wire-level style. func TestSweepStaleSiteHeartbeatsMarksUnreachable(t *testing.T) { fc, d, orch, _ := newConfigTestHarness(t) ctx := context.Background() fc.do(http.MethodPut, "/api/v1/admin/config/sites/1", putSiteRequest{SiteID: "site-1"}) resp, body := fc.do(http.MethodPost, "/api/v1/probers/register", registerProberRequest{SiteID: "site-1", Hostname: "probe-host-1"}) if resp.StatusCode != http.StatusOK { t.Fatalf("register prober: status=%d body=%s", resp.StatusCode, body) } // Backdate last_heartbeat_at past HeartbeatTimeoutSeconds (30s, per // newConfigTestHarness) directly in the DB, rather than sleeping 30+ // real seconds in the test. old := db.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano) if _, err := d.ExecContext(ctx, `UPDATE sites SET last_heartbeat_at=? WHERE site_id=?`, old, "site-1"); err != nil { t.Fatalf("backdate heartbeat: %v", err) } if err := orch.SweepStaleSiteHeartbeats(ctx); err != nil { t.Fatalf("sweep stale site heartbeats: %v", err) } resp, body = fc.do(http.MethodGet, "/api/v1/admin/config/sites", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("get sites: status=%d body=%s", resp.StatusCode, body) } var sites []siteDTO if err := json.Unmarshal(body, &sites); err != nil { t.Fatalf("unmarshal sites: %v", err) } if len(sites) != 1 || sites[0].State != "unreachable" { t.Fatalf("expected site-1 marked unreachable after sweep, got %+v", sites) } // A fresh heartbeat brings it back to idle. resp, body = fc.do(http.MethodPost, "/api/v1/probers/site-1/heartbeat", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("heartbeat: status=%d body=%s", resp.StatusCode, body) } resp, body = fc.do(http.MethodGet, "/api/v1/admin/config/sites", nil) if resp.StatusCode != http.StatusOK { t.Fatalf("get sites after heartbeat: status=%d body=%s", resp.StatusCode, body) } if err := json.Unmarshal(body, &sites); err != nil { t.Fatalf("unmarshal sites: %v", err) } if sites[0].State != "idle" { t.Fatalf("expected site-1 back to idle after heartbeat, got %+v", sites) } }