A validator that failed the self-check of an address no longer gets that address
again in the current round (ClaimNextQueued skips it); the validator itself stays
in service and takes all other addresses. The verdict fail is set when the number
of failed self-checks of an address reaches settings.self_check_max_attempts
(1..50, default 5, independent of the number of validators); max_retries and
retry_count are no longer used for self-check. If every working validator has
already failed the address, a new round starts and the exclusions lapse.
Migration 0012: ip_self_check_failures (permanent history per registry address),
ip_queue.sc_failures and sc_round_start_cycle (cycle_id is used instead of
attempt_number, which restarts when a queue row is recreated), the setting.
db.FailSelfCheck does it in one transaction; re-submission starts a new series.
API: self_check_max_attempts in GET/PUT /admin/config/orchestrator,
self_check_failed_on in /admin/ips/{ip} and /admin/registry/{ip}. Dashboard: the
field on /settings and the line "Self-check не прошёл на: ..." on the address
pages. Docs, plan and summary in docs/changes/.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
270 lines
10 KiB
Go
270 lines
10 KiB
Go
package httpapi
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
|
|
"cloudipvalidator/internal/db"
|
|
)
|
|
|
|
// TestScanFloatingIPsEndpoint proves POST /api/v1/admin/ips/scan?wait=true
|
|
// (the synchronous form) only queues floating IPs that are currently
|
|
// unassociated in OpenStack and answers with the classic counters body.
|
|
func TestScanFloatingIPsEndpoint(t *testing.T) {
|
|
fc, _, _, mock := newConfigTestHarness(t)
|
|
mock.Seed("fip-free", "5.5.5.5", "svc-project")
|
|
mock.SeedWithPort("fip-occupied", "6.6.6.6", "svc-project", "some-port")
|
|
|
|
resp, body := fc.do(http.MethodPost, "/api/v1/admin/ips/scan?wait=true", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("scan: status=%d body=%s", resp.StatusCode, body)
|
|
}
|
|
var scanResp scanIPsResponse
|
|
if err := json.Unmarshal(body, &scanResp); err != nil {
|
|
t.Fatalf("unmarshal scan response: %v", err)
|
|
}
|
|
if scanResp.ScannedFree != 1 {
|
|
t.Fatalf("expected 1 free fip scanned, got %+v", scanResp)
|
|
}
|
|
if len(scanResp.Added) != 1 || scanResp.Added[0] != "5.5.5.5" {
|
|
t.Fatalf("expected only 5.5.5.5 added, got %+v", scanResp)
|
|
}
|
|
|
|
_, body = fc.do(http.MethodGet, "/api/v1/admin/ips", nil)
|
|
var ips []db.IPQueueItem
|
|
if err := json.Unmarshal(body, &ips); err != nil {
|
|
t.Fatalf("unmarshal ips: %v", err)
|
|
}
|
|
if len(ips) != 1 || ips[0].IPAddress != "5.5.5.5" {
|
|
t.Fatalf("expected only the free address queued, got %+v", ips)
|
|
}
|
|
}
|
|
|
|
// TestRegistryHistoryOutlivesIPDeletion proves that after an address
|
|
// completes a full check cycle and is then deleted from the queue, its
|
|
// history is still reachable via the registry endpoints (though the plain
|
|
// /ips/{ip} endpoint now 404s), and that resubmitting the same address adds
|
|
// a second, distinct cycle to the same registry entry.
|
|
func TestRegistryHistoryOutlivesIPDeletion(t *testing.T) {
|
|
fc, d, orch, mock := newConfigTestHarness(t)
|
|
ctx := context.Background()
|
|
mock.Seed("fip-1", "9.9.9.9", "svc-project")
|
|
|
|
fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ValidatorID: "validator-1", OSPortID: "port-1"})
|
|
fc.do(http.MethodPut, "/api/v1/admin/config/targets/web", putTargetGroupRequest{Targets: []string{"https://example.test"}})
|
|
fc.do(http.MethodPut, "/api/v1/admin/config/check-types/https", putCheckTypeRequest{Enabled: true, Targets: []string{"web"}})
|
|
fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"})
|
|
|
|
runOneCycle := func() {
|
|
orch.Tick(ctx)
|
|
_, body := fc.do(http.MethodGet, "/api/v1/agents/validator-1/assignment", nil)
|
|
var assignment assignmentResponse
|
|
if err := json.Unmarshal(body, &assignment); err != nil {
|
|
t.Fatalf("unmarshal assignment: %v", err)
|
|
}
|
|
fc.do(http.MethodPost, "/api/v1/agents/validator-1/self-check", selfCheckRequest{
|
|
IPID: assignment.IPID, DetectedEgress: "9.9.9.9", Success: true,
|
|
})
|
|
fc.do(http.MethodPost, "/api/v1/agents/validator-1/results", agentResultsRequest{
|
|
Results: []checkResultDTO{{
|
|
IPID: assignment.IPID, CheckType: "https", Target: "https://example.test",
|
|
Success: true, CheckedAt: time.Now().Format(time.RFC3339Nano),
|
|
}},
|
|
})
|
|
fc.do(http.MethodPost, "/api/v1/agents/validator-1/complete", agentCompleteRequest{IPID: assignment.IPID})
|
|
orch.Tick(ctx)
|
|
}
|
|
|
|
fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}})
|
|
runOneCycle()
|
|
|
|
item, err := d.GetIPByAddress(ctx, "9.9.9.9")
|
|
if err != nil {
|
|
t.Fatalf("get ip: %v", err)
|
|
}
|
|
if item.State != db.IPDone {
|
|
t.Fatalf("expected done after first cycle, got %s", item.State)
|
|
}
|
|
|
|
resp, _ := fc.do(http.MethodDelete, "/api/v1/admin/ips/9.9.9.9", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("delete ip: status=%d", resp.StatusCode)
|
|
}
|
|
|
|
resp, _ = fc.do(http.MethodGet, "/api/v1/admin/ips/9.9.9.9", nil)
|
|
if resp.StatusCode != http.StatusNotFound {
|
|
t.Fatalf("expected 404 for deleted address on /ips, got %d", resp.StatusCode)
|
|
}
|
|
|
|
resp, body := fc.do(http.MethodGet, "/api/v1/admin/registry/9.9.9.9", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("registry history: status=%d body=%s", resp.StatusCode, body)
|
|
}
|
|
var hist struct {
|
|
Registry registryDTO `json:"registry"`
|
|
Checks []db.Check `json:"checks"`
|
|
}
|
|
if err := json.Unmarshal(body, &hist); err != nil {
|
|
t.Fatalf("unmarshal registry history: %v", err)
|
|
}
|
|
if hist.Registry.TotalCycles != 1 {
|
|
t.Fatalf("expected 1 retained cycle, got %+v", hist.Registry)
|
|
}
|
|
if len(hist.Checks) == 0 {
|
|
t.Fatalf("expected retained check history, got none")
|
|
}
|
|
if hist.Registry.InQueue {
|
|
t.Fatalf("expected registry entry to report not-in-queue after delete, got %+v", hist.Registry)
|
|
}
|
|
|
|
// Resubmitting starts a second, distinct cycle on the same registry
|
|
// entry rather than colliding with the first.
|
|
mock.Seed("fip-1", "9.9.9.9", "svc-project")
|
|
fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}})
|
|
runOneCycle()
|
|
|
|
resp, body = fc.do(http.MethodGet, "/api/v1/admin/registry/9.9.9.9", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("registry history (2nd): status=%d body=%s", resp.StatusCode, body)
|
|
}
|
|
if err := json.Unmarshal(body, &hist); err != nil {
|
|
t.Fatalf("unmarshal registry history (2nd): %v", err)
|
|
}
|
|
if hist.Registry.TotalCycles != 2 {
|
|
t.Fatalf("expected 2 retained cycles after resubmission, got %+v", hist.Registry)
|
|
}
|
|
if !hist.Registry.InQueue {
|
|
t.Fatalf("expected registry entry to report in-queue again, got %+v", hist.Registry)
|
|
}
|
|
|
|
_, body = fc.do(http.MethodGet, "/api/v1/admin/registry", nil)
|
|
var all []registryDTO
|
|
if err := json.Unmarshal(body, &all); err != nil {
|
|
t.Fatalf("unmarshal registry list: %v", err)
|
|
}
|
|
if len(all) != 1 || all[0].IPAddress != "9.9.9.9" {
|
|
t.Fatalf("expected a single registry entry for the address, got %+v", all)
|
|
}
|
|
}
|
|
|
|
// TestAdminRegistryLevels proves the registry endpoints report the last
|
|
// cycle's recorded checks as "ok of total" per level (egress / ingress) and
|
|
// per check family, and that by_type is an empty list, not null, when a
|
|
// level has no checks.
|
|
func TestAdminRegistryLevels(t *testing.T) {
|
|
fc, d, _, _ := newConfigTestHarness(t)
|
|
ctx := context.Background()
|
|
if _, err := d.SubmitIPs(ctx, []string{"7.7.7.7", "8.8.8.8"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ip, err := d.GetIPByAddress(ctx, "7.7.7.7")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, c := range []struct {
|
|
source, typ, target string
|
|
ok bool
|
|
}{
|
|
{db.SourceEgress, "https", "https://a.test", true},
|
|
{db.SourceEgress, "https", "https://b.test", false},
|
|
{db.SourceEgress, "icmp", "a.test", true},
|
|
{db.InboundSource(1), "tcp-22", "7.7.7.7", true},
|
|
{db.InboundSource(1), "tcp-443", "7.7.7.7", true},
|
|
{db.InboundSource(2), "tcp-22", "7.7.7.7", false},
|
|
} {
|
|
if err := d.UpsertCheck(ctx, db.Check{
|
|
IPID: ip.ID, IPAddress: "7.7.7.7", AttemptNumber: ip.AttemptNumber,
|
|
Source: c.source, CheckType: c.typ, Target: c.target, Success: c.ok, CheckedAt: db.Now(),
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
resp, body := fc.do(http.MethodGet, "/api/v1/admin/registry/7.7.7.7", nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("detail: %d %s", resp.StatusCode, body)
|
|
}
|
|
var detail struct {
|
|
Registry registryDTO `json:"registry"`
|
|
}
|
|
if err := json.Unmarshal(body, &detail); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r := detail.Registry
|
|
if r.LastCycleID != 1 {
|
|
t.Errorf("last_cycle_id = %d", r.LastCycleID)
|
|
}
|
|
wantEgress := levelResultDTO{Total: 3, OK: 2, ByType: []typeStatDTO{{"https", 2, 1}, {"icmp", 1, 1}}}
|
|
wantIngress := levelResultDTO{Total: 3, OK: 2, ByType: []typeStatDTO{{"tcp", 3, 2}}}
|
|
if !reflect.DeepEqual(r.Egress, wantEgress) || !reflect.DeepEqual(r.Ingress, wantIngress) {
|
|
t.Errorf("egress=%+v ingress=%+v", r.Egress, r.Ingress)
|
|
}
|
|
|
|
// The paginated list carries the same fields; an address without checks
|
|
// serialises empty levels as {"total":0,"ok":0,"by_type":[]}.
|
|
_, body = fc.do(http.MethodGet, "/api/v1/admin/registry?limit=10", nil)
|
|
var page registryPageResponse
|
|
if err := json.Unmarshal(body, &page); err != nil || len(page.Items) != 2 {
|
|
t.Fatalf("list: %v %s", err, body)
|
|
}
|
|
if !reflect.DeepEqual(page.Items[0].Egress, wantEgress) {
|
|
t.Errorf("list egress = %+v", page.Items[0].Egress)
|
|
}
|
|
if !bytes.Contains(body, []byte(`"egress":{"total":0,"ok":0,"by_type":[]}`)) {
|
|
t.Errorf("empty level must serialise by_type as [], got %s", body)
|
|
}
|
|
}
|
|
|
|
// TestSelfCheckFailedOnInAddressEndpoints proves GET /admin/ips/{ip} and GET
|
|
// /admin/registry/{ip} list the validators whose self-check of the address
|
|
// failed ([] when none).
|
|
func TestSelfCheckFailedOnInAddressEndpoints(t *testing.T) {
|
|
fc, d, orch, mock := newConfigTestHarness(t)
|
|
ctx := context.Background()
|
|
mock.Seed("fip-1", "9.9.9.9", "svc-project")
|
|
fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ValidatorID: "validator-1", OSPortID: "port-1"})
|
|
fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"})
|
|
fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}})
|
|
|
|
failedOn := func(path string) []string {
|
|
t.Helper()
|
|
resp, body := fc.do(http.MethodGet, path, nil)
|
|
if resp.StatusCode != http.StatusOK {
|
|
t.Fatalf("get %s: status=%d body=%s", path, resp.StatusCode, body)
|
|
}
|
|
var got struct {
|
|
SelfCheckFailedOn []string `json:"self_check_failed_on"`
|
|
}
|
|
if err := json.Unmarshal(body, &got); err != nil {
|
|
t.Fatalf("unmarshal %s: %v", path, err)
|
|
}
|
|
if got.SelfCheckFailedOn == nil {
|
|
t.Fatalf("%s: self_check_failed_on must be [] rather than null, body=%s", path, body)
|
|
}
|
|
return got.SelfCheckFailedOn
|
|
}
|
|
if got := failedOn("/api/v1/admin/ips/9.9.9.9"); len(got) != 0 {
|
|
t.Fatalf("expected no failures yet, got %v", got)
|
|
}
|
|
|
|
orch.Tick(ctx) // claim + associate -> awaiting_self_check
|
|
ip, err := d.GetIPByAddress(ctx, "9.9.9.9")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := orch.SelfCheckResult(ctx, "validator-1", ip.ID, false, "ip echo timeout"); err != nil {
|
|
t.Fatalf("self-check result: %v", err)
|
|
}
|
|
|
|
for _, path := range []string{"/api/v1/admin/ips/9.9.9.9", "/api/v1/admin/registry/9.9.9.9"} {
|
|
if got := failedOn(path); !reflect.DeepEqual(got, []string{"validator-1"}) {
|
|
t.Fatalf("%s: expected [validator-1], got %v", path, got)
|
|
}
|
|
}
|
|
}
|