Retry a failed self-check on another validator; add the self-check failure ceiling

A validator that failed the self-check of an address no longer gets that address
again in the current round (ClaimNextQueued skips it); the validator itself stays
in service and takes all other addresses. The verdict fail is set when the number
of failed self-checks of an address reaches settings.self_check_max_attempts
(1..50, default 5, independent of the number of validators); max_retries and
retry_count are no longer used for self-check. If every working validator has
already failed the address, a new round starts and the exclusions lapse.

Migration 0012: ip_self_check_failures (permanent history per registry address),
ip_queue.sc_failures and sc_round_start_cycle (cycle_id is used instead of
attempt_number, which restarts when a queue row is recreated), the setting.
db.FailSelfCheck does it in one transaction; re-submission starts a new series.
API: self_check_max_attempts in GET/PUT /admin/config/orchestrator,
self_check_failed_on in /admin/ips/{ip} and /admin/registry/{ip}. Dashboard: the
field on /settings and the line "Self-check не прошёл на: ..." on the address
pages. Docs, plan and summary in docs/changes/.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ayurishchevandClaude Sonnet 5.5 committed 2026-10-04 09:50:12 +03:00
1 parent b7669c9e41
commit e95b5eb7d5
34 files changed
+1092 -82

No files matched your search

@@ -219,3 +219,51 @@ func TestAdminRegistryLevels(t *testing.T) {
t.Errorf("empty level must serialise by_type as [], got %s", body)
}
}
// TestSelfCheckFailedOnInAddressEndpoints proves GET /admin/ips/{ip} and GET
// /admin/registry/{ip} list the validators whose self-check of the address
// failed ([] when none).
func TestSelfCheckFailedOnInAddressEndpoints(t *testing.T) {
fc, d, orch, mock := newConfigTestHarness(t)
ctx := context.Background()
mock.Seed("fip-1", "9.9.9.9", "svc-project")
fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ValidatorID: "validator-1", OSPortID: "port-1"})
fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"})
fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}})
failedOn := func(path string) []string {
t.Helper()
resp, body := fc.do(http.MethodGet, path, nil)
if resp.StatusCode != http.StatusOK {
t.Fatalf("get %s: status=%d body=%s", path, resp.StatusCode, body)
}
var got struct {
SelfCheckFailedOn []string `json:"self_check_failed_on"`
}
if err := json.Unmarshal(body, &got); err != nil {
t.Fatalf("unmarshal %s: %v", path, err)
}
if got.SelfCheckFailedOn == nil {
t.Fatalf("%s: self_check_failed_on must be [] rather than null, body=%s", path, body)
}
return got.SelfCheckFailedOn
}
if got := failedOn("/api/v1/admin/ips/9.9.9.9"); len(got) != 0 {
t.Fatalf("expected no failures yet, got %v", got)
}
orch.Tick(ctx) // claim + associate -> awaiting_self_check
ip, err := d.GetIPByAddress(ctx, "9.9.9.9")
if err != nil {
t.Fatal(err)
}
if err := orch.SelfCheckResult(ctx, "validator-1", ip.ID, false, "ip echo timeout"); err != nil {
t.Fatalf("self-check result: %v", err)
}
for _, path := range []string{"/api/v1/admin/ips/9.9.9.9", "/api/v1/admin/registry/9.9.9.9"} {
if got := failedOn(path); !reflect.DeepEqual(got, []string{"validator-1"}) {
t.Fatalf("%s: expected [validator-1], got %v", path, got)
}
}
}