Retry a failed self-check on another validator; add the self-check failure ceiling
A validator that failed the self-check of an address no longer gets that address
again in the current round (ClaimNextQueued skips it); the validator itself stays
in service and takes all other addresses. The verdict fail is set when the number
of failed self-checks of an address reaches settings.self_check_max_attempts
(1..50, default 5, independent of the number of validators); max_retries and
retry_count are no longer used for self-check. If every working validator has
already failed the address, a new round starts and the exclusions lapse.
Migration 0012: ip_self_check_failures (permanent history per registry address),
ip_queue.sc_failures and sc_round_start_cycle (cycle_id is used instead of
attempt_number, which restarts when a queue row is recreated), the setting.
db.FailSelfCheck does it in one transaction; re-submission starts a new series.
API: self_check_max_attempts in GET/PUT /admin/config/orchestrator,
self_check_failed_on in /admin/ips/{ip} and /admin/registry/{ip}. Dashboard: the
field on /settings and the line "Self-check не прошёл на: ..." on the address
pages. Docs, plan and summary in docs/changes/.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b7669c9e41
commit
e95b5eb7d5
34 files changed
+1092
-82
No files matched your search
@@ -426,19 +426,32 @@ func TestOrchestratorSettingsGetPut(t *testing.T) {
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatalf("unmarshal get response: %v", err)
|
||||
}
|
||||
if got.FIPSettleSeconds != 0 {
|
||||
t.Fatalf("expected default 0, got %+v", got)
|
||||
if got.FIPSettleSeconds != 0 || got.SelfCheckMaxAttempts != 5 {
|
||||
t.Fatalf("expected defaults 0 and 5, got %+v", got)
|
||||
}
|
||||
|
||||
resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", orchestratorSettingsDTO{FIPSettleSeconds: 20})
|
||||
resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", putOrchestratorSettingsRequest{FIPSettleSeconds: 20})
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("put settings: status=%d body=%s", resp.StatusCode, body)
|
||||
}
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatalf("unmarshal put response: %v", err)
|
||||
}
|
||||
if got.FIPSettleSeconds != 20 {
|
||||
t.Fatalf("expected 20, got %+v", got)
|
||||
if got.FIPSettleSeconds != 20 || got.SelfCheckMaxAttempts != 5 {
|
||||
t.Fatalf("expected 20 with the omitted ceiling kept at 5, got %+v", got)
|
||||
}
|
||||
|
||||
eight := 8
|
||||
resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", putOrchestratorSettingsRequest{FIPSettleSeconds: 20, SelfCheckMaxAttempts: &eight})
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("put settings with ceiling: status=%d body=%s", resp.StatusCode, body)
|
||||
}
|
||||
for _, bad := range []int{0, 51} {
|
||||
bad := bad
|
||||
resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", putOrchestratorSettingsRequest{FIPSettleSeconds: 20, SelfCheckMaxAttempts: &bad})
|
||||
if resp.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("expected 400 for self_check_max_attempts=%d, status=%d body=%s", bad, resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
resp, body = fc.do(http.MethodGet, "/api/v1/admin/config/orchestrator", nil)
|
||||
@@ -448,8 +461,8 @@ func TestOrchestratorSettingsGetPut(t *testing.T) {
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatalf("unmarshal get-after-put response: %v", err)
|
||||
}
|
||||
if got.FIPSettleSeconds != 20 {
|
||||
t.Fatalf("expected 20 to persist, got %+v", got)
|
||||
if got.FIPSettleSeconds != 20 || got.SelfCheckMaxAttempts != 8 {
|
||||
t.Fatalf("expected 20 and 8 to persist, got %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -459,12 +472,12 @@ func TestOrchestratorSettingsPutValidation(t *testing.T) {
|
||||
fc, _, _, _ := newConfigTestHarness(t)
|
||||
// newConfigTestHarness: LeaseTTLSeconds=180, SelfCheckTimeoutSeconds=10.
|
||||
|
||||
resp, body := fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", orchestratorSettingsDTO{FIPSettleSeconds: 175})
|
||||
resp, body := fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", putOrchestratorSettingsRequest{FIPSettleSeconds: 175})
|
||||
if resp.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("expected 400 for settle seconds too close to lease ttl, status=%d body=%s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", orchestratorSettingsDTO{FIPSettleSeconds: -1})
|
||||
resp, body = fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", putOrchestratorSettingsRequest{FIPSettleSeconds: -1})
|
||||
if resp.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("expected 400 for negative settle seconds, status=%d body=%s", resp.StatusCode, body)
|
||||
}
|
||||
@@ -479,7 +492,7 @@ func TestFIPSettleDelayGatesAssignmentEndpoint(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mock.Seed("fip-1", "9.9.9.9", "svc-project")
|
||||
|
||||
resp, body := fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", orchestratorSettingsDTO{FIPSettleSeconds: 1})
|
||||
resp, body := fc.do(http.MethodPut, "/api/v1/admin/config/orchestrator", putOrchestratorSettingsRequest{FIPSettleSeconds: 1})
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("put settings: status=%d body=%s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user