Retry a failed self-check on another validator; add the self-check failure ceiling

A validator that failed the self-check of an address no longer gets that address
again in the current round (ClaimNextQueued skips it); the validator itself stays
in service and takes all other addresses. The verdict fail is set when the number
of failed self-checks of an address reaches settings.self_check_max_attempts
(1..50, default 5, independent of the number of validators); max_retries and
retry_count are no longer used for self-check. If every working validator has
already failed the address, a new round starts and the exclusions lapse.

Migration 0012: ip_self_check_failures (permanent history per registry address),
ip_queue.sc_failures and sc_round_start_cycle (cycle_id is used instead of
attempt_number, which restarts when a queue row is recreated), the setting.
db.FailSelfCheck does it in one transaction; re-submission starts a new series.
API: self_check_max_attempts in GET/PUT /admin/config/orchestrator,
self_check_failed_on in /admin/ips/{ip} and /admin/registry/{ip}. Dashboard: the
field on /settings and the line "Self-check не прошёл на: ..." on the address
pages. Docs, plan and summary in docs/changes/.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

internal/orchestrator/orchestrator_test.go

@@ -1162,3 +1162,169 @@ func TestLateFailedSelfCheckIsIgnored(t *testing.T) {
 		t.Fatalf("a late report changed the address state to %s", cur.State)
 	}
 }
+
+// reportSelfChecks answers the self-check of every address that is waiting for
+// one: a validator in failing reports a failure, any other a success.
+func reportSelfChecks(t *testing.T, ctx context.Context, o *Orchestrator, d *db.DB, failing ...string) {
+	t.Helper()
+	fails := map[string]bool{}
+	for _, v := range failing {
+		fails[v] = true
+	}
+	validators, err := d.ListValidators(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, v := range validators {
+		if v.CurrentIPID == nil {
+			continue
+		}
+		ip, err := d.GetIP(ctx, *v.CurrentIPID)
+		if err != nil || ip.State != db.IPAwaitingSelfCheck {
+			continue
+		}
+		if err := o.SelfCheckResult(ctx, v.ValidatorID, ip.ID, !fails[v.ValidatorID], "ip echo timeout"); err != nil {
+			t.Fatalf("self-check result of %s: %v", v.ValidatorID, err)
+		}
+	}
+}
+
+func registerValidators(t *testing.T, ctx context.Context, d *db.DB, ids ...string) {
+	t.Helper()
+	for i, id := range ids {
+		if err := d.RegisterValidator(ctx, id, "h", fmt.Sprintf("port-%d", i+1), "v"); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func eventTypes(t *testing.T, ctx context.Context, d *db.DB, ipID int64) map[string]int {
+	t.Helper()
+	events, err := d.ListEventsForIP(ctx, ipID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	out := map[string]int{}
+	for _, e := range events {
+		out[e.EventType]++
+	}
+	return out
+}
+
+// The incident: v1 fails the self-check of 1.1.1.1. The address must not go
+// back to v1; another validator takes it and passes, while v1 keeps taking
+// other addresses.
+func TestSelfCheckFailureExcludesValidatorForThatAddress(t *testing.T) {
+	ctx := context.Background()
+	o, d, mock := newTestOrchestrator(t, 180)
+	for i, a := range []string{"1.1.1.1", "2.2.2.2", "3.3.3.3"} {
+		mock.Seed(fmt.Sprintf("fip-%d", i+1), a, "svc-project")
+	}
+	registerValidators(t, ctx, d, "v1", "v2", "v3")
+	if err := d.SeedQueue(ctx, []string{"1.1.1.1", "2.2.2.2"}); err != nil {
+		t.Fatal(err)
+	}
+
+	o.Tick(ctx) // v1 -> 1.1.1.1, v2 -> 2.2.2.2, v3 stays idle
+	reportSelfChecks(t, ctx, o, d, "v1")
+	a1, _ := d.GetIPByAddress(ctx, "1.1.1.1")
+	if a1.State != db.IPQueued || a1.RetryCount != 0 {
+		t.Fatalf("expected 1.1.1.1 back in the queue with retry_count 0, got %s/%d", a1.State, a1.RetryCount)
+	}
+	if v1, _ := d.GetValidator(ctx, "v1"); v1.State != db.ValidatorIdle {
+		t.Fatalf("v1 must stay in service, got %s", v1.State)
+	}
+
+	if _, err := d.SubmitIPs(ctx, []string{"3.3.3.3"}); err != nil {
+		t.Fatal(err)
+	}
+	o.Tick(ctx) // v1 skips 1.1.1.1 and takes 3.3.3.3; v3 takes 1.1.1.1
+	a1, _ = d.GetIPByAddress(ctx, "1.1.1.1")
+	a3, _ := d.GetIPByAddress(ctx, "3.3.3.3")
+	if a1.OwnerValidatorID == nil || *a1.OwnerValidatorID != "v3" {
+		t.Fatalf("expected 1.1.1.1 on v3, got %v", a1.OwnerValidatorID)
+	}
+	if a3.OwnerValidatorID == nil || *a3.OwnerValidatorID != "v1" {
+		t.Fatalf("expected 3.3.3.3 on v1, got %v", a3.OwnerValidatorID)
+	}
+
+	reportSelfChecks(t, ctx, o, d) // v1 passes this time: the others are fine
+	a1, _ = d.GetIPByAddress(ctx, "1.1.1.1")
+	a3, _ = d.GetIPByAddress(ctx, "3.3.3.3")
+	if a1.State != db.IPChecking || a3.State != db.IPChecking {
+		t.Fatalf("expected both addresses to pass the self-check, got %s and %s", a1.State, a3.State)
+	}
+	if got, _ := d.ListSelfCheckFailedOn(ctx, a1.RegistryID, 0); len(got) != 1 || got[0] != "v1" {
+		t.Fatalf("expected self-check failures on v1 only, got %v", got)
+	}
+	if ev := eventTypes(t, ctx, d, a1.ID); ev["validator_excluded"] != 1 {
+		t.Fatalf("expected one validator_excluded event, got %v", ev)
+	}
+}
+
+// The ceiling does not depend on the number of validators and is read at each
+// failure: the address fails once it is reached, not earlier, even though the
+// third validator never tried it.
+func TestSelfCheckCeilingGivesFailAndFollowsSetting(t *testing.T) {
+	ctx := context.Background()
+	o, d, mock := newTestOrchestrator(t, 180)
+	mock.Seed("fip-1", "1.1.1.1", "svc-project")
+	registerValidators(t, ctx, d, "v1", "v2", "v3")
+	if err := d.SeedQueue(ctx, []string{"1.1.1.1"}); err != nil {
+		t.Fatal(err)
+	}
+
+	o.Tick(ctx)
+	reportSelfChecks(t, ctx, o, d, "v1", "v2", "v3")
+	if ip, _ := d.GetIPByAddress(ctx, "1.1.1.1"); ip.State != db.IPQueued {
+		t.Fatalf("expected queued after the first failure (ceiling 5), got %s", ip.State)
+	}
+
+	if err := d.SetSelfCheckMaxAttempts(ctx, 2); err != nil {
+		t.Fatal(err)
+	}
+	o.Tick(ctx)
+	reportSelfChecks(t, ctx, o, d, "v1", "v2", "v3")
+	ip, _ := d.GetIPByAddress(ctx, "1.1.1.1")
+	if ip.State != db.IPFailed || ip.OverallResult != db.ResultFail || ip.RetryCount != 0 {
+		t.Fatalf("expected failed/fail at the lowered ceiling 2, got %s/%s retry_count=%d", ip.State, ip.OverallResult, ip.RetryCount)
+	}
+	if got, _ := d.ListSelfCheckFailedOn(ctx, ip.RegistryID, 0); len(got) != 2 {
+		t.Fatalf("expected failures on two validators, got %v", got)
+	}
+}
+
+// With fewer working validators than the ceiling (down to a single one) the
+// retries continue on the validators in a new round, up to the ceiling; the
+// address never gets stuck in the queue.
+func TestSelfCheckFewerValidatorsThanCeilingRetriesUntilCeiling(t *testing.T) {
+	for _, validators := range [][]string{{"v1"}, {"v1", "v2"}} {
+		ctx := context.Background()
+		o, d, mock := newTestOrchestrator(t, 180)
+		mock.Seed("fip-1", "1.1.1.1", "svc-project")
+		registerValidators(t, ctx, d, validators...)
+		if err := d.SeedQueue(ctx, []string{"1.1.1.1"}); err != nil {
+			t.Fatal(err)
+		}
+		if err := d.SetSelfCheckMaxAttempts(ctx, 5); err != nil {
+			t.Fatal(err)
+		}
+
+		for attempt := 1; attempt <= 5; attempt++ {
+			o.Tick(ctx)
+			ip, _ := d.GetIPByAddress(ctx, "1.1.1.1")
+			if ip.State != db.IPAwaitingSelfCheck {
+				t.Fatalf("%d validators, attempt %d: expected the address to be handed out, got %s", len(validators), attempt, ip.State)
+			}
+			reportSelfChecks(t, ctx, o, d, validators...)
+			ip, _ = d.GetIPByAddress(ctx, "1.1.1.1")
+			want := db.IPQueued
+			if attempt == 5 {
+				want = db.IPFailed
+			}
+			if ip.State != want {
+				t.Fatalf("%d validators, after failure %d: expected %s, got %s", len(validators), attempt, want, ip.State)
+			}
+		}
+	}
+}