Retry a failed self-check on another validator; add the self-check failure ceiling

A validator that failed the self-check of an address no longer gets that address
again in the current round (ClaimNextQueued skips it); the validator itself stays
in service and takes all other addresses. The verdict fail is set when the number
of failed self-checks of an address reaches settings.self_check_max_attempts
(1..50, default 5, independent of the number of validators); max_retries and
retry_count are no longer used for self-check. If every working validator has
already failed the address, a new round starts and the exclusions lapse.

Migration 0012: ip_self_check_failures (permanent history per registry address),
ip_queue.sc_failures and sc_round_start_cycle (cycle_id is used instead of
attempt_number, which restarts when a queue row is recreated), the setting.
db.FailSelfCheck does it in one transaction; re-submission starts a new series.
API: self_check_max_attempts in GET/PUT /admin/config/orchestrator,
self_check_failed_on in /admin/ips/{ip} and /admin/registry/{ip}. Dashboard: the
field on /settings and the line "Self-check не прошёл на: ..." on the address
pages. Docs, plan and summary in docs/changes/.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

internal/dashboard/handlers_test.go

@@ -307,9 +307,12 @@ func TestSettingsGetAndPut(t *testing.T) {
 	if !strings.Contains(page, `value="0"`) {
 		t.Fatalf("expected default 0 in the form, got:\n%s", page)
 	}
+	if !strings.Contains(page, "Потолок провалов self-check на адрес") || !strings.Contains(page, `id="self_check_max_attempts" name="self_check_max_attempts" min="1" max="50" step="1" value="5"`) {
+		t.Fatalf("expected the self-check ceiling field with the default 5, got:\n%s", page)
+	}
 
 	body := postForm(t, ts, "PUT", "/settings", map[string][]string{
-		"fip_settle_seconds": {"15"}, "history_retention_cycles": {"10"},
+		"fip_settle_seconds": {"15"}, "history_retention_cycles": {"10"}, "self_check_max_attempts": {"8"},
 	})
 	if !strings.Contains(body, `value="15"`) || !strings.Contains(body, `value="10"`) {
 		t.Fatalf("expected updated values 15/10 in re-rendered form, got:\n%s", body)
@@ -320,11 +323,14 @@ func TestSettingsGetAndPut(t *testing.T) {
 	if fake.historyRetentionCycles != 10 {
 		t.Fatalf("expected fake control-api history_retention_cycles updated, got %d", fake.historyRetentionCycles)
 	}
+	if fake.selfCheckMaxAttempts != 8 || !strings.Contains(body, `name="self_check_max_attempts" min="1" max="50" step="1" value="8"`) {
+		t.Fatalf("expected the self-check ceiling 8 saved and shown, got %d:\n%s", fake.selfCheckMaxAttempts, body)
+	}
 
 	// A control-api validation error (negative value here) surfaces via
 	// the banner, not a crash.
 	body = postForm(t, ts, "PUT", "/settings", map[string][]string{
-		"fip_settle_seconds": {"-1"}, "history_retention_cycles": {"10"},
+		"fip_settle_seconds": {"-1"}, "history_retention_cycles": {"10"}, "self_check_max_attempts": {"8"},
 	})
 	if !strings.Contains(body, "alert-warning") {
 		t.Fatalf("expected client error banner for invalid value, got:\n%s", body)
@@ -333,7 +339,7 @@ func TestSettingsGetAndPut(t *testing.T) {
 	// A non-numeric value is caught by the dashboard itself before it ever
 	// reaches control-api.
 	body = postForm(t, ts, "PUT", "/settings", map[string][]string{
-		"fip_settle_seconds": {"not-a-number"}, "history_retention_cycles": {"10"},
+		"fip_settle_seconds": {"not-a-number"}, "history_retention_cycles": {"10"}, "self_check_max_attempts": {"8"},
 	})
 	if !strings.Contains(body, "alert-warning") {
 		t.Fatalf("expected client error banner for non-numeric value, got:\n%s", body)
@@ -341,11 +347,49 @@ func TestSettingsGetAndPut(t *testing.T) {
 
 	// Same for a non-numeric retention value.
 	body = postForm(t, ts, "PUT", "/settings", map[string][]string{
-		"fip_settle_seconds": {"15"}, "history_retention_cycles": {"not-a-number"},
+		"fip_settle_seconds": {"15"}, "history_retention_cycles": {"not-a-number"}, "self_check_max_attempts": {"8"},
 	})
 	if !strings.Contains(body, "alert-warning") {
 		t.Fatalf("expected client error banner for non-numeric retention value, got:\n%s", body)
 	}
+
+	// A ceiling outside 1..50 is refused by control-api and shown as a
+	// warning; the form keeps the stored value.
+	body = postForm(t, ts, "PUT", "/settings", map[string][]string{
+		"fip_settle_seconds": {"15"}, "history_retention_cycles": {"10"}, "self_check_max_attempts": {"51"},
+	})
+	if !strings.Contains(body, "alert-warning") || fake.selfCheckMaxAttempts != 8 {
+		t.Fatalf("expected a warning and the stored ceiling 8 kept, got %d:\n%s", fake.selfCheckMaxAttempts, body)
+	}
+
+	// A non-numeric ceiling is caught by the dashboard itself.
+	body = postForm(t, ts, "PUT", "/settings", map[string][]string{
+		"fip_settle_seconds": {"15"}, "history_retention_cycles": {"10"}, "self_check_max_attempts": {"many"},
+	})
+	if !strings.Contains(body, "alert-warning") {
+		t.Fatalf("expected client error banner for non-numeric ceiling, got:\n%s", body)
+	}
+}
+
+// The address pages say on which validators the self-check failed.
+func TestAddressPagesShowSelfCheckFailedOn(t *testing.T) {
+	fake, caURL := newFakeControlAPI(t)
+	now := time.Now()
+	fake.ips = []ipQueueItem{{IPAddress: "5.5.5.5", State: "checking", UpdatedAt: now, CreatedAt: now}}
+	fake.registry["5.5.5.5"] = registryItem{IPAddress: "5.5.5.5", FirstSeenAt: now, LastSeenAt: now}
+	ts := newTestServer(t, caURL)
+
+	for _, path := range []string{"/ips/5.5.5.5", "/registry/5.5.5.5"} {
+		if page := get(t, ts, path); strings.Contains(page, "Self-check не прошёл на") {
+			t.Fatalf("%s: no failures, but the line is shown:\n%s", path, page)
+		}
+	}
+	fake.selfCheckFailedOn = []string{"vkiplab-v17", "vkiplab-v13"}
+	for _, path := range []string{"/ips/5.5.5.5", "/registry/5.5.5.5"} {
+		if page := get(t, ts, path); !strings.Contains(page, "Self-check не прошёл на: vkiplab-v17, vkiplab-v13") {
+			t.Fatalf("%s: expected the failed validators line, got:\n%s", path, page)
+		}
+	}
 }
 
 func TestSettingsPageShowsInboundChecks(t *testing.T) {