Retry a failed self-check on another validator; add the self-check failure ceiling

A validator that failed the self-check of an address no longer gets that address
again in the current round (ClaimNextQueued skips it); the validator itself stays
in service and takes all other addresses. The verdict fail is set when the number
of failed self-checks of an address reaches settings.self_check_max_attempts
(1..50, default 5, independent of the number of validators); max_retries and
retry_count are no longer used for self-check. If every working validator has
already failed the address, a new round starts and the exclusions lapse.

Migration 0012: ip_self_check_failures (permanent history per registry address),
ip_queue.sc_failures and sc_round_start_cycle (cycle_id is used instead of
attempt_number, which restarts when a queue row is recreated), the setting.
db.FailSelfCheck does it in one transaction; re-submission starts a new series.
API: self_check_max_attempts in GET/PUT /admin/config/orchestrator,
self_check_failed_on in /admin/ips/{ip} and /admin/registry/{ip}. Dashboard: the
field on /settings and the line "Self-check не прошёл на: ..." on the address
pages. Docs, plan and summary in docs/changes/.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

internal/dashboard/client.go

@@ -356,10 +356,10 @@ func (c *client) GetOrchestratorSettings(ctx context.Context) (orchestratorSetti
 	return out, err
 }
 
-func (c *client) PutOrchestratorSettings(ctx context.Context, fipSettleSeconds, historyRetentionCycles int) (orchestratorSettingsDTO, error) {
+func (c *client) PutOrchestratorSettings(ctx context.Context, fipSettleSeconds, historyRetentionCycles, selfCheckMaxAttempts int) (orchestratorSettingsDTO, error) {
 	var out orchestratorSettingsDTO
 	err := c.do(ctx, http.MethodPut, "/api/v1/admin/config/orchestrator",
-		orchestratorSettingsDTO{FIPSettleSeconds: fipSettleSeconds, HistoryRetentionCycles: historyRetentionCycles}, &out)
+		orchestratorSettingsDTO{FIPSettleSeconds: fipSettleSeconds, HistoryRetentionCycles: historyRetentionCycles, SelfCheckMaxAttempts: selfCheckMaxAttempts}, &out)
 	return out, err
 }
 

internal/dashboard/dashboard_test.go

@@ -37,6 +37,10 @@ type fakeControlAPI struct {
 	inboundICMP      bool
 
 	historyRetentionCycles int
+	selfCheckMaxAttempts   int
+	// selfCheckFailedOn is served as self_check_failed_on of the address
+	// detail and registry history endpoints.
+	selfCheckFailedOn []string
 
 	// Analytics: the run selector, the report JSON per run, the lists per
 	// "run/kind[/class]" and the subnet list of /settings.
@@ -94,6 +98,8 @@ func newFakeControlAPI(t *testing.T) (*fakeControlAPI, string) {
 		registry:       map[string]registryItem{},
 		registryChecks: map[string][]check{},
 		autoCycle:      autoCycleDTO{IntervalSeconds: 3600, Phase: "idle"},
+
+		selfCheckMaxAttempts: 5,
 	}
 	ts := httptest.NewServer(f.handler())
 	t.Cleanup(ts.Close)
@@ -191,7 +197,7 @@ func (f *fakeControlAPI) handler() http.Handler {
 		addr := r.PathValue("ip")
 		for _, ip := range f.ips {
 			if ip.IPAddress == addr {
-				writeJSON(w, http.StatusOK, ipDetailResponse{IP: ip, Checks: []check{}, Events: []event{}})
+				writeJSON(w, http.StatusOK, ipDetailResponse{IP: ip, Checks: []check{}, Events: []event{}, SelfCheckFailedOn: f.selfCheckFailedOn})
 				return
 			}
 		}
@@ -310,6 +316,7 @@ func (f *fakeControlAPI) handler() http.Handler {
 		writeJSON(w, http.StatusOK, orchestratorSettingsDTO{
 			FIPSettleSeconds:       f.fipSettleSeconds,
 			HistoryRetentionCycles: f.historyRetentionCycles,
+			SelfCheckMaxAttempts:   f.selfCheckMaxAttempts,
 		})
 	})
 	mux.HandleFunc("PUT /api/v1/admin/config/orchestrator", func(w http.ResponseWriter, r *http.Request) {
@@ -325,11 +332,17 @@ func (f *fakeControlAPI) handler() http.Handler {
 			writeAPIErr(w, http.StatusBadRequest, "history_retention_cycles must be >= 0")
 			return
 		}
+		if req.SelfCheckMaxAttempts < 1 || req.SelfCheckMaxAttempts > 50 {
+			writeAPIErr(w, http.StatusBadRequest, "self_check_max_attempts must be in 1..50")
+			return
+		}
 		f.fipSettleSeconds = req.FIPSettleSeconds
 		f.historyRetentionCycles = req.HistoryRetentionCycles
+		f.selfCheckMaxAttempts = req.SelfCheckMaxAttempts
 		writeJSON(w, http.StatusOK, orchestratorSettingsDTO{
 			FIPSettleSeconds:       f.fipSettleSeconds,
 			HistoryRetentionCycles: f.historyRetentionCycles,
+			SelfCheckMaxAttempts:   f.selfCheckMaxAttempts,
 		})
 	})
 
@@ -474,7 +487,7 @@ func (f *fakeControlAPI) handler() http.Handler {
 			writeAPIErr(w, http.StatusNotFound, "unknown ip: "+addr)
 			return
 		}
-		writeJSON(w, http.StatusOK, registryHistoryResponse{Registry: item, Checks: f.registryChecks[addr]})
+		writeJSON(w, http.StatusOK, registryHistoryResponse{Registry: item, Checks: f.registryChecks[addr], SelfCheckFailedOn: f.selfCheckFailedOn})
 	})
 
 	mux.HandleFunc("GET /api/v1/admin/config/subnets", func(w http.ResponseWriter, r *http.Request) {

internal/dashboard/dto.go

@@ -95,6 +95,9 @@ type ipDetailResponse struct {
 	IP     ipQueueItem `json:"ip"`
 	Checks []check     `json:"checks"`
 	Events []event     `json:"events"`
+	// SelfCheckFailedOn lists the validators whose self-check of this address
+	// failed (in its current run).
+	SelfCheckFailedOn []string `json:"self_check_failed_on"`
 }
 
 type validator struct {
@@ -308,6 +311,9 @@ func (t typeStat) Class() string { return statClass(t.OK, t.Total) }
 type registryHistoryResponse struct {
 	Registry registryItem `json:"registry"`
 	Checks   []check      `json:"checks"`
+	// SelfCheckFailedOn lists the validators whose self-check of this address
+	// failed, over every run.
+	SelfCheckFailedOn []string `json:"self_check_failed_on"`
 }
 
 type validatorDTO struct {
@@ -344,6 +350,7 @@ type errorResponse struct {
 type orchestratorSettingsDTO struct {
 	FIPSettleSeconds       int `json:"fip_settle_seconds"`
 	HistoryRetentionCycles int `json:"history_retention_cycles"`
+	SelfCheckMaxAttempts   int `json:"self_check_max_attempts"`
 }
 
 type inboundChecksDTO struct {

internal/dashboard/handlers_settings.go

@@ -77,7 +77,12 @@ func (s *Server) handleSettingsPut(w http.ResponseWriter, r *http.Request) {
 		s.renderSettingsForm(w, r, &apiErr{Status: http.StatusBadRequest, Message: "глубина истории должна быть целым числом циклов"})
 		return
 	}
-	_, err = s.CA.PutOrchestratorSettings(r.Context(), seconds, retentionCycles)
+	selfCheckMax, err := strconv.Atoi(r.PostFormValue("self_check_max_attempts"))
+	if err != nil {
+		s.renderSettingsForm(w, r, &apiErr{Status: http.StatusBadRequest, Message: "потолок провалов self-check должен быть целым числом"})
+		return
+	}
+	_, err = s.CA.PutOrchestratorSettings(r.Context(), seconds, retentionCycles, selfCheckMax)
 	s.renderSettingsForm(w, r, err)
 }
 

internal/dashboard/handlers_test.go

@@ -307,9 +307,12 @@ func TestSettingsGetAndPut(t *testing.T) {
 	if !strings.Contains(page, `value="0"`) {
 		t.Fatalf("expected default 0 in the form, got:\n%s", page)
 	}
+	if !strings.Contains(page, "Потолок провалов self-check на адрес") || !strings.Contains(page, `id="self_check_max_attempts" name="self_check_max_attempts" min="1" max="50" step="1" value="5"`) {
+		t.Fatalf("expected the self-check ceiling field with the default 5, got:\n%s", page)
+	}
 
 	body := postForm(t, ts, "PUT", "/settings", map[string][]string{
-		"fip_settle_seconds": {"15"}, "history_retention_cycles": {"10"},
+		"fip_settle_seconds": {"15"}, "history_retention_cycles": {"10"}, "self_check_max_attempts": {"8"},
 	})
 	if !strings.Contains(body, `value="15"`) || !strings.Contains(body, `value="10"`) {
 		t.Fatalf("expected updated values 15/10 in re-rendered form, got:\n%s", body)
@@ -320,11 +323,14 @@ func TestSettingsGetAndPut(t *testing.T) {
 	if fake.historyRetentionCycles != 10 {
 		t.Fatalf("expected fake control-api history_retention_cycles updated, got %d", fake.historyRetentionCycles)
 	}
+	if fake.selfCheckMaxAttempts != 8 || !strings.Contains(body, `name="self_check_max_attempts" min="1" max="50" step="1" value="8"`) {
+		t.Fatalf("expected the self-check ceiling 8 saved and shown, got %d:\n%s", fake.selfCheckMaxAttempts, body)
+	}
 
 	// A control-api validation error (negative value here) surfaces via
 	// the banner, not a crash.
 	body = postForm(t, ts, "PUT", "/settings", map[string][]string{
-		"fip_settle_seconds": {"-1"}, "history_retention_cycles": {"10"},
+		"fip_settle_seconds": {"-1"}, "history_retention_cycles": {"10"}, "self_check_max_attempts": {"8"},
 	})
 	if !strings.Contains(body, "alert-warning") {
 		t.Fatalf("expected client error banner for invalid value, got:\n%s", body)
@@ -333,7 +339,7 @@ func TestSettingsGetAndPut(t *testing.T) {
 	// A non-numeric value is caught by the dashboard itself before it ever
 	// reaches control-api.
 	body = postForm(t, ts, "PUT", "/settings", map[string][]string{
-		"fip_settle_seconds": {"not-a-number"}, "history_retention_cycles": {"10"},
+		"fip_settle_seconds": {"not-a-number"}, "history_retention_cycles": {"10"}, "self_check_max_attempts": {"8"},
 	})
 	if !strings.Contains(body, "alert-warning") {
 		t.Fatalf("expected client error banner for non-numeric value, got:\n%s", body)
@@ -341,11 +347,49 @@ func TestSettingsGetAndPut(t *testing.T) {
 
 	// Same for a non-numeric retention value.
 	body = postForm(t, ts, "PUT", "/settings", map[string][]string{
-		"fip_settle_seconds": {"15"}, "history_retention_cycles": {"not-a-number"},
+		"fip_settle_seconds": {"15"}, "history_retention_cycles": {"not-a-number"}, "self_check_max_attempts": {"8"},
 	})
 	if !strings.Contains(body, "alert-warning") {
 		t.Fatalf("expected client error banner for non-numeric retention value, got:\n%s", body)
 	}
+
+	// A ceiling outside 1..50 is refused by control-api and shown as a
+	// warning; the form keeps the stored value.
+	body = postForm(t, ts, "PUT", "/settings", map[string][]string{
+		"fip_settle_seconds": {"15"}, "history_retention_cycles": {"10"}, "self_check_max_attempts": {"51"},
+	})
+	if !strings.Contains(body, "alert-warning") || fake.selfCheckMaxAttempts != 8 {
+		t.Fatalf("expected a warning and the stored ceiling 8 kept, got %d:\n%s", fake.selfCheckMaxAttempts, body)
+	}
+
+	// A non-numeric ceiling is caught by the dashboard itself.
+	body = postForm(t, ts, "PUT", "/settings", map[string][]string{
+		"fip_settle_seconds": {"15"}, "history_retention_cycles": {"10"}, "self_check_max_attempts": {"many"},
+	})
+	if !strings.Contains(body, "alert-warning") {
+		t.Fatalf("expected client error banner for non-numeric ceiling, got:\n%s", body)
+	}
+}
+
+// The address pages say on which validators the self-check failed.
+func TestAddressPagesShowSelfCheckFailedOn(t *testing.T) {
+	fake, caURL := newFakeControlAPI(t)
+	now := time.Now()
+	fake.ips = []ipQueueItem{{IPAddress: "5.5.5.5", State: "checking", UpdatedAt: now, CreatedAt: now}}
+	fake.registry["5.5.5.5"] = registryItem{IPAddress: "5.5.5.5", FirstSeenAt: now, LastSeenAt: now}
+	ts := newTestServer(t, caURL)
+
+	for _, path := range []string{"/ips/5.5.5.5", "/registry/5.5.5.5"} {
+		if page := get(t, ts, path); strings.Contains(page, "Self-check не прошёл на") {
+			t.Fatalf("%s: no failures, but the line is shown:\n%s", path, page)
+		}
+	}
+	fake.selfCheckFailedOn = []string{"vkiplab-v17", "vkiplab-v13"}
+	for _, path := range []string{"/ips/5.5.5.5", "/registry/5.5.5.5"} {
+		if page := get(t, ts, path); !strings.Contains(page, "Self-check не прошёл на: vkiplab-v17, vkiplab-v13") {
+			t.Fatalf("%s: expected the failed validators line, got:\n%s", path, page)
+		}
+	}
 }
 
 func TestSettingsPageShowsInboundChecks(t *testing.T) {

internal/dashboard/templates/ip_detail.html

@@ -34,6 +34,7 @@
 создан: {{fmtTime .Detail.IP.CreatedAt}} · назначен: {{fmtTime .Detail.IP.AssignedAt}} ·
 агрегирован: {{fmtTime .Detail.IP.AggregatedAt}} · FIP освобождён: {{fmtTime .Detail.IP.FIPReleasedAt}}
 </p>
+{{if .Detail.SelfCheckFailedOn}}<p class="muted">Self-check не прошёл на: {{join .Detail.SelfCheckFailedOn ", "}}</p>{{end}}
 
 <h2 class="section-title">Проверки (попытка {{.Detail.IP.AttemptNumber}})</h2>
 {{if .Detail.Checks}}

internal/dashboard/templates/registry_detail.html

@@ -34,6 +34,7 @@
 впервые замечен: {{fmtTime .History.Registry.FirstSeenAt}} · последний раз замечен: {{fmtTime .History.Registry.LastSeenAt}}
 · всего циклов проверки: {{.History.Registry.TotalCycles}}
 </p>
+{{if .History.SelfCheckFailedOn}}<p class="muted">Self-check не прошёл на: {{join .History.SelfCheckFailedOn ", "}}</p>{{end}}
 
 <h2 class="section-title">История проверок (все сохранённые циклы)</h2>
 {{if .History.Checks}}

internal/dashboard/templates/settings.html

@@ -80,6 +80,10 @@
 <a href="/registry">реестре</a> для каждого адреса. Адрес и его накопленная статистика остаются в реестре даже
 после удаления из очереди — эта настройка ограничивает только глубину истории конкретных проверок, не сам реестр.
 <code>0</code> — хранить без ограничения.</p>
+<p class="muted" style="margin-bottom:16px">Сколько раз self-check может не пройти у одного адреса, прежде чем адрес
+получит итог <code>fail</code> (допустимо 1–50, по умолчанию 5). Повторы идут на других валидаторах: валидатор, на
+котором адрес не прошёл self-check, этому адресу больше не выдаётся (на остальные адреса это не влияет).
+Значение действует на следующих повторах, без перезапуска.</p>
 <form hx-put="/settings" hx-target="#settings-form-wrap" hx-swap="innerHTML">
 <div class="field-row">
 <div class="field">
@@ -90,6 +94,10 @@
 <label for="history_retention_cycles">Глубина истории проверок (циклов на адрес, 0 = не ограничено)</label>
 <input type="number" id="history_retention_cycles" name="history_retention_cycles" min="0" step="1" value="{{.Settings.HistoryRetentionCycles}}" required>
 </div>
+<div class="field">
+<label for="self_check_max_attempts">Потолок провалов self-check на адрес</label>
+<input type="number" id="self_check_max_attempts" name="self_check_max_attempts" min="1" max="50" step="1" value="{{.Settings.SelfCheckMaxAttempts}}" required>
+</div>
 <button type="submit" class="btn btn-primary">Сохранить</button>
 </div>
 </form>