Files
cloud-ip-validator/internal/dashboard/dto.go
T
ayurishchevandClaude Sonnet 5.5 e95b5eb7d5 Retry a failed self-check on another validator; add the self-check failure ceiling
A validator that failed the self-check of an address no longer gets that address
again in the current round (ClaimNextQueued skips it); the validator itself stays
in service and takes all other addresses. The verdict fail is set when the number
of failed self-checks of an address reaches settings.self_check_max_attempts
(1..50, default 5, independent of the number of validators); max_retries and
retry_count are no longer used for self-check. If every working validator has
already failed the address, a new round starts and the exclusions lapse.

Migration 0012: ip_self_check_failures (permanent history per registry address),
ip_queue.sc_failures and sc_round_start_cycle (cycle_id is used instead of
attempt_number, which restarts when a queue row is recreated), the setting.
db.FailSelfCheck does it in one transaction; re-submission starts a new series.
API: self_check_max_attempts in GET/PUT /admin/config/orchestrator,
self_check_failed_on in /admin/ips/{ip} and /admin/registry/{ip}. Dashboard: the
field on /settings and the line "Self-check не прошёл на: ..." on the address
pages. Docs, plan and summary in docs/changes/.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-04 09:50:12 +03:00

438 lines
13 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package dashboard
import (
"fmt"
"strconv"
"time"
)
// Wire shapes for control-api's /api/v1/admin/* surface, defined locally
// rather than importing internal/httpapi's (unexported) DTOs or
// internal/db's models — the same "each binary owns the wire shapes it
// needs" pattern already used by internal/probercore and internal/agentcore.
//
// The read-only list/detail endpoints (ipQueueItem, validator, check,
// event below) mirror internal/db model structs field-for-field: those
// endpoints marshal Go structs with no json tags, so encoding/json matches
// fields by name (case-insensitively) with no tags needed here either.
// Everything else mirrors internal/httpapi/dto_admin.go's snake_case tags.
type statusResponse struct {
TotalIPs int `json:"total_ips"`
IPsByState map[string]int `json:"ips_by_state"`
TotalValidators int `json:"total_validators"`
// ResultsByOverall counts finished addresses by overall result
// (pass/partial/fail/cancelled).
ResultsByOverall map[string]int `json:"results_by_overall"`
}
// Terminal queue states: the address needs no further processing. Shared by
// the overview progress indicator; "occupied" counts as terminal too (the
// check cycle never ran because the floating IP was already bound).
var terminalStates = []string{"done", "failed", "occupied"}
// activeStates are the states of an address that is being worked on right
// now (everything between "queued" and a terminal state).
var activeStates = []string{"assigning_fip", "awaiting_self_check", "checking", "aggregating"}
// sumStates adds up the counts of the given states in a status breakdown.
func sumStates(byState map[string]int, states []string) int {
n := 0
for _, s := range states {
n += byState[s]
}
return n
}
type ipQueueItem struct {
ID int64
IPAddress string
Sequence int
State string
OwnerValidatorID *string
FIPID string
AttemptNumber int
RetryCount int
LeaseExpiresAt *time.Time
EgressComplete bool
OverallResult string
AssignedAt *time.Time
FIPAssociatedAt *time.Time
AggregatedAt *time.Time
FIPReleasedAt *time.Time
CreatedAt time.Time
UpdatedAt time.Time
}
type check struct {
ID int64
RegistryID int64
CycleID int
IPID int64
IPAddress string
AttemptNumber int
ValidatorID string
Source string
CheckType string
Target string
Success bool
LatencyMS int64
Detail string
CheckedAt time.Time
}
type event struct {
ID int64
SourceType string
SourceID string
IPID *int64
EventType string
Payload string
OccurredAt time.Time
}
type ipDetailResponse struct {
IP ipQueueItem `json:"ip"`
Checks []check `json:"checks"`
Events []event `json:"events"`
// SelfCheckFailedOn lists the validators whose self-check of this address
// failed (in its current run).
SelfCheckFailedOn []string `json:"self_check_failed_on"`
}
type validator struct {
ValidatorID string
Hostname string
OSPortID string
State string
CurrentIPID *int64
AgentVersion string
LastHeartbeatAt *time.Time
}
type submitIPsResponse struct {
Added []string `json:"added"`
Requeued []string `json:"requeued"`
Reordered []string `json:"reordered"`
SkippedInProgress []string `json:"skipped_in_progress"`
}
type deleteIPsResponse struct {
Deleted []string `json:"deleted"`
NotFound []string `json:"not_found"`
}
type clearQueueResponse struct {
Deleted []string `json:"deleted"`
Count int `json:"count"`
}
// ipsPage is the paginated envelope of GET /admin/ips (sent when the request
// carries `limit`).
type ipsPage struct {
Items []ipQueueItem `json:"items"`
Total int `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
// registryPage is the paginated envelope of GET /admin/registry.
type registryPage struct {
Items []registryItem `json:"items"`
Total int `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
// scanStatusDTO is the state of control-api's background floating-IP scan job
// (POST/GET /api/v1/admin/ips/scan).
type scanStatusDTO struct {
State string `json:"state"`
Running bool `json:"running"`
DryRun bool `json:"dry_run"`
Pages int `json:"pages"`
Discovered int `json:"discovered"`
Free int `json:"free"`
Added int `json:"added"`
Requeued int `json:"requeued"`
Reordered int `json:"reordered"`
SkippedInProgress int `json:"skipped_in_progress"`
StartedAt *time.Time `json:"started_at"`
FinishedAt *time.Time `json:"finished_at"`
Error string `json:"error"`
}
// Finished reports a job that has run to a terminal state (as opposed to
// "idle" = never started, or still running).
func (s scanStatusDTO) Finished() bool {
if s.Running {
return false
}
switch s.State {
case "done", "error", "cancelled":
return true
}
return false
}
// StateLabel is the Russian description of the job's state.
func (s scanStatusDTO) StateLabel() string {
switch s.State {
case "clearing":
return "очистка"
case "listing":
return "читаются страницы"
case "enqueuing":
return "ставятся в очередь"
case "done":
return "готово"
case "error":
return "ошибка"
case "cancelled":
return "отменено"
case "idle", "":
return "нет активного сканирования"
default:
return s.State
}
}
// PillClass picks the pill style for the state.
func (s scanStatusDTO) PillClass() string {
switch s.State {
case "done":
return "pill-success"
case "error":
return "pill-danger"
case "cancelled":
return "pill-cancel"
case "clearing", "listing", "enqueuing":
return "pill-info"
default:
return "pill-neutral"
}
}
// Handled is how many of the free addresses the enqueuing phase has already
// processed.
func (s scanStatusDTO) Handled() int {
return s.Added + s.Requeued + s.Reordered + s.SkippedInProgress
}
// Indeterminate is true while the amount of work is not known yet.
func (s scanStatusDTO) Indeterminate() bool {
return s.State == "clearing" || s.State == "listing" || (s.State == "enqueuing" && s.Free <= 0)
}
// Elapsed is the human-readable run time: until now while running, until
// finished_at afterwards; empty when the job never started.
func (s scanStatusDTO) Elapsed() string {
if s.StartedAt == nil {
return ""
}
end := time.Now()
if !s.Running && s.FinishedAt != nil {
end = *s.FinishedAt
}
return fmtDuration(end.Sub(*s.StartedAt))
}
// fmtDuration renders a duration in Russian as "2 ч 05 мин", "3 мин 07 с" or
// "42 с" — coarse on purpose (progress/ETA display).
func fmtDuration(d time.Duration) string {
if d < 0 {
d = 0
}
sec := int(d.Round(time.Second) / time.Second)
h, m, sc := sec/3600, (sec%3600)/60, sec%60
switch {
case h > 0:
return fmt.Sprintf("%d ч %02d мин", h, m)
case m > 0:
return fmt.Sprintf("%d мин %02d с", m, sc)
default:
return fmt.Sprintf("%d с", sc)
}
}
// registryItem is one row of the durable per-address registry — see
// httpapi's registryDTO. Survives an address being deleted from the check
// queue and later re-added, unlike ipQueueItem above.
type registryItem struct {
IPAddress string `json:"ip_address"`
FirstSeenAt time.Time `json:"first_seen_at"`
LastSeenAt time.Time `json:"last_seen_at"`
TotalCycles int `json:"total_cycles"`
LastResult string `json:"last_result"`
LastCheckedAt *time.Time `json:"last_checked_at"`
InQueue bool `json:"in_queue"`
CurrentState string `json:"current_state"`
// LastCycleID is the cycle counted by Egress and Ingress (0 = no checks).
LastCycleID int `json:"last_cycle_id"`
Egress levelResult `json:"egress"`
Ingress levelResult `json:"ingress"`
}
// levelResult is "ok of total" recorded checks of one level (egress or
// ingress) in the last cycle, split by check family — see httpapi's
// levelResultDTO.
type levelResult struct {
Total int `json:"total"`
OK int `json:"ok"`
ByType []typeStat `json:"by_type"`
}
type typeStat struct {
Type string `json:"type"`
Total int `json:"total"`
OK int `json:"ok"`
}
// statClass picks the colour of an "ok of total" figure: all checks passed,
// none passed, some passed, or nothing recorded.
func statClass(ok, total int) string {
switch {
case total == 0:
return "none"
case ok == total:
return "ok"
case ok == 0:
return "fail"
}
return "part"
}
// Class is the CSS modifier for the level's total.
func (l levelResult) Class() string { return statClass(l.OK, l.Total) }
// Class is the CSS modifier for one check family.
func (t typeStat) Class() string { return statClass(t.OK, t.Total) }
type registryHistoryResponse struct {
Registry registryItem `json:"registry"`
Checks []check `json:"checks"`
// SelfCheckFailedOn lists the validators whose self-check of this address
// failed, over every run.
SelfCheckFailedOn []string `json:"self_check_failed_on"`
}
type validatorDTO struct {
ValidatorID string `json:"validator_id"`
Hostname string `json:"hostname"`
OSPortID string `json:"os_port_id"`
State string `json:"state"`
LastHeartbeatAt *time.Time `json:"last_heartbeat_at"`
}
type siteDTO struct {
Index int `json:"index"`
SiteID string `json:"site_id"`
Hostname string `json:"hostname"`
State string `json:"state"`
LastHeartbeatAt *time.Time `json:"last_heartbeat_at"`
}
type targetGroupDTO struct {
Name string `json:"name"`
Targets []string `json:"targets"`
}
type checkTypeDTO struct {
Name string `json:"name"`
Enabled bool `json:"enabled"`
Targets []string `json:"targets"`
}
type errorResponse struct {
Error string `json:"error"`
}
type orchestratorSettingsDTO struct {
FIPSettleSeconds int `json:"fip_settle_seconds"`
HistoryRetentionCycles int `json:"history_retention_cycles"`
SelfCheckMaxAttempts int `json:"self_check_max_attempts"`
}
type inboundChecksDTO struct {
Ports []int `json:"ports"`
ICMP bool `json:"icmp"`
}
// autoCycleDTO mirrors internal/httpapi's autoCycleDTO — the status and
// parameters of the automatic check cycle (/api/v1/admin/auto-cycle).
type autoCycleDTO struct {
Enabled bool `json:"enabled"`
IntervalSeconds int `json:"interval_seconds"`
MaxRunSeconds int `json:"max_run_seconds"`
Phase string `json:"phase"`
RunStartedAt *time.Time `json:"run_started_at"`
NextRunAt *time.Time `json:"next_run_at"`
LastRunStartedAt *time.Time `json:"last_run_started_at"`
LastRunFinishedAt *time.Time `json:"last_run_finished_at"`
LastOutcome string `json:"last_outcome"`
LastError string `json:"last_error"`
LastScannedFree int `json:"last_scanned_free"`
RunsTotal int `json:"runs_total"`
}
// secondsToMinutes renders seconds as minutes for the settings form: a
// whole number when divisible by 60, otherwise a decimal ("1.5").
func secondsToMinutes(sec int) string {
return strconv.FormatFloat(float64(sec)/60, 'f', -1, 64)
}
// IntervalMinutes and MaxRunMinutes are used by the settings template: the
// UI works in minutes, the API in seconds.
func (a autoCycleDTO) IntervalMinutes() string { return secondsToMinutes(a.IntervalSeconds) }
func (a autoCycleDTO) MaxRunMinutes() string { return secondsToMinutes(a.MaxRunSeconds) }
// PhaseLabel is the Russian description of the current phase.
func (a autoCycleDTO) PhaseLabel() string {
switch a.Phase {
case "scanning":
return "сканирование Floating IP"
case "running":
return "идёт проверка"
case "waiting":
return "пауза между циклами"
case "idle":
return "ожидает запуска"
default:
return a.Phase
}
}
// OutcomeLabel is the Russian description of the last cycle's outcome.
func (a autoCycleDTO) OutcomeLabel() string {
switch a.LastOutcome {
case "completed":
return "завершён"
case "no_free_ips":
return "нет свободных IP"
case "timeout":
return "превышено время ожидания"
case "error":
return "ошибка"
case "stopped":
return "остановлен"
case "":
return "—"
default:
return a.LastOutcome
}
}
// OutcomePillClass picks the pill style for the last outcome.
func (a autoCycleDTO) OutcomePillClass() string {
switch a.LastOutcome {
case "completed":
return "pill-success"
case "no_free_ips", "timeout", "stopped":
return "pill-warning"
case "error":
return "pill-danger"
default:
return "pill-neutral"
}
}