Files
cloud-ip-validator/internal/db/models.go
T
ayurishchevandClaude Sonnet 5.5 e95b5eb7d5 Retry a failed self-check on another validator; add the self-check failure ceiling
A validator that failed the self-check of an address no longer gets that address
again in the current round (ClaimNextQueued skips it); the validator itself stays
in service and takes all other addresses. The verdict fail is set when the number
of failed self-checks of an address reaches settings.self_check_max_attempts
(1..50, default 5, independent of the number of validators); max_retries and
retry_count are no longer used for self-check. If every working validator has
already failed the address, a new round starts and the exclusions lapse.

Migration 0012: ip_self_check_failures (permanent history per registry address),
ip_queue.sc_failures and sc_round_start_cycle (cycle_id is used instead of
attempt_number, which restarts when a queue row is recreated), the setting.
db.FailSelfCheck does it in one transaction; re-submission starts a new series.
API: self_check_max_attempts in GET/PUT /admin/config/orchestrator,
self_check_failed_on in /admin/ips/{ip} and /admin/registry/{ip}. Dashboard: the
field on /settings and the line "Self-check не прошёл на: ..." on the address
pages. Docs, plan and summary in docs/changes/.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-04 09:50:12 +03:00

351 lines
9.6 KiB
Go

package db
import (
"strings"
"time"
)
// Validator and IP lifecycle states. Kept as typed string constants rather
// than a Go enum type so they round-trip through SQLite TEXT columns and
// JSON without conversion.
const (
ValidatorUnregistered = "unregistered"
ValidatorIdle = "idle"
ValidatorAssigned = "assigned"
ValidatorChecking = "checking"
ValidatorUnreachable = "unreachable"
// Site (prober) availability states — no assigned/checking equivalent,
// since a prober isn't bound to one IP at a time.
SiteUnregistered = "unregistered"
SiteIdle = "idle"
SiteUnreachable = "unreachable"
IPQueued = "queued"
IPAssigningFIP = "assigning_fip"
IPAwaitingSelfCheck = "awaiting_self_check"
IPChecking = "checking"
IPAggregating = "aggregating"
IPDone = "done"
IPFailed = "failed"
// IPOccupied is a terminal state distinct from IPFailed: the floating IP
// was found already associated to a different port at claim time, so the
// check cycle never started for this attempt. See
// Orchestrator.associateFIP and db.MarkFIPOccupied.
IPOccupied = "occupied"
ResultPass = "pass"
ResultPartial = "partial"
ResultFail = "fail"
ResultCancelled = "cancelled"
SourceEgress = "egress"
)
// IPStates lists every valid ip_queue state, in lifecycle order.
var IPStates = []string{
IPQueued, IPAssigningFIP, IPAwaitingSelfCheck, IPChecking, IPAggregating,
IPDone, IPFailed, IPOccupied,
}
// IsValidIPState reports whether s is one of IPStates.
func IsValidIPState(s string) bool {
for _, v := range IPStates {
if v == s {
return true
}
}
return false
}
// IsValidResult reports whether s is a valid overall result value
// (pass | partial | fail | cancelled).
func IsValidResult(s string) bool {
switch s {
case ResultPass, ResultPartial, ResultFail, ResultCancelled:
return true
}
return false
}
// InboundSource returns the checks.source value for the given prober site
// index (1-based), e.g. InboundSource(1) == "inbound-site-1".
func InboundSource(siteIndex int) string {
return "inbound-site-" + itoa(siteIndex)
}
// Check levels: the two directions a check can run in, derived from
// checks.source (there is no separate direction column).
const (
LevelEgress = "egress"
LevelIngress = "ingress"
)
// CheckLevel maps a checks.source value to its level: "egress" for the
// validator's own outbound checks, "ingress" for any prober site
// ("inbound-site-N"). Any other source yields "".
func CheckLevel(source string) string {
switch {
case source == SourceEgress:
return LevelEgress
case strings.HasPrefix(source, "inbound-site-"):
return LevelIngress
}
return ""
}
// CheckFamily maps a checks.check_type value to its family for grouping:
// the part before the first "-", so tcp-22 and tcp-443 are both "tcp" while
// https, icmp, ssh and tls-443 ("tls") stay distinct. A check type added in
// the future is grouped by its own name without code changes.
func CheckFamily(checkType string) string {
if i := strings.IndexByte(checkType, '-'); i > 0 {
return checkType[:i]
}
return checkType
}
func itoa(n int) string {
if n == 0 {
return "0"
}
neg := n < 0
if neg {
n = -n
}
var buf [20]byte
i := len(buf)
for n > 0 {
i--
buf[i] = byte('0' + n%10)
n /= 10
}
if neg {
i--
buf[i] = '-'
}
return string(buf[i:])
}
type Validator struct {
ValidatorID string
Hostname string
OSPortID string
State string
CurrentIPID *int64
AgentVersion string
LastHeartbeatAt *time.Time
CreatedAt time.Time
UpdatedAt time.Time
}
type IPQueueItem struct {
ID int64
IPAddress string
Sequence int
State string
OwnerValidatorID *string
FIPID string
AttemptNumber int
RetryCount int
LeaseExpiresAt *time.Time
EgressComplete bool
OverallResult string
AssignedAt *time.Time
// CheckingStartedAt is when the address entered the checking state; the
// aggregation window counts from it. nil on rows that predate it.
CheckingStartedAt *time.Time
FIPAssociatedAt *time.Time
AggregatedAt *time.Time
FIPReleasedAt *time.Time
RegistryID int64
CycleID int
CreatedAt time.Time
UpdatedAt time.Time
}
type Check struct {
ID int64
RegistryID int64
CycleID int
// IPID is the ip_queue row this check was originally recorded against.
// It's cleared to 0 (SQL NULL) if that row was later deleted — history
// stays reachable via RegistryID/CycleID regardless (see
// migrations/0007_ip_registry.sql). 0 is never a valid ip_queue id
// (AUTOINCREMENT starts at 1), so it unambiguously means "orphaned."
IPID int64
IPAddress string
AttemptNumber int
ValidatorID string
Source string
CheckType string
Target string
Success bool
LatencyMS int64
Detail string
CheckedAt time.Time
CreatedAt time.Time
}
// RegistryItem is a durable per-address record that survives an address
// being removed from ip_queue and later re-added — see
// migrations/0007_ip_registry.sql. NextCycle is the cycle_id that will be
// assigned the next time this address is (re)submitted; it only ever
// increases, so cycle_id stays unique for this address even across
// ip_queue row deletion/recreation.
type RegistryItem struct {
ID int64
IPAddress string
FirstSeenAt time.Time
LastSeenAt time.Time
NextCycle int
CreatedAt time.Time
UpdatedAt time.Time
}
type Event struct {
ID int64
SourceType string
SourceID string
IPID *int64
// RegistryID/CycleID are resolved from IPID at insert time (see
// InsertEvent) and stay set even after the ip_queue row IPID pointed to
// is later deleted, so retention pruning can cut events at the same
// cycle boundary as checks — see migrations/0007_ip_registry.sql.
RegistryID int64
CycleID int
EventType string
Payload string
OccurredAt time.Time
CreatedAt time.Time
}
type Site struct {
Index int
SiteID string
Hostname string
State string
LastHeartbeatAt *time.Time
CreatedAt time.Time
UpdatedAt time.Time
}
type TargetGroup struct {
Name string
Targets []string
CreatedAt time.Time
UpdatedAt time.Time
}
type CheckType struct {
Name string
Enabled bool
TargetGroups []string
CreatedAt time.Time
UpdatedAt time.Time
}
// ResolvedCheckType is a check type with its target groups already expanded
// into a flat target list — what AssignmentForValidator hands to an agent.
type ResolvedCheckType struct {
Type string
Targets []string
}
// SubmitIPsResult categorizes how each address in a SubmitIPs call was
// handled.
type SubmitIPsResult struct {
Added []string
Requeued []string
Reordered []string
SkippedInProgress []string
}
// DeleteIPsResult categorizes how each address in a DeleteIPs call (or a
// ClearQueue call, which is DeleteIPs given every currently queued address)
// was handled.
type DeleteIPsResult struct {
Deleted []string
NotFound []string
}
// Settings is the singleton row of orchestrator-wide scalar knobs that are
// admin-configurable at runtime (see queries_settings.go).
type Settings struct {
FIPSettleSeconds int
// HistoryRetentionCycles caps how many recent check cycles are kept per
// registry address (see PruneRegistryHistory); 0 means unlimited.
HistoryRetentionCycles int
// SelfCheckMaxAttempts is the ceiling of failed self-checks per address
// (1..MaxSelfCheckMaxAttempts); reaching it gives the address the verdict
// fail (see FailSelfCheck).
SelfCheckMaxAttempts int
CreatedAt time.Time
UpdatedAt time.Time
}
// Bounds of Settings.SelfCheckMaxAttempts.
const (
MinSelfCheckMaxAttempts = 1
MaxSelfCheckMaxAttempts = 50
)
// InboundChecksSettings is the singleton row describing what the prober
// checks on every site for every in-flight IP (TCP ports + optional ICMP).
// Admin-configurable at runtime (see queries_inbound.go).
type InboundChecksSettings struct {
Ports []int
ICMP bool
CreatedAt time.Time
UpdatedAt time.Time
}
// Auto-cycle phases and outcomes (see AutoCycle).
const (
AutoCyclePhaseIdle = "idle"
AutoCyclePhaseRunning = "running"
AutoCyclePhaseWaiting = "waiting"
// AutoCyclePhaseScanning: the queue is being cleared and the floating IPs
// are being (re)discovered and enqueued by the background scan job.
AutoCyclePhaseScanning = "scanning"
AutoCycleOutcomeCompleted = "completed"
AutoCycleOutcomeNoFreeIPs = "no_free_ips"
AutoCycleOutcomeTimeout = "timeout"
AutoCycleOutcomeError = "error"
AutoCycleOutcomeStopped = "stopped"
)
// AutoCycle is the singleton row describing the automatic check cycle:
// its configuration (Enabled, IntervalSeconds, MaxRunSeconds) and its
// persisted runtime state (Phase and timestamps). MaxRunSeconds == 0 means
// no limit on how long a run may wait for checks to finish.
type AutoCycle struct {
Enabled bool
IntervalSeconds int
MaxRunSeconds int
Phase string
RunStartedAt *time.Time
NextRunAt *time.Time
LastRunStartedAt *time.Time
LastRunFinishedAt *time.Time
LastOutcome string
LastError string
LastScannedFree int
RunsTotal int
}
// AutoCycleState is a full replacement of the runtime-state columns of the
// auto_cycle row (everything except configuration and enabled flag).
type AutoCycleState struct {
Phase string
RunStartedAt *time.Time
NextRunAt *time.Time
LastRunStartedAt *time.Time
LastRunFinishedAt *time.Time
LastOutcome string
LastError string
LastScannedFree int
RunsTotal int
}