Scan floating IPs in the background, page by page, so thousands of addresses work

The "Scan Floating IP" button failed with a client timeout: the project now
holds ~6.4k floating IPs and the scan listed them all in one unpaginated,
timeout-less Neutron request on the HTTP request context.

openstack: ListFreeFloatingIPs reads marker-based pages (fields= keeps them
small) with per-page retry/backoff on transport errors, 5xx and 429, and every
request now has a timeout (also ends hangs inside the orchestrator tick).

orchestrator: the scan is a single-flight background job on the process
context with progress (clearing/listing/enqueuing/done/error), dry_run, full
discovery before anything is enqueued, then SubmitIPs in chunks of 500 in
ascending IP order; a failed read leaves the queue untouched. The auto-cycle
gets a "scanning" phase that polls the job, so the control loop and
autoCycleMu are never held across OpenStack/DB work; it recovers after a
restart and waits for (instead of adopting) a scan started by someone else.

db: migration 0009 (indexes), paged ListIPsPage/ListRegistryPage, GROUP BY
counters, EXISTS completion check, set-based ClearAllIPs.

API: POST /admin/ips/scan -> 202 (dry_run, wait), GET /admin/ips/scan, paging
and filters on /admin/ips and /admin/registry (bare arrays without limit),
results_by_overall in /admin/status.

dashboard: scan progress panel and dry-run button, paginated /ips and
/registry with server-side filters, Overview on counters and capped lists
with progress/ETA, "select all N by filter", hx-params fix for per-row
buttons, real counts in confirmations.

Also: docs (API, USAGE, DASHBOARD, README), plan and review under
docs/changes/, bin/ rebuilt with new SHA256SUMS.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ayurishchevandClaude Sonnet 5.5 committed 2026-10-01 19:31:11 +03:00
1 parent debf2afed2
commit aff8fe38b5
61 files changed
+5833 -536

No files matched your search

+151 -6
View File
@@ -1,6 +1,7 @@
package dashboard
import (
"fmt"
"strconv"
"time"
)
@@ -20,6 +21,27 @@ type statusResponse struct {
TotalIPs int `json:"total_ips"`
IPsByState map[string]int `json:"ips_by_state"`
TotalValidators int `json:"total_validators"`
// ResultsByOverall counts finished addresses by overall result
// (pass/partial/fail/cancelled).
ResultsByOverall map[string]int `json:"results_by_overall"`
}
// Terminal queue states: the address needs no further processing. Shared by
// the overview progress indicator; "occupied" counts as terminal too (the
// check cycle never ran because the floating IP was already bound).
var terminalStates = []string{"done", "failed", "occupied"}
// activeStates are the states of an address that is being worked on right
// now (everything between "queued" and a terminal state).
var activeStates = []string{"assigning_fip", "awaiting_self_check", "checking", "aggregating"}
// sumStates adds up the counts of the given states in a status breakdown.
func sumStates(byState map[string]int, states []string) int {
n := 0
for _, s := range states {
n += byState[s]
}
return n
}
type ipQueueItem struct {
@@ -99,14 +121,135 @@ type deleteIPsResponse struct {
type clearQueueResponse struct {
Deleted []string `json:"deleted"`
Count int `json:"count"`
}
type scanIPsResponse struct {
ScannedFree int `json:"scanned_free"`
Added []string `json:"added"`
Requeued []string `json:"requeued"`
Reordered []string `json:"reordered"`
SkippedInProgress []string `json:"skipped_in_progress"`
// ipsPage is the paginated envelope of GET /admin/ips (sent when the request
// carries `limit`).
type ipsPage struct {
Items []ipQueueItem `json:"items"`
Total int `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
// registryPage is the paginated envelope of GET /admin/registry.
type registryPage struct {
Items []registryItem `json:"items"`
Total int `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
// scanStatusDTO is the state of control-api's background floating-IP scan job
// (POST/GET /api/v1/admin/ips/scan).
type scanStatusDTO struct {
State string `json:"state"`
Running bool `json:"running"`
DryRun bool `json:"dry_run"`
Pages int `json:"pages"`
Discovered int `json:"discovered"`
Free int `json:"free"`
Added int `json:"added"`
Requeued int `json:"requeued"`
Reordered int `json:"reordered"`
SkippedInProgress int `json:"skipped_in_progress"`
StartedAt *time.Time `json:"started_at"`
FinishedAt *time.Time `json:"finished_at"`
Error string `json:"error"`
}
// Finished reports a job that has run to a terminal state (as opposed to
// "idle" = never started, or still running).
func (s scanStatusDTO) Finished() bool {
if s.Running {
return false
}
switch s.State {
case "done", "error", "cancelled":
return true
}
return false
}
// StateLabel is the Russian description of the job's state.
func (s scanStatusDTO) StateLabel() string {
switch s.State {
case "clearing":
return "очистка"
case "listing":
return "читаются страницы"
case "enqueuing":
return "ставятся в очередь"
case "done":
return "готово"
case "error":
return "ошибка"
case "cancelled":
return "отменено"
case "idle", "":
return "нет активного сканирования"
default:
return s.State
}
}
// PillClass picks the pill style for the state.
func (s scanStatusDTO) PillClass() string {
switch s.State {
case "done":
return "pill-success"
case "error":
return "pill-danger"
case "cancelled":
return "pill-cancel"
case "clearing", "listing", "enqueuing":
return "pill-info"
default:
return "pill-neutral"
}
}
// Handled is how many of the free addresses the enqueuing phase has already
// processed.
func (s scanStatusDTO) Handled() int {
return s.Added + s.Requeued + s.Reordered + s.SkippedInProgress
}
// Indeterminate is true while the amount of work is not known yet.
func (s scanStatusDTO) Indeterminate() bool {
return s.State == "clearing" || s.State == "listing" || (s.State == "enqueuing" && s.Free <= 0)
}
// Elapsed is the human-readable run time: until now while running, until
// finished_at afterwards; empty when the job never started.
func (s scanStatusDTO) Elapsed() string {
if s.StartedAt == nil {
return ""
}
end := time.Now()
if !s.Running && s.FinishedAt != nil {
end = *s.FinishedAt
}
return fmtDuration(end.Sub(*s.StartedAt))
}
// fmtDuration renders a duration in Russian as "2 ч 05 мин", "3 мин 07 с" or
// "42 с" — coarse on purpose (progress/ETA display).
func fmtDuration(d time.Duration) string {
if d < 0 {
d = 0
}
sec := int(d.Round(time.Second) / time.Second)
h, m, sc := sec/3600, (sec%3600)/60, sec%60
switch {
case h > 0:
return fmt.Sprintf("%d ч %02d мин", h, m)
case m > 0:
return fmt.Sprintf("%d мин %02d с", m, sc)
default:
return fmt.Sprintf("%d с", sc)
}
}
// registryItem is one row of the durable per-address registry — see
@@ -200,6 +343,8 @@ func (a autoCycleDTO) MaxRunMinutes() string { return secondsToMinutes(a.MaxRu
// PhaseLabel is the Russian description of the current phase.
func (a autoCycleDTO) PhaseLabel() string {
switch a.Phase {
case "scanning":
return "сканирование Floating IP"
case "running":
return "идёт проверка"
case "waiting":