Add optional automatic check cycle (clear queue -> scan FIPs -> wait -> repeat)

An admin-controlled scenario that repeats what the operator does by hand:
clear the IP queue, scan and enqueue all free Floating IPs, wait until every
queued address reaches a terminal state (so results are in the Registry),
then wait a configurable interval and start over.

- control-api: new auto_cycle singleton table (migration 0008) holding
  enabled/interval/max-run settings and persisted phase state, so the cycle
  survives restarts; engine in orchestrator/autocycle.go driven from the
  existing loop tick with an injectable "now" for deterministic tests.
- Interval (default 1h, min 60s) and max wait (default unlimited, timeout
  outcome) are runtime settings, never hardcoded.
- The periodic fip_scan_interval_seconds scan is skipped while the cycle is
  enabled. An emptied queue mid-cycle counts as finished; stopping during
  the pause keeps the last cycle's outcome.
- API: GET/PUT /api/v1/admin/auto-cycle, POST .../start, POST .../stop.
- admin-dashboard: "Автоматический цикл" panel on /settings and an
  "Автоцикл активен" indicator on /overview.
- Tests for db, orchestrator, httpapi and dashboard; run-local-e2e.sh now
  exercises a full auto cycle; docs updated; bin/ rebuilt with refreshed
  SHA256SUMS.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

internal/db/db.go

@@ -34,6 +34,9 @@ var proberHeartbeatSchema string
 //go:embed migrations/0007_ip_registry.sql
 var ipRegistrySchema string
 
+//go:embed migrations/0008_auto_cycle.sql
+var autoCycleSchema string
+
 // migrations is the ordered list of schema versions. Each entry's SQL is
 // applied, in order, for any version greater than the database's current
 // PRAGMA user_version — so a fresh database walks the whole list and an
@@ -49,6 +52,7 @@ var migrations = []struct {
 	{5, unboundedSitesSchema},
 	{6, proberHeartbeatSchema},
 	{7, ipRegistrySchema},
+	{8, autoCycleSchema},
 }
 
 type DB struct {

internal/db/migrations/0008_auto_cycle.sql

@@ -0,0 +1,24 @@
+-- Automatic check cycle (see docs/USAGE.md): optionally repeats the
+-- "clear queue -> scan floating IPs -> wait for all checks to finish ->
+-- wait interval" scenario. Singleton row, kept apart from `settings` so that
+-- a full-overwrite PUT /config/orchestrator never resets these fields.
+-- State is persisted so the cycle survives a control-api restart.
+
+CREATE TABLE auto_cycle (
+  id                   INTEGER PRIMARY KEY CHECK (id = 1),
+  enabled              INTEGER NOT NULL DEFAULT 0,
+  interval_seconds     INTEGER NOT NULL DEFAULT 3600,
+  max_run_seconds      INTEGER NOT NULL DEFAULT 0,
+  phase                TEXT NOT NULL DEFAULT 'idle',
+  run_started_at       TIMESTAMP,
+  next_run_at          TIMESTAMP,
+  last_run_started_at  TIMESTAMP,
+  last_run_finished_at TIMESTAMP,
+  last_outcome         TEXT NOT NULL DEFAULT '',
+  last_error           TEXT NOT NULL DEFAULT '',
+  last_scanned_free    INTEGER NOT NULL DEFAULT 0,
+  runs_total           INTEGER NOT NULL DEFAULT 0,
+  updated_at           TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+INSERT INTO auto_cycle (id, enabled, interval_seconds, max_run_seconds, phase) VALUES (1, 0, 3600, 0, 'idle');

internal/db/models.go

@@ -226,3 +226,49 @@ type InboundChecksSettings struct {
 	CreatedAt time.Time
 	UpdatedAt time.Time
 }
+
+// Auto-cycle phases and outcomes (see AutoCycle).
+const (
+	AutoCyclePhaseIdle    = "idle"
+	AutoCyclePhaseRunning = "running"
+	AutoCyclePhaseWaiting = "waiting"
+
+	AutoCycleOutcomeCompleted = "completed"
+	AutoCycleOutcomeNoFreeIPs = "no_free_ips"
+	AutoCycleOutcomeTimeout   = "timeout"
+	AutoCycleOutcomeError     = "error"
+	AutoCycleOutcomeStopped   = "stopped"
+)
+
+// AutoCycle is the singleton row describing the automatic check cycle:
+// its configuration (Enabled, IntervalSeconds, MaxRunSeconds) and its
+// persisted runtime state (Phase and timestamps). MaxRunSeconds == 0 means
+// no limit on how long a run may wait for checks to finish.
+type AutoCycle struct {
+	Enabled           bool
+	IntervalSeconds   int
+	MaxRunSeconds     int
+	Phase             string
+	RunStartedAt      *time.Time
+	NextRunAt         *time.Time
+	LastRunStartedAt  *time.Time
+	LastRunFinishedAt *time.Time
+	LastOutcome       string
+	LastError         string
+	LastScannedFree   int
+	RunsTotal         int
+}
+
+// AutoCycleState is a full replacement of the runtime-state columns of the
+// auto_cycle row (everything except configuration and enabled flag).
+type AutoCycleState struct {
+	Phase             string
+	RunStartedAt      *time.Time
+	NextRunAt         *time.Time
+	LastRunStartedAt  *time.Time
+	LastRunFinishedAt *time.Time
+	LastOutcome       string
+	LastError         string
+	LastScannedFree   int
+	RunsTotal         int
+}

internal/db/queries_autocycle.go

@@ -0,0 +1,121 @@
+package db
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+)
+
+// MinAutoCycleIntervalSeconds is the smallest allowed pause between
+// automatic cycles; it protects the OpenStack API from overly frequent
+// floating-IP scans.
+const MinAutoCycleIntervalSeconds = 60
+
+// GetAutoCycle returns the singleton auto_cycle row. Migration 0008 inserts
+// it, so sql.ErrNoRows would indicate a broken database, not a normal case.
+func (d *DB) GetAutoCycle(ctx context.Context) (AutoCycle, error) {
+	var a AutoCycle
+	var enabled int
+	var runStarted, nextRun, lastStarted, lastFinished sql.NullString
+	err := d.QueryRowContext(ctx, `
+		SELECT enabled, interval_seconds, max_run_seconds, phase,
+		       run_started_at, next_run_at, last_run_started_at, last_run_finished_at,
+		       last_outcome, last_error, last_scanned_free, runs_total
+		FROM auto_cycle WHERE id=1
+	`).Scan(&enabled, &a.IntervalSeconds, &a.MaxRunSeconds, &a.Phase,
+		&runStarted, &nextRun, &lastStarted, &lastFinished,
+		&a.LastOutcome, &a.LastError, &a.LastScannedFree, &a.RunsTotal)
+	if err != nil {
+		return AutoCycle{}, err
+	}
+	a.Enabled = enabled != 0
+	if a.RunStartedAt, err = nullStringToTimePtr(runStarted); err != nil {
+		return AutoCycle{}, err
+	}
+	if a.NextRunAt, err = nullStringToTimePtr(nextRun); err != nil {
+		return AutoCycle{}, err
+	}
+	if a.LastRunStartedAt, err = nullStringToTimePtr(lastStarted); err != nil {
+		return AutoCycle{}, err
+	}
+	if a.LastRunFinishedAt, err = nullStringToTimePtr(lastFinished); err != nil {
+		return AutoCycle{}, err
+	}
+	return a, nil
+}
+
+// SetAutoCycleParams updates the auto-cycle configuration. A nil argument
+// leaves the corresponding field unchanged (partial update). Validation:
+// interval >= MinAutoCycleIntervalSeconds, maxRun >= 0 (0 = no limit).
+func (d *DB) SetAutoCycleParams(ctx context.Context, intervalSeconds, maxRunSeconds *int) error {
+	if intervalSeconds != nil && *intervalSeconds < MinAutoCycleIntervalSeconds {
+		return fmt.Errorf("interval_seconds must be >= %d: %w", MinAutoCycleIntervalSeconds, ErrValidation)
+	}
+	if maxRunSeconds != nil && *maxRunSeconds < 0 {
+		return fmt.Errorf("max_run_seconds must be >= 0: %w", ErrValidation)
+	}
+	now := timeToDB(Now())
+	if intervalSeconds != nil {
+		if _, err := d.ExecContext(ctx, `
+			UPDATE auto_cycle SET interval_seconds=?, updated_at=? WHERE id=1
+		`, *intervalSeconds, now); err != nil {
+			return err
+		}
+	}
+	if maxRunSeconds != nil {
+		if _, err := d.ExecContext(ctx, `
+			UPDATE auto_cycle SET max_run_seconds=?, updated_at=? WHERE id=1
+		`, *maxRunSeconds, now); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// SetAutoCycleEnabled flips the enabled flag and resets the runtime phase to
+// idle (clearing run_started_at). nextRunAt sets next_run_at (nil clears it).
+// A non-empty outcome is stored as last_outcome (and last_error is cleared);
+// an empty one leaves the last outcome untouched.
+func (d *DB) SetAutoCycleEnabled(ctx context.Context, enabled bool, nextRunAt *time.Time, outcome string) error {
+	en := 0
+	if enabled {
+		en = 1
+	}
+	now := timeToDB(Now())
+	_, err := d.ExecContext(ctx, `
+		UPDATE auto_cycle SET
+			enabled=?,
+			phase=?,
+			run_started_at=NULL,
+			next_run_at=?,
+			last_outcome=CASE WHEN ?='' THEN last_outcome ELSE ? END,
+			last_error=CASE WHEN ?='' THEN last_error ELSE '' END,
+			updated_at=?
+		WHERE id=1
+	`, en, AutoCyclePhaseIdle, timePtrToDB(nextRunAt), outcome, outcome, outcome, now)
+	return err
+}
+
+// UpdateAutoCycleState replaces all runtime-state columns (phase,
+// timestamps, last outcome/error, counters) in one statement. It does not
+// touch the configuration or the enabled flag.
+func (d *DB) UpdateAutoCycleState(ctx context.Context, s AutoCycleState) error {
+	switch s.Phase {
+	case AutoCyclePhaseIdle, AutoCyclePhaseRunning, AutoCyclePhaseWaiting:
+	default:
+		return fmt.Errorf("invalid auto-cycle phase %q: %w", s.Phase, ErrValidation)
+	}
+	now := timeToDB(Now())
+	_, err := d.ExecContext(ctx, `
+		UPDATE auto_cycle SET
+			phase=?, run_started_at=?, next_run_at=?,
+			last_run_started_at=?, last_run_finished_at=?,
+			last_outcome=?, last_error=?, last_scanned_free=?, runs_total=?,
+			updated_at=?
+		WHERE id=1
+	`, s.Phase, timePtrToDB(s.RunStartedAt), timePtrToDB(s.NextRunAt),
+		timePtrToDB(s.LastRunStartedAt), timePtrToDB(s.LastRunFinishedAt),
+		s.LastOutcome, s.LastError, s.LastScannedFree, s.RunsTotal, now)
+	return err
+}

internal/db/queries_autocycle_test.go

@@ -0,0 +1,156 @@
+package db
+
+import (
+	"errors"
+	"testing"
+	"time"
+)
+
+func TestAutoCycleDefaultsFromMigration(t *testing.T) {
+	d, ctx := newTestDB(t)
+
+	a, err := d.GetAutoCycle(ctx)
+	if err != nil {
+		t.Fatalf("get auto cycle: %v", err)
+	}
+	if a.Enabled {
+		t.Fatalf("expected disabled by default")
+	}
+	if a.IntervalSeconds != 3600 {
+		t.Fatalf("expected default interval 3600, got %d", a.IntervalSeconds)
+	}
+	if a.MaxRunSeconds != 0 {
+		t.Fatalf("expected default max_run_seconds 0, got %d", a.MaxRunSeconds)
+	}
+	if a.Phase != AutoCyclePhaseIdle {
+		t.Fatalf("expected phase idle, got %q", a.Phase)
+	}
+	if a.RunStartedAt != nil || a.NextRunAt != nil || a.LastRunStartedAt != nil || a.LastRunFinishedAt != nil {
+		t.Fatalf("expected all timestamps unset, got %+v", a)
+	}
+	if a.LastOutcome != "" || a.LastError != "" || a.LastScannedFree != 0 || a.RunsTotal != 0 {
+		t.Fatalf("expected empty last-run info, got %+v", a)
+	}
+}
+
+func TestAutoCycleParamsRoundTripAndPartialUpdate(t *testing.T) {
+	d, ctx := newTestDB(t)
+
+	interval, maxRun := 120, 900
+	if err := d.SetAutoCycleParams(ctx, &interval, &maxRun); err != nil {
+		t.Fatalf("set params: %v", err)
+	}
+	a, err := d.GetAutoCycle(ctx)
+	if err != nil {
+		t.Fatalf("get: %v", err)
+	}
+	if a.IntervalSeconds != 120 || a.MaxRunSeconds != 900 {
+		t.Fatalf("expected 120/900, got %d/%d", a.IntervalSeconds, a.MaxRunSeconds)
+	}
+
+	// Partial: only max_run_seconds changes.
+	newMax := 0
+	if err := d.SetAutoCycleParams(ctx, nil, &newMax); err != nil {
+		t.Fatalf("partial set: %v", err)
+	}
+	a, _ = d.GetAutoCycle(ctx)
+	if a.IntervalSeconds != 120 || a.MaxRunSeconds != 0 {
+		t.Fatalf("expected 120/0 after partial update, got %d/%d", a.IntervalSeconds, a.MaxRunSeconds)
+	}
+
+	// Partial: only interval changes.
+	newInterval := 60
+	if err := d.SetAutoCycleParams(ctx, &newInterval, nil); err != nil {
+		t.Fatalf("partial set interval: %v", err)
+	}
+	a, _ = d.GetAutoCycle(ctx)
+	if a.IntervalSeconds != 60 || a.MaxRunSeconds != 0 {
+		t.Fatalf("expected 60/0, got %d/%d", a.IntervalSeconds, a.MaxRunSeconds)
+	}
+}
+
+func TestAutoCycleParamsValidation(t *testing.T) {
+	d, ctx := newTestDB(t)
+
+	low := 59
+	if err := d.SetAutoCycleParams(ctx, &low, nil); !errors.Is(err, ErrValidation) {
+		t.Fatalf("expected ErrValidation for interval 59, got %v", err)
+	}
+	neg := -1
+	if err := d.SetAutoCycleParams(ctx, nil, &neg); !errors.Is(err, ErrValidation) {
+		t.Fatalf("expected ErrValidation for negative max_run_seconds, got %v", err)
+	}
+	// A rejected request must not partially apply the valid half.
+	ok := 300
+	if err := d.SetAutoCycleParams(ctx, &ok, &neg); !errors.Is(err, ErrValidation) {
+		t.Fatalf("expected ErrValidation, got %v", err)
+	}
+	a, _ := d.GetAutoCycle(ctx)
+	if a.IntervalSeconds != 3600 {
+		t.Fatalf("interval must stay 3600 after rejected update, got %d", a.IntervalSeconds)
+	}
+}
+
+func TestAutoCycleEnabledAndStateRoundTrip(t *testing.T) {
+	d, ctx := newTestDB(t)
+
+	next := Now()
+	if err := d.SetAutoCycleEnabled(ctx, true, &next, ""); err != nil {
+		t.Fatalf("enable: %v", err)
+	}
+	a, _ := d.GetAutoCycle(ctx)
+	if !a.Enabled || a.Phase != AutoCyclePhaseIdle {
+		t.Fatalf("expected enabled+idle, got %+v", a)
+	}
+	if a.NextRunAt == nil || !a.NextRunAt.Equal(next) {
+		t.Fatalf("expected next_run_at=%v, got %v", next, a.NextRunAt)
+	}
+
+	started := next.Add(time.Second)
+	finished := next.Add(time.Minute)
+	nextRun := finished.Add(time.Hour)
+	st := AutoCycleState{
+		Phase:             AutoCyclePhaseWaiting,
+		NextRunAt:         &nextRun,
+		LastRunStartedAt:  &started,
+		LastRunFinishedAt: &finished,
+		LastOutcome:       AutoCycleOutcomeCompleted,
+		LastError:         "boom",
+		LastScannedFree:   3,
+		RunsTotal:         7,
+	}
+	if err := d.UpdateAutoCycleState(ctx, st); err != nil {
+		t.Fatalf("update state: %v", err)
+	}
+	a, _ = d.GetAutoCycle(ctx)
+	if !a.Enabled {
+		t.Fatalf("UpdateAutoCycleState must not touch the enabled flag")
+	}
+	if a.Phase != AutoCyclePhaseWaiting || a.LastOutcome != AutoCycleOutcomeCompleted ||
+		a.LastError != "boom" || a.LastScannedFree != 3 || a.RunsTotal != 7 {
+		t.Fatalf("state mismatch: %+v", a)
+	}
+	if a.RunStartedAt != nil {
+		t.Fatalf("expected run_started_at nil, got %v", a.RunStartedAt)
+	}
+	if a.NextRunAt == nil || !a.NextRunAt.Equal(nextRun) ||
+		a.LastRunStartedAt == nil || !a.LastRunStartedAt.Equal(started) ||
+		a.LastRunFinishedAt == nil || !a.LastRunFinishedAt.Equal(finished) {
+		t.Fatalf("timestamp mismatch: %+v", a)
+	}
+
+	// Disable with an outcome: phase back to idle, outcome recorded,
+	// last_error cleared, counters preserved.
+	if err := d.SetAutoCycleEnabled(ctx, false, nil, AutoCycleOutcomeStopped); err != nil {
+		t.Fatalf("disable: %v", err)
+	}
+	a, _ = d.GetAutoCycle(ctx)
+	if a.Enabled || a.Phase != AutoCyclePhaseIdle || a.LastOutcome != AutoCycleOutcomeStopped ||
+		a.LastError != "" || a.NextRunAt != nil || a.RunsTotal != 7 {
+		t.Fatalf("unexpected state after disable: %+v", a)
+	}
+
+	if err := d.UpdateAutoCycleState(ctx, AutoCycleState{Phase: "bogus"}); !errors.Is(err, ErrValidation) {
+		t.Fatalf("expected ErrValidation for bogus phase, got %v", err)
+	}
+}