package orchestrator import ( "context" "errors" "strings" "testing" "time" "cloudipvalidator/internal/db" ) func getAutoCycle(t *testing.T, d *db.DB) db.AutoCycle { t.Helper() ac, err := d.GetAutoCycle(context.Background()) if err != nil { t.Fatalf("get auto cycle: %v", err) } return ac } func setAutoCycleParams(t *testing.T, d *db.DB, interval, maxRun int) { t.Helper() if err := d.SetAutoCycleParams(context.Background(), &interval, &maxRun); err != nil { t.Fatalf("set auto cycle params: %v", err) } } // finishAllIPs simulates completed checks by moving every queued address to // the given terminal state directly (the full check pipeline is covered by // TestHappyPath). func finishAllIPs(t *testing.T, d *db.DB, state string) { t.Helper() if _, err := d.ExecContext(context.Background(), `UPDATE ip_queue SET state=?`, state); err != nil { t.Fatalf("finish ips: %v", err) } } func queuedAddresses(t *testing.T, d *db.DB) map[string]string { t.Helper() items, err := d.ListIPs(context.Background()) if err != nil { t.Fatalf("list ips: %v", err) } out := make(map[string]string, len(items)) for _, it := range items { out[it.IPAddress] = it.State } return out } func countEvents(t *testing.T, d *db.DB, eventType string) int { t.Helper() var n int if err := d.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM events WHERE event_type=?`, eventType).Scan(&n); err != nil { t.Fatalf("count events: %v", err) } return n } func TestAutoCycleDisabledIsNoOp(t *testing.T) { ctx := context.Background() o, d, mock := newTestOrchestrator(t, 180) mock.Seed("fip-1", "1.1.1.1", "svc") if err := d.SeedQueue(ctx, []string{"9.9.9.9"}); err != nil { t.Fatalf("seed queue: %v", err) } o.autoCycleStep(ctx, db.Now()) ac := getAutoCycle(t, d) if ac.Enabled || ac.Phase != db.AutoCyclePhaseIdle { t.Fatalf("expected disabled+idle, got %+v", ac) } if got := queuedAddresses(t, d); len(got) != 1 || got["9.9.9.9"] != db.IPQueued { t.Fatalf("queue must be untouched while disabled, got %v", got) } } func TestAutoCycleDisabledResetsStalePhase(t *testing.T) { ctx := context.Background() o, d, _ := newTestOrchestrator(t, 180) now := db.Now() if err := d.UpdateAutoCycleState(ctx, db.AutoCycleState{Phase: db.AutoCyclePhaseRunning, RunStartedAt: &now}); err != nil { t.Fatalf("update state: %v", err) } o.autoCycleStep(ctx, now) if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseIdle || ac.RunStartedAt != nil { t.Fatalf("expected phase reset to idle, got %+v", ac) } } func TestAutoCycleStartClearsQueueAndScans(t *testing.T) { ctx := context.Background() o, d, mock := newTestOrchestrator(t, 180) mock.Seed("fip-1", "1.1.1.1", "svc") mock.Seed("fip-2", "2.2.2.2", "svc") mock.SeedWithPort("fip-3", "3.3.3.3", "svc", "someone-elses-port") if err := d.SeedQueue(ctx, []string{"9.9.9.9"}); err != nil { t.Fatalf("seed queue: %v", err) } if err := o.StartAutoCycle(ctx); err != nil { t.Fatalf("start: %v", err) } ac := getAutoCycle(t, d) if !ac.Enabled || ac.Phase != db.AutoCyclePhaseIdle || ac.NextRunAt == nil { t.Fatalf("expected enabled+idle with next_run_at set after start, got %+v", ac) } now := db.Now() o.autoCycleStep(ctx, now) got := queuedAddresses(t, d) if _, stale := got["9.9.9.9"]; stale { t.Fatalf("old queue entry must be cleared, got %v", got) } if len(got) != 2 || got["1.1.1.1"] != db.IPQueued || got["2.2.2.2"] != db.IPQueued { t.Fatalf("expected the two free FIPs queued, got %v", got) } ac = getAutoCycle(t, d) if ac.Phase != db.AutoCyclePhaseRunning { t.Fatalf("expected running, got %s", ac.Phase) } if ac.RunStartedAt == nil || !ac.RunStartedAt.Equal(now) { t.Fatalf("expected run_started_at=%v, got %v", now, ac.RunStartedAt) } if ac.LastRunStartedAt == nil || !ac.LastRunStartedAt.Equal(now) { t.Fatalf("expected last_run_started_at=%v, got %v", now, ac.LastRunStartedAt) } if ac.LastScannedFree != 2 { t.Fatalf("expected last_scanned_free=2, got %d", ac.LastScannedFree) } if countEvents(t, d, "queue_cleared") != 1 || countEvents(t, d, "fip_scan") != 1 { t.Fatalf("expected queue_cleared and fip_scan events") } if countEvents(t, d, "auto_cycle_started") != 2 { // enable + cycle start t.Fatalf("expected two auto_cycle_started events, got %d", countEvents(t, d, "auto_cycle_started")) } } func TestAutoCycleWaitsWhileChecksInProgressAndTickPicksUpQueue(t *testing.T) { ctx := context.Background() o, d, mock := newTestOrchestrator(t, 180) mock.Seed("fip-1", "1.1.1.1", "svc") mock.Seed("fip-2", "2.2.2.2", "svc") if err := d.RegisterValidator(ctx, "validator-1", "host-1", "port-1", "v0.1"); err != nil { t.Fatalf("register validator: %v", err) } if err := o.StartAutoCycle(ctx); err != nil { t.Fatalf("start: %v", err) } t0 := db.Now() o.autoCycleStep(ctx, t0) // Existing Tick logic starts the checks on its own. o.Tick(ctx) states := queuedAddresses(t, d) inProgress := 0 for _, s := range states { if s == db.IPAwaitingSelfCheck { inProgress++ } } if inProgress != 1 { t.Fatalf("expected exactly one address claimed by Tick, got %v", states) } o.autoCycleStep(ctx, t0.Add(time.Minute)) if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseRunning || ac.RunsTotal != 0 { t.Fatalf("expected still running, got %+v", ac) } // One address done, the other still queued: still not finished. if _, err := d.ExecContext(ctx, `UPDATE ip_queue SET state=? WHERE ip_address=?`, db.IPDone, "1.1.1.1"); err != nil { t.Fatalf("update: %v", err) } if _, err := d.ExecContext(ctx, `UPDATE ip_queue SET state=? WHERE ip_address=?`, db.IPQueued, "2.2.2.2"); err != nil { t.Fatalf("update: %v", err) } o.autoCycleStep(ctx, t0.Add(2*time.Minute)) if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseRunning { t.Fatalf("expected still running with a queued address left, got %+v", ac) } } func TestAutoCycleCompletesAndRepeatsAfterInterval(t *testing.T) { ctx := context.Background() o, d, mock := newTestOrchestrator(t, 180) mock.Seed("fip-1", "1.1.1.1", "svc") mock.Seed("fip-2", "2.2.2.2", "svc") setAutoCycleParams(t, d, 600, 0) if err := o.StartAutoCycle(ctx); err != nil { t.Fatalf("start: %v", err) } t0 := db.Now() o.autoCycleStep(ctx, t0) // done, failed and occupied are all terminal. if _, err := d.ExecContext(ctx, `UPDATE ip_queue SET state=? WHERE ip_address=?`, db.IPDone, "1.1.1.1"); err != nil { t.Fatalf("update: %v", err) } if _, err := d.ExecContext(ctx, `UPDATE ip_queue SET state=? WHERE ip_address=?`, db.IPOccupied, "2.2.2.2"); err != nil { t.Fatalf("update: %v", err) } t1 := t0.Add(5 * time.Minute) o.autoCycleStep(ctx, t1) ac := getAutoCycle(t, d) if ac.Phase != db.AutoCyclePhaseWaiting || ac.LastOutcome != db.AutoCycleOutcomeCompleted { t.Fatalf("expected waiting/completed, got %+v", ac) } if ac.RunsTotal != 1 { t.Fatalf("expected runs_total=1, got %d", ac.RunsTotal) } wantNext := t1.Add(600 * time.Second) if ac.NextRunAt == nil || !ac.NextRunAt.Equal(wantNext) { t.Fatalf("expected next_run_at=%v (completion + interval), got %v", wantNext, ac.NextRunAt) } if ac.LastRunFinishedAt == nil || !ac.LastRunFinishedAt.Equal(t1) { t.Fatalf("expected last_run_finished_at=%v, got %v", t1, ac.LastRunFinishedAt) } if ac.RunStartedAt != nil { t.Fatalf("expected run_started_at cleared, got %v", ac.RunStartedAt) } if countEvents(t, d, "auto_cycle_completed") != 1 { t.Fatalf("expected one auto_cycle_completed event") } // Before next_run_at: nothing happens, the finished queue is kept. o.autoCycleStep(ctx, wantNext.Add(-time.Second)) if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseWaiting || ac.RunsTotal != 1 { t.Fatalf("expected still waiting, got %+v", ac) } if got := queuedAddresses(t, d); got["1.1.1.1"] != db.IPDone { t.Fatalf("queue must not be touched while waiting, got %v", got) } // At next_run_at: new cycle starts, queue is rebuilt from scratch. o.autoCycleStep(ctx, wantNext) ac = getAutoCycle(t, d) if ac.Phase != db.AutoCyclePhaseRunning { t.Fatalf("expected running again, got %+v", ac) } if ac.LastRunStartedAt == nil || !ac.LastRunStartedAt.Equal(wantNext) { t.Fatalf("expected last_run_started_at=%v, got %v", wantNext, ac.LastRunStartedAt) } if got := queuedAddresses(t, d); got["1.1.1.1"] != db.IPQueued || got["2.2.2.2"] != db.IPQueued { t.Fatalf("expected both addresses re-queued, got %v", got) } if ac.RunsTotal != 1 { t.Fatalf("runs_total only counts completed cycles, got %d", ac.RunsTotal) } } func TestAutoCycleTimeout(t *testing.T) { ctx := context.Background() o, d, mock := newTestOrchestrator(t, 180) mock.Seed("fip-1", "1.1.1.1", "svc") setAutoCycleParams(t, d, 60, 300) if err := o.StartAutoCycle(ctx); err != nil { t.Fatalf("start: %v", err) } t0 := db.Now() o.autoCycleStep(ctx, t0) o.autoCycleStep(ctx, t0.Add(300*time.Second)) if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseRunning { t.Fatalf("expected still running exactly at the limit, got %+v", ac) } t1 := t0.Add(301 * time.Second) o.autoCycleStep(ctx, t1) ac := getAutoCycle(t, d) if ac.Phase != db.AutoCyclePhaseWaiting || ac.LastOutcome != db.AutoCycleOutcomeTimeout { t.Fatalf("expected waiting/timeout, got %+v", ac) } if ac.RunsTotal != 0 { t.Fatalf("timeout must not count as completed, got runs_total=%d", ac.RunsTotal) } if ac.NextRunAt == nil || !ac.NextRunAt.Equal(t1.Add(60*time.Second)) { t.Fatalf("expected next_run_at=now+interval, got %v", ac.NextRunAt) } if got := queuedAddresses(t, d); got["1.1.1.1"] != db.IPQueued { t.Fatalf("queue must be left untouched on timeout, got %v", got) } if countEvents(t, d, "auto_cycle_timeout") != 1 { t.Fatalf("expected one auto_cycle_timeout event") } } func TestAutoCycleNoLimitNeverTimesOut(t *testing.T) { ctx := context.Background() o, d, mock := newTestOrchestrator(t, 180) mock.Seed("fip-1", "1.1.1.1", "svc") if err := o.StartAutoCycle(ctx); err != nil { t.Fatalf("start: %v", err) } t0 := db.Now() o.autoCycleStep(ctx, t0) o.autoCycleStep(ctx, t0.Add(1000*time.Hour)) if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseRunning { t.Fatalf("max_run_seconds=0 means no limit, got %+v", ac) } } // An operator pressing «Очистить всё» in the middle of a cycle leaves nothing // to wait for; with max_run_seconds=0 the cycle would otherwise hang forever. func TestAutoCycleManualClearMidCycleCompletes(t *testing.T) { ctx := context.Background() o, d, mock := newTestOrchestrator(t, 180) mock.Seed("fip-1", "1.1.1.1", "svc") setAutoCycleParams(t, d, 60, 0) if err := o.StartAutoCycle(ctx); err != nil { t.Fatalf("start: %v", err) } t0 := db.Now() o.autoCycleStep(ctx, t0) if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseRunning { t.Fatalf("expected running after start, got %+v", ac) } if _, err := o.ClearQueue(ctx); err != nil { t.Fatalf("manual clear: %v", err) } t1 := t0.Add(5 * time.Second) o.autoCycleStep(ctx, t1) ac := getAutoCycle(t, d) if ac.Phase != db.AutoCyclePhaseWaiting || ac.LastOutcome != db.AutoCycleOutcomeCompleted { t.Fatalf("expected waiting/completed after the queue was emptied, got %+v", ac) } if ac.NextRunAt == nil || !ac.NextRunAt.Equal(t1.Add(60*time.Second)) { t.Fatalf("expected next_run_at=now+interval, got %v", ac.NextRunAt) } } func TestAutoCycleNoFreeIPs(t *testing.T) { ctx := context.Background() o, d, mock := newTestOrchestrator(t, 180) mock.SeedWithPort("fip-1", "1.1.1.1", "svc", "someone-elses-port") setAutoCycleParams(t, d, 120, 0) if err := o.StartAutoCycle(ctx); err != nil { t.Fatalf("start: %v", err) } now := db.Now() o.autoCycleStep(ctx, now) ac := getAutoCycle(t, d) if ac.Phase != db.AutoCyclePhaseWaiting || ac.LastOutcome != db.AutoCycleOutcomeNoFreeIPs { t.Fatalf("expected waiting/no_free_ips, got %+v", ac) } if ac.LastScannedFree != 0 { t.Fatalf("expected last_scanned_free=0, got %d", ac.LastScannedFree) } if ac.NextRunAt == nil || !ac.NextRunAt.Equal(now.Add(120*time.Second)) { t.Fatalf("expected next_run_at=now+interval, got %v", ac.NextRunAt) } if ac.RunStartedAt != nil { t.Fatalf("run_started_at must stay unset, got %v", ac.RunStartedAt) } } func TestAutoCycleOpenStackErrorRetriesNextInterval(t *testing.T) { ctx := context.Background() o, d, mock := newTestOrchestrator(t, 180) mock.Seed("fip-1", "1.1.1.1", "svc") mock.ListFailure = errors.New("neutron is down") setAutoCycleParams(t, d, 60, 0) if err := o.StartAutoCycle(ctx); err != nil { t.Fatalf("start: %v", err) } t0 := db.Now() o.autoCycleStep(ctx, t0) ac := getAutoCycle(t, d) if ac.Phase != db.AutoCyclePhaseWaiting || ac.LastOutcome != db.AutoCycleOutcomeError { t.Fatalf("expected waiting/error, got %+v", ac) } if !strings.Contains(ac.LastError, "neutron is down") { t.Fatalf("expected last_error to mention the cause, got %q", ac.LastError) } if ac.NextRunAt == nil || !ac.NextRunAt.Equal(t0.Add(60*time.Second)) { t.Fatalf("expected next_run_at=now+interval, got %v", ac.NextRunAt) } if countEvents(t, d, "auto_cycle_error") != 1 { t.Fatalf("expected one auto_cycle_error event") } if !ac.Enabled { t.Fatalf("an error must not disable the auto-cycle") } // OpenStack recovers: the next interval starts a normal cycle and the // stale error is cleared. mock.ListFailure = nil o.autoCycleStep(ctx, t0.Add(60*time.Second)) ac = getAutoCycle(t, d) if ac.Phase != db.AutoCyclePhaseRunning || ac.LastError != "" { t.Fatalf("expected running with cleared error, got %+v", ac) } } func TestAutoCycleStartIsIdempotent(t *testing.T) { ctx := context.Background() o, d, mock := newTestOrchestrator(t, 180) mock.Seed("fip-1", "1.1.1.1", "svc") if err := o.StartAutoCycle(ctx); err != nil { t.Fatalf("start: %v", err) } t0 := db.Now() o.autoCycleStep(ctx, t0) if err := o.StartAutoCycle(ctx); err != nil { t.Fatalf("second start: %v", err) } ac := getAutoCycle(t, d) if !ac.Enabled || ac.Phase != db.AutoCyclePhaseRunning || ac.RunStartedAt == nil || !ac.RunStartedAt.Equal(t0) { t.Fatalf("second start must not restart a running cycle, got %+v", ac) } if countEvents(t, d, "auto_cycle_started") != 2 { // enable + cycle start, not a third t.Fatalf("expected no extra started event, got %d", countEvents(t, d, "auto_cycle_started")) } } func TestAutoCycleStopMidCycle(t *testing.T) { ctx := context.Background() o, d, mock := newTestOrchestrator(t, 180) mock.Seed("fip-1", "1.1.1.1", "svc") if err := d.RegisterValidator(ctx, "validator-1", "host-1", "port-1", "v0.1"); err != nil { t.Fatalf("register validator: %v", err) } if err := o.StartAutoCycle(ctx); err != nil { t.Fatalf("start: %v", err) } t0 := db.Now() o.autoCycleStep(ctx, t0) o.Tick(ctx) // check in flight if err := o.StopAutoCycle(ctx); err != nil { t.Fatalf("stop: %v", err) } ac := getAutoCycle(t, d) if ac.Enabled || ac.Phase != db.AutoCyclePhaseIdle || ac.LastOutcome != db.AutoCycleOutcomeStopped { t.Fatalf("expected disabled/idle/stopped, got %+v", ac) } if got := queuedAddresses(t, d); got["1.1.1.1"] != db.IPAwaitingSelfCheck { t.Fatalf("in-flight check must not be cancelled, got %v", got) } if countEvents(t, d, "auto_cycle_stopped") != 1 { t.Fatalf("expected one auto_cycle_stopped event") } // Further steps do nothing, even far in the future. o.autoCycleStep(ctx, t0.Add(100*time.Hour)) ac = getAutoCycle(t, d) if ac.Enabled || ac.Phase != db.AutoCyclePhaseIdle || ac.RunsTotal != 0 { t.Fatalf("expected no activity after stop, got %+v", ac) } // Stopping again is a harmless no-op. if err := o.StopAutoCycle(ctx); err != nil { t.Fatalf("second stop: %v", err) } if countEvents(t, d, "auto_cycle_stopped") != 1 { t.Fatalf("second stop must not emit another event") } } // Stopping during the pause between cycles must keep the result of the last // finished cycle visible instead of replacing it with "stopped". func TestAutoCycleStopWhileWaitingKeepsLastOutcome(t *testing.T) { ctx := context.Background() o, d, mock := newTestOrchestrator(t, 180) mock.Seed("fip-1", "1.1.1.1", "svc") setAutoCycleParams(t, d, 60, 0) if err := o.StartAutoCycle(ctx); err != nil { t.Fatalf("start: %v", err) } t0 := db.Now() o.autoCycleStep(ctx, t0) finishAllIPs(t, d, db.IPDone) o.autoCycleStep(ctx, t0.Add(5*time.Second)) if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseWaiting || ac.LastOutcome != db.AutoCycleOutcomeCompleted { t.Fatalf("precondition: expected waiting/completed, got %+v", ac) } if err := o.StopAutoCycle(ctx); err != nil { t.Fatalf("stop: %v", err) } ac := getAutoCycle(t, d) if ac.Enabled || ac.Phase != db.AutoCyclePhaseIdle { t.Fatalf("expected disabled/idle, got %+v", ac) } if ac.LastOutcome != db.AutoCycleOutcomeCompleted || ac.RunsTotal != 1 { t.Fatalf("stop in the pause must keep last_outcome=completed, got %+v", ac) } } func TestAutoCycleSurvivesRestart(t *testing.T) { ctx := context.Background() o, d, mock := newTestOrchestrator(t, 180) mock.Seed("fip-1", "1.1.1.1", "svc") setAutoCycleParams(t, d, 300, 0) if err := o.StartAutoCycle(ctx); err != nil { t.Fatalf("start: %v", err) } t0 := db.Now() o.autoCycleStep(ctx, t0) // A fresh Orchestrator on the same database (a control-api restart) // continues the running phase instead of starting over. o2 := &Orchestrator{DB: d, OS: mock, Cfg: o.Cfg, Agg: o.Agg, Log: o.Log} o2.autoCycleStep(ctx, t0.Add(time.Minute)) ac := getAutoCycle(t, d) if ac.Phase != db.AutoCyclePhaseRunning || ac.RunStartedAt == nil || !ac.RunStartedAt.Equal(t0) { t.Fatalf("expected the running phase to continue, got %+v", ac) } finishAllIPs(t, d, db.IPDone) t1 := t0.Add(2 * time.Minute) o2.autoCycleStep(ctx, t1) ac = getAutoCycle(t, d) if ac.Phase != db.AutoCyclePhaseWaiting || ac.LastOutcome != db.AutoCycleOutcomeCompleted { t.Fatalf("expected waiting/completed, got %+v", ac) } // A third instance still honours the persisted next_run_at. o3 := &Orchestrator{DB: d, OS: mock, Cfg: o.Cfg, Agg: o.Agg, Log: o.Log} o3.autoCycleStep(ctx, t1.Add(299*time.Second)) if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseWaiting { t.Fatalf("expected waiting until next_run_at, got %+v", ac) } o3.autoCycleStep(ctx, t1.Add(300*time.Second)) if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseRunning { t.Fatalf("expected a new cycle at next_run_at, got %+v", ac) } }