cloud-ip-validator/internal/orchestrator/autocycle_test.go

package orchestrator

import (
	"context"
	"errors"
	"strings"
	"testing"
	"time"

	"cloudipvalidator/internal/db"
)

func getAutoCycle(t *testing.T, d *db.DB) db.AutoCycle {
	t.Helper()
	ac, err := d.GetAutoCycle(context.Background())
	if err != nil {
		t.Fatalf("get auto cycle: %v", err)
	}
	return ac
}

func setAutoCycleParams(t *testing.T, d *db.DB, interval, maxRun int) {
	t.Helper()
	if err := d.SetAutoCycleParams(context.Background(), &interval, &maxRun); err != nil {
		t.Fatalf("set auto cycle params: %v", err)
	}
}

// finishAllIPs simulates completed checks by moving every queued address to
// the given terminal state directly (the full check pipeline is covered by
// TestHappyPath).
func finishAllIPs(t *testing.T, d *db.DB, state string) {
	t.Helper()
	if _, err := d.ExecContext(context.Background(), `UPDATE ip_queue SET state=?`, state); err != nil {
		t.Fatalf("finish ips: %v", err)
	}
}

func queuedAddresses(t *testing.T, d *db.DB) map[string]string {
	t.Helper()
	items, err := d.ListIPs(context.Background())
	if err != nil {
		t.Fatalf("list ips: %v", err)
	}
	out := make(map[string]string, len(items))
	for _, it := range items {
		out[it.IPAddress] = it.State
	}
	return out
}

func countEvents(t *testing.T, d *db.DB, eventType string) int {
	t.Helper()
	var n int
	if err := d.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM events WHERE event_type=?`, eventType).Scan(&n); err != nil {
		t.Fatalf("count events: %v", err)
	}
	return n
}

func TestAutoCycleDisabledIsNoOp(t *testing.T) {
	ctx := context.Background()
	o, d, mock := newTestOrchestrator(t, 180)
	mock.Seed("fip-1", "1.1.1.1", "svc")
	if err := d.SeedQueue(ctx, []string{"9.9.9.9"}); err != nil {
		t.Fatalf("seed queue: %v", err)
	}

	o.autoCycleStep(ctx, db.Now())

	ac := getAutoCycle(t, d)
	if ac.Enabled || ac.Phase != db.AutoCyclePhaseIdle {
		t.Fatalf("expected disabled+idle, got %+v", ac)
	}
	if got := queuedAddresses(t, d); len(got) != 1 || got["9.9.9.9"] != db.IPQueued {
		t.Fatalf("queue must be untouched while disabled, got %v", got)
	}
}

func TestAutoCycleDisabledResetsStalePhase(t *testing.T) {
	ctx := context.Background()
	o, d, _ := newTestOrchestrator(t, 180)
	now := db.Now()
	if err := d.UpdateAutoCycleState(ctx, db.AutoCycleState{Phase: db.AutoCyclePhaseRunning, RunStartedAt: &now}); err != nil {
		t.Fatalf("update state: %v", err)
	}

	o.autoCycleStep(ctx, now)

	if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseIdle || ac.RunStartedAt != nil {
		t.Fatalf("expected phase reset to idle, got %+v", ac)
	}
}

func TestAutoCycleStartClearsQueueAndScans(t *testing.T) {
	ctx := context.Background()
	o, d, mock := newTestOrchestrator(t, 180)
	mock.Seed("fip-1", "1.1.1.1", "svc")
	mock.Seed("fip-2", "2.2.2.2", "svc")
	mock.SeedWithPort("fip-3", "3.3.3.3", "svc", "someone-elses-port")
	if err := d.SeedQueue(ctx, []string{"9.9.9.9"}); err != nil {
		t.Fatalf("seed queue: %v", err)
	}

	if err := o.StartAutoCycle(ctx); err != nil {
		t.Fatalf("start: %v", err)
	}
	ac := getAutoCycle(t, d)
	if !ac.Enabled || ac.Phase != db.AutoCyclePhaseIdle || ac.NextRunAt == nil {
		t.Fatalf("expected enabled+idle with next_run_at set after start, got %+v", ac)
	}

	now := db.Now()
	o.autoCycleStep(ctx, now)

	got := queuedAddresses(t, d)
	if _, stale := got["9.9.9.9"]; stale {
		t.Fatalf("old queue entry must be cleared, got %v", got)
	}
	if len(got) != 2 || got["1.1.1.1"] != db.IPQueued || got["2.2.2.2"] != db.IPQueued {
		t.Fatalf("expected the two free FIPs queued, got %v", got)
	}
	ac = getAutoCycle(t, d)
	if ac.Phase != db.AutoCyclePhaseRunning {
		t.Fatalf("expected running, got %s", ac.Phase)
	}
	if ac.RunStartedAt == nil || !ac.RunStartedAt.Equal(now) {
		t.Fatalf("expected run_started_at=%v, got %v", now, ac.RunStartedAt)
	}
	if ac.LastRunStartedAt == nil || !ac.LastRunStartedAt.Equal(now) {
		t.Fatalf("expected last_run_started_at=%v, got %v", now, ac.LastRunStartedAt)
	}
	if ac.LastScannedFree != 2 {
		t.Fatalf("expected last_scanned_free=2, got %d", ac.LastScannedFree)
	}
	if countEvents(t, d, "queue_cleared") != 1 || countEvents(t, d, "fip_scan") != 1 {
		t.Fatalf("expected queue_cleared and fip_scan events")
	}
	if countEvents(t, d, "auto_cycle_started") != 2 { // enable + cycle start
		t.Fatalf("expected two auto_cycle_started events, got %d", countEvents(t, d, "auto_cycle_started"))
	}
}

func TestAutoCycleWaitsWhileChecksInProgressAndTickPicksUpQueue(t *testing.T) {
	ctx := context.Background()
	o, d, mock := newTestOrchestrator(t, 180)
	mock.Seed("fip-1", "1.1.1.1", "svc")
	mock.Seed("fip-2", "2.2.2.2", "svc")
	if err := d.RegisterValidator(ctx, "validator-1", "host-1", "port-1", "v0.1"); err != nil {
		t.Fatalf("register validator: %v", err)
	}
	if err := o.StartAutoCycle(ctx); err != nil {
		t.Fatalf("start: %v", err)
	}

	t0 := db.Now()
	o.autoCycleStep(ctx, t0)

	// Existing Tick logic starts the checks on its own.
	o.Tick(ctx)
	states := queuedAddresses(t, d)
	inProgress := 0
	for _, s := range states {
		if s == db.IPAwaitingSelfCheck {
			inProgress++
		}
	}
	if inProgress != 1 {
		t.Fatalf("expected exactly one address claimed by Tick, got %v", states)
	}

	o.autoCycleStep(ctx, t0.Add(time.Minute))
	if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseRunning || ac.RunsTotal != 0 {
		t.Fatalf("expected still running, got %+v", ac)
	}

	// One address done, the other still queued: still not finished.
	if _, err := d.ExecContext(ctx, `UPDATE ip_queue SET state=? WHERE ip_address=?`, db.IPDone, "1.1.1.1"); err != nil {
		t.Fatalf("update: %v", err)
	}
	if _, err := d.ExecContext(ctx, `UPDATE ip_queue SET state=? WHERE ip_address=?`, db.IPQueued, "2.2.2.2"); err != nil {
		t.Fatalf("update: %v", err)
	}
	o.autoCycleStep(ctx, t0.Add(2*time.Minute))
	if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseRunning {
		t.Fatalf("expected still running with a queued address left, got %+v", ac)
	}
}

func TestAutoCycleCompletesAndRepeatsAfterInterval(t *testing.T) {
	ctx := context.Background()
	o, d, mock := newTestOrchestrator(t, 180)
	mock.Seed("fip-1", "1.1.1.1", "svc")
	mock.Seed("fip-2", "2.2.2.2", "svc")
	setAutoCycleParams(t, d, 600, 0)
	if err := o.StartAutoCycle(ctx); err != nil {
		t.Fatalf("start: %v", err)
	}

	t0 := db.Now()
	o.autoCycleStep(ctx, t0)

	// done, failed and occupied are all terminal.
	if _, err := d.ExecContext(ctx, `UPDATE ip_queue SET state=? WHERE ip_address=?`, db.IPDone, "1.1.1.1"); err != nil {
		t.Fatalf("update: %v", err)
	}
	if _, err := d.ExecContext(ctx, `UPDATE ip_queue SET state=? WHERE ip_address=?`, db.IPOccupied, "2.2.2.2"); err != nil {
		t.Fatalf("update: %v", err)
	}
	t1 := t0.Add(5 * time.Minute)
	o.autoCycleStep(ctx, t1)

	ac := getAutoCycle(t, d)
	if ac.Phase != db.AutoCyclePhaseWaiting || ac.LastOutcome != db.AutoCycleOutcomeCompleted {
		t.Fatalf("expected waiting/completed, got %+v", ac)
	}
	if ac.RunsTotal != 1 {
		t.Fatalf("expected runs_total=1, got %d", ac.RunsTotal)
	}
	wantNext := t1.Add(600 * time.Second)
	if ac.NextRunAt == nil || !ac.NextRunAt.Equal(wantNext) {
		t.Fatalf("expected next_run_at=%v (completion + interval), got %v", wantNext, ac.NextRunAt)
	}
	if ac.LastRunFinishedAt == nil || !ac.LastRunFinishedAt.Equal(t1) {
		t.Fatalf("expected last_run_finished_at=%v, got %v", t1, ac.LastRunFinishedAt)
	}
	if ac.RunStartedAt != nil {
		t.Fatalf("expected run_started_at cleared, got %v", ac.RunStartedAt)
	}
	if countEvents(t, d, "auto_cycle_completed") != 1 {
		t.Fatalf("expected one auto_cycle_completed event")
	}

	// Before next_run_at: nothing happens, the finished queue is kept.
	o.autoCycleStep(ctx, wantNext.Add(-time.Second))
	if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseWaiting || ac.RunsTotal != 1 {
		t.Fatalf("expected still waiting, got %+v", ac)
	}
	if got := queuedAddresses(t, d); got["1.1.1.1"] != db.IPDone {
		t.Fatalf("queue must not be touched while waiting, got %v", got)
	}

	// At next_run_at: new cycle starts, queue is rebuilt from scratch.
	o.autoCycleStep(ctx, wantNext)
	ac = getAutoCycle(t, d)
	if ac.Phase != db.AutoCyclePhaseRunning {
		t.Fatalf("expected running again, got %+v", ac)
	}
	if ac.LastRunStartedAt == nil || !ac.LastRunStartedAt.Equal(wantNext) {
		t.Fatalf("expected last_run_started_at=%v, got %v", wantNext, ac.LastRunStartedAt)
	}
	if got := queuedAddresses(t, d); got["1.1.1.1"] != db.IPQueued || got["2.2.2.2"] != db.IPQueued {
		t.Fatalf("expected both addresses re-queued, got %v", got)
	}
	if ac.RunsTotal != 1 {
		t.Fatalf("runs_total only counts completed cycles, got %d", ac.RunsTotal)
	}
}

func TestAutoCycleTimeout(t *testing.T) {
	ctx := context.Background()
	o, d, mock := newTestOrchestrator(t, 180)
	mock.Seed("fip-1", "1.1.1.1", "svc")
	setAutoCycleParams(t, d, 60, 300)
	if err := o.StartAutoCycle(ctx); err != nil {
		t.Fatalf("start: %v", err)
	}

	t0 := db.Now()
	o.autoCycleStep(ctx, t0)

	o.autoCycleStep(ctx, t0.Add(300*time.Second))
	if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseRunning {
		t.Fatalf("expected still running exactly at the limit, got %+v", ac)
	}

	t1 := t0.Add(301 * time.Second)
	o.autoCycleStep(ctx, t1)
	ac := getAutoCycle(t, d)
	if ac.Phase != db.AutoCyclePhaseWaiting || ac.LastOutcome != db.AutoCycleOutcomeTimeout {
		t.Fatalf("expected waiting/timeout, got %+v", ac)
	}
	if ac.RunsTotal != 0 {
		t.Fatalf("timeout must not count as completed, got runs_total=%d", ac.RunsTotal)
	}
	if ac.NextRunAt == nil || !ac.NextRunAt.Equal(t1.Add(60*time.Second)) {
		t.Fatalf("expected next_run_at=now+interval, got %v", ac.NextRunAt)
	}
	if got := queuedAddresses(t, d); got["1.1.1.1"] != db.IPQueued {
		t.Fatalf("queue must be left untouched on timeout, got %v", got)
	}
	if countEvents(t, d, "auto_cycle_timeout") != 1 {
		t.Fatalf("expected one auto_cycle_timeout event")
	}
}

func TestAutoCycleNoLimitNeverTimesOut(t *testing.T) {
	ctx := context.Background()
	o, d, mock := newTestOrchestrator(t, 180)
	mock.Seed("fip-1", "1.1.1.1", "svc")
	if err := o.StartAutoCycle(ctx); err != nil {
		t.Fatalf("start: %v", err)
	}
	t0 := db.Now()
	o.autoCycleStep(ctx, t0)
	o.autoCycleStep(ctx, t0.Add(1000*time.Hour))
	if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseRunning {
		t.Fatalf("max_run_seconds=0 means no limit, got %+v", ac)
	}
}

// An operator pressing «Очистить всё» in the middle of a cycle leaves nothing
// to wait for; with max_run_seconds=0 the cycle would otherwise hang forever.
func TestAutoCycleManualClearMidCycleCompletes(t *testing.T) {
	ctx := context.Background()
	o, d, mock := newTestOrchestrator(t, 180)
	mock.Seed("fip-1", "1.1.1.1", "svc")
	setAutoCycleParams(t, d, 60, 0)
	if err := o.StartAutoCycle(ctx); err != nil {
		t.Fatalf("start: %v", err)
	}
	t0 := db.Now()
	o.autoCycleStep(ctx, t0)
	if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseRunning {
		t.Fatalf("expected running after start, got %+v", ac)
	}

	if _, err := o.ClearQueue(ctx); err != nil {
		t.Fatalf("manual clear: %v", err)
	}

	t1 := t0.Add(5 * time.Second)
	o.autoCycleStep(ctx, t1)
	ac := getAutoCycle(t, d)
	if ac.Phase != db.AutoCyclePhaseWaiting || ac.LastOutcome != db.AutoCycleOutcomeCompleted {
		t.Fatalf("expected waiting/completed after the queue was emptied, got %+v", ac)
	}
	if ac.NextRunAt == nil || !ac.NextRunAt.Equal(t1.Add(60*time.Second)) {
		t.Fatalf("expected next_run_at=now+interval, got %v", ac.NextRunAt)
	}
}

func TestAutoCycleNoFreeIPs(t *testing.T) {
	ctx := context.Background()
	o, d, mock := newTestOrchestrator(t, 180)
	mock.SeedWithPort("fip-1", "1.1.1.1", "svc", "someone-elses-port")
	setAutoCycleParams(t, d, 120, 0)
	if err := o.StartAutoCycle(ctx); err != nil {
		t.Fatalf("start: %v", err)
	}

	now := db.Now()
	o.autoCycleStep(ctx, now)

	ac := getAutoCycle(t, d)
	if ac.Phase != db.AutoCyclePhaseWaiting || ac.LastOutcome != db.AutoCycleOutcomeNoFreeIPs {
		t.Fatalf("expected waiting/no_free_ips, got %+v", ac)
	}
	if ac.LastScannedFree != 0 {
		t.Fatalf("expected last_scanned_free=0, got %d", ac.LastScannedFree)
	}
	if ac.NextRunAt == nil || !ac.NextRunAt.Equal(now.Add(120*time.Second)) {
		t.Fatalf("expected next_run_at=now+interval, got %v", ac.NextRunAt)
	}
	if ac.RunStartedAt != nil {
		t.Fatalf("run_started_at must stay unset, got %v", ac.RunStartedAt)
	}
}

func TestAutoCycleOpenStackErrorRetriesNextInterval(t *testing.T) {
	ctx := context.Background()
	o, d, mock := newTestOrchestrator(t, 180)
	mock.Seed("fip-1", "1.1.1.1", "svc")
	mock.ListFailure = errors.New("neutron is down")
	setAutoCycleParams(t, d, 60, 0)
	if err := o.StartAutoCycle(ctx); err != nil {
		t.Fatalf("start: %v", err)
	}

	t0 := db.Now()
	o.autoCycleStep(ctx, t0)

	ac := getAutoCycle(t, d)
	if ac.Phase != db.AutoCyclePhaseWaiting || ac.LastOutcome != db.AutoCycleOutcomeError {
		t.Fatalf("expected waiting/error, got %+v", ac)
	}
	if !strings.Contains(ac.LastError, "neutron is down") {
		t.Fatalf("expected last_error to mention the cause, got %q", ac.LastError)
	}
	if ac.NextRunAt == nil || !ac.NextRunAt.Equal(t0.Add(60*time.Second)) {
		t.Fatalf("expected next_run_at=now+interval, got %v", ac.NextRunAt)
	}
	if countEvents(t, d, "auto_cycle_error") != 1 {
		t.Fatalf("expected one auto_cycle_error event")
	}
	if !ac.Enabled {
		t.Fatalf("an error must not disable the auto-cycle")
	}

	// OpenStack recovers: the next interval starts a normal cycle and the
	// stale error is cleared.
	mock.ListFailure = nil
	o.autoCycleStep(ctx, t0.Add(60*time.Second))
	ac = getAutoCycle(t, d)
	if ac.Phase != db.AutoCyclePhaseRunning || ac.LastError != "" {
		t.Fatalf("expected running with cleared error, got %+v", ac)
	}
}

func TestAutoCycleStartIsIdempotent(t *testing.T) {
	ctx := context.Background()
	o, d, mock := newTestOrchestrator(t, 180)
	mock.Seed("fip-1", "1.1.1.1", "svc")
	if err := o.StartAutoCycle(ctx); err != nil {
		t.Fatalf("start: %v", err)
	}
	t0 := db.Now()
	o.autoCycleStep(ctx, t0)

	if err := o.StartAutoCycle(ctx); err != nil {
		t.Fatalf("second start: %v", err)
	}
	ac := getAutoCycle(t, d)
	if !ac.Enabled || ac.Phase != db.AutoCyclePhaseRunning || ac.RunStartedAt == nil || !ac.RunStartedAt.Equal(t0) {
		t.Fatalf("second start must not restart a running cycle, got %+v", ac)
	}
	if countEvents(t, d, "auto_cycle_started") != 2 { // enable + cycle start, not a third
		t.Fatalf("expected no extra started event, got %d", countEvents(t, d, "auto_cycle_started"))
	}
}

func TestAutoCycleStopMidCycle(t *testing.T) {
	ctx := context.Background()
	o, d, mock := newTestOrchestrator(t, 180)
	mock.Seed("fip-1", "1.1.1.1", "svc")
	if err := d.RegisterValidator(ctx, "validator-1", "host-1", "port-1", "v0.1"); err != nil {
		t.Fatalf("register validator: %v", err)
	}
	if err := o.StartAutoCycle(ctx); err != nil {
		t.Fatalf("start: %v", err)
	}
	t0 := db.Now()
	o.autoCycleStep(ctx, t0)
	o.Tick(ctx) // check in flight

	if err := o.StopAutoCycle(ctx); err != nil {
		t.Fatalf("stop: %v", err)
	}
	ac := getAutoCycle(t, d)
	if ac.Enabled || ac.Phase != db.AutoCyclePhaseIdle || ac.LastOutcome != db.AutoCycleOutcomeStopped {
		t.Fatalf("expected disabled/idle/stopped, got %+v", ac)
	}
	if got := queuedAddresses(t, d); got["1.1.1.1"] != db.IPAwaitingSelfCheck {
		t.Fatalf("in-flight check must not be cancelled, got %v", got)
	}
	if countEvents(t, d, "auto_cycle_stopped") != 1 {
		t.Fatalf("expected one auto_cycle_stopped event")
	}

	// Further steps do nothing, even far in the future.
	o.autoCycleStep(ctx, t0.Add(100*time.Hour))
	ac = getAutoCycle(t, d)
	if ac.Enabled || ac.Phase != db.AutoCyclePhaseIdle || ac.RunsTotal != 0 {
		t.Fatalf("expected no activity after stop, got %+v", ac)
	}

	// Stopping again is a harmless no-op.
	if err := o.StopAutoCycle(ctx); err != nil {
		t.Fatalf("second stop: %v", err)
	}
	if countEvents(t, d, "auto_cycle_stopped") != 1 {
		t.Fatalf("second stop must not emit another event")
	}
}

// Stopping during the pause between cycles must keep the result of the last
// finished cycle visible instead of replacing it with "stopped".
func TestAutoCycleStopWhileWaitingKeepsLastOutcome(t *testing.T) {
	ctx := context.Background()
	o, d, mock := newTestOrchestrator(t, 180)
	mock.Seed("fip-1", "1.1.1.1", "svc")
	setAutoCycleParams(t, d, 60, 0)
	if err := o.StartAutoCycle(ctx); err != nil {
		t.Fatalf("start: %v", err)
	}
	t0 := db.Now()
	o.autoCycleStep(ctx, t0)
	finishAllIPs(t, d, db.IPDone)
	o.autoCycleStep(ctx, t0.Add(5*time.Second))
	if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseWaiting || ac.LastOutcome != db.AutoCycleOutcomeCompleted {
		t.Fatalf("precondition: expected waiting/completed, got %+v", ac)
	}

	if err := o.StopAutoCycle(ctx); err != nil {
		t.Fatalf("stop: %v", err)
	}
	ac := getAutoCycle(t, d)
	if ac.Enabled || ac.Phase != db.AutoCyclePhaseIdle {
		t.Fatalf("expected disabled/idle, got %+v", ac)
	}
	if ac.LastOutcome != db.AutoCycleOutcomeCompleted || ac.RunsTotal != 1 {
		t.Fatalf("stop in the pause must keep last_outcome=completed, got %+v", ac)
	}
}

func TestAutoCycleSurvivesRestart(t *testing.T) {
	ctx := context.Background()
	o, d, mock := newTestOrchestrator(t, 180)
	mock.Seed("fip-1", "1.1.1.1", "svc")
	setAutoCycleParams(t, d, 300, 0)
	if err := o.StartAutoCycle(ctx); err != nil {
		t.Fatalf("start: %v", err)
	}
	t0 := db.Now()
	o.autoCycleStep(ctx, t0)

	// A fresh Orchestrator on the same database (a control-api restart)
	// continues the running phase instead of starting over.
	o2 := &Orchestrator{DB: d, OS: mock, Cfg: o.Cfg, Agg: o.Agg, Log: o.Log}
	o2.autoCycleStep(ctx, t0.Add(time.Minute))
	ac := getAutoCycle(t, d)
	if ac.Phase != db.AutoCyclePhaseRunning || ac.RunStartedAt == nil || !ac.RunStartedAt.Equal(t0) {
		t.Fatalf("expected the running phase to continue, got %+v", ac)
	}

	finishAllIPs(t, d, db.IPDone)
	t1 := t0.Add(2 * time.Minute)
	o2.autoCycleStep(ctx, t1)
	ac = getAutoCycle(t, d)
	if ac.Phase != db.AutoCyclePhaseWaiting || ac.LastOutcome != db.AutoCycleOutcomeCompleted {
		t.Fatalf("expected waiting/completed, got %+v", ac)
	}

	// A third instance still honours the persisted next_run_at.
	o3 := &Orchestrator{DB: d, OS: mock, Cfg: o.Cfg, Agg: o.Agg, Log: o.Log}
	o3.autoCycleStep(ctx, t1.Add(299*time.Second))
	if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseWaiting {
		t.Fatalf("expected waiting until next_run_at, got %+v", ac)
	}
	o3.autoCycleStep(ctx, t1.Add(300*time.Second))
	if ac := getAutoCycle(t, d); ac.Phase != db.AutoCyclePhaseRunning {
		t.Fatalf("expected a new cycle at next_run_at, got %+v", ac)
	}
}