cloud-ip-validator/internal/orchestrator/autocycle.go

package orchestrator

import (
	"context"
	"encoding/json"
	"fmt"
	"time"

	"cloudipvalidator/internal/db"
)

// This file implements the automatic check cycle: an optional, repeating
// "clear queue -> scan floating IPs -> wait until every queued address has
// reached a terminal state -> wait interval" scenario. Steps 1-2 reuse
// ClearQueue/ScanFloatingIPs verbatim; step 3 needs no code at all because
// Tick already picks up `queued` addresses. All state lives in the database
// (db.AutoCycle), so the cycle survives a control-api restart and the
// interval/limits can be changed at runtime.

// GetAutoCycle returns the current auto-cycle configuration and state.
func (o *Orchestrator) GetAutoCycle(ctx context.Context) (db.AutoCycle, error) {
	return o.DB.GetAutoCycle(ctx)
}

// StartAutoCycle enables the auto-cycle; the first cycle begins on the next
// AutoCycleStep. It is idempotent: if the cycle is already enabled nothing
// changes (in particular a running cycle is not restarted).
func (o *Orchestrator) StartAutoCycle(ctx context.Context) error {
	o.autoCycleMu.Lock()
	defer o.autoCycleMu.Unlock()

	ac, err := o.DB.GetAutoCycle(ctx)
	if err != nil {
		return fmt.Errorf("get auto cycle: %w", err)
	}
	if ac.Enabled {
		return nil
	}
	now := db.Now()
	if err := o.DB.SetAutoCycleEnabled(ctx, true, &now, ""); err != nil {
		return fmt.Errorf("enable auto cycle: %w", err)
	}
	o.event(ctx, "control-api", "", nil, "auto_cycle_started", autoCyclePayload(map[string]any{
		"reason":           "enabled",
		"interval_seconds": ac.IntervalSeconds,
		"max_run_seconds":  ac.MaxRunSeconds,
	}))
	return nil
}

// StopAutoCycle disables the auto-cycle and returns it to idle. Checks that
// are already in flight are NOT cancelled — they finish normally and land in
// the registry; only the repetition stops.
func (o *Orchestrator) StopAutoCycle(ctx context.Context) error {
	o.autoCycleMu.Lock()
	defer o.autoCycleMu.Unlock()

	ac, err := o.DB.GetAutoCycle(ctx)
	if err != nil {
		return fmt.Errorf("get auto cycle: %w", err)
	}
	if !ac.Enabled {
		return nil
	}
	// "stopped" describes an interrupted run. Stopping during the pause
	// between cycles must not overwrite the result of the last finished one.
	outcome := ""
	if ac.Phase == db.AutoCyclePhaseRunning {
		outcome = db.AutoCycleOutcomeStopped
	}
	if err := o.DB.SetAutoCycleEnabled(ctx, false, nil, outcome); err != nil {
		return fmt.Errorf("disable auto cycle: %w", err)
	}
	o.event(ctx, "control-api", "", nil, "auto_cycle_stopped", autoCyclePayload(map[string]any{
		"phase": ac.Phase,
	}))
	return nil
}

// AutoCycleStep advances the auto-cycle state machine by one step. It is
// called by the control-api loop right after Tick.
func (o *Orchestrator) AutoCycleStep(ctx context.Context) {
	o.autoCycleStep(ctx, db.Now())
}

// autoCycleStep is AutoCycleStep with an explicit "now", so tests can drive
// the state machine deterministically without sleeping.
func (o *Orchestrator) autoCycleStep(ctx context.Context, now time.Time) {
	o.autoCycleMu.Lock()
	defer o.autoCycleMu.Unlock()

	// Read inside the lock: Start/Stop may have changed the row since the
	// caller last looked.
	ac, err := o.DB.GetAutoCycle(ctx)
	if err != nil {
		o.Log.Error("auto cycle: read state", "err", err)
		return
	}

	if !ac.Enabled {
		if ac.Phase != db.AutoCyclePhaseIdle {
			if err := o.DB.SetAutoCycleEnabled(ctx, false, nil, ""); err != nil {
				o.Log.Error("auto cycle: reset phase to idle", "err", err)
			}
		}
		return
	}

	if ac.Phase == db.AutoCyclePhaseRunning {
		o.autoCycleCheckRun(ctx, ac, now)
		return
	}

	// idle or waiting: start a new cycle once next_run_at has come.
	if ac.NextRunAt != nil && now.Before(*ac.NextRunAt) {
		return
	}
	o.autoCycleStartRun(ctx, ac, now)
}

// autoCycleStartRun performs steps 1-2 of the scenario (clear the queue,
// scan floating IPs) and moves the state to running, or straight to waiting
// if there is nothing to wait for.
func (o *Orchestrator) autoCycleStartRun(ctx context.Context, ac db.AutoCycle, now time.Time) {
	interval := time.Duration(ac.IntervalSeconds) * time.Second
	next := now.Add(interval)

	st := autoCycleStateOf(ac)
	st.LastRunStartedAt = &now
	st.RunStartedAt = nil

	fail := func(step string, err error) {
		o.Log.Error("auto cycle: step failed", "step", step, "err", err)
		st.Phase = db.AutoCyclePhaseWaiting
		st.NextRunAt = &next
		st.LastRunFinishedAt = &now
		st.LastOutcome = db.AutoCycleOutcomeError
		st.LastError = fmt.Sprintf("%s: %v", step, err)
		if uerr := o.DB.UpdateAutoCycleState(ctx, st); uerr != nil {
			o.Log.Error("auto cycle: save state", "err", uerr)
		}
		o.event(ctx, "control-api", "", nil, "auto_cycle_error", autoCyclePayload(map[string]any{
			"step":  step,
			"error": err.Error(),
		}))
	}

	if _, err := o.ClearQueue(ctx); err != nil {
		fail("clear queue", err)
		return
	}
	_, scanned, err := o.ScanFloatingIPs(ctx)
	if err != nil {
		fail("scan floating ips", err)
		return
	}
	st.LastScannedFree = scanned
	st.LastError = ""

	if scanned == 0 {
		// Nothing was queued; waiting for completion would never end.
		o.Log.Info("auto cycle: no free floating ips, waiting for next interval")
		st.Phase = db.AutoCyclePhaseWaiting
		st.NextRunAt = &next
		st.LastRunFinishedAt = &now
		st.LastOutcome = db.AutoCycleOutcomeNoFreeIPs
		if err := o.DB.UpdateAutoCycleState(ctx, st); err != nil {
			o.Log.Error("auto cycle: save state", "err", err)
		}
		return
	}

	st.Phase = db.AutoCyclePhaseRunning
	st.RunStartedAt = &now
	st.NextRunAt = nil
	if err := o.DB.UpdateAutoCycleState(ctx, st); err != nil {
		o.Log.Error("auto cycle: save state", "err", err)
		return
	}
	o.event(ctx, "control-api", "", nil, "auto_cycle_started", autoCyclePayload(map[string]any{
		"reason":       "cycle",
		"scanned_free": scanned,
	}))
}

// autoCycleCheckRun handles the running phase: finish the cycle once every
// queued address is terminal, or give up after max_run_seconds.
func (o *Orchestrator) autoCycleCheckRun(ctx context.Context, ac db.AutoCycle, now time.Time) {
	items, err := o.DB.ListIPs(ctx)
	if err != nil {
		o.Log.Error("auto cycle: list ips", "err", err)
		return
	}

	interval := time.Duration(ac.IntervalSeconds) * time.Second
	next := now.Add(interval)

	// An empty queue counts as finished: right after the scan it cannot be
	// empty (scanned > 0), so it only happens when an operator cleared or
	// deleted every address mid-cycle — and then there is nothing to wait for
	// (with max_run_seconds=0 the cycle would otherwise hang forever).
	allTerminal := true
	for _, it := range items {
		if !isTerminalIPState(it.State) {
			allTerminal = false
			break
		}
	}
	if allTerminal {
		st := autoCycleStateOf(ac)
		st.Phase = db.AutoCyclePhaseWaiting
		st.RunStartedAt = nil
		st.NextRunAt = &next
		st.LastRunFinishedAt = &now
		st.LastOutcome = db.AutoCycleOutcomeCompleted
		st.LastError = ""
		st.RunsTotal++
		if err := o.DB.UpdateAutoCycleState(ctx, st); err != nil {
			o.Log.Error("auto cycle: save state", "err", err)
			return
		}
		o.event(ctx, "control-api", "", nil, "auto_cycle_completed", autoCyclePayload(map[string]any{
			"addresses": len(items),
			"runs":      st.RunsTotal,
		}))
		return
	}

	if ac.MaxRunSeconds > 0 && ac.RunStartedAt != nil &&
		now.Sub(*ac.RunStartedAt) > time.Duration(ac.MaxRunSeconds)*time.Second {
		// The queue is left untouched: the next cycle clears it anyway, and
		// an operator can still inspect what got stuck.
		st := autoCycleStateOf(ac)
		st.Phase = db.AutoCyclePhaseWaiting
		st.RunStartedAt = nil
		st.NextRunAt = &next
		st.LastRunFinishedAt = &now
		st.LastOutcome = db.AutoCycleOutcomeTimeout
		st.LastError = fmt.Sprintf("checks did not finish within %d seconds", ac.MaxRunSeconds)
		if err := o.DB.UpdateAutoCycleState(ctx, st); err != nil {
			o.Log.Error("auto cycle: save state", "err", err)
			return
		}
		o.event(ctx, "control-api", "", nil, "auto_cycle_timeout", autoCyclePayload(map[string]any{
			"max_run_seconds": ac.MaxRunSeconds,
			"addresses":       len(items),
		}))
	}
}

// isTerminalIPState reports whether an address has finished its check cycle
// for good (its result, if any, is already written to the registry).
// db.IPOccupied counts: such an address never enters the check cycle.
func isTerminalIPState(state string) bool {
	switch state {
	case db.IPDone, db.IPFailed, db.IPOccupied:
		return true
	}
	return false
}

func autoCycleStateOf(ac db.AutoCycle) db.AutoCycleState {
	return db.AutoCycleState{
		Phase:             ac.Phase,
		RunStartedAt:      ac.RunStartedAt,
		NextRunAt:         ac.NextRunAt,
		LastRunStartedAt:  ac.LastRunStartedAt,
		LastRunFinishedAt: ac.LastRunFinishedAt,
		LastOutcome:       ac.LastOutcome,
		LastError:         ac.LastError,
		LastScannedFree:   ac.LastScannedFree,
		RunsTotal:         ac.RunsTotal,
	}
}

func autoCyclePayload(m map[string]any) string {
	b, err := json.Marshal(m)
	if err != nil {
		return "{}"
	}
	return string(b)
}