added FIP Cooldown before start

internal/orchestrator/orchestrator.go

@@ -170,6 +170,15 @@ func (o *Orchestrator) AssignmentForValidator(ctx context.Context, validatorID s
 	if item.State != db.IPAwaitingSelfCheck && item.State != db.IPChecking {
 		return nil, nil, nil
 	}
+	if item.State == db.IPAwaitingSelfCheck {
+		settled, err := o.isFIPSettled(ctx, item)
+		if err != nil {
+			return nil, nil, err
+		}
+		if !settled {
+			return nil, nil, nil
+		}
+	}
 	resolved, err := o.DB.ListResolvedCheckTypes(ctx)
 	if err != nil {
 		return nil, nil, err
@@ -181,6 +190,47 @@ func (o *Orchestrator) AssignmentForValidator(ctx context.Context, validatorID s
 	return item, checks, nil
 }
 
+// isFIPSettled reports whether an address currently awaiting_self_check
+// has cleared the configured fip_settle_seconds pause since its floating
+// IP was associated — withholding the assignment until then is how the
+// pause is enforced, with zero changes needed to the agent's poll loop or
+// the assignment endpoint's wire contract (it just keeps seeing 204s).
+// Settings are read fresh on every call, same as check_types/sites
+// elsewhere in this file, so an admin change applies immediately even to
+// an address already mid-wait. A nil FIPAssociatedAt (an in-flight row
+// from before this feature's migration) is always treated as settled —
+// upgrading control-api must never newly strand an address that was
+// already awaiting self-check.
+func (o *Orchestrator) isFIPSettled(ctx context.Context, item *db.IPQueueItem) (bool, error) {
+	settings, err := o.DB.GetSettings(ctx)
+	if err != nil {
+		return false, err
+	}
+	if settings.FIPSettleSeconds <= 0 || item.FIPAssociatedAt == nil {
+		return true, nil
+	}
+	deadline := item.FIPAssociatedAt.Add(time.Duration(settings.FIPSettleSeconds) * time.Second)
+	return !db.Now().Before(deadline), nil
+}
+
+// SetFIPSettleSeconds validates and persists a new fip_settle_seconds
+// value. Lives here rather than internal/db because the cross-field rule
+// below needs o.Cfg, which the db package has no access to: the pause plus
+// self-check's own timeout must leave room inside the claim lease, or the
+// lease sweep would reclaim the address before self-check ever gets a
+// chance to run, producing a perpetual requeue loop.
+func (o *Orchestrator) SetFIPSettleSeconds(ctx context.Context, seconds int) error {
+	if seconds < 0 {
+		return fmt.Errorf("fip_settle_seconds must be >= 0: %w", db.ErrValidation)
+	}
+	if seconds+o.Cfg.SelfCheckTimeoutSeconds >= o.Cfg.LeaseTTLSeconds {
+		return fmt.Errorf(
+			"fip_settle_seconds (%d) + self_check_timeout_seconds (%d) must be < lease_ttl_seconds (%d): %w",
+			seconds, o.Cfg.SelfCheckTimeoutSeconds, o.Cfg.LeaseTTLSeconds, db.ErrValidation)
+	}
+	return o.DB.SetFIPSettleSeconds(ctx, seconds)
+}
+
 // SiteIndexForID resolves a configured site_id to its 1/2/3 index, or
 // (0, nil) if unconfigured.
 func (o *Orchestrator) SiteIndexForID(ctx context.Context, siteID string) (int, error) {

internal/orchestrator/orchestrator_test.go

@@ -409,6 +409,124 @@ func TestClearQueueDisassociatesAllFIPs(t *testing.T) {
 	}
 }
 
+// TestFIPSettleDelayWithholdsAssignment proves a nonzero fip_settle_seconds
+// keeps AssignmentForValidator returning nil (agent keeps polling and
+// getting nothing) until the configured pause has elapsed since the
+// floating IP was associated.
+func TestFIPSettleDelayWithholdsAssignment(t *testing.T) {
+	ctx := context.Background()
+	o, d, mock := newTestOrchestrator(t, 180)
+	if err := o.SetFIPSettleSeconds(ctx, 1); err != nil {
+		t.Fatalf("set fip settle seconds: %v", err)
+	}
+	mock.Seed("fip-1", "1.2.3.4", "svc-project")
+	_ = d.RegisterValidator(ctx, "validator-1", "host-1", "port-1", "v0.1")
+	_ = d.SeedQueue(ctx, []string{"1.2.3.4"})
+
+	o.Tick(ctx) // claim + associate -> awaiting_self_check, fip attached
+
+	item, checks, err := o.AssignmentForValidator(ctx, "validator-1")
+	if err != nil {
+		t.Fatalf("assignment for validator: %v", err)
+	}
+	if item != nil || checks != nil {
+		t.Fatalf("expected no assignment during settle window, got item=%+v checks=%+v", item, checks)
+	}
+
+	time.Sleep(1100 * time.Millisecond)
+
+	item, checks, err = o.AssignmentForValidator(ctx, "validator-1")
+	if err != nil {
+		t.Fatalf("assignment for validator after settle: %v", err)
+	}
+	if item == nil {
+		t.Fatalf("expected assignment to be available once settle window elapsed")
+	}
+	if len(checks) == 0 {
+		t.Fatalf("expected check config to be returned alongside the item")
+	}
+}
+
+// TestFIPSettleDelayZeroIsNoOp proves the default fip_settle_seconds=0
+// never withholds an assignment — no behavior change for upgraded-but-
+// unconfigured installs.
+func TestFIPSettleDelayZeroIsNoOp(t *testing.T) {
+	ctx := context.Background()
+	o, d, mock := newTestOrchestrator(t, 180)
+	mock.Seed("fip-1", "1.2.3.4", "svc-project")
+	_ = d.RegisterValidator(ctx, "validator-1", "host-1", "port-1", "v0.1")
+	_ = d.SeedQueue(ctx, []string{"1.2.3.4"})
+
+	o.Tick(ctx)
+
+	item, _, err := o.AssignmentForValidator(ctx, "validator-1")
+	if err != nil {
+		t.Fatalf("assignment for validator: %v", err)
+	}
+	if item == nil {
+		t.Fatalf("expected assignment immediately available with fip_settle_seconds=0")
+	}
+}
+
+// TestFIPSettleDelayIgnoresNilFIPAssociatedAt proves an in-flight row from
+// before this feature's migration (fip_associated_at NULL) is never newly
+// blocked, even with a nonzero configured pause — upgrading control-api
+// must not strand an address already awaiting self-check.
+func TestFIPSettleDelayIgnoresNilFIPAssociatedAt(t *testing.T) {
+	ctx := context.Background()
+	o, d, mock := newTestOrchestrator(t, 180)
+	if err := o.SetFIPSettleSeconds(ctx, 60); err != nil {
+		t.Fatalf("set fip settle seconds: %v", err)
+	}
+	mock.Seed("fip-1", "1.2.3.4", "svc-project")
+	_ = d.RegisterValidator(ctx, "validator-1", "host-1", "port-1", "v0.1")
+	_ = d.SeedQueue(ctx, []string{"1.2.3.4"})
+
+	o.Tick(ctx)
+
+	item, err := d.GetIPByAddress(ctx, "1.2.3.4")
+	if err != nil {
+		t.Fatalf("get ip: %v", err)
+	}
+	if _, err := d.ExecContext(ctx, `UPDATE ip_queue SET fip_associated_at=NULL WHERE id=?`, item.ID); err != nil {
+		t.Fatalf("clear fip_associated_at: %v", err)
+	}
+
+	assigned, _, err := o.AssignmentForValidator(ctx, "validator-1")
+	if err != nil {
+		t.Fatalf("assignment for validator: %v", err)
+	}
+	if assigned == nil {
+		t.Fatalf("expected assignment available for a row with nil FIPAssociatedAt despite nonzero settle delay")
+	}
+}
+
+// TestSetFIPSettleSecondsValidation proves the cross-field rule against
+// lease_ttl_seconds/self_check_timeout_seconds is enforced.
+func TestSetFIPSettleSecondsValidation(t *testing.T) {
+	ctx := context.Background()
+	o, _, _ := newTestOrchestrator(t, 180) // SelfCheckTimeoutSeconds: 10 (see newTestOrchestratorWithSites)
+
+	cases := []struct {
+		seconds int
+		wantErr bool
+	}{
+		{-1, true},
+		{170, true}, // 170+10 >= 180
+		{169, false},
+		{0, false},
+	}
+	for _, c := range cases {
+		err := o.SetFIPSettleSeconds(ctx, c.seconds)
+		if c.wantErr && !errors.Is(err, db.ErrValidation) {
+			t.Errorf("seconds=%d: expected ErrValidation, got %v", c.seconds, err)
+		}
+		if !c.wantErr && err != nil {
+			t.Errorf("seconds=%d: expected success, got %v", c.seconds, err)
+		}
+	}
+}
+
 // TestInboundChecksPartialSites confirms a partially-configured sites list
 // (fewer than 3 slots assigned) only waits on the sites actually
 // configured — the two unassigned slots are never expected.