Keep one address per validator; fix heartbeat handling and queue clear

A mass check on 2026-10-02 stalled 7 of 20 validators and sent 42
addresses to fail without a single check. A validator busy with slow
checks went silent, was marked unreachable, and its next heartbeat put it
back to idle while it still held the address; it was handed a second one,
whose association never ran (the in-flight guard was keyed by validator),
and both waited for their leases to expire.

- Heartbeat/re-register return an unreachable validator to assigned when
  it still holds an address, else idle.
- A validator is released only from the address it currently holds
  (ReleaseFIP, RequeueOrFail, MarkFIPOccupied, FreeValidator); an
  unreachable validator stays unreachable until its next heartbeat, so a
  dead validator is no longer handed a new address every lease period.
- ClaimNextQueued refuses a validator that still has an address; a
  ReconcileValidators pass on every tick repairs rows that disagree with
  the queue.
- Association guard is keyed by address, not validator.
- The agent sends heartbeats from their own goroutine.
- Clear queue / delete: detach only floating IPs of unfinished rows (done,
  failed and occupied rows kept their fip_id and made a clear issue >1000
  sequential cloud calls: 256 s), at most 8 in parallel; the operation no
  longer dies with the client connection (10 minute limit).

Includes the incident analysis and the plan under analysis/ and
docs/changes/, and rebuilt bin/control-api and bin/validator-agent.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

internal/db/queries_ipqueue.go

@@ -101,7 +101,7 @@ func (d *DB) ClaimNextQueued(ctx context.Context, validatorID string, leaseTTL t
 
 	res, err = tx.ExecContext(ctx, `
 		UPDATE validators SET state=?, current_ip_id=?, updated_at=?
-		WHERE validator_id=? AND state=?
+		WHERE validator_id=? AND state=? AND current_ip_id IS NULL
 	`, ValidatorAssigned, item.ID, timeToDB(now), validatorID, ValidatorIdle)
 	if err != nil {
 		return nil, err
@@ -207,7 +207,8 @@ func (d *DB) FinishIP(ctx context.Context, ipID int64, result string) error {
 }
 
 // ReleaseFIP records that the floating IP has been disassociated and frees
-// the owning validator back to idle, in one transaction.
+// the owning validator (if this address is still its current one, see
+// freeValidatorSQL), in one transaction.
 func (d *DB) ReleaseFIP(ctx context.Context, ipID int64, validatorID string) error {
 	tx, err := d.BeginTx(ctx, nil)
 	if err != nil {
@@ -219,10 +220,7 @@ func (d *DB) ReleaseFIP(ctx context.Context, ipID int64, validatorID string) err
 	if _, err := tx.ExecContext(ctx, `UPDATE ip_queue SET fip_released_at=?, updated_at=? WHERE id=?`, now, now, ipID); err != nil {
 		return err
 	}
-	if _, err := tx.ExecContext(ctx, `
-		UPDATE validators SET state=?, current_ip_id=NULL, updated_at=?
-		WHERE validator_id=?
-	`, ValidatorIdle, now, validatorID); err != nil {
+	if _, err := tx.ExecContext(ctx, freeValidatorSQL, now, validatorID, ipID); err != nil {
 		return err
 	}
 	return tx.Commit()
@@ -252,10 +250,7 @@ func (d *DB) MarkFIPOccupied(ctx context.Context, ipID int64, validatorID string
 		return err
 	}
 	if validatorID != "" {
-		if _, err := tx.ExecContext(ctx, `
-			UPDATE validators SET state=?, current_ip_id=NULL, updated_at=?
-			WHERE validator_id=?
-		`, ValidatorIdle, now, validatorID); err != nil {
+		if _, err := tx.ExecContext(ctx, freeValidatorSQL, now, validatorID, ipID); err != nil {
 			return err
 		}
 	}
@@ -315,10 +310,7 @@ func (d *DB) RequeueOrFail(ctx context.Context, ipID int64, validatorID string,
 	}
 
 	if validatorID != "" {
-		if _, err := tx.ExecContext(ctx, `
-			UPDATE validators SET state=?, current_ip_id=NULL, updated_at=?
-			WHERE validator_id=?
-		`, ValidatorIdle, now, validatorID); err != nil {
+		if _, err := tx.ExecContext(ctx, freeValidatorSQL, now, validatorID, ipID); err != nil {
 			return err
 		}
 	}
@@ -520,9 +512,11 @@ func (d *DB) DeleteIPs(ctx context.Context, addresses []string) (DeleteIPsResult
 func deleteIPTx(ctx context.Context, tx *sql.Tx, ipID int64) error {
 	now := timeToDB(Now())
 	if _, err := tx.ExecContext(ctx, `
-		UPDATE validators SET state=?, current_ip_id=NULL, updated_at=?
+		UPDATE validators SET current_ip_id=NULL,
+			state = CASE WHEN state=? THEN state ELSE ? END,
+			updated_at=?
 		WHERE current_ip_id=?
-	`, ValidatorIdle, now, ipID); err != nil {
+	`, ValidatorUnreachable, ValidatorIdle, now, ipID); err != nil {
 		return fmt.Errorf("free owning validator: %w", err)
 	}
 	if _, err := tx.ExecContext(ctx, `UPDATE checks SET ip_id=NULL WHERE ip_id=?`, ipID); err != nil {
@@ -579,9 +573,11 @@ func (d *DB) ClearAllIPs(ctx context.Context) ([]string, error) {
 
 	now := timeToDB(Now())
 	if _, err := tx.ExecContext(ctx, `
-		UPDATE validators SET state=?, current_ip_id=NULL, updated_at=?
+		UPDATE validators SET current_ip_id=NULL,
+			state = CASE WHEN state=? THEN state ELSE ? END,
+			updated_at=?
 		WHERE current_ip_id IS NOT NULL
-	`, ValidatorIdle, now); err != nil {
+	`, ValidatorUnreachable, ValidatorIdle, now); err != nil {
 		return nil, fmt.Errorf("free owning validators: %w", err)
 	}
 	if _, err := tx.ExecContext(ctx, `UPDATE checks SET ip_id=NULL WHERE ip_id IS NOT NULL`); err != nil {
@@ -610,11 +606,15 @@ type FIPRef struct {
 	FIPID     string
 }
 
-// ListFIPRefs returns every queue row with an attached floating IP (fip_id
-// set) — typically at most one per validator — so a bulk clear can
-// disassociate them without loading the whole queue.
+// ListFIPRefs returns the queue rows that may still hold a floating IP: an
+// fip_id is set and the row is not finished. A finished row (done, failed,
+// occupied) keeps its fip_id for display, but its floating IP was already
+// disassociated before the final state was written, so listing it would only
+// make a bulk clear issue thousands of pointless cloud calls. At most one row
+// per validator qualifies, so a clear does not need to load the whole queue.
 func (d *DB) ListFIPRefs(ctx context.Context) ([]FIPRef, error) {
-	rows, err := d.QueryContext(ctx, `SELECT id, ip_address, fip_id FROM ip_queue WHERE fip_id<>'' ORDER BY id`)
+	rows, err := d.QueryContext(ctx, `SELECT id, ip_address, fip_id FROM ip_queue WHERE fip_id<>'' AND state NOT IN (?, ?, ?) ORDER BY id`,
+		IPDone, IPFailed, IPOccupied)
 	if err != nil {
 		return nil, err
 	}
@@ -631,8 +631,7 @@ func (d *DB) ListFIPRefs(ctx context.Context) ([]FIPRef, error) {
 }
 
 // ListFIPRefsByAddresses is ListFIPRefs restricted to the given addresses
-// (unknown addresses and rows without an attached floating IP are simply
-// absent), using a handful of IN (...) queries instead of one lookup per
+// (unknown addresses and rows that hold no floating IP are simply absent), using a handful of IN (...) queries instead of one lookup per
 // address.
 func (d *DB) ListFIPRefsByAddresses(ctx context.Context, addresses []string) ([]FIPRef, error) {
 	const chunk = 500
@@ -643,12 +642,12 @@ func (d *DB) ListFIPRefsByAddresses(ctx context.Context, addresses []string) ([]
 			end = len(addresses)
 		}
 		part := addresses[start:end]
-		args := make([]any, len(part))
-		for i, a := range part {
-			args[i] = a
+		args := []any{IPDone, IPFailed, IPOccupied}
+		for _, a := range part {
+			args = append(args, a)
 		}
 		rows, err := d.QueryContext(ctx,
-			`SELECT id, ip_address, fip_id FROM ip_queue WHERE fip_id<>'' AND ip_address IN (`+placeholders(len(part))+`)`, args...)
+			`SELECT id, ip_address, fip_id FROM ip_queue WHERE fip_id<>'' AND state NOT IN (?, ?, ?) AND ip_address IN (`+placeholders(len(part))+`)`, args...)
 		if err != nil {
 			return nil, err
 		}

internal/db/queries_validator_state_test.go

@@ -0,0 +1,186 @@
+package db
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+)
+
+func validatorState(t *testing.T, d *DB, id string) *Validator {
+	t.Helper()
+	v, err := d.GetValidator(testCtx(t), id)
+	if err != nil {
+		t.Fatalf("get validator %s: %v", id, err)
+	}
+	return v
+}
+
+func testCtx(t *testing.T) context.Context {
+	t.Helper()
+	return context.Background()
+}
+
+// claimFor2 seeds an address and returns its id without claiming it.
+func claimFor2(t *testing.T, d *DB, addr string) int64 {
+	t.Helper()
+	ctx := testCtx(t)
+	if err := d.SeedQueue(ctx, []string{addr}); err != nil {
+		t.Fatalf("seed %s: %v", addr, err)
+	}
+	ip, err := d.GetIPByAddress(ctx, addr)
+	if err != nil {
+		t.Fatalf("get %s: %v", addr, err)
+	}
+	return ip.ID
+}
+
+// claimFor seeds an address and claims it for the validator.
+func claimFor(t *testing.T, d *DB, addr, validatorID string) *IPQueueItem {
+	t.Helper()
+	ctx := testCtx(t)
+	if err := d.SeedQueue(ctx, []string{addr}); err != nil {
+		t.Fatalf("seed %s: %v", addr, err)
+	}
+	item, err := d.ClaimNextQueued(ctx, validatorID, time.Minute)
+	if err != nil || item == nil {
+		t.Fatalf("claim %s for %s: item=%v err=%v", addr, validatorID, item, err)
+	}
+	return item
+}
+
+func TestHeartbeatKeepsAssignedWhenValidatorStillHoldsAnAddress(t *testing.T) {
+	d, ctx := newTestDB(t)
+	_ = d.AdminCreateValidator(ctx, "v1", "p1")
+	_ = d.AdminCreateValidator(ctx, "v2", "p2")
+	item := claimFor(t, d, "1.1.1.1", "v1")
+	_ = d.MarkValidatorUnreachable(ctx, "v1")
+	_ = d.MarkValidatorUnreachable(ctx, "v2")
+
+	if err := d.Heartbeat(ctx, "v1"); err != nil {
+		t.Fatal(err)
+	}
+	if v := validatorState(t, d, "v1"); v.State != ValidatorAssigned || v.CurrentIPID == nil || *v.CurrentIPID != item.ID {
+		t.Fatalf("v1 after heartbeat: state=%s current_ip=%v, want assigned to %d", v.State, v.CurrentIPID, item.ID)
+	}
+	if err := d.Heartbeat(ctx, "v2"); err != nil {
+		t.Fatal(err)
+	}
+	if v := validatorState(t, d, "v2"); v.State != ValidatorIdle {
+		t.Fatalf("v2 after heartbeat: state=%s, want idle", v.State)
+	}
+}
+
+func TestRegisterValidatorReactivationKeepsAssignedAddress(t *testing.T) {
+	d, ctx := newTestDB(t)
+	_ = d.AdminCreateValidator(ctx, "v1", "p1")
+	item := claimFor(t, d, "1.1.1.1", "v1")
+	_ = d.MarkValidatorUnreachable(ctx, "v1")
+
+	if err := d.RegisterValidator(ctx, "v1", "host", "p1", "v"); err != nil {
+		t.Fatal(err)
+	}
+	if v := validatorState(t, d, "v1"); v.State != ValidatorAssigned || v.CurrentIPID == nil || *v.CurrentIPID != item.ID {
+		t.Fatalf("after re-register: state=%s current_ip=%v, want assigned to %d", v.State, v.CurrentIPID, item.ID)
+	}
+}
+
+func TestStaleReleasesLeaveTheCurrentAddressAlone(t *testing.T) {
+	cases := map[string]func(d *DB, ipID int64) error{
+		"ReleaseFIP":      func(d *DB, id int64) error { return d.ReleaseFIP(context.Background(), id, "v1") },
+		"RequeueOrFail":   func(d *DB, id int64) error { return d.RequeueOrFail(context.Background(), id, "v1", 3) },
+		"MarkFIPOccupied": func(d *DB, id int64) error { return d.MarkFIPOccupied(context.Background(), id, "v1") },
+		"FreeValidator":   func(d *DB, id int64) error { return d.FreeValidator(context.Background(), "v1", id) },
+	}
+	for name, release := range cases {
+		t.Run(name, func(t *testing.T) {
+			d, ctx := newTestDB(t)
+			_ = d.AdminCreateValidator(ctx, "v1", "p1")
+			stale := claimFor(t, d, "1.1.1.1", "v1")
+			// v1 has moved on to another address (set directly: the claim path
+			// itself refuses a validator that is still busy).
+			cur := claimFor2(t, d, "2.2.2.2")
+			if _, err := d.ExecContext(ctx, `UPDATE ip_queue SET state='checking', owner_validator_id='v1' WHERE id=?`, cur); err != nil {
+				t.Fatal(err)
+			}
+			if _, err := d.ExecContext(ctx, `UPDATE validators SET state='assigned', current_ip_id=? WHERE validator_id='v1'`, cur); err != nil {
+				t.Fatal(err)
+			}
+
+			if err := release(d, stale.ID); err != nil {
+				t.Fatalf("%s: %v", name, err)
+			}
+			v := validatorState(t, d, "v1")
+			if v.State != ValidatorAssigned || v.CurrentIPID == nil || *v.CurrentIPID != cur {
+				t.Fatalf("%s freed a validator that holds another address: state=%s current_ip=%v", name, v.State, v.CurrentIPID)
+			}
+		})
+	}
+}
+
+// The right address frees the validator, but an unreachable one stays so.
+func TestReleaseKeepsUnreachableValidatorUnreachable(t *testing.T) {
+	d, ctx := newTestDB(t)
+	_ = d.AdminCreateValidator(ctx, "v1", "p1")
+	_ = d.AdminCreateValidator(ctx, "v2", "p2")
+	a := claimFor(t, d, "1.1.1.1", "v1")
+	b := claimFor(t, d, "2.2.2.2", "v2")
+	_ = d.MarkValidatorUnreachable(ctx, "v1")
+
+	if err := d.ReleaseFIP(ctx, a.ID, "v1"); err != nil {
+		t.Fatal(err)
+	}
+	if v := validatorState(t, d, "v1"); v.State != ValidatorUnreachable || v.CurrentIPID != nil {
+		t.Fatalf("v1: state=%s current_ip=%v, want unreachable and empty", v.State, v.CurrentIPID)
+	}
+	if err := d.RequeueOrFail(ctx, b.ID, "v2", 3); err != nil {
+		t.Fatal(err)
+	}
+	if v := validatorState(t, d, "v2"); v.State != ValidatorIdle || v.CurrentIPID != nil {
+		t.Fatalf("v2: state=%s current_ip=%v, want idle and empty", v.State, v.CurrentIPID)
+	}
+}
+
+func TestClaimRefusesValidatorThatStillHoldsAnAddress(t *testing.T) {
+	d, ctx := newTestDB(t)
+	_ = d.AdminCreateValidator(ctx, "v1", "p1")
+	first := claimFor(t, d, "1.1.1.1", "v1")
+	// An old version could leave a validator idle while it still pointed at an address.
+	if _, err := d.ExecContext(ctx, `UPDATE validators SET state='idle' WHERE validator_id='v1'`); err != nil {
+		t.Fatal(err)
+	}
+	_ = d.SeedQueue(ctx, []string{"2.2.2.2"})
+	got, err := d.ClaimNextQueued(ctx, "v1", time.Minute)
+	if err != nil || got != nil {
+		t.Fatalf("claim for a validator that holds %d: item=%v err=%v, want nothing", first.ID, got, err)
+	}
+	if ip, _ := d.GetIPByAddress(ctx, "2.2.2.2"); ip.State != IPQueued {
+		t.Fatalf("2.2.2.2 is %s, want queued", ip.State)
+	}
+}
+
+func TestListFIPRefsSkipsFinishedAddresses(t *testing.T) {
+	d, ctx := newTestDB(t)
+	var addrs []string
+	for i := 0; i < 6; i++ {
+		addrs = append(addrs, fmt.Sprintf("10.0.0.%d", i+1))
+	}
+	_ = d.SeedQueue(ctx, addrs)
+	states := []string{"done", "failed", "occupied", "awaiting_self_check", "checking", "aggregating"}
+	for i, a := range addrs {
+		if _, err := d.ExecContext(ctx, `UPDATE ip_queue SET state=?, fip_id=? WHERE ip_address=?`, states[i], fmt.Sprintf("fip-%d", i), a); err != nil {
+			t.Fatal(err)
+		}
+	}
+	refs, err := d.ListFIPRefs(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(refs) != 3 {
+		t.Fatalf("ListFIPRefs returned %d rows, want the 3 unfinished ones: %+v", len(refs), refs)
+	}
+	byAddr, err := d.ListFIPRefsByAddresses(ctx, addrs)
+	if err != nil || len(byAddr) != 3 {
+		t.Fatalf("ListFIPRefsByAddresses returned %d rows (err %v), want 3", len(byAddr), err)
+	}
+}

internal/db/queries_validators.go

@@ -27,24 +27,32 @@ func (d *DB) RegisterValidator(ctx context.Context, validatorID, hostname, osPor
 	}
 	// A brand-new row already lands in ValidatorIdle via the INSERT branch;
 	// a re-registering validator that was 'unregistered' or 'unreachable'
-	// (but not mid-assignment) should also come back to idle.
+	// comes back: to idle when it holds no address, to assigned when it still
+	// does (its current_ip_id is kept, so it must not be handed another).
 	_, err = d.ExecContext(ctx, `
-		UPDATE validators SET state=?, updated_at=?
+		UPDATE validators SET
+			state = CASE WHEN current_ip_id IS NULL THEN ? ELSE ? END,
+			updated_at=?
 		WHERE validator_id=? AND state IN (?, ?)
-	`, ValidatorIdle, now, validatorID, ValidatorUnregistered, ValidatorUnreachable)
+	`, ValidatorIdle, ValidatorAssigned, now, validatorID, ValidatorUnregistered, ValidatorUnreachable)
 	if err != nil {
 		return fmt.Errorf("register validator (reactivate): %w", err)
 	}
 	return nil
 }
 
+// Heartbeat records a sign of life. A validator that was marked unreachable
+// returns to idle if it holds no address, but to assigned if it still does:
+// it was only silent (for example busy with slow checks), and handing it a
+// second address while it works on the first would leave the second one
+// without an owner that can ever pick it up.
 func (d *DB) Heartbeat(ctx context.Context, validatorID string) error {
 	now := timeToDB(Now())
 	res, err := d.ExecContext(ctx, `
 		UPDATE validators SET last_heartbeat_at=?, updated_at=?,
-			state = CASE WHEN state=? THEN ? ELSE state END
+			state = CASE WHEN state=? THEN (CASE WHEN current_ip_id IS NULL THEN ? ELSE ? END) ELSE state END
 		WHERE validator_id=?
-	`, now, now, ValidatorUnreachable, ValidatorIdle, validatorID)
+	`, now, now, ValidatorUnreachable, ValidatorIdle, ValidatorAssigned, validatorID)
 	if err != nil {
 		return fmt.Errorf("heartbeat: %w", err)
 	}
@@ -138,16 +146,59 @@ func (d *DB) MarkValidatorUnreachable(ctx context.Context, validatorID string) e
 	return err
 }
 
-// FreeValidator returns a validator to idle with no assigned IP. Used after
-// an IP finishes (success or failure) or is reclaimed by the lease sweep.
-func (d *DB) FreeValidator(ctx context.Context, validatorID string) error {
-	_, err := d.ExecContext(ctx, `
-		UPDATE validators SET state=?, current_ip_id=NULL, updated_at=?
-		WHERE validator_id=?
-	`, ValidatorIdle, timeToDB(Now()), validatorID)
+// freeValidatorSQL releases a validator from the address it holds. It only
+// applies when the validator's current address is the one being released
+// (args: now, validator id, ip id): a late release of an old address must not
+// free a validator that has already moved on to another one. An unreachable
+// validator stays unreachable until its next heartbeat, so a dead validator
+// is not handed new addresses just because its lease was reclaimed.
+var freeValidatorSQL = fmt.Sprintf(`
+	UPDATE validators SET current_ip_id=NULL,
+		state = CASE WHEN state='%s' THEN state ELSE '%s' END,
+		updated_at=?
+	WHERE validator_id=? AND current_ip_id=?`, ValidatorUnreachable, ValidatorIdle)
+
+// FreeValidator releases a validator from the given address (see
+// freeValidatorSQL). Used after an IP finishes (success or failure) or is
+// reclaimed by the lease sweep.
+func (d *DB) FreeValidator(ctx context.Context, validatorID string, ipID int64) error {
+	_, err := d.ExecContext(ctx, freeValidatorSQL, timeToDB(Now()), validatorID, ipID)
 	return err
 }
 
+// ReconcileValidators repairs validators whose state disagrees with the
+// queue: a validator pointing at an address that no longer exists, is
+// finished, or belongs to another validator is released; an "assigned"
+// validator that holds nothing goes back to idle. The invariants normally
+// hold by construction (every change is one transaction); this heals what a
+// crash or an older version left behind. It returns the number of validators
+// repaired.
+func (d *DB) ReconcileValidators(ctx context.Context) (int64, error) {
+	now := timeToDB(Now())
+	res, err := d.ExecContext(ctx, fmt.Sprintf(`
+		UPDATE validators SET current_ip_id=NULL,
+			state = CASE WHEN state='%s' THEN state ELSE '%s' END,
+			updated_at=?
+		WHERE current_ip_id IS NOT NULL AND NOT EXISTS (
+			SELECT 1 FROM ip_queue q
+			WHERE q.id = validators.current_ip_id
+			  AND q.owner_validator_id = validators.validator_id
+			  AND q.state NOT IN ('%s','%s','%s'))`,
+		ValidatorUnreachable, ValidatorIdle, IPDone, IPFailed, IPOccupied), now)
+	if err != nil {
+		return 0, fmt.Errorf("reconcile validators: %w", err)
+	}
+	n, _ := res.RowsAffected()
+	res, err = d.ExecContext(ctx, `
+		UPDATE validators SET state=?, updated_at=?
+		WHERE state=? AND current_ip_id IS NULL`, ValidatorIdle, now, ValidatorAssigned)
+	if err != nil {
+		return n, fmt.Errorf("reconcile validators: %w", err)
+	}
+	m, _ := res.RowsAffected()
+	return n + m, nil
+}
+
 // AdminCreateValidator registers a brand-new validator via the admin API.
 // Unlike RegisterValidator (used by the agent's self-registration call),
 // this refuses to upsert over an existing row.