Scan floating IPs in the background, page by page, so thousands of addresses work

The "Scan Floating IP" button failed with a client timeout: the project now
holds ~6.4k floating IPs and the scan listed them all in one unpaginated,
timeout-less Neutron request on the HTTP request context.

openstack: ListFreeFloatingIPs reads marker-based pages (fields= keeps them
small) with per-page retry/backoff on transport errors, 5xx and 429, and every
request now has a timeout (also ends hangs inside the orchestrator tick).

orchestrator: the scan is a single-flight background job on the process
context with progress (clearing/listing/enqueuing/done/error), dry_run, full
discovery before anything is enqueued, then SubmitIPs in chunks of 500 in
ascending IP order; a failed read leaves the queue untouched. The auto-cycle
gets a "scanning" phase that polls the job, so the control loop and
autoCycleMu are never held across OpenStack/DB work; it recovers after a
restart and waits for (instead of adopting) a scan started by someone else.

db: migration 0009 (indexes), paged ListIPsPage/ListRegistryPage, GROUP BY
counters, EXISTS completion check, set-based ClearAllIPs.

API: POST /admin/ips/scan -> 202 (dry_run, wait), GET /admin/ips/scan, paging
and filters on /admin/ips and /admin/registry (bare arrays without limit),
results_by_overall in /admin/status.

dashboard: scan progress panel and dry-run button, paginated /ips and
/registry with server-side filters, Overview on counters and capped lists
with progress/ETA, "select all N by filter", hx-params fix for per-row
buttons, real counts in confirmations.

Also: docs (API, USAGE, DASHBOARD, README), plan and review under
docs/changes/, bin/ rebuilt with new SHA256SUMS.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

internal/db/db.go

@@ -37,6 +37,9 @@ var ipRegistrySchema string
 //go:embed migrations/0008_auto_cycle.sql
 var autoCycleSchema string
 
+//go:embed migrations/0009_scale_indexes.sql
+var scaleIndexesSchema string
+
 // migrations is the ordered list of schema versions. Each entry's SQL is
 // applied, in order, for any version greater than the database's current
 // PRAGMA user_version — so a fresh database walks the whole list and an
@@ -53,6 +56,7 @@ var migrations = []struct {
 	{6, proberHeartbeatSchema},
 	{7, ipRegistrySchema},
 	{8, autoCycleSchema},
+	{9, scaleIndexesSchema},
 }
 
 type DB struct {

internal/db/migrations/0009_scale_indexes.sql

@@ -0,0 +1,11 @@
+-- Scale indexes (see docs/changes: FIP scan at scale).
+--
+-- idx_ip_queue_registry: ListRegistry/ListRegistryPage look up the live
+-- ip_queue row of every registry address by registry_id; without an index
+-- that was a full scan per address, i.e. O(n^2) with thousands of addresses.
+--
+-- idx_ip_queue_state_aggregated: paged "recently finished" listings
+-- (state filter, newest aggregated_at first).
+
+CREATE INDEX idx_ip_queue_registry ON ip_queue(registry_id);
+CREATE INDEX idx_ip_queue_state_aggregated ON ip_queue(state, aggregated_at);

internal/db/models.go

@@ -39,6 +39,32 @@ const (
 	SourceEgress = "egress"
 )
 
+// IPStates lists every valid ip_queue state, in lifecycle order.
+var IPStates = []string{
+	IPQueued, IPAssigningFIP, IPAwaitingSelfCheck, IPChecking, IPAggregating,
+	IPDone, IPFailed, IPOccupied,
+}
+
+// IsValidIPState reports whether s is one of IPStates.
+func IsValidIPState(s string) bool {
+	for _, v := range IPStates {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+// IsValidResult reports whether s is a valid overall result value
+// (pass | partial | fail | cancelled).
+func IsValidResult(s string) bool {
+	switch s {
+	case ResultPass, ResultPartial, ResultFail, ResultCancelled:
+		return true
+	}
+	return false
+}
+
 // InboundSource returns the checks.source value for the given prober site
 // index (1-based), e.g. InboundSource(1) == "inbound-site-1".
 func InboundSource(siteIndex int) string {
@@ -232,6 +258,9 @@ const (
 	AutoCyclePhaseIdle    = "idle"
 	AutoCyclePhaseRunning = "running"
 	AutoCyclePhaseWaiting = "waiting"
+	// AutoCyclePhaseScanning: the queue is being cleared and the floating IPs
+	// are being (re)discovered and enqueued by the background scan job.
+	AutoCyclePhaseScanning = "scanning"
 
 	AutoCycleOutcomeCompleted = "completed"
 	AutoCycleOutcomeNoFreeIPs = "no_free_ips"

internal/db/queries_autocycle.go

@@ -102,7 +102,7 @@ func (d *DB) SetAutoCycleEnabled(ctx context.Context, enabled bool, nextRunAt *t
 // touch the configuration or the enabled flag.
 func (d *DB) UpdateAutoCycleState(ctx context.Context, s AutoCycleState) error {
 	switch s.Phase {
-	case AutoCyclePhaseIdle, AutoCyclePhaseRunning, AutoCyclePhaseWaiting:
+	case AutoCyclePhaseIdle, AutoCyclePhaseScanning, AutoCyclePhaseRunning, AutoCyclePhaseWaiting:
 	default:
 		return fmt.Errorf("invalid auto-cycle phase %q: %w", s.Phase, ErrValidation)
 	}

internal/db/queries_ipqueue.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"strings"
 	"time"
 )
 
@@ -497,6 +498,263 @@ func deleteIPTx(ctx context.Context, tx *sql.Tx, ipID int64) error {
 	return nil
 }
 
+// ClearAllIPs deletes every ip_queue row in one short, set-based
+// transaction (five statements regardless of the queue size) and returns the
+// deleted addresses in queue order. It does exactly what deleteIPTx does per
+// row: frees every validator that still points at a doomed row, detaches
+// checks/events (their registry_id keeps the history), drops the ephemeral
+// ip_site_checks progress flags and finally the rows themselves.
+// Disassociating attached floating IPs is the caller's (orchestrator's) job.
+func (d *DB) ClearAllIPs(ctx context.Context) ([]string, error) {
+	tx, err := d.BeginTx(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer tx.Rollback()
+
+	rows, err := tx.QueryContext(ctx, `SELECT ip_address FROM ip_queue ORDER BY sequence`)
+	if err != nil {
+		return nil, err
+	}
+	deleted := []string{}
+	for rows.Next() {
+		var addr string
+		if err := rows.Scan(&addr); err != nil {
+			rows.Close()
+			return nil, err
+		}
+		deleted = append(deleted, addr)
+	}
+	if err := rows.Err(); err != nil {
+		rows.Close()
+		return nil, err
+	}
+	rows.Close()
+
+	now := timeToDB(Now())
+	if _, err := tx.ExecContext(ctx, `
+		UPDATE validators SET state=?, current_ip_id=NULL, updated_at=?
+		WHERE current_ip_id IS NOT NULL
+	`, ValidatorIdle, now); err != nil {
+		return nil, fmt.Errorf("free owning validators: %w", err)
+	}
+	if _, err := tx.ExecContext(ctx, `UPDATE checks SET ip_id=NULL WHERE ip_id IS NOT NULL`); err != nil {
+		return nil, fmt.Errorf("detach checks: %w", err)
+	}
+	if _, err := tx.ExecContext(ctx, `UPDATE events SET ip_id=NULL WHERE ip_id IS NOT NULL`); err != nil {
+		return nil, fmt.Errorf("detach events: %w", err)
+	}
+	if _, err := tx.ExecContext(ctx, `DELETE FROM ip_site_checks`); err != nil {
+		return nil, fmt.Errorf("delete ip_site_checks: %w", err)
+	}
+	if _, err := tx.ExecContext(ctx, `DELETE FROM ip_queue`); err != nil {
+		return nil, fmt.Errorf("delete ip_queue rows: %w", err)
+	}
+	if err := tx.Commit(); err != nil {
+		return nil, err
+	}
+	return deleted, nil
+}
+
+// FIPRef identifies a queue row that currently holds a Neutron floating-IP
+// association.
+type FIPRef struct {
+	IPID      int64
+	IPAddress string
+	FIPID     string
+}
+
+// ListFIPRefs returns every queue row with an attached floating IP (fip_id
+// set) — typically at most one per validator — so a bulk clear can
+// disassociate them without loading the whole queue.
+func (d *DB) ListFIPRefs(ctx context.Context) ([]FIPRef, error) {
+	rows, err := d.QueryContext(ctx, `SELECT id, ip_address, fip_id FROM ip_queue WHERE fip_id<>'' ORDER BY id`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []FIPRef
+	for rows.Next() {
+		var r FIPRef
+		if err := rows.Scan(&r.IPID, &r.IPAddress, &r.FIPID); err != nil {
+			return nil, err
+		}
+		out = append(out, r)
+	}
+	return out, rows.Err()
+}
+
+// ListFIPRefsByAddresses is ListFIPRefs restricted to the given addresses
+// (unknown addresses and rows without an attached floating IP are simply
+// absent), using a handful of IN (...) queries instead of one lookup per
+// address.
+func (d *DB) ListFIPRefsByAddresses(ctx context.Context, addresses []string) ([]FIPRef, error) {
+	const chunk = 500
+	var out []FIPRef
+	for start := 0; start < len(addresses); start += chunk {
+		end := start + chunk
+		if end > len(addresses) {
+			end = len(addresses)
+		}
+		part := addresses[start:end]
+		args := make([]any, len(part))
+		for i, a := range part {
+			args[i] = a
+		}
+		rows, err := d.QueryContext(ctx,
+			`SELECT id, ip_address, fip_id FROM ip_queue WHERE fip_id<>'' AND ip_address IN (`+placeholders(len(part))+`)`, args...)
+		if err != nil {
+			return nil, err
+		}
+		for rows.Next() {
+			var r FIPRef
+			if err := rows.Scan(&r.IPID, &r.IPAddress, &r.FIPID); err != nil {
+				rows.Close()
+				return nil, err
+			}
+			out = append(out, r)
+		}
+		if err := rows.Err(); err != nil {
+			rows.Close()
+			return nil, err
+		}
+		rows.Close()
+	}
+	return out, nil
+}
+
+func placeholders(n int) string {
+	if n <= 0 {
+		return ""
+	}
+	return strings.TrimSuffix(strings.Repeat("?,", n), ",")
+}
+
+// CountIPsByState returns how many ip_queue rows are in each state, plus the
+// grand total, via a single GROUP BY (no row loading).
+func (d *DB) CountIPsByState(ctx context.Context) (map[string]int, int, error) {
+	rows, err := d.QueryContext(ctx, `SELECT state, COUNT(*) FROM ip_queue GROUP BY state`)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer rows.Close()
+	counts := map[string]int{}
+	total := 0
+	for rows.Next() {
+		var state string
+		var n int
+		if err := rows.Scan(&state, &n); err != nil {
+			return nil, 0, err
+		}
+		counts[state] = n
+		total += n
+	}
+	return counts, total, rows.Err()
+}
+
+// CountIPsByResult returns how many ip_queue rows carry each non-empty
+// overall_result (pass/partial/fail/cancelled), via a single GROUP BY.
+func (d *DB) CountIPsByResult(ctx context.Context) (map[string]int, error) {
+	rows, err := d.QueryContext(ctx, `
+		SELECT overall_result, COUNT(*) FROM ip_queue WHERE overall_result<>'' GROUP BY overall_result
+	`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	counts := map[string]int{}
+	for rows.Next() {
+		var res string
+		var n int
+		if err := rows.Scan(&res, &n); err != nil {
+			return nil, err
+		}
+		counts[res] = n
+	}
+	return counts, rows.Err()
+}
+
+// AnyNonTerminalIP reports whether any ip_queue row is still unfinished (its
+// state is none of done/failed/occupied).
+func (d *DB) AnyNonTerminalIP(ctx context.Context) (bool, error) {
+	var any bool
+	err := d.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM ip_queue WHERE state NOT IN (?, ?, ?))`,
+		IPDone, IPFailed, IPOccupied).Scan(&any)
+	return any, err
+}
+
+// Order values for IPFilter.Order.
+const (
+	IPOrderSequence         = "sequence"
+	IPOrderAggregatedAtDesc = "aggregated_at_desc"
+)
+
+// IPFilter narrows ListIPsPage. The zero value matches everything, ordered by
+// queue sequence.
+type IPFilter struct {
+	States []string // any of these states (empty = all)
+	Query  string   // substring of ip_address
+	Result string   // overall_result equals this (pass|partial|fail|cancelled)
+	Order  string   // IPOrderSequence (default) | IPOrderAggregatedAtDesc
+}
+
+// ListIPsPage returns one page (limit/offset) of ip_queue rows matching f,
+// plus the total number of matching rows. limit <= 0 means no limit.
+func (d *DB) ListIPsPage(ctx context.Context, f IPFilter, limit, offset int) ([]IPQueueItem, int, error) {
+	var conds []string
+	var args []any
+	if len(f.States) > 0 {
+		conds = append(conds, "state IN ("+placeholders(len(f.States))+")")
+		for _, s := range f.States {
+			args = append(args, s)
+		}
+	}
+	if f.Query != "" {
+		conds = append(conds, "instr(ip_address, ?) > 0")
+		args = append(args, f.Query)
+	}
+	if f.Result != "" {
+		conds = append(conds, "overall_result = ?")
+		args = append(args, f.Result)
+	}
+	where := ""
+	if len(conds) > 0 {
+		where = "WHERE " + strings.Join(conds, " AND ") + " "
+	}
+
+	var total int
+	if err := d.QueryRowContext(ctx, `SELECT COUNT(*) FROM ip_queue `+where, args...).Scan(&total); err != nil {
+		return nil, 0, err
+	}
+
+	order := "ORDER BY sequence"
+	if f.Order == IPOrderAggregatedAtDesc {
+		// Timestamps are stored as RFC3339Nano, which trims trailing zeros
+		// and so does not sort correctly as text; strftime normalizes them to
+		// a fixed millisecond layout. NULL aggregated_at sorts last in DESC.
+		order = "ORDER BY strftime('%Y-%m-%d %H:%M:%f', aggregated_at) DESC, sequence"
+	}
+	q := ipQueueSelect + where + order
+	qargs := append([]any(nil), args...)
+	if limit > 0 {
+		q += " LIMIT ? OFFSET ?"
+		qargs = append(qargs, limit, offset)
+	}
+	rows, err := d.QueryContext(ctx, q, qargs...)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer rows.Close()
+	items, err := scanIPQueueItems(rows)
+	if err != nil {
+		return nil, 0, err
+	}
+	if items == nil {
+		items = []IPQueueItem{}
+	}
+	return items, total, nil
+}
+
 func (d *DB) SetEgressComplete(ctx context.Context, ipID int64) error {
 	_, err := d.ExecContext(ctx, `UPDATE ip_queue SET egress_complete=1, updated_at=? WHERE id=?`, timeToDB(Now()), ipID)
 	return err

internal/db/queries_registry.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"strings"
 	"time"
 )
 
@@ -66,7 +67,7 @@ type RegistrySummary struct {
 func (d *DB) ListRegistry(ctx context.Context) ([]RegistrySummary, error) {
 	rows, err := d.QueryContext(ctx, `
 		SELECT id, ip_address, first_seen_at, last_seen_at, next_cycle, created_at, updated_at
-		FROM ip_registry ORDER BY first_seen_at
+		FROM ip_registry ORDER BY first_seen_at, id
 	`)
 	if err != nil {
 		return nil, err
@@ -89,6 +90,84 @@ func (d *DB) ListRegistry(ctx context.Context) ([]RegistrySummary, error) {
 	return out, nil
 }
 
+// RegistryFilter narrows ListRegistryPage. The zero value matches everything.
+type RegistryFilter struct {
+	Query      string // substring of ip_address
+	LastResult string // pass|partial|fail|cancelled — same meaning as RegistrySummary.LastResult
+}
+
+// lastResultCond is the SQL form of fillRegistrySummary's LastResult rule,
+// over `ip_registry r LEFT JOIN ip_queue q ON q.registry_id=r.id`: a live
+// queue row's overall_result is authoritative (and a live row without one has
+// no verdict); with no live row the latest recorded cycle is classified from
+// its checks (pass if all succeeded, fail if none, partial otherwise).
+const lastResultCond = `(
+	(q.id IS NOT NULL AND q.overall_result = ?)
+	OR (q.id IS NULL AND (
+		SELECT CASE WHEN SUM(c.success) = 0 THEN 'fail'
+		            WHEN SUM(c.success) = COUNT(*) THEN 'pass'
+		            ELSE 'partial' END
+		FROM checks c
+		WHERE c.registry_id = r.id
+		  AND c.cycle_id = (SELECT MAX(c2.cycle_id) FROM checks c2 WHERE c2.registry_id = r.id)
+		HAVING COUNT(*) > 0
+	) = ?)
+)`
+
+// ListRegistryPage returns one page (limit/offset) of the registry in the same
+// order as ListRegistry, filtered by f, plus the total number of matching
+// rows. LIMIT/OFFSET are applied in SQL before the per-row summary queries,
+// so only the rows of the page pay for them. limit <= 0 means no limit.
+func (d *DB) ListRegistryPage(ctx context.Context, f RegistryFilter, limit, offset int) ([]RegistrySummary, int, error) {
+	var conds []string
+	var args []any
+	if f.Query != "" {
+		conds = append(conds, "instr(r.ip_address, ?) > 0")
+		args = append(args, f.Query)
+	}
+	if f.LastResult != "" {
+		conds = append(conds, lastResultCond)
+		args = append(args, f.LastResult, f.LastResult)
+	}
+	from := ` FROM ip_registry r LEFT JOIN ip_queue q ON q.registry_id = r.id `
+	where := ""
+	if len(conds) > 0 {
+		where = "WHERE " + strings.Join(conds, " AND ") + " "
+	}
+
+	var total int
+	if err := d.QueryRowContext(ctx, `SELECT COUNT(*)`+from+where, args...).Scan(&total); err != nil {
+		return nil, 0, err
+	}
+
+	q := `SELECT r.id, r.ip_address, r.first_seen_at, r.last_seen_at, r.next_cycle, r.created_at, r.updated_at` +
+		from + where + `ORDER BY r.first_seen_at, r.id`
+	qargs := append([]any(nil), args...)
+	if limit > 0 {
+		q += " LIMIT ? OFFSET ?"
+		qargs = append(qargs, limit, offset)
+	}
+	rows, err := d.QueryContext(ctx, q, qargs...)
+	if err != nil {
+		return nil, 0, err
+	}
+	items, err := scanRegistryItems(rows)
+	rows.Close()
+	if err != nil {
+		return nil, 0, err
+	}
+
+	out := make([]RegistrySummary, len(items))
+	for i, item := range items {
+		s := RegistrySummary{RegistryItem: item}
+		if err := d.fillRegistrySummary(ctx, &s); err != nil {
+			return nil, 0, err
+		}
+		out[i] = s
+	}
+	return out, total, nil
+}
+
 // GetRegistryByAddress returns the registry row (with summary) for a single
 // address, or ErrNotFound if it has never been submitted.
 func (d *DB) GetRegistryByAddress(ctx context.Context, address string) (*RegistrySummary, error) {

internal/db/queries_scale_test.go

@@ -0,0 +1,405 @@
+package db
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+)
+
+// scaleAddrs returns n distinct addresses 10.<a>.<b>.<c> in ascending order.
+func scaleAddrs(n int) []string {
+	out := make([]string, n)
+	for i := 0; i < n; i++ {
+		out[i] = fmt.Sprintf("10.%d.%d.%d", (i/65536)%256, (i/256)%256, i%256)
+	}
+	return out
+}
+
+// finishWithChecks submits the address, records ok successes and bad
+// failures as the current cycle's checks and, when finish != "", finishes the
+// row with that overall result.
+func finishWithChecks(t *testing.T, d *DB, addr string, ok, bad int, finish string) {
+	t.Helper()
+	ctx := context.Background()
+	ip, err := d.GetIPByAddress(ctx, addr)
+	if err != nil {
+		t.Fatalf("get %s: %v", addr, err)
+	}
+	for i := 0; i < ok+bad; i++ {
+		if err := d.UpsertCheck(ctx, Check{
+			IPID: ip.ID, IPAddress: addr, AttemptNumber: ip.AttemptNumber,
+			Source: SourceEgress, CheckType: "https", Target: fmt.Sprintf("https://t%d.test", i),
+			Success: i < ok, CheckedAt: Now(),
+		}); err != nil {
+			t.Fatalf("upsert check: %v", err)
+		}
+	}
+	if finish != "" {
+		if err := d.FinishIP(ctx, ip.ID, finish); err != nil {
+			t.Fatalf("finish: %v", err)
+		}
+	}
+}
+
+func TestListIPsPageFiltersTotalOrder(t *testing.T) {
+	d, ctx := newTestDB(t)
+	addrs := []string{"10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.1.1", "10.0.1.2", "192.168.0.10"}
+	if _, err := d.SubmitIPs(ctx, addrs); err != nil {
+		t.Fatalf("submit: %v", err)
+	}
+	// 10.0.0.1 pass, 10.0.0.2 fail, 10.0.0.3 checking, 10.0.1.1 occupied.
+	finishWithChecks(t, d, "10.0.0.1", 1, 0, ResultPass)
+	time.Sleep(3 * time.Millisecond)
+	finishWithChecks(t, d, "10.0.0.2", 0, 1, ResultFail)
+	ip3, _ := d.GetIPByAddress(ctx, "10.0.0.3")
+	if err := d.SetChecking(ctx, ip3.ID, time.Minute); err != nil {
+		t.Fatal(err)
+	}
+	ip4, _ := d.GetIPByAddress(ctx, "10.0.1.1")
+	if err := d.MarkFIPOccupied(ctx, ip4.ID, ""); err != nil {
+		t.Fatal(err)
+	}
+
+	addrsOf := func(items []IPQueueItem) []string {
+		out := []string{}
+		for _, it := range items {
+			out = append(out, it.IPAddress)
+		}
+		return out
+	}
+
+	items, total, err := d.ListIPsPage(ctx, IPFilter{}, 4, 0)
+	if err != nil || total != 6 || !reflect.DeepEqual(addrsOf(items), addrs[:4]) {
+		t.Fatalf("page 1: total=%d items=%v err=%v", total, addrsOf(items), err)
+	}
+	items, total, _ = d.ListIPsPage(ctx, IPFilter{}, 4, 4)
+	if total != 6 || !reflect.DeepEqual(addrsOf(items), addrs[4:]) {
+		t.Fatalf("page 2: total=%d items=%v", total, addrsOf(items))
+	}
+	items, total, _ = d.ListIPsPage(ctx, IPFilter{}, 4, 100)
+	if total != 6 || len(items) != 0 || items == nil {
+		t.Fatalf("offset past end: total=%d items=%v", total, items)
+	}
+
+	items, total, _ = d.ListIPsPage(ctx, IPFilter{States: []string{IPDone, IPFailed}}, 50, 0)
+	if total != 2 || !reflect.DeepEqual(addrsOf(items), []string{"10.0.0.1", "10.0.0.2"}) {
+		t.Fatalf("states filter: total=%d items=%v", total, addrsOf(items))
+	}
+	items, total, _ = d.ListIPsPage(ctx, IPFilter{States: []string{IPQueued}, Query: "10.0.1."}, 50, 0)
+	if total != 1 || addrsOf(items)[0] != "10.0.1.2" {
+		t.Fatalf("state+q filter: total=%d items=%v", total, addrsOf(items))
+	}
+	items, total, _ = d.ListIPsPage(ctx, IPFilter{Result: ResultFail}, 50, 0)
+	if total != 1 || addrsOf(items)[0] != "10.0.0.2" {
+		t.Fatalf("result filter: total=%d items=%v", total, addrsOf(items))
+	}
+	// q is a plain substring, not a LIKE pattern: % and _ match literally.
+	if _, total, _ = d.ListIPsPage(ctx, IPFilter{Query: "%"}, 50, 0); total != 0 {
+		t.Fatalf("expected literal substring match, total=%d", total)
+	}
+
+	// Newest aggregated first; never-aggregated rows last.
+	items, _, _ = d.ListIPsPage(ctx, IPFilter{Order: IPOrderAggregatedAtDesc}, 50, 0)
+	got := addrsOf(items)
+	if got[0] != "10.0.1.1" && got[0] != "10.0.0.2" {
+		t.Fatalf("expected a finished row first, got %v", got)
+	}
+	if items[0].AggregatedAt == nil || items[len(items)-1].AggregatedAt != nil {
+		t.Fatalf("expected aggregated rows first and unaggregated last: %v", got)
+	}
+	for i := 1; i < len(items); i++ {
+		a, b := items[i-1].AggregatedAt, items[i].AggregatedAt
+		if a != nil && b != nil && a.Before(*b) {
+			t.Fatalf("not sorted by aggregated_at desc at %d: %v", i, got)
+		}
+	}
+}
+
+func TestCountAndNonTerminal(t *testing.T) {
+	d, ctx := newTestDB(t)
+
+	byState, total, err := d.CountIPsByState(ctx)
+	if err != nil || total != 0 || len(byState) != 0 {
+		t.Fatalf("empty: %v %d %v", byState, total, err)
+	}
+	if any, err := d.AnyNonTerminalIP(ctx); err != nil || any {
+		t.Fatalf("empty queue must not report non-terminal: %v %v", any, err)
+	}
+
+	if _, err := d.SubmitIPs(ctx, []string{"1.1.1.1", "1.1.1.2", "1.1.1.3", "1.1.1.4"}); err != nil {
+		t.Fatal(err)
+	}
+	finishWithChecks(t, d, "1.1.1.1", 1, 0, ResultPass)
+	finishWithChecks(t, d, "1.1.1.2", 1, 1, ResultPartial)
+	ip3, _ := d.GetIPByAddress(ctx, "1.1.1.3")
+	if err := d.CancelIP(ctx, ip3.ID); err != nil {
+		t.Fatal(err)
+	}
+
+	byState, total, err = d.CountIPsByState(ctx)
+	if err != nil || total != 4 || byState[IPDone] != 2 || byState[IPFailed] != 1 || byState[IPQueued] != 1 {
+		t.Fatalf("by state: %v total=%d err=%v", byState, total, err)
+	}
+	byResult, err := d.CountIPsByResult(ctx)
+	if err != nil || len(byResult) != 3 || byResult[ResultPass] != 1 || byResult[ResultPartial] != 1 || byResult[ResultCancelled] != 1 {
+		t.Fatalf("by result: %v err=%v", byResult, err)
+	}
+	if any, _ := d.AnyNonTerminalIP(ctx); !any {
+		t.Fatalf("a queued row is non-terminal")
+	}
+	ip4, _ := d.GetIPByAddress(ctx, "1.1.1.4")
+	if err := d.MarkFIPOccupied(ctx, ip4.ID, ""); err != nil {
+		t.Fatal(err)
+	}
+	if any, _ := d.AnyNonTerminalIP(ctx); any {
+		t.Fatalf("done/failed/occupied only: expected terminal")
+	}
+}
+
+// TestListRegistryPageMatchesListRegistry builds a mixed dataset (live rows
+// with every overall result, an in-progress live row, deleted rows whose
+// last cycle classifies as pass/partial/fail, and an address without checks)
+// and verifies that ListRegistryPage's SQL filter selects exactly the rows
+// ListRegistry+fillRegistrySummary labels with the same LastResult.
+func TestListRegistryPageMatchesListRegistry(t *testing.T) {
+	d, ctx := newTestDB(t)
+	addrs := scaleAddrs(14)
+	if _, err := d.SubmitIPs(ctx, addrs); err != nil {
+		t.Fatal(err)
+	}
+	// live rows with an aggregated result
+	finishWithChecks(t, d, addrs[0], 2, 0, ResultPass)
+	finishWithChecks(t, d, addrs[1], 1, 1, ResultPartial)
+	finishWithChecks(t, d, addrs[2], 0, 2, ResultFail)
+	ip3, _ := d.GetIPByAddress(ctx, addrs[3])
+	if err := d.CancelIP(ctx, ip3.ID); err != nil {
+		t.Fatal(err)
+	}
+	// live row, passing checks recorded, but cycle unfinished -> no verdict
+	finishWithChecks(t, d, addrs[4], 2, 0, "")
+	// rows to be deleted: result derived from the checks of the last cycle
+	finishWithChecks(t, d, addrs[5], 2, 0, ResultPass)    // -> pass
+	finishWithChecks(t, d, addrs[6], 1, 2, ResultPartial) // -> partial
+	finishWithChecks(t, d, addrs[7], 0, 2, ResultFail)    // -> fail
+	finishWithChecks(t, d, addrs[8], 0, 0, "")            // deleted, no checks -> ""
+	// a deleted row whose first cycle failed but whose last passed
+	finishWithChecks(t, d, addrs[9], 0, 1, ResultFail)
+	if _, err := d.DeleteIPs(ctx, []string{addrs[5], addrs[6], addrs[7], addrs[8], addrs[9]}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := d.SubmitIPs(ctx, []string{addrs[9]}); err != nil {
+		t.Fatal(err)
+	}
+	finishWithChecks(t, d, addrs[9], 1, 0, ResultPass)
+	if _, err := d.DeleteIPs(ctx, []string{addrs[9]}); err != nil {
+		t.Fatal(err)
+	}
+
+	all, err := d.ListRegistry(ctx)
+	if err != nil || len(all) != 14 {
+		t.Fatalf("list registry: n=%d err=%v", len(all), err)
+	}
+
+	// No filter: same rows in the same order as ListRegistry, paged.
+	var paged []RegistrySummary
+	for off := 0; ; off += 5 {
+		page, total, err := d.ListRegistryPage(ctx, RegistryFilter{}, 5, off)
+		if err != nil || total != 14 {
+			t.Fatalf("page off=%d total=%d err=%v", off, total, err)
+		}
+		if len(page) == 0 {
+			break
+		}
+		paged = append(paged, page...)
+	}
+	if len(paged) != len(all) {
+		t.Fatalf("paged %d rows, want %d", len(paged), len(all))
+	}
+	for i := range all {
+		if all[i].IPAddress != paged[i].IPAddress || all[i].LastResult != paged[i].LastResult {
+			t.Fatalf("row %d differs: %+v vs %+v", i, all[i], paged[i])
+		}
+	}
+
+	for _, res := range []string{ResultPass, ResultPartial, ResultFail, ResultCancelled} {
+		var want []string
+		for _, s := range all {
+			if s.LastResult == res {
+				want = append(want, s.IPAddress)
+			}
+		}
+		page, total, err := d.ListRegistryPage(ctx, RegistryFilter{LastResult: res}, 100, 0)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var got []string
+		for _, s := range page {
+			got = append(got, s.IPAddress)
+			if s.LastResult != res {
+				t.Fatalf("%s: row %s has LastResult %q", res, s.IPAddress, s.LastResult)
+			}
+		}
+		sort.Strings(want)
+		sort.Strings(got)
+		if total != len(want) || !reflect.DeepEqual(got, want) || len(want) == 0 {
+			t.Fatalf("last_result=%s: total=%d got=%v want=%v", res, total, got, want)
+		}
+	}
+
+	// pass: addrs[0] (live) + addrs[5] + addrs[9] (deleted, latest cycle).
+	if _, total, _ := d.ListRegistryPage(ctx, RegistryFilter{LastResult: ResultPass}, 100, 0); total != 3 {
+		t.Fatalf("expected 3 pass rows, got %d", total)
+	}
+
+	// q filter + LIMIT applies after the filter, total is the filtered count.
+	page, total, err := d.ListRegistryPage(ctx, RegistryFilter{Query: "10.0.0.1"}, 2, 0)
+	if err != nil || total != 5 || len(page) != 2 { // 10.0.0.1, .10-.13
+		t.Fatalf("q filter: total=%d len=%d err=%v", total, len(page), err)
+	}
+	page, total, _ = d.ListRegistryPage(ctx, RegistryFilter{Query: "10.0.0.1", LastResult: ResultPass}, 10, 0)
+	if total != 0 || len(page) != 0 {
+		// 10.0.0.1 is partial; 10.0.0.10-13 have no verdict or fail.
+		t.Fatalf("q+last_result: total=%d page=%+v", total, page)
+	}
+}
+
+func TestClearAllIPsKeepsHistoryAndFreesValidators(t *testing.T) {
+	d, ctx := newTestDB(t)
+	if err := d.AdminCreateValidator(ctx, "validator-1", "port-1"); err != nil {
+		t.Fatal(err)
+	}
+	addrs := []string{"5.5.5.1", "5.5.5.2", "5.5.5.3"}
+	if _, err := d.SubmitIPs(ctx, addrs); err != nil {
+		t.Fatal(err)
+	}
+	finishWithChecks(t, d, "5.5.5.2", 1, 1, ResultPartial)
+	claimed, err := d.ClaimNextQueued(ctx, "validator-1", time.Minute)
+	if err != nil || claimed == nil {
+		t.Fatalf("claim: %v %v", claimed, err)
+	}
+	if err := d.SetFIPAssociated(ctx, claimed.ID, "fip-9", time.Minute); err != nil {
+		t.Fatal(err)
+	}
+	if err := d.SetEgressComplete(ctx, claimed.ID); err != nil {
+		t.Fatal(err)
+	}
+	if err := d.SetSiteComplete(ctx, claimed.ID, 1); err != nil {
+		t.Fatal(err)
+	}
+
+	refs, err := d.ListFIPRefs(ctx)
+	if err != nil || len(refs) != 1 || refs[0].FIPID != "fip-9" || refs[0].IPAddress != claimed.IPAddress {
+		t.Fatalf("fip refs: %+v err=%v", refs, err)
+	}
+	refs, err = d.ListFIPRefsByAddresses(ctx, []string{claimed.IPAddress, "nope"})
+	if err != nil || len(refs) != 1 {
+		t.Fatalf("fip refs by address: %+v err=%v", refs, err)
+	}
+	if refs, _ := d.ListFIPRefsByAddresses(ctx, []string{"5.5.5.3"}); len(refs) != 0 {
+		t.Fatalf("row without fip must not be listed: %+v", refs)
+	}
+
+	deleted, err := d.ClearAllIPs(ctx)
+	if err != nil {
+		t.Fatalf("clear: %v", err)
+	}
+	sort.Strings(deleted)
+	if !reflect.DeepEqual(deleted, addrs) {
+		t.Fatalf("deleted %v, want %v", deleted, addrs)
+	}
+	if items, _ := d.ListIPs(ctx); len(items) != 0 {
+		t.Fatalf("queue not empty: %+v", items)
+	}
+	v, err := d.GetValidator(ctx, "validator-1")
+	if err != nil || v.State != ValidatorIdle || v.CurrentIPID != nil {
+		t.Fatalf("validator not freed: %+v err=%v", v, err)
+	}
+	// History and registry rows survive, detached from the queue.
+	reg, err := d.GetRegistryByAddress(ctx, "5.5.5.2")
+	if err != nil || reg.TotalCycles != 1 || reg.LastResult != ResultPartial || reg.InQueue {
+		t.Fatalf("registry after clear: %+v err=%v", reg, err)
+	}
+	checks, err := d.ListChecksForRegistry(ctx, reg.ID, nil)
+	if err != nil || len(checks) != 2 || checks[0].IPID != 0 {
+		t.Fatalf("checks after clear: %+v err=%v", checks, err)
+	}
+	// Clearing an empty queue is fine and returns an empty (non-nil) list.
+	if deleted, err := d.ClearAllIPs(ctx); err != nil || deleted == nil || len(deleted) != 0 {
+		t.Fatalf("second clear: %v %v", deleted, err)
+	}
+	// The same addresses can be re-submitted afterwards.
+	if res, err := d.SubmitIPs(ctx, addrs); err != nil || len(res.Added) != 3 {
+		t.Fatalf("resubmit: %+v err=%v", res, err)
+	}
+}
+
+func TestMigration0009Indexes(t *testing.T) {
+	d, ctx := newTestDB(t)
+	for _, name := range []string{"idx_ip_queue_registry", "idx_ip_queue_state_aggregated"} {
+		var n int
+		if err := d.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name=?`, name).Scan(&n); err != nil || n != 1 {
+			t.Fatalf("index %s missing (n=%d err=%v)", name, n, err)
+		}
+	}
+	var ver int
+	if err := d.QueryRowContext(ctx, `PRAGMA user_version`).Scan(&ver); err != nil || ver < 9 {
+		t.Fatalf("user_version=%d err=%v", ver, err)
+	}
+}
+
+// TestScaleSmoke6440 pushes a realistic project size through the hot paths
+// with a loose time bound: the point is the absence of O(n^2) / N+1 work, not
+// a benchmark.
+func TestScaleSmoke6440(t *testing.T) {
+	if testing.Short() {
+		t.Skip("scale smoke test skipped in -short mode")
+	}
+	d, ctx := newTestDB(t)
+	addrs := scaleAddrs(6440)
+
+	start := time.Now()
+	for off := 0; off < len(addrs); off += 500 {
+		end := min(off+500, len(addrs))
+		if _, err := d.SubmitIPs(ctx, addrs[off:end]); err != nil {
+			t.Fatalf("submit chunk: %v", err)
+		}
+	}
+	submitDur := time.Since(start)
+
+	start = time.Now()
+	page, total, err := d.ListRegistryPage(ctx, RegistryFilter{}, 100, 3000)
+	if err != nil || total != 6440 || len(page) != 100 {
+		t.Fatalf("registry page: total=%d len=%d err=%v", total, len(page), err)
+	}
+	if _, total, err = d.ListRegistryPage(ctx, RegistryFilter{LastResult: ResultPass}, 100, 0); err != nil || total != 0 {
+		t.Fatalf("registry last_result filter: total=%d err=%v", total, err)
+	}
+	registryDur := time.Since(start)
+
+	start = time.Now()
+	if items, total, err := d.ListIPsPage(ctx, IPFilter{States: []string{IPQueued}, Query: "10.0.1."}, 50, 0); err != nil || total != 256 || len(items) != 50 {
+		t.Fatalf("ips page: total=%d len=%d err=%v", total, len(items), err)
+	}
+	if by, total, err := d.CountIPsByState(ctx); err != nil || total != 6440 || by[IPQueued] != 6440 {
+		t.Fatalf("count: %v %d %v", by, total, err)
+	}
+	if any, err := d.AnyNonTerminalIP(ctx); err != nil || !any {
+		t.Fatalf("any non terminal: %v %v", any, err)
+	}
+	queryDur := time.Since(start)
+
+	start = time.Now()
+	deleted, err := d.ClearAllIPs(ctx)
+	if err != nil || len(deleted) != 6440 {
+		t.Fatalf("clear: n=%d err=%v", len(deleted), err)
+	}
+	clearDur := time.Since(start)
+
+	t.Logf("submit=%v registry=%v queries=%v clear=%v", submitDur, registryDur, queryDur, clearDur)
+	if registryDur > 10*time.Second || queryDur > 5*time.Second || clearDur > 10*time.Second {
+		t.Fatalf("too slow: registry=%v queries=%v clear=%v", registryDur, queryDur, clearDur)
+	}
+}