Scan floating IPs in the background, page by page, so thousands of addresses work

The "Scan Floating IP" button failed with a client timeout: the project now holds ~6.4k floating IPs and the scan listed them all in one unpaginated, timeout-less Neutron request on the HTTP request context. openstack: ListFreeFloatingIPs reads marker-based pages (fields= keeps them small) with per-page retry/backoff on transport errors, 5xx and 429, and every request now has a timeout (also ends hangs inside the orchestrator tick). orchestrator: the scan is a single-flight background job on the process context with progress (clearing/listing/enqueuing/done/error), dry_run, full discovery before anything is enqueued, then SubmitIPs in chunks of 500 in ascending IP order; a failed read leaves the queue untouched. The auto-cycle gets a "scanning" phase that polls the job, so the control loop and autoCycleMu are never held across OpenStack/DB work; it recovers after a restart and waits for (instead of adopting) a scan started by someone else. db: migration 0009 (indexes), paged ListIPsPage/ListRegistryPage, GROUP BY counters, EXISTS completion check, set-based ClearAllIPs. API: POST /admin/ips/scan -> 202 (dry_run, wait), GET /admin/ips/scan, paging and filters on /admin/ips and /admin/registry (bare arrays without limit), results_by_overall in /admin/status. dashboard: scan progress panel and dry-run button, paginated /ips and /registry with server-side filters, Overview on counters and capped lists with progress/ETA, "select all N by filter", hx-params fix for per-row buttons, real counts in confirmations. Also: docs (API, USAGE, DASHBOARD, README), plan and review under docs/changes/, bin/ rebuilt with new SHA256SUMS. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-01 19:31:11 +03:00
1 parent debf2afed2
commit aff8fe38b5
@@ -9,6 +9,8 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"os"
+	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"testing"
@@ -42,6 +44,27 @@ type fakeControlAPI struct {
 	registry          map[string]registryItem
 	registryChecks    map[string][]check

+	// Scan job state machine (see the scan handlers): POST starts a job that
+	// stays "running" for scanRunPolls GET polls (0 = finishes at once), then
+	// ends as done — or as error when scanFinalError is set. scan is the status
+	// served by GET; tests may also set it directly (with scanPollsLeft == 0 it
+	// stays as is). scanStartStatus != 0 makes POST fail with that HTTP status.
+	scan            scanStatusDTO
+	scanRunPolls    int
+	scanPollsLeft   int
+	scanFinalError  string
+	scanStartStatus int
+
+	// Requests seen on the list endpoints, for "never loads everything" checks:
+	// the raw query of every GET /ips (ipsQueries) and the number of GET
+	// /registry calls without `limit` (bare), plus the sizes of the DeleteIPs /
+	// SubmitIPs bulk calls.
+	ipsQueries      []string
+	registryQueries []string
+	bareIPsCalls    int
+	deleteChunks    []int
+	submitChunks    []int
+
 	// autoCycle is the state served by /api/v1/admin/auto-cycle*;
 	// autoCycleDown makes all four endpoints answer 500 (unavailable API).
 	autoCycle     autoCycleDTO
@@ -86,16 +109,72 @@ func (f *fakeControlAPI) handler() http.Handler {
 		f.mu.Lock()
 		defer f.mu.Unlock()
 		byState := map[string]int{}
+		results := map[string]int{"pass": 0, "partial": 0, "fail": 0, "cancelled": 0}
 		for _, ip := range f.ips {
 			byState[ip.State]++
+			if ip.OverallResult != "" {
+				results[ip.OverallResult]++
+			}
 		}
-		writeJSON(w, http.StatusOK, statusResponse{TotalIPs: len(f.ips), IPsByState: byState, TotalValidators: len(f.validators)})
+		writeJSON(w, http.StatusOK, statusResponse{TotalIPs: len(f.ips), IPsByState: byState, TotalValidators: len(f.validators), ResultsByOverall: results})
 	})

 	mux.HandleFunc("GET /api/v1/admin/ips", func(w http.ResponseWriter, r *http.Request) {
 		f.mu.Lock()
 		defer f.mu.Unlock()
-		writeJSON(w, http.StatusOK, f.ips)
+		f.ipsQueries = append(f.ipsQueries, r.URL.RawQuery)
+		qv := r.URL.Query()
+		if qv.Get("limit") == "" {
+			// Legacy shape: the whole queue as a bare array.
+			f.bareIPsCalls++
+			writeJSON(w, http.StatusOK, f.ips)
+			return
+		}
+		limit, err := strconv.Atoi(qv.Get("limit"))
+		if err != nil || limit < 1 || limit > 1000 {
+			writeAPIErr(w, http.StatusBadRequest, "limit must be 1..1000")
+			return
+		}
+		offset, _ := strconv.Atoi(qv.Get("offset"))
+		var states map[string]bool
+		if st := qv.Get("state"); st != "" {
+			states = map[string]bool{}
+			for _, x := range strings.Split(st, ",") {
+				states[x] = true
+			}
+		}
+		var matched []ipQueueItem
+		for _, ip := range f.ips {
+			if states != nil && !states[ip.State] {
+				continue
+			}
+			if q := qv.Get("q"); q != "" && !strings.Contains(strings.ToLower(ip.IPAddress), strings.ToLower(q)) {
+				continue
+			}
+			if res := qv.Get("result"); res != "" && ip.OverallResult != res {
+				continue
+			}
+			matched = append(matched, ip)
+		}
+		if qv.Get("order") == "aggregated_at_desc" {
+			sort.SliceStable(matched, func(i, j int) bool {
+				a, b := matched[i].AggregatedAt, matched[j].AggregatedAt
+				if a == nil || b == nil {
+					return a != nil && b == nil
+				}
+				return a.After(*b)
+			})
+		} else {
+			sort.SliceStable(matched, func(i, j int) bool { return matched[i].Sequence < matched[j].Sequence })
+		}
+		page := []ipQueueItem{}
+		if offset < len(matched) {
+			page = matched[offset:]
+			if len(page) > limit {
+				page = page[:limit]
+			}
+		}
+		writeJSON(w, http.StatusOK, ipsPage{Items: page, Total: len(matched), Limit: limit, Offset: offset})
 	})

 	mux.HandleFunc("GET /api/v1/admin/ips/{ip}", func(w http.ResponseWriter, r *http.Request) {
@@ -122,6 +201,7 @@ func (f *fakeControlAPI) handler() http.Handler {
 			writeAPIErr(w, http.StatusBadRequest, "addresses must not be empty")
 			return
 		}
+		f.submitChunks = append(f.submitChunks, len(req.Addresses))
 		resp := submitIPsResponse{}
 		for _, addr := range req.Addresses {
 			idx := f.findIP(addr)
@@ -190,6 +270,7 @@ func (f *fakeControlAPI) handler() http.Handler {
 			writeAPIErr(w, http.StatusBadRequest, "addresses must not be empty")
 			return
 		}
+		f.deleteChunks = append(f.deleteChunks, len(req.Addresses))
 		resp := deleteIPsResponse{}
 		for _, addr := range req.Addresses {
 			idx := f.findIP(addr)
@@ -210,6 +291,7 @@ func (f *fakeControlAPI) handler() http.Handler {
 		for _, ip := range f.ips {
 			resp.Deleted = append(resp.Deleted, ip.IPAddress)
 		}
+		resp.Count = len(resp.Deleted)
 		f.ips = nil
 		writeJSON(w, http.StatusOK, resp)
 	})
@@ -246,18 +328,32 @@ func (f *fakeControlAPI) handler() http.Handler {
 	mux.HandleFunc("POST /api/v1/admin/ips/scan", func(w http.ResponseWriter, r *http.Request) {
 		f.mu.Lock()
 		defer f.mu.Unlock()
-		resp := scanIPsResponse{ScannedFree: len(f.scanFreeAddresses)}
-		for _, addr := range f.scanFreeAddresses {
-			idx := f.findIP(addr)
-			if idx < 0 {
-				now := time.Now()
-				f.ips = append(f.ips, ipQueueItem{IPAddress: addr, State: "queued", CreatedAt: now, UpdatedAt: now})
-				resp.Added = append(resp.Added, addr)
-				continue
-			}
-			resp.Reordered = append(resp.Reordered, addr)
+		if f.scanStartStatus != 0 {
+			writeAPIErr(w, f.scanStartStatus, "scan refused")
+			return
 		}
-		writeJSON(w, http.StatusOK, resp)
+		if !f.scan.Running {
+			now := time.Now()
+			f.scan = scanStatusDTO{State: "listing", Running: true, DryRun: r.URL.Query().Get("dry_run") == "true", StartedAt: &now}
+			f.scanPollsLeft = f.scanRunPolls
+			if f.scanPollsLeft == 0 {
+				f.finishScan()
+			}
+		}
+		writeJSON(w, http.StatusAccepted, f.scan)
+	})
+	mux.HandleFunc("GET /api/v1/admin/ips/scan", func(w http.ResponseWriter, r *http.Request) {
+		f.mu.Lock()
+		defer f.mu.Unlock()
+		if f.scan.Running && f.scanPollsLeft > 0 {
+			f.scanPollsLeft--
+			if f.scanPollsLeft == 0 {
+				f.finishScan()
+			} else {
+				f.scan.Pages++
+			}
+		}
+		writeJSON(w, http.StatusOK, f.scan)
 	})

 	mux.HandleFunc("GET /api/v1/admin/auto-cycle", func(w http.ResponseWriter, r *http.Request) {
@@ -329,11 +425,37 @@ func (f *fakeControlAPI) handler() http.Handler {
 	mux.HandleFunc("GET /api/v1/admin/registry", func(w http.ResponseWriter, r *http.Request) {
 		f.mu.Lock()
 		defer f.mu.Unlock()
-		out := make([]registryItem, 0, len(f.registry))
+		qv := r.URL.Query()
+		f.registryQueries = append(f.registryQueries, r.URL.RawQuery)
+		matched := make([]registryItem, 0, len(f.registry))
 		for _, item := range f.registry {
-			out = append(out, item)
+			if q := qv.Get("q"); q != "" && !strings.Contains(strings.ToLower(item.IPAddress), strings.ToLower(q)) {
+				continue
+			}
+			if lr := qv.Get("last_result"); lr != "" && item.LastResult != lr {
+				continue
+			}
+			matched = append(matched, item)
 		}
-		writeJSON(w, http.StatusOK, out)
+		sort.Slice(matched, func(i, j int) bool { return matched[i].IPAddress < matched[j].IPAddress })
+		if qv.Get("limit") == "" {
+			writeJSON(w, http.StatusOK, matched)
+			return
+		}
+		limit, err := strconv.Atoi(qv.Get("limit"))
+		if err != nil || limit < 1 || limit > 1000 {
+			writeAPIErr(w, http.StatusBadRequest, "limit must be 1..1000")
+			return
+		}
+		offset, _ := strconv.Atoi(qv.Get("offset"))
+		page := []registryItem{}
+		if offset < len(matched) {
+			page = matched[offset:]
+			if len(page) > limit {
+				page = page[:limit]
+			}
+		}
+		writeJSON(w, http.StatusOK, registryPage{Items: page, Total: len(matched), Limit: limit, Offset: offset})
 	})
 	mux.HandleFunc("GET /api/v1/admin/registry/{ip}", func(w http.ResponseWriter, r *http.Request) {
 		f.mu.Lock()
@@ -544,6 +666,51 @@ func (f *fakeControlAPI) handler() http.Handler {
 	})
 }

+// finishScan ends the running scan job (caller holds f.mu): the discovered
+// free addresses are queued unless it was a dry run, or the job fails with
+// scanFinalError.
+func (f *fakeControlAPI) finishScan() {
+	now := time.Now()
+	f.scan.Running = false
+	f.scan.FinishedAt = &now
+	f.scan.Discovered = len(f.scanFreeAddresses)
+	f.scan.Free = len(f.scanFreeAddresses)
+	if f.scanFinalError != "" {
+		f.scan.State = "error"
+		f.scan.Error = f.scanFinalError
+		return
+	}
+	f.scan.State = "done"
+	if f.scan.DryRun {
+		return
+	}
+	for _, addr := range f.scanFreeAddresses {
+		if f.findIP(addr) < 0 {
+			f.ips = append(f.ips, ipQueueItem{IPAddress: addr, State: "queued", Sequence: len(f.ips) + 1, CreatedAt: now, UpdatedAt: now})
+			f.scan.Added++
+		} else {
+			f.scan.Reordered++
+		}
+	}
+}
+
+// seedIPs appends n queued addresses 10.x.y.z (distinct, in sequence order)
+// and returns them. Use it for tests that need pages' worth of rows.
+func (f *fakeControlAPI) seedIPs(n int) []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	now := time.Now()
+	out := make([]string, 0, n)
+	base := len(f.ips)
+	for i := 0; i < n; i++ {
+		k := base + i + 1
+		addr := fmt.Sprintf("10.%d.%d.%d", k/65536, (k/256)%256, k%256)
+		f.ips = append(f.ips, ipQueueItem{IPAddress: addr, State: "queued", Sequence: k, CreatedAt: now, UpdatedAt: now})
+		out = append(out, addr)
+	}
+	return out
+}
+
 func (f *fakeControlAPI) findIP(addr string) int {
 	for i, ip := range f.ips {
 		if ip.IPAddress == addr {
@@ -596,3 +763,14 @@ func postForm(t *testing.T, ts *httptest.Server, method, path string, form url.V
 	body, _ := io.ReadAll(resp.Body)
 	return string(body)
 }
+
+// newSlowAPI serves an empty JSON object for every request after delay.
+func newSlowAPI(t *testing.T, delay time.Duration) string {
+	t.Helper()
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(delay)
+		writeJSON(w, http.StatusOK, map[string]interface{}{})
+	}))
+	t.Cleanup(ts.Close)
+	return ts.URL
+}