Scan floating IPs in the background, page by page, so thousands of addresses work

The "Scan Floating IP" button failed with a client timeout: the project now holds ~6.4k floating IPs and the scan listed them all in one unpaginated, timeout-less Neutron request on the HTTP request context. openstack: ListFreeFloatingIPs reads marker-based pages (fields= keeps them small) with per-page retry/backoff on transport errors, 5xx and 429, and every request now has a timeout (also ends hangs inside the orchestrator tick). orchestrator: the scan is a single-flight background job on the process context with progress (clearing/listing/enqueuing/done/error), dry_run, full discovery before anything is enqueued, then SubmitIPs in chunks of 500 in ascending IP order; a failed read leaves the queue untouched. The auto-cycle gets a "scanning" phase that polls the job, so the control loop and autoCycleMu are never held across OpenStack/DB work; it recovers after a restart and waits for (instead of adopting) a scan started by someone else. db: migration 0009 (indexes), paged ListIPsPage/ListRegistryPage, GROUP BY counters, EXISTS completion check, set-based ClearAllIPs. API: POST /admin/ips/scan -> 202 (dry_run, wait), GET /admin/ips/scan, paging and filters on /admin/ips and /admin/registry (bare arrays without limit), results_by_overall in /admin/status. dashboard: scan progress panel and dry-run button, paginated /ips and /registry with server-side filters, Overview on counters and capped lists with progress/ETA, "select all N by filter", hx-params fix for per-row buttons, real counts in confirmations. Also: docs (API, USAGE, DASHBOARD, README), plan and review under docs/changes/, bin/ rebuilt with new SHA256SUMS. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-01 19:31:11 +03:00
1 parent debf2afed2
commit aff8fe38b5
@@ -1,14 +1,122 @@
 package dashboard

 import (
+	"context"
+	"errors"
 	"fmt"
 	"net/http"
+	"net/url"
+	"strings"
 )

+// bulkChunk is how many addresses go into one DeleteIPs/SubmitIPs call when
+// an operation spans a whole filter ("scope=all").
+const bulkChunk = 500
+
+// ipStates are the queue states control-api accepts in the `state` filter.
+var ipStates = []string{"queued", "assigning_fip", "awaiting_self_check", "checking", "aggregating", "done", "failed", "occupied"}
+
+var ipResults = []string{"pass", "partial", "fail", "cancelled"}
+
+func containsStr(list []string, s string) bool {
+	for _, v := range list {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+// ipsFilter is the server-side filter of the /ips list. State is a filter
+// token: one of "", "queued", "active" (every in-progress state), "done",
+// "failed", "occupied", or a comma-separated list of raw queue states; the
+// result filter is separate (Result) and may be combined with it.
+type ipsFilter struct {
+	Q      string
+	State  string
+	Result string
+}
+
+// parseIPsFilter reads q/state/result from the request (query or form body).
+// The state <select> posts a single `state` value, so "result:pass" selects a
+// result filter; anything unrecognised is dropped rather than forwarded.
+func parseIPsFilter(r *http.Request) ipsFilter {
+	f := ipsFilter{
+		Q:      strings.TrimSpace(r.FormValue("q")),
+		State:  strings.TrimSpace(r.FormValue("state")),
+		Result: strings.TrimSpace(r.FormValue("result")),
+	}
+	if res, ok := strings.CutPrefix(f.State, "result:"); ok {
+		f.State, f.Result = "", res
+	}
+	if !containsStr(ipResults, f.Result) {
+		f.Result = ""
+	}
+	switch f.State {
+	case "", "active":
+	default:
+		for _, st := range strings.Split(f.State, ",") {
+			if !containsStr(ipStates, st) {
+				f.State = ""
+				break
+			}
+		}
+	}
+	return f
+}
+
+// States expands the state token into the list sent to control-api.
+func (f ipsFilter) States() []string {
+	switch f.State {
+	case "":
+		return nil
+	case "active":
+		return activeStates
+	default:
+		return strings.Split(f.State, ",")
+	}
+}
+
+// Active reports whether any filter is applied.
+func (f ipsFilter) Active() bool { return f.Q != "" || f.State != "" || f.Result != "" }
+
+// Token is the value of the state <select> option matching the filter.
+func (f ipsFilter) Token() string {
+	if f.State == "" && f.Result != "" {
+		return "result:" + f.Result
+	}
+	return f.State
+}
+
+func (f ipsFilter) values() url.Values {
+	v := url.Values{}
+	if f.Q != "" {
+		v.Set("q", f.Q)
+	}
+	if f.State != "" {
+		v.Set("state", f.State)
+	}
+	if f.Result != "" {
+		v.Set("result", f.Result)
+	}
+	return v
+}
+
 type ipsPageData struct {
 	PageData
 	Items            []ipQueueItem
 	FIPSettleSeconds int
+	Filter           ipsFilter
+	Page, PerPage    int
+	// Total is the number of rows matching the filter; QueueTotal is the whole
+	// queue (what "Очистить всё" would remove).
+	Total, QueueTotal int
+	Pager             pagerData
+	// SelfURL is this page's own URL (filter + page), re-requested to reload
+	// the table when a scan finishes.
+	SelfURL        string
+	PerPageOptions []int
+	Scan           scanProgressData
 }

 type ipDetailData struct {
@@ -16,13 +124,67 @@ type ipDetailData struct {
 	Detail ipDetailResponse
 }

-func (s *Server) handleIPsPage(w http.ResponseWriter, r *http.Request) {
-	items, err := s.CA.ListIPs(r.Context())
-	settings, settingsErr := s.CA.GetOrchestratorSettings(r.Context())
+// loadIPsData fetches the page of the queue selected by the request's
+// page/per_page/q/state/result params. A page that became empty (e.g. after
+// deleting its last rows) is clamped to the last non-empty page.
+func (s *Server) loadIPsData(r *http.Request) (ipsPageData, error) {
+	ctx := r.Context()
+	f := parseIPsFilter(r)
+	perPage := parsePerPage(r.FormValue("per_page"))
+	page := parsePage(r.FormValue("page"))
+	q := ipsQuery{States: f.States(), Q: f.Q, Result: f.Result, Order: "sequence", Limit: perPage, Offset: (page - 1) * perPage}
+
+	res, err := s.CA.ListIPsPage(ctx, q)
 	if err == nil {
-		err = settingsErr
+		if clamped := clampPage(page, res.Total, perPage); clamped != page {
+			page = clamped
+			q.Offset = (page - 1) * perPage
+			res, err = s.CA.ListIPsPage(ctx, q)
+		}
+	}
+	data := ipsPageData{
+		Items:          res.Items,
+		Filter:         f,
+		Page:           page,
+		PerPage:        perPage,
+		Total:          res.Total,
+		QueueTotal:     res.Total,
+		PerPageOptions: perPageOptions,
+	}
+	data.Pager = newPager("/ips", "ips-table-wrap", f.values(), page, perPage, res.Total)
+	data.SelfURL = pageURL("/ips", f.values(), page, perPage)
+
+	if settings, settingsErr := s.CA.GetOrchestratorSettings(ctx); settingsErr != nil {
+		if err == nil {
+			err = settingsErr
+		}
+	} else {
+		data.FIPSettleSeconds = settings.FIPSettleSeconds
+	}
+	if err == nil && f.Active() {
+		// "Очистить всё" ignores the filter: show the real queue size in its
+		// confirmation. Non-fatal — the label just falls back to the filtered total.
+		if st, stErr := s.CA.Status(ctx); stErr == nil {
+			data.QueueTotal = st.TotalIPs
+		}
+	}
+	return data, err
+}
+
+func (s *Server) handleIPsPage(w http.ResponseWriter, r *http.Request) {
+	data, err := s.loadIPsData(r)
+	// A filter/pager request from htmx swaps only #ips-table-wrap (hx-select),
+	// so there is no need to re-render the whole page (and re-query the scan
+	// status). A history-restore fetch needs the full page.
+	if r.Header.Get("HX-Request") == "true" && r.Header.Get("HX-History-Restore-Request") != "true" {
+		s.renderFragment(w, "ips_table_wrap", data, err)
+		return
+	}
+	if st, scanErr := s.CA.ScanStatus(r.Context()); scanErr != nil {
+		s.Log.Warn("ips: scan status unavailable", "err", scanErr)
+	} else {
+		data.Scan = newScanProgress(st)
 	}
-	data := ipsPageData{Items: items, FIPSettleSeconds: settings.FIPSettleSeconds}
 	data.ActiveNav = "ips"
 	data.Banner = bannerFor(err)
 	s.renderPage(w, r, "ips_page", data)
@@ -37,20 +199,18 @@ func (s *Server) handleIPDetail(w http.ResponseWriter, r *http.Request) {
 	s.renderPage(w, r, "ip_detail_page", data)
 }

-// renderIPsTable re-fetches the current queue and renders the ips_table
-// fragment, tagging actionErr (if any) on the shared error banner. Called
-// after every mutating /ips/* request so the table always reflects true
-// current state regardless of whether the mutation itself succeeded.
+// renderIPsTable re-fetches the current page of the queue (same page/filter as
+// the request, carried in hidden #ips-form inputs) and renders the
+// ips_table_wrap fragment, tagging actionErr (if any) on the shared error
+// banner. Called after every mutating /ips/* request so the table always
+// reflects true current state regardless of whether the mutation itself
+// succeeded.
 func (s *Server) renderIPsTable(w http.ResponseWriter, r *http.Request, actionErr error) {
-	items, listErr := s.CA.ListIPs(r.Context())
+	data, err := s.loadIPsData(r)
 	if actionErr == nil {
-		actionErr = listErr
+		actionErr = err
 	}
-	settings, settingsErr := s.CA.GetOrchestratorSettings(r.Context())
-	if actionErr == nil {
-		actionErr = settingsErr
-	}
-	s.renderFragment(w, "ips_table", ipsPageData{Items: items, FIPSettleSeconds: settings.FIPSettleSeconds}, actionErr)
+	s.renderFragment(w, "ips_table_wrap", data, actionErr)
 }

 func (s *Server) handleIPsSubmit(w http.ResponseWriter, r *http.Request) {
@@ -63,7 +223,7 @@ func (s *Server) handleIPsSubmit(w http.ResponseWriter, r *http.Request) {
 		s.renderIPsTable(w, r, &apiErr{Status: http.StatusBadRequest, Message: "укажите хотя бы один адрес"})
 		return
 	}
-	_, err := s.CA.SubmitIPs(r.Context(), addresses)
+	err := s.submitChunked(r.Context(), addresses)
 	s.renderIPsTable(w, r, err)
 }

@@ -85,31 +245,76 @@ func (s *Server) handleIPDelete(w http.ResponseWriter, r *http.Request) {
 	s.renderIPsTable(w, r, err)
 }

-func (s *Server) handleIPsDeleteSelected(w http.ResponseWriter, r *http.Request) {
+// submitChunked feeds addresses to SubmitIPs in chunks of bulkChunk so one
+// huge list never becomes one huge request/transaction.
+func (s *Server) submitChunked(ctx context.Context, addresses []string) error {
+	for _, part := range chunk(addresses, bulkChunk) {
+		if _, err := s.CA.SubmitIPs(ctx, part); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// resolveFilterAddresses lists every address matching filter by paging
+// ListIPsPage (≤ maxPageLimit rows per call), for "select all N by filter".
+func (s *Server) resolveFilterAddresses(ctx context.Context, f ipsFilter) ([]string, error) {
+	var out []string
+	for offset := 0; ; {
+		page, err := s.CA.ListIPsPage(ctx, ipsQuery{States: f.States(), Q: f.Q, Result: f.Result, Order: "sequence", Limit: maxPageLimit, Offset: offset})
+		if err != nil {
+			return nil, err
+		}
+		for _, it := range page.Items {
+			out = append(out, it.IPAddress)
+		}
+		offset += len(page.Items)
+		if len(page.Items) == 0 || offset >= page.Total {
+			return out, nil
+		}
+	}
+}
+
+// bulkAddresses returns the addresses a bulk delete/recheck acts on: the
+// checked rows of the current page, or — with scope=all — everything that
+// matches the current filter, resolved server-side.
+func (s *Server) bulkAddresses(r *http.Request) ([]string, error) {
 	if err := r.ParseForm(); err != nil {
-		s.renderIPsTable(w, r, fmt.Errorf("invalid form: %w", err))
-		return
+		return nil, fmt.Errorf("invalid form: %w", err)
+	}
+	var addresses []string
+	if r.FormValue("scope") == "all" {
+		var err error
+		addresses, err = s.resolveFilterAddresses(r.Context(), parseIPsFilter(r))
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		addresses = r.Form["addresses"]
 	}
-	addresses := r.Form["addresses"]
 	if len(addresses) == 0 {
-		s.renderIPsTable(w, r, &apiErr{Status: http.StatusBadRequest, Message: "ничего не выбрано"})
-		return
+		return nil, &apiErr{Status: http.StatusBadRequest, Message: "ничего не выбрано"}
+	}
+	return addresses, nil
+}
+
+func (s *Server) handleIPsDeleteSelected(w http.ResponseWriter, r *http.Request) {
+	addresses, err := s.bulkAddresses(r)
+	if err == nil {
+		for _, part := range chunk(addresses, bulkChunk) {
+			if _, err = s.CA.DeleteIPs(r.Context(), part); err != nil {
+				break
+			}
+		}
 	}
-	_, err := s.CA.DeleteIPs(r.Context(), addresses)
 	s.renderIPsTable(w, r, err)
 }

 func (s *Server) handleIPsRecheckSelected(w http.ResponseWriter, r *http.Request) {
-	if err := r.ParseForm(); err != nil {
-		s.renderIPsTable(w, r, fmt.Errorf("invalid form: %w", err))
-		return
+	addresses, err := s.bulkAddresses(r)
+	if err == nil {
+		err = s.submitChunked(r.Context(), addresses)
 	}
-	addresses := r.Form["addresses"]
-	if len(addresses) == 0 {
-		s.renderIPsTable(w, r, &apiErr{Status: http.StatusBadRequest, Message: "ничего не выбрано"})
-		return
-	}
-	_, err := s.CA.SubmitIPs(r.Context(), addresses)
 	s.renderIPsTable(w, r, err)
 }

@@ -118,10 +323,46 @@ func (s *Server) handleIPsClear(w http.ResponseWriter, r *http.Request) {
 	s.renderIPsTable(w, r, err)
 }

-// handleIPsScan lists the OpenStack project's free (unassociated) floating
-// IPs and submits them to the check queue in one step — see
-// client.ScanFloatingIPs.
+// scanProgressData drives the scan_progress partial. Poll keeps the partial's
+// own hx-trigger="every 2s" alive: while the job runs, or while control-api is
+// transiently unreachable (so one failed poll doesn't freeze the panel).
+type scanProgressData struct {
+	Status scanStatusDTO
+	Poll   bool
+}
+
+func newScanProgress(st scanStatusDTO) scanProgressData {
+	return scanProgressData{Status: st, Poll: st.Running}
+}
+
+// renderScanProgress renders the scan panel fragment. A finished job also
+// sends `HX-Trigger: scan-finished`, which makes #ips-table-wrap reload.
+// Errors go to the shared banner; only transient ones (transport/5xx) keep
+// polling when keepPolling is set.
+func (s *Server) renderScanProgress(w http.ResponseWriter, st scanStatusDTO, err error, keepPolling bool) {
+	data := newScanProgress(st)
+	if err != nil {
+		data = scanProgressData{}
+		var ae *apiErr
+		if keepPolling && errors.As(err, &ae) && (ae.Status == 0 || ae.Status >= 500) {
+			data.Poll = true
+		}
+	} else if st.Finished() {
+		w.Header().Set("HX-Trigger", "scan-finished")
+	}
+	s.renderFragment(w, "scan_progress", data, err)
+}
+
+// handleIPsScan starts the background floating-IP scan (or joins the running
+// one) and returns the progress panel at once — the job itself runs in
+// control-api, so this never waits for OpenStack. ?dry_run=true only counts.
 func (s *Server) handleIPsScan(w http.ResponseWriter, r *http.Request) {
-	_, err := s.CA.ScanFloatingIPs(r.Context())
-	s.renderIPsTable(w, r, err)
+	st, err := s.CA.StartScan(r.Context(), r.URL.Query().Get("dry_run") == "true")
+	s.renderScanProgress(w, st, err, false)
+}
+
+// handleIPsScanStatus is the progress panel's poll target.
+func (s *Server) handleIPsScanStatus(w http.ResponseWriter, r *http.Request) {
+	st, err := s.CA.ScanStatus(r.Context())
+	s.renderScanProgress(w, st, err, true)
 }