Scan floating IPs in the background, page by page, so thousands of addresses work
The "Scan Floating IP" button failed with a client timeout: the project now holds ~6.4k floating IPs and the scan listed them all in one unpaginated, timeout-less Neutron request on the HTTP request context. openstack: ListFreeFloatingIPs reads marker-based pages (fields= keeps them small) with per-page retry/backoff on transport errors, 5xx and 429, and every request now has a timeout (also ends hangs inside the orchestrator tick). orchestrator: the scan is a single-flight background job on the process context with progress (clearing/listing/enqueuing/done/error), dry_run, full discovery before anything is enqueued, then SubmitIPs in chunks of 500 in ascending IP order; a failed read leaves the queue untouched. The auto-cycle gets a "scanning" phase that polls the job, so the control loop and autoCycleMu are never held across OpenStack/DB work; it recovers after a restart and waits for (instead of adopting) a scan started by someone else. db: migration 0009 (indexes), paged ListIPsPage/ListRegistryPage, GROUP BY counters, EXISTS completion check, set-based ClearAllIPs. API: POST /admin/ips/scan -> 202 (dry_run, wait), GET /admin/ips/scan, paging and filters on /admin/ips and /admin/registry (bare arrays without limit), results_by_overall in /admin/status. dashboard: scan progress panel and dry-run button, paginated /ips and /registry with server-side filters, Overview on counters and capped lists with progress/ETA, "select all N by filter", hx-params fix for per-row buttons, real counts in confirmations. Also: docs (API, USAGE, DASHBOARD, README), plan and review under docs/changes/, bin/ rebuilt with new SHA256SUMS. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
debf2afed2
commit
aff8fe38b5
61 files changed
+5833
-536
No files matched your search
@@ -3,10 +3,13 @@ package openstack
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/gophercloud/gophercloud/v2"
|
||||
osauth "github.com/gophercloud/gophercloud/v2/openstack"
|
||||
"github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/layer3/floatingips"
|
||||
"github.com/gophercloud/gophercloud/v2/pagination"
|
||||
)
|
||||
|
||||
// AuthMethod selects how the client obtains the token it uses for Neutron
|
||||
@@ -51,10 +54,20 @@ type ClientConfig struct {
|
||||
ProjectID string
|
||||
Region string
|
||||
Interface string // "public" | "internal" | "admin"; "" defaults to "public"
|
||||
|
||||
// RequestTimeout bounds every single HTTP request to Keystone/Neutron
|
||||
// (provider.HTTPClient.Timeout). Zero means no timeout (not recommended:
|
||||
// a hung Neutron call would otherwise block the caller forever).
|
||||
RequestTimeout time.Duration
|
||||
// ListPageRetries is how many times one failed page of the floating-IP
|
||||
// listing is retried (exponential backoff 1s,2s,4s,...). Zero disables
|
||||
// retries; negative values mean "use DefaultListPageRetries".
|
||||
ListPageRetries int
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
networking *gophercloud.ServiceClient
|
||||
retry pageRetry
|
||||
}
|
||||
|
||||
// buildAuthOptions translates ClientConfig into gophercloud.AuthOptions. It
|
||||
@@ -119,6 +132,10 @@ func NewClient(ctx context.Context, cfg ClientConfig) (*Client, error) {
|
||||
return nil, fmt.Errorf("openstack: authenticate: %w", err)
|
||||
}
|
||||
|
||||
if cfg.RequestTimeout > 0 {
|
||||
provider.HTTPClient.Timeout = cfg.RequestTimeout
|
||||
}
|
||||
|
||||
iface := cfg.Interface
|
||||
if iface == "" {
|
||||
iface = string(gophercloud.AvailabilityPublic)
|
||||
@@ -131,7 +148,11 @@ func NewClient(ctx context.Context, cfg ClientConfig) (*Client, error) {
|
||||
return nil, fmt.Errorf("openstack: networking client: %w", err)
|
||||
}
|
||||
|
||||
return &Client{networking: networking}, nil
|
||||
retries := cfg.ListPageRetries
|
||||
if retries < 0 {
|
||||
retries = DefaultListPageRetries
|
||||
}
|
||||
return &Client{networking: networking, retry: pageRetry{Retries: retries}}, nil
|
||||
}
|
||||
|
||||
func (c *Client) GetFloatingIPByAddress(ctx context.Context, address string) (*FloatingIP, error) {
|
||||
@@ -150,22 +171,78 @@ func (c *Client) GetFloatingIPByAddress(ctx context.Context, address string) (*F
|
||||
return &FloatingIP{ID: f.ID, Address: f.FloatingIP, PortID: f.PortID, ProjectID: f.TenantID}, nil
|
||||
}
|
||||
|
||||
func (c *Client) ListFloatingIPs(ctx context.Context) ([]FloatingIP, error) {
|
||||
pages, err := floatingips.List(c.networking, floatingips.ListOpts{}).AllPages(ctx)
|
||||
// fipListFields is the set of attributes requested from Neutron when listing:
|
||||
// everything FloatingIP needs and nothing more (the full resource is several
|
||||
// times larger, which matters with thousands of floating IPs).
|
||||
var fipListFields = []string{"id", "floating_ip_address", "port_id", "project_id"}
|
||||
|
||||
// pagedListOpts wraps floatingips.ListOpts to add the `fields` query
|
||||
// parameter, which gophercloud's ListOpts does not expose.
|
||||
type pagedListOpts struct {
|
||||
floatingips.ListOpts
|
||||
fields []string
|
||||
}
|
||||
|
||||
func (o pagedListOpts) ToFloatingIPListQuery() (string, error) {
|
||||
q, err := o.ListOpts.ToFloatingIPListQuery()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(o.fields) == 0 {
|
||||
return q, nil
|
||||
}
|
||||
v := url.Values{}
|
||||
for _, f := range o.fields {
|
||||
v.Add("fields", f)
|
||||
}
|
||||
if q == "" {
|
||||
return "?" + v.Encode(), nil
|
||||
}
|
||||
return q + "&" + v.Encode(), nil
|
||||
}
|
||||
|
||||
// fetchPage requests exactly one page (limit entries after marker). It uses
|
||||
// marker pagination driven by us rather than gophercloud's `next` link: behind
|
||||
// a proxy that link may point at an internal host.
|
||||
func (c *Client) fetchPage(ctx context.Context, marker string, limit int) ([]FloatingIP, error) {
|
||||
opts := pagedListOpts{
|
||||
ListOpts: floatingips.ListOpts{Limit: limit, Marker: marker},
|
||||
fields: fipListFields,
|
||||
}
|
||||
var out []FloatingIP
|
||||
err := floatingips.List(c.networking, opts).EachPage(ctx, func(_ context.Context, page pagination.Page) (bool, error) {
|
||||
list, err := floatingips.ExtractFloatingIPs(page)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("openstack: extract floating ips: %w", err)
|
||||
}
|
||||
out = make([]FloatingIP, 0, len(list))
|
||||
for _, f := range list {
|
||||
proj := f.TenantID
|
||||
if proj == "" {
|
||||
proj = f.ProjectID // Neutron may return only project_id when `fields` is used
|
||||
}
|
||||
out = append(out, FloatingIP{ID: f.ID, Address: f.FloatingIP, PortID: f.PortID, ProjectID: proj})
|
||||
}
|
||||
return false, nil // one page per request
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("openstack: list floating ips: %w", err)
|
||||
}
|
||||
list, err := floatingips.ExtractFloatingIPs(pages)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("openstack: extract floating ips: %w", err)
|
||||
}
|
||||
out := make([]FloatingIP, 0, len(list))
|
||||
for _, f := range list {
|
||||
out = append(out, FloatingIP{ID: f.ID, Address: f.FloatingIP, PortID: f.PortID, ProjectID: f.TenantID})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ListFreeFloatingIPs pages through every floating IP of the project (page by
|
||||
// page, retrying transient failures per page) and hands each page to onPage in
|
||||
// server order. All entries of a page are passed — free and associated; the
|
||||
// caller filters. It returns the number of non-empty pages read.
|
||||
func (c *Client) ListFreeFloatingIPs(ctx context.Context, pageSize int, onPage func([]FloatingIP) error) (int, error) {
|
||||
return paginate(ctx, pageSize, c.retry, c.fetchPage, onPage)
|
||||
}
|
||||
|
||||
func (c *Client) ListFloatingIPs(ctx context.Context) ([]FloatingIP, error) {
|
||||
return listAll(ctx, c)
|
||||
}
|
||||
|
||||
func (c *Client) AssociateFloatingIP(ctx context.Context, fipID, portID string) error {
|
||||
_, err := floatingips.Update(ctx, c.networking, fipID, floatingips.UpdateOpts{
|
||||
PortID: &portID,
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
package openstack
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gophercloud/gophercloud/v2"
|
||||
)
|
||||
|
||||
// TestClientListFreeFloatingIPsAgainstFakeNeutron drives the real Client
|
||||
// against an httptest server: marker pagination, the `fields` projection, a
|
||||
// transient 503 that is retried and an empty port_id meaning "free".
|
||||
func TestClientListFreeFloatingIPsAgainstFakeNeutron(t *testing.T) {
|
||||
const total = 5
|
||||
var requests int
|
||||
failedOnce := false
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requests++
|
||||
q := r.URL.Query()
|
||||
if got := q["fields"]; len(got) != 4 {
|
||||
t.Errorf("expected 4 fields params, got %v", got)
|
||||
}
|
||||
if q.Get("limit") != "2" {
|
||||
t.Errorf("expected limit=2, got %q", q.Get("limit"))
|
||||
}
|
||||
if q.Get("marker") == "id-1" && !failedOnce {
|
||||
failedOnce = true
|
||||
http.Error(w, "busy", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
start := 0
|
||||
if m := q.Get("marker"); m != "" {
|
||||
fmt.Sscanf(m, "id-%d", &start)
|
||||
start++
|
||||
}
|
||||
type fip struct {
|
||||
ID string `json:"id"`
|
||||
Addr string `json:"floating_ip_address"`
|
||||
PortID *string `json:"port_id"`
|
||||
TenantID string `json:"tenant_id"`
|
||||
}
|
||||
var items []fip
|
||||
for i := start; i < total && len(items) < 2; i++ {
|
||||
f := fip{ID: fmt.Sprintf("id-%d", i), Addr: fmt.Sprintf("203.0.113.%d", i+1), TenantID: "p"}
|
||||
if i == 1 {
|
||||
p := "port-1"
|
||||
f.PortID = &p
|
||||
}
|
||||
items = append(items, f)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"floatingips": items})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := &Client{
|
||||
networking: &gophercloud.ServiceClient{
|
||||
ProviderClient: &gophercloud.ProviderClient{HTTPClient: *srv.Client()},
|
||||
Endpoint: srv.URL + "/",
|
||||
ResourceBase: srv.URL + "/v2.0/",
|
||||
},
|
||||
retry: pageRetry{Retries: 3, Sleep: func(context.Context, time.Duration) error { return nil }},
|
||||
}
|
||||
var all []FloatingIP
|
||||
pages, err := c.ListFreeFloatingIPs(context.Background(), 2, func(p []FloatingIP) error {
|
||||
all = append(all, p...)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if pages != 3 || len(all) != total {
|
||||
t.Fatalf("expected 3 pages / 5 fips, got %d / %d", pages, len(all))
|
||||
}
|
||||
if all[1].PortID != "port-1" || all[0].PortID != "" || all[4].Address != "203.0.113.5" {
|
||||
t.Fatalf("unexpected mapping: %+v", all)
|
||||
}
|
||||
if requests != 4 { // 3 pages + 1 retried 503
|
||||
t.Fatalf("expected 4 requests, got %d", requests)
|
||||
}
|
||||
}
|
||||
@@ -30,6 +30,15 @@ type FloatingIPClient interface {
|
||||
// the check queue without an admin having to enumerate them by hand.
|
||||
ListFloatingIPs(ctx context.Context) ([]FloatingIP, error)
|
||||
|
||||
// ListFreeFloatingIPs reads the project's floating IPs page by page
|
||||
// (pageSize entries per request; <=0 means DefaultListPageSize), calling
|
||||
// onPage with every page in server order — transient per-page failures
|
||||
// are retried inside, so a long listing survives a flaky Neutron. Every
|
||||
// entry of the page is passed, associated or free: the caller filters on
|
||||
// PortID. It returns the number of non-empty pages read. An error from
|
||||
// onPage stops the walk and is returned as-is.
|
||||
ListFreeFloatingIPs(ctx context.Context, pageSize int, onPage func(page []FloatingIP) error) (pages int, err error)
|
||||
|
||||
// AssociateFloatingIP attaches the floating IP to the given Neutron
|
||||
// port (the validator's primary NIC port).
|
||||
AssociateFloatingIP(ctx context.Context, fipID, portID string) error
|
||||
@@ -53,3 +62,16 @@ func IsNotFound(err error) bool {
|
||||
_, ok := err.(*notFoundError)
|
||||
return ok
|
||||
}
|
||||
|
||||
// listAll collects every page of c.ListFreeFloatingIPs into one slice; it is
|
||||
// the shared implementation of ListFloatingIPs on top of the paged method.
|
||||
func listAll(ctx context.Context, c FloatingIPClient) ([]FloatingIP, error) {
|
||||
var all []FloatingIP
|
||||
if _, err := c.ListFreeFloatingIPs(ctx, DefaultListPageSize, func(page []FloatingIP) error {
|
||||
all = append(all, page...)
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
@@ -3,7 +3,10 @@ package openstack
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MockClient is an in-memory FloatingIPClient used by unit tests and the
|
||||
@@ -22,6 +25,24 @@ type MockClient struct {
|
||||
// ListFailure, when non-nil, is returned by every ListFloatingIPs call
|
||||
// until the test resets it to nil — to exercise the scan-error path.
|
||||
ListFailure error
|
||||
|
||||
// ListFailures is a queue of errors consumed one per page request (each
|
||||
// request, including retries, pops the head; an empty queue means no
|
||||
// failure), on top of the sticky ListFailure. Exercises per-page retry.
|
||||
ListFailures []error
|
||||
|
||||
// PageSize, when > 0, overrides the page size requested by the caller.
|
||||
PageSize int
|
||||
// PageDelay is an artificial latency added to every page request
|
||||
// (honours ctx cancellation) — to simulate a slow Neutron.
|
||||
PageDelay time.Duration
|
||||
// PageRetries / Sleep configure per-page retry like the real client;
|
||||
// zero retries by default so a failure surfaces immediately.
|
||||
PageRetries int
|
||||
Sleep func(ctx context.Context, d time.Duration) error
|
||||
|
||||
// PageCalls counts page requests served (successful or failed).
|
||||
PageCalls int
|
||||
}
|
||||
|
||||
func NewMockClient() *MockClient {
|
||||
@@ -60,19 +81,84 @@ func (m *MockClient) GetFloatingIPByAddress(ctx context.Context, address string)
|
||||
return &f, nil
|
||||
}
|
||||
|
||||
func (m *MockClient) ListFloatingIPs(ctx context.Context) ([]FloatingIP, error) {
|
||||
// SeedMany registers n free floating IPs with IDs "<prefix>-00000"… and
|
||||
// ascending addresses starting at 198.18.0.1 (TEST-NET benchmarking range),
|
||||
// returning the addresses in seed order.
|
||||
func (m *MockClient) SeedMany(prefix string, n int) []string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
base := netip.MustParseAddr("198.18.0.1")
|
||||
addrs := make([]string, 0, n)
|
||||
a := base
|
||||
for i := 0; i < n; i++ {
|
||||
id := fmt.Sprintf("%s-%05d", prefix, i)
|
||||
addr := a.String()
|
||||
m.fips[id] = &FloatingIP{ID: id, Address: addr, ProjectID: "mock-project"}
|
||||
m.byIP[addr] = id
|
||||
addrs = append(addrs, addr)
|
||||
a = a.Next()
|
||||
}
|
||||
return addrs
|
||||
}
|
||||
|
||||
func (m *MockClient) fetchPage(ctx context.Context, marker string, limit int) ([]FloatingIP, error) {
|
||||
m.mu.Lock()
|
||||
delay := m.PageDelay
|
||||
m.mu.Unlock()
|
||||
if delay > 0 {
|
||||
if err := sleepCtx(ctx, delay); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.PageCalls++
|
||||
if m.ListFailure != nil {
|
||||
return nil, m.ListFailure
|
||||
}
|
||||
out := make([]FloatingIP, 0, len(m.fips))
|
||||
for _, f := range m.fips {
|
||||
out = append(out, *f)
|
||||
if len(m.ListFailures) > 0 {
|
||||
err := m.ListFailures[0]
|
||||
m.ListFailures = m.ListFailures[1:]
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if m.PageSize > 0 {
|
||||
limit = m.PageSize
|
||||
}
|
||||
ids := make([]string, 0, len(m.fips))
|
||||
for id := range m.fips {
|
||||
if id > marker {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
if len(ids) > limit {
|
||||
ids = ids[:limit]
|
||||
}
|
||||
out := make([]FloatingIP, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
out = append(out, *m.fips[id])
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ListFreeFloatingIPs pages through the seeded floating IPs sorted by ID, like
|
||||
// the real client does against Neutron (see FloatingIPClient).
|
||||
func (m *MockClient) ListFreeFloatingIPs(ctx context.Context, pageSize int, onPage func([]FloatingIP) error) (int, error) {
|
||||
m.mu.Lock()
|
||||
rp := pageRetry{Retries: m.PageRetries, Sleep: m.Sleep}
|
||||
if m.PageSize > 0 {
|
||||
pageSize = m.PageSize
|
||||
}
|
||||
m.mu.Unlock()
|
||||
return paginate(ctx, pageSize, rp, m.fetchPage, onPage)
|
||||
}
|
||||
|
||||
func (m *MockClient) ListFloatingIPs(ctx context.Context) ([]FloatingIP, error) {
|
||||
return listAll(ctx, m)
|
||||
}
|
||||
|
||||
func (m *MockClient) AssociateFloatingIP(ctx context.Context, fipID, portID string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
package openstack
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultListPageSize is the page size used when the caller passes a
|
||||
// non-positive one.
|
||||
const DefaultListPageSize = 200
|
||||
|
||||
// DefaultListPageRetries is how many times a single failed page request is
|
||||
// retried (so up to DefaultListPageRetries+1 attempts in total).
|
||||
const DefaultListPageRetries = 5
|
||||
|
||||
// pageFetcher fetches one page of floating IPs: at most limit entries that
|
||||
// follow the entry with ID marker ("" = from the start), in a stable server
|
||||
// order. A page shorter than limit is the last one.
|
||||
type pageFetcher func(ctx context.Context, marker string, limit int) ([]FloatingIP, error)
|
||||
|
||||
// pageRetry configures per-page retries. The zero value retries nothing.
|
||||
type pageRetry struct {
|
||||
// Retries is the number of retries after the first failed attempt.
|
||||
Retries int
|
||||
// Sleep waits for d or until ctx is done; nil uses a real timer. Tests
|
||||
// inject a fake to avoid real backoff delays.
|
||||
Sleep func(ctx context.Context, d time.Duration) error
|
||||
}
|
||||
|
||||
// retryBackoff is the delay before retry number attempt (1-based):
|
||||
// 1s, 2s, 4s, 8s, 16s, then capped at 30s.
|
||||
func retryBackoff(attempt int) time.Duration {
|
||||
if attempt < 1 {
|
||||
attempt = 1
|
||||
}
|
||||
d := time.Second << uint(attempt-1)
|
||||
if d > 30*time.Second || d <= 0 {
|
||||
d = 30 * time.Second
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func sleepCtx(ctx context.Context, d time.Duration) error {
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-t.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// statusCoder is implemented by gophercloud.ErrUnexpectedResponseCode (and
|
||||
// anything wrapping it): it carries the HTTP status of a failed request.
|
||||
type statusCoder interface{ GetStatusCode() int }
|
||||
|
||||
// IsRetryableListError reports whether a failed page request is worth
|
||||
// repeating: transport-level failures (timeouts, resets, EOF /
|
||||
// RemoteDisconnected), HTTP 429 and HTTP 5xx. Any other 4xx, context
|
||||
// cancellation and unknown errors are not retried.
|
||||
func IsRetryableListError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return false
|
||||
}
|
||||
var sc statusCoder
|
||||
if errors.As(err, &sc) {
|
||||
code := sc.GetStatusCode()
|
||||
return code == 429 || code >= 500
|
||||
}
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return true
|
||||
}
|
||||
// A deadline is retryable only when it is a per-request network timeout
|
||||
// (net.Error), which is checked below; the caller's own ctx deadline is
|
||||
// filtered out by the paginator via ctx.Err().
|
||||
var ne net.Error
|
||||
if errors.As(err, &ne) {
|
||||
return true
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
for _, s := range []string{
|
||||
"remotedisconnected", "remote end closed connection",
|
||||
"connection reset", "connection refused", "broken pipe",
|
||||
"unexpected eof", "eof", "timeout", "tls handshake",
|
||||
} {
|
||||
if strings.Contains(msg, s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// paginate walks all pages with marker-based pagination, retrying each failed
|
||||
// page per rp, and hands every non-empty page to onPage in server order. It
|
||||
// returns the number of non-empty pages delivered. An error from onPage, a
|
||||
// non-retryable error, exhausted retries or ctx cancellation stop the walk.
|
||||
func paginate(ctx context.Context, pageSize int, rp pageRetry, fetch pageFetcher, onPage func([]FloatingIP) error) (int, error) {
|
||||
if pageSize <= 0 {
|
||||
pageSize = DefaultListPageSize
|
||||
}
|
||||
sleep := rp.Sleep
|
||||
if sleep == nil {
|
||||
sleep = sleepCtx
|
||||
}
|
||||
marker := ""
|
||||
pages := 0
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return pages, err
|
||||
}
|
||||
var page []FloatingIP
|
||||
var err error
|
||||
for attempt := 0; ; attempt++ {
|
||||
page, err = fetch(ctx, marker, pageSize)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return pages, ctx.Err()
|
||||
}
|
||||
if attempt >= rp.Retries || !IsRetryableListError(err) {
|
||||
return pages, err
|
||||
}
|
||||
if serr := sleep(ctx, retryBackoff(attempt+1)); serr != nil {
|
||||
return pages, serr
|
||||
}
|
||||
}
|
||||
if len(page) == 0 {
|
||||
return pages, nil
|
||||
}
|
||||
pages++
|
||||
if err := onPage(page); err != nil {
|
||||
return pages, err
|
||||
}
|
||||
if len(page) < pageSize {
|
||||
return pages, nil
|
||||
}
|
||||
marker = page[len(page)-1].ID
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
package openstack
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gophercloud/gophercloud/v2"
|
||||
"github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/layer3/floatingips"
|
||||
)
|
||||
|
||||
func TestMockPaginationReturnsAllPages(t *testing.T) {
|
||||
m := NewMockClient()
|
||||
m.PageSize = 200
|
||||
m.SeedMany("fip", 2500)
|
||||
|
||||
seen := map[string]bool{}
|
||||
var sizes []int
|
||||
pages, err := m.ListFreeFloatingIPs(context.Background(), 200, func(page []FloatingIP) error {
|
||||
sizes = append(sizes, len(page))
|
||||
for _, f := range page {
|
||||
if seen[f.ID] {
|
||||
t.Fatalf("duplicate id %s", f.ID)
|
||||
}
|
||||
seen[f.ID] = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if len(seen) != 2500 {
|
||||
t.Fatalf("expected 2500 unique fips, got %d", len(seen))
|
||||
}
|
||||
if pages != 13 || len(sizes) != 13 || sizes[12] != 100 {
|
||||
t.Fatalf("expected 13 pages with a short last one, got pages=%d sizes=%v", pages, sizes)
|
||||
}
|
||||
if m.PageCalls != 13 {
|
||||
t.Fatalf("expected 13 page requests, got %d", m.PageCalls)
|
||||
}
|
||||
|
||||
// ListFloatingIPs stays available on top of the paged method.
|
||||
all, err := m.ListFloatingIPs(context.Background())
|
||||
if err != nil || len(all) != 2500 {
|
||||
t.Fatalf("ListFloatingIPs: n=%d err=%v", len(all), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockPaginationExactMultipleOfPageSize(t *testing.T) {
|
||||
m := NewMockClient()
|
||||
m.SeedMany("fip", 400)
|
||||
pages, err := m.ListFreeFloatingIPs(context.Background(), 200, func([]FloatingIP) error { return nil })
|
||||
if err != nil || pages != 2 {
|
||||
t.Fatalf("expected 2 non-empty pages, got pages=%d err=%v", pages, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPageRetryOnTransientErrors(t *testing.T) {
|
||||
m := NewMockClient()
|
||||
m.PageSize = 100
|
||||
m.SeedMany("fip", 250)
|
||||
m.PageRetries = 5
|
||||
var slept []time.Duration
|
||||
m.Sleep = func(ctx context.Context, d time.Duration) error {
|
||||
slept = append(slept, d)
|
||||
return nil
|
||||
}
|
||||
// Fail the first request twice, then the 2nd page once.
|
||||
m.ListFailures = []error{io.ErrUnexpectedEOF, errors.New("RemoteDisconnected('Remote end closed connection')"), nil, io.EOF}
|
||||
|
||||
n := 0
|
||||
pages, err := m.ListFreeFloatingIPs(context.Background(), 100, func(p []FloatingIP) error {
|
||||
n += len(p)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected retries to succeed, got %v", err)
|
||||
}
|
||||
if n != 250 || pages != 3 {
|
||||
t.Fatalf("expected 250 fips in 3 pages, got %d in %d", n, pages)
|
||||
}
|
||||
want := []time.Duration{time.Second, 2 * time.Second, time.Second}
|
||||
if len(slept) != len(want) {
|
||||
t.Fatalf("expected %d backoffs, got %v", len(want), slept)
|
||||
}
|
||||
for i := range want {
|
||||
if slept[i] != want[i] {
|
||||
t.Fatalf("backoff[%d]=%v want %v (all %v)", i, slept[i], want[i], slept)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPageRetryGivesUpAfterRetries(t *testing.T) {
|
||||
m := NewMockClient()
|
||||
m.SeedMany("fip", 10)
|
||||
m.PageRetries = 2
|
||||
m.Sleep = func(context.Context, time.Duration) error { return nil }
|
||||
m.ListFailure = io.ErrUnexpectedEOF // sticky
|
||||
_, err := m.ListFreeFloatingIPs(context.Background(), 5, func([]FloatingIP) error { return nil })
|
||||
if !errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
t.Fatalf("expected the page error after retries, got %v", err)
|
||||
}
|
||||
if m.PageCalls != 3 {
|
||||
t.Fatalf("expected 1+2 attempts, got %d", m.PageCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPageNonRetryableAborts(t *testing.T) {
|
||||
m := NewMockClient()
|
||||
m.SeedMany("fip", 10)
|
||||
m.PageRetries = 5
|
||||
m.Sleep = func(context.Context, time.Duration) error {
|
||||
t.Fatal("must not back off for a non-retryable error")
|
||||
return nil
|
||||
}
|
||||
boom := errors.New("403 forbidden")
|
||||
m.ListFailures = []error{boom}
|
||||
called := false
|
||||
_, err := m.ListFreeFloatingIPs(context.Background(), 5, func([]FloatingIP) error { called = true; return nil })
|
||||
if !errors.Is(err, boom) || called || m.PageCalls != 1 {
|
||||
t.Fatalf("expected immediate abort: err=%v called=%v calls=%d", err, called, m.PageCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPageOnPageErrorStops(t *testing.T) {
|
||||
m := NewMockClient()
|
||||
m.SeedMany("fip", 50)
|
||||
stop := errors.New("stop")
|
||||
pages, err := m.ListFreeFloatingIPs(context.Background(), 10, func([]FloatingIP) error { return stop })
|
||||
if !errors.Is(err, stop) || pages != 1 || m.PageCalls != 1 {
|
||||
t.Fatalf("expected stop after first page: pages=%d err=%v calls=%d", pages, err, m.PageCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPageContextCancel(t *testing.T) {
|
||||
m := NewMockClient()
|
||||
m.SeedMany("fip", 100)
|
||||
m.PageDelay = 5 * time.Second
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
_, err := m.ListFreeFloatingIPs(ctx, 10, func([]FloatingIP) error { return nil })
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("expected deadline error, got %v", err)
|
||||
}
|
||||
if time.Since(start) > 2*time.Second {
|
||||
t.Fatalf("cancel was not honoured promptly")
|
||||
}
|
||||
|
||||
// Cancellation during the backoff sleep is honoured too.
|
||||
m2 := NewMockClient()
|
||||
m2.SeedMany("fip", 10)
|
||||
m2.PageRetries = 3
|
||||
m2.ListFailure = io.EOF
|
||||
ctx2, cancel2 := context.WithCancel(context.Background())
|
||||
m2.Sleep = func(ctx context.Context, d time.Duration) error { cancel2(); return ctx.Err() }
|
||||
_, err = m2.ListFreeFloatingIPs(ctx2, 5, func([]FloatingIP) error { return nil })
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected canceled, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRetryableListError(t *testing.T) {
|
||||
status := func(code int) error {
|
||||
return gophercloud.ErrUnexpectedResponseCode{Actual: code, Expected: []int{200}}
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{"nil", nil, false},
|
||||
{"eof", io.EOF, true},
|
||||
{"unexpected eof", io.ErrUnexpectedEOF, true},
|
||||
{"wrapped eof", fmt.Errorf("list: %w", io.EOF), true},
|
||||
{"remote disconnected text", errors.New("Get x: RemoteDisconnected('Remote end closed connection without response')"), true},
|
||||
{"connection reset", errors.New("read tcp: connection reset by peer"), true},
|
||||
{"connection refused", errors.New("dial tcp: connection refused"), true},
|
||||
{"net timeout", &url.Error{Op: "Get", URL: "http://x", Err: &net.DNSError{IsTimeout: true}}, true},
|
||||
{"500", status(http.StatusInternalServerError), true},
|
||||
{"503 wrapped", fmt.Errorf("openstack: list floating ips: %w", status(503)), true},
|
||||
{"429", status(http.StatusTooManyRequests), true},
|
||||
{"400", status(http.StatusBadRequest), false},
|
||||
{"401", status(http.StatusUnauthorized), false},
|
||||
{"404", status(http.StatusNotFound), false},
|
||||
{"canceled", context.Canceled, false},
|
||||
{"plain", errors.New("boom"), false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := IsRetryableListError(c.err); got != c.want {
|
||||
t.Errorf("%s: IsRetryableListError=%v want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPagedListOptsQuery(t *testing.T) {
|
||||
o := pagedListOpts{ListOpts: floatingipsListOpts("m1", 200), fields: fipListFields}
|
||||
q, err := o.ToFloatingIPListQuery()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := url.ParseQuery(q[1:])
|
||||
if err != nil {
|
||||
t.Fatalf("parse %q: %v", q, err)
|
||||
}
|
||||
if v.Get("limit") != "200" || v.Get("marker") != "m1" {
|
||||
t.Fatalf("limit/marker missing in %q", q)
|
||||
}
|
||||
if got := v["fields"]; len(got) != 4 || got[0] != "id" || got[1] != "floating_ip_address" || got[2] != "port_id" || got[3] != "project_id" {
|
||||
t.Fatalf("unexpected fields %v in %q", got, q)
|
||||
}
|
||||
// Without any other option the query must still start with '?'.
|
||||
q, _ = pagedListOpts{fields: []string{"id"}}.ToFloatingIPListQuery()
|
||||
if q != "?fields=id" {
|
||||
t.Fatalf("got %q", q)
|
||||
}
|
||||
}
|
||||
|
||||
func floatingipsListOpts(marker string, limit int) floatingips.ListOpts {
|
||||
return floatingips.ListOpts{Marker: marker, Limit: limit}
|
||||
}
|
||||
Reference in new issue
Block a user