ayurishchevandClaude Sonnet 5.5
e95b5eb7d5
A validator that failed the self-check of an address no longer gets that address
again in the current round (ClaimNextQueued skips it); the validator itself stays
in service and takes all other addresses. The verdict fail is set when the number
of failed self-checks of an address reaches settings.self_check_max_attempts
(1..50, default 5, independent of the number of validators); max_retries and
retry_count are no longer used for self-check. If every working validator has
already failed the address, a new round starts and the exclusions lapse.
Migration 0012: ip_self_check_failures (permanent history per registry address),
ip_queue.sc_failures and sc_round_start_cycle (cycle_id is used instead of
attempt_number, which restarts when a queue row is recreated), the setting.
db.FailSelfCheck does it in one transaction; re-submission starts a new series.
API: self_check_max_attempts in GET/PUT /admin/config/orchestrator,
self_check_failed_on in /admin/ips/{ip} and /admin/registry/{ip}. Dashboard: the
field on /settings and the line "Self-check не прошёл на: ..." on the address
pages. Docs, plan and summary in docs/changes/.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
504 lines
17 KiB
Go
504 lines
17 KiB
Go
package dashboard
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// apiErr is returned by every client method for a non-2xx response from
|
|
// control-api, or for a transport-level failure (control-api unreachable/
|
|
// timeout, Status==0). Handlers use it (via errors.As) to render an error
|
|
// banner with the right status/severity instead of a raw 500 — see
|
|
// writeErrorBanner in render.go.
|
|
type apiErr struct {
|
|
Status int
|
|
Message string
|
|
}
|
|
|
|
func (e *apiErr) Error() string {
|
|
if e.Status == 0 {
|
|
return fmt.Sprintf("control-api недоступен: %s", e.Message)
|
|
}
|
|
return fmt.Sprintf("control-api: %d %s", e.Status, e.Message)
|
|
}
|
|
|
|
// client is a thin, dashboard-local JSON client for control-api's
|
|
// /api/v1/admin/* surface. It intentionally doesn't reuse
|
|
// internal/apiclient.Client (shared by validator-agent/prober): that
|
|
// client's Do only returns an opaque error, with no way to recover the
|
|
// HTTP status code — which the dashboard needs to render 400 vs 404 vs 409
|
|
// vs 5xx differently. Duplicating ~30 lines here avoids widening
|
|
// apiclient's contract (and its blast radius on the other two binaries)
|
|
// for a need only this package has.
|
|
type client struct {
|
|
baseURL string
|
|
http *http.Client
|
|
// long is used for clear / bulk operations, which legitimately take far
|
|
// longer than a plain read (thousands of rows in one transaction): same
|
|
// transport, but a longer whole-call timeout.
|
|
long *http.Client
|
|
// token, when non-empty, is sent to control-api as a Bearer credential.
|
|
token string
|
|
}
|
|
|
|
// longCallTimeout is the minimum whole-call timeout for clear and bulk
|
|
// operations (ClearQueue, DeleteIPs, SubmitIPs).
|
|
const longCallTimeout = 120 * time.Second
|
|
|
|
func newClient(baseURL string, timeout time.Duration) *client {
|
|
longT := longCallTimeout
|
|
if timeout > longT {
|
|
longT = timeout
|
|
}
|
|
return &client{
|
|
baseURL: baseURL,
|
|
http: &http.Client{Timeout: timeout},
|
|
long: &http.Client{Timeout: longT},
|
|
}
|
|
}
|
|
|
|
func (c *client) do(ctx context.Context, method, path string, body, out interface{}) error {
|
|
return c.doWith(ctx, c.http, method, path, body, out)
|
|
}
|
|
|
|
// doLong is do with the long (clear/bulk) timeout.
|
|
func (c *client) doLong(ctx context.Context, method, path string, body, out interface{}) error {
|
|
hc := c.long
|
|
if hc == nil {
|
|
hc = c.http
|
|
}
|
|
return c.doWith(ctx, hc, method, path, body, out)
|
|
}
|
|
|
|
func (c *client) doWith(ctx context.Context, hc *http.Client, method, path string, body, out interface{}) error {
|
|
var reader io.Reader
|
|
if body != nil {
|
|
b, err := json.Marshal(body)
|
|
if err != nil {
|
|
return fmt.Errorf("marshal request: %w", err)
|
|
}
|
|
reader = bytes.NewReader(b)
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, reader)
|
|
if err != nil {
|
|
return fmt.Errorf("build request: %w", err)
|
|
}
|
|
if body != nil {
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
if c.token != "" {
|
|
req.Header.Set("Authorization", "Bearer "+c.token)
|
|
}
|
|
|
|
resp, err := hc.Do(req)
|
|
if err != nil {
|
|
return &apiErr{Status: 0, Message: err.Error()}
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
respBody, _ := io.ReadAll(resp.Body)
|
|
if resp.StatusCode >= 300 {
|
|
msg := string(respBody)
|
|
var er errorResponse
|
|
if json.Unmarshal(respBody, &er) == nil && er.Error != "" {
|
|
msg = er.Error
|
|
}
|
|
return &apiErr{Status: resp.StatusCode, Message: msg}
|
|
}
|
|
if out != nil && len(respBody) > 0 {
|
|
if err := json.Unmarshal(respBody, out); err != nil {
|
|
return fmt.Errorf("decode response from %s %s: %w", method, path, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *client) Status(ctx context.Context) (statusResponse, error) {
|
|
var out statusResponse
|
|
err := c.do(ctx, http.MethodGet, "/api/v1/admin/status", nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
// maxPageLimit is control-api's cap on `limit`.
|
|
const maxPageLimit = 1000
|
|
|
|
// clampLimit keeps limit within 1..maxPageLimit: a request without `limit`
|
|
// would make control-api answer with the legacy unbounded bare array.
|
|
func clampLimit(limit int) int {
|
|
if limit < 1 {
|
|
return 1
|
|
}
|
|
if limit > maxPageLimit {
|
|
return maxPageLimit
|
|
}
|
|
return limit
|
|
}
|
|
|
|
// ipsQuery selects one page of GET /admin/ips: server-side filters plus
|
|
// limit/offset. Order is "sequence" (default) or "aggregated_at_desc".
|
|
type ipsQuery struct {
|
|
States []string
|
|
Q string
|
|
Result string
|
|
Order string
|
|
Limit int
|
|
Offset int
|
|
}
|
|
|
|
func (q ipsQuery) values() url.Values {
|
|
v := url.Values{}
|
|
v.Set("limit", strconv.Itoa(clampLimit(q.Limit)))
|
|
if q.Offset > 0 {
|
|
v.Set("offset", strconv.Itoa(q.Offset))
|
|
}
|
|
if len(q.States) > 0 {
|
|
v.Set("state", strings.Join(q.States, ","))
|
|
}
|
|
if q.Q != "" {
|
|
v.Set("q", q.Q)
|
|
}
|
|
if q.Result != "" {
|
|
v.Set("result", q.Result)
|
|
}
|
|
if q.Order != "" {
|
|
v.Set("order", q.Order)
|
|
}
|
|
return v
|
|
}
|
|
|
|
// ListIPsPage returns one page of the check queue plus the total number of
|
|
// rows matching the filter. Never loads the whole queue.
|
|
func (c *client) ListIPsPage(ctx context.Context, q ipsQuery) (ipsPage, error) {
|
|
var out ipsPage
|
|
err := c.do(ctx, http.MethodGet, "/api/v1/admin/ips?"+q.values().Encode(), nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
func (c *client) GetIP(ctx context.Context, ip string) (ipDetailResponse, error) {
|
|
var out ipDetailResponse
|
|
err := c.do(ctx, http.MethodGet, "/api/v1/admin/ips/"+url.PathEscape(ip), nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
// SubmitIPs is the single entry point for both adding new addresses and
|
|
// forcing a recheck of already-finished ones — see docs/API.md.
|
|
func (c *client) SubmitIPs(ctx context.Context, addresses []string) (submitIPsResponse, error) {
|
|
var out submitIPsResponse
|
|
err := c.doLong(ctx, http.MethodPost, "/api/v1/admin/ips", map[string][]string{"addresses": addresses}, &out)
|
|
return out, err
|
|
}
|
|
|
|
func (c *client) CancelIP(ctx context.Context, ip string) error {
|
|
return c.do(ctx, http.MethodPost, "/api/v1/admin/ips/"+url.PathEscape(ip)+"/cancel", nil, nil)
|
|
}
|
|
|
|
// DeleteIP permanently removes one address and its full history — unlike
|
|
// CancelIP, there's nothing left to look up afterward.
|
|
func (c *client) DeleteIP(ctx context.Context, ip string) error {
|
|
return c.do(ctx, http.MethodDelete, "/api/v1/admin/ips/"+url.PathEscape(ip), nil, nil)
|
|
}
|
|
|
|
// DeleteIPs permanently removes a specific list of addresses in one call.
|
|
func (c *client) DeleteIPs(ctx context.Context, addresses []string) (deleteIPsResponse, error) {
|
|
var out deleteIPsResponse
|
|
err := c.doLong(ctx, http.MethodPost, "/api/v1/admin/ips/delete", map[string][]string{"addresses": addresses}, &out)
|
|
return out, err
|
|
}
|
|
|
|
// ClearQueue permanently removes every address currently in the queue,
|
|
// including those actively being checked.
|
|
func (c *client) ClearQueue(ctx context.Context) (clearQueueResponse, error) {
|
|
var out clearQueueResponse
|
|
err := c.doLong(ctx, http.MethodPost, "/api/v1/admin/ips/clear", nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
// StartScan starts control-api's background floating-IP scan (or joins the
|
|
// one already running) and returns immediately with its current status.
|
|
func (c *client) StartScan(ctx context.Context, dryRun bool) (scanStatusDTO, error) {
|
|
var out scanStatusDTO
|
|
path := "/api/v1/admin/ips/scan"
|
|
if dryRun {
|
|
path += "?dry_run=true"
|
|
}
|
|
err := c.do(ctx, http.MethodPost, path, nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
// ScanStatus returns the progress of the background scan job.
|
|
func (c *client) ScanStatus(ctx context.Context) (scanStatusDTO, error) {
|
|
var out scanStatusDTO
|
|
err := c.do(ctx, http.MethodGet, "/api/v1/admin/ips/scan", nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
// registryQuery selects one page of GET /admin/registry.
|
|
type registryQuery struct {
|
|
Q string
|
|
LastResult string
|
|
Run int64 // only addresses with a result in this run
|
|
Subnet string // only addresses inside this CIDR
|
|
Limit int
|
|
Offset int
|
|
}
|
|
|
|
// ListRegistryPage returns one page of the registry (every address ever
|
|
// submitted to the check queue, each with a summary of its accumulated check
|
|
// history — survives an address being deleted from the queue and later
|
|
// re-added) plus the total number of rows matching the filter.
|
|
func (c *client) ListRegistryPage(ctx context.Context, q registryQuery) (registryPage, error) {
|
|
v := url.Values{}
|
|
v.Set("limit", strconv.Itoa(clampLimit(q.Limit)))
|
|
if q.Offset > 0 {
|
|
v.Set("offset", strconv.Itoa(q.Offset))
|
|
}
|
|
if q.Q != "" {
|
|
v.Set("q", q.Q)
|
|
}
|
|
if q.LastResult != "" {
|
|
v.Set("last_result", q.LastResult)
|
|
}
|
|
if q.Run > 0 {
|
|
v.Set("run", strconv.FormatInt(q.Run, 10))
|
|
}
|
|
if q.Subnet != "" {
|
|
v.Set("subnet", q.Subnet)
|
|
}
|
|
var out registryPage
|
|
err := c.do(ctx, http.MethodGet, "/api/v1/admin/registry?"+v.Encode(), nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
// GetRegistryHistory returns one address's registry record plus its full
|
|
// retained check history across every cycle still kept.
|
|
func (c *client) GetRegistryHistory(ctx context.Context, ip string) (registryHistoryResponse, error) {
|
|
var out registryHistoryResponse
|
|
err := c.do(ctx, http.MethodGet, "/api/v1/admin/registry/"+url.PathEscape(ip), nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
func (c *client) ListValidators(ctx context.Context) ([]validatorDTO, error) {
|
|
var out []validatorDTO
|
|
err := c.do(ctx, http.MethodGet, "/api/v1/admin/config/validators", nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
func (c *client) CreateValidator(ctx context.Context, id, osPortID string) error {
|
|
body := map[string]string{"validator_id": id, "os_port_id": osPortID}
|
|
return c.do(ctx, http.MethodPost, "/api/v1/admin/config/validators", body, nil)
|
|
}
|
|
|
|
func (c *client) UpdateValidator(ctx context.Context, id, osPortID string) error {
|
|
body := map[string]string{"os_port_id": osPortID}
|
|
return c.do(ctx, http.MethodPut, "/api/v1/admin/config/validators/"+url.PathEscape(id), body, nil)
|
|
}
|
|
|
|
func (c *client) DeleteValidator(ctx context.Context, id string) error {
|
|
return c.do(ctx, http.MethodDelete, "/api/v1/admin/config/validators/"+url.PathEscape(id), nil, nil)
|
|
}
|
|
|
|
func (c *client) ListSites(ctx context.Context) ([]siteDTO, error) {
|
|
var out []siteDTO
|
|
err := c.do(ctx, http.MethodGet, "/api/v1/admin/config/sites", nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
func (c *client) PutSite(ctx context.Context, index int, siteID string) error {
|
|
body := map[string]string{"site_id": siteID}
|
|
return c.do(ctx, http.MethodPut, fmt.Sprintf("/api/v1/admin/config/sites/%d", index), body, nil)
|
|
}
|
|
|
|
func (c *client) DeleteSite(ctx context.Context, index int) error {
|
|
return c.do(ctx, http.MethodDelete, fmt.Sprintf("/api/v1/admin/config/sites/%d", index), nil, nil)
|
|
}
|
|
|
|
func (c *client) ListTargetGroups(ctx context.Context) ([]targetGroupDTO, error) {
|
|
var out []targetGroupDTO
|
|
err := c.do(ctx, http.MethodGet, "/api/v1/admin/config/targets", nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
func (c *client) PutTargetGroup(ctx context.Context, name string, targets []string) error {
|
|
body := map[string][]string{"targets": targets}
|
|
return c.do(ctx, http.MethodPut, "/api/v1/admin/config/targets/"+url.PathEscape(name), body, nil)
|
|
}
|
|
|
|
func (c *client) DeleteTargetGroup(ctx context.Context, name string) error {
|
|
return c.do(ctx, http.MethodDelete, "/api/v1/admin/config/targets/"+url.PathEscape(name), nil, nil)
|
|
}
|
|
|
|
func (c *client) ListCheckTypes(ctx context.Context) ([]checkTypeDTO, error) {
|
|
var out []checkTypeDTO
|
|
err := c.do(ctx, http.MethodGet, "/api/v1/admin/config/check-types", nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
func (c *client) PutCheckType(ctx context.Context, name string, enabled bool, targets []string) error {
|
|
body := map[string]interface{}{"enabled": enabled, "targets": targets}
|
|
return c.do(ctx, http.MethodPut, "/api/v1/admin/config/check-types/"+url.PathEscape(name), body, nil)
|
|
}
|
|
|
|
func (c *client) DeleteCheckType(ctx context.Context, name string) error {
|
|
return c.do(ctx, http.MethodDelete, "/api/v1/admin/config/check-types/"+url.PathEscape(name), nil, nil)
|
|
}
|
|
|
|
func (c *client) GetOrchestratorSettings(ctx context.Context) (orchestratorSettingsDTO, error) {
|
|
var out orchestratorSettingsDTO
|
|
err := c.do(ctx, http.MethodGet, "/api/v1/admin/config/orchestrator", nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
func (c *client) PutOrchestratorSettings(ctx context.Context, fipSettleSeconds, historyRetentionCycles, selfCheckMaxAttempts int) (orchestratorSettingsDTO, error) {
|
|
var out orchestratorSettingsDTO
|
|
err := c.do(ctx, http.MethodPut, "/api/v1/admin/config/orchestrator",
|
|
orchestratorSettingsDTO{FIPSettleSeconds: fipSettleSeconds, HistoryRetentionCycles: historyRetentionCycles, SelfCheckMaxAttempts: selfCheckMaxAttempts}, &out)
|
|
return out, err
|
|
}
|
|
|
|
func (c *client) GetInboundChecks(ctx context.Context) (inboundChecksDTO, error) {
|
|
var out inboundChecksDTO
|
|
err := c.do(ctx, http.MethodGet, "/api/v1/admin/config/inbound-checks", nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
func (c *client) PutInboundChecks(ctx context.Context, ports []int, icmp bool) (inboundChecksDTO, error) {
|
|
var out inboundChecksDTO
|
|
err := c.do(ctx, http.MethodPut, "/api/v1/admin/config/inbound-checks",
|
|
inboundChecksDTO{Ports: ports, ICMP: icmp}, &out)
|
|
return out, err
|
|
}
|
|
|
|
func (c *client) GetAutoCycle(ctx context.Context) (autoCycleDTO, error) {
|
|
var out autoCycleDTO
|
|
err := c.do(ctx, http.MethodGet, "/api/v1/admin/auto-cycle", nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
func (c *client) PutAutoCycle(ctx context.Context, intervalSeconds, maxRunSeconds int) (autoCycleDTO, error) {
|
|
var out autoCycleDTO
|
|
err := c.do(ctx, http.MethodPut, "/api/v1/admin/auto-cycle",
|
|
map[string]int{"interval_seconds": intervalSeconds, "max_run_seconds": maxRunSeconds}, &out)
|
|
return out, err
|
|
}
|
|
|
|
func (c *client) StartAutoCycle(ctx context.Context) (autoCycleDTO, error) {
|
|
var out autoCycleDTO
|
|
err := c.do(ctx, http.MethodPost, "/api/v1/admin/auto-cycle/start", nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
func (c *client) StopAutoCycle(ctx context.Context) (autoCycleDTO, error) {
|
|
var out autoCycleDTO
|
|
err := c.do(ctx, http.MethodPost, "/api/v1/admin/auto-cycle/stop", nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
// analyticsRun is one entry of GET /admin/analytics/runs (the run selector).
|
|
type analyticsRun struct {
|
|
ID int64 `json:"id"`
|
|
Kind string `json:"kind"`
|
|
State string `json:"state"`
|
|
StartedAt time.Time `json:"started_at"`
|
|
FinalizedAt *time.Time `json:"finalized_at"`
|
|
Addresses int `json:"addresses"`
|
|
Pass int `json:"pass"`
|
|
Partial int `json:"partial"`
|
|
Fail int `json:"fail"`
|
|
Cancelled int `json:"cancelled"`
|
|
Total int `json:"total"`
|
|
Pending int `json:"pending"`
|
|
}
|
|
|
|
func (c *client) ListAnalyticsRuns(ctx context.Context) ([]analyticsRun, error) {
|
|
var out []analyticsRun
|
|
err := c.do(ctx, http.MethodGet, "/api/v1/admin/analytics/runs", nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
// GetAnalyticsReport returns the analytics of one finished run as the raw
|
|
// JSON control-api computed; the page's script reads it as it is.
|
|
func (c *client) GetAnalyticsReport(ctx context.Context, runID int64) (json.RawMessage, error) {
|
|
var out json.RawMessage
|
|
err := c.do(ctx, http.MethodGet, "/api/v1/admin/analytics/runs/"+strconv.FormatInt(runID, 10), nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
func analyticsListPath(runID int64, kind, class string, csv bool) string {
|
|
v := url.Values{}
|
|
if class != "" {
|
|
v.Set("class", class)
|
|
}
|
|
if csv {
|
|
v.Set("format", "csv")
|
|
}
|
|
p := "/api/v1/admin/analytics/runs/" + strconv.FormatInt(runID, 10) + "/lists/" + url.PathEscape(kind)
|
|
if len(v) > 0 {
|
|
p += "?" + v.Encode()
|
|
}
|
|
return p
|
|
}
|
|
|
|
// GetAnalyticsList returns one address table (JSON) of a run.
|
|
func (c *client) GetAnalyticsList(ctx context.Context, runID int64, kind, class string) (json.RawMessage, error) {
|
|
var out json.RawMessage
|
|
err := c.do(ctx, http.MethodGet, analyticsListPath(runID, kind, class, false), nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
// GetAnalyticsListCSV returns the CSV file of one address table, with the
|
|
// file name control-api proposed.
|
|
func (c *client) GetAnalyticsListCSV(ctx context.Context, runID int64, kind, class string) ([]byte, string, error) {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+analyticsListPath(runID, kind, class, true), nil)
|
|
if err != nil {
|
|
return nil, "", fmt.Errorf("build request: %w", err)
|
|
}
|
|
if c.token != "" {
|
|
req.Header.Set("Authorization", "Bearer "+c.token)
|
|
}
|
|
resp, err := c.http.Do(req)
|
|
if err != nil {
|
|
return nil, "", &apiErr{Status: 0, Message: err.Error()}
|
|
}
|
|
defer resp.Body.Close()
|
|
body, _ := io.ReadAll(resp.Body)
|
|
if resp.StatusCode >= 300 {
|
|
msg := string(body)
|
|
var er errorResponse
|
|
if json.Unmarshal(body, &er) == nil && er.Error != "" {
|
|
msg = er.Error
|
|
}
|
|
return nil, "", &apiErr{Status: resp.StatusCode, Message: msg}
|
|
}
|
|
return body, resp.Header.Get("Content-Disposition"), nil
|
|
}
|
|
|
|
// subnetEntry is one line of the subnet list (GET/PUT /admin/config/subnets).
|
|
type subnetEntry struct {
|
|
CIDR string `json:"cidr"`
|
|
Label string `json:"label,omitempty"`
|
|
}
|
|
|
|
type subnetList struct {
|
|
Subnets []subnetEntry `json:"subnets"`
|
|
}
|
|
|
|
func (c *client) GetSubnets(ctx context.Context) (subnetList, error) {
|
|
var out subnetList
|
|
err := c.do(ctx, http.MethodGet, "/api/v1/admin/config/subnets", nil, &out)
|
|
return out, err
|
|
}
|
|
|
|
func (c *client) PutSubnets(ctx context.Context, in subnetList) (subnetList, error) {
|
|
var out subnetList
|
|
err := c.do(ctx, http.MethodPut, "/api/v1/admin/config/subnets", in, &out)
|
|
return out, err
|
|
}
|