Registry and Analytics: run, subnet, direction and protocol filters, successes-by-target chart
Registry (/registry):
- filters by run (slice by the address's cycle in that run), subnet
(drop-down of configured subnets), direction (egress/ingress) and
protocol (icmp, tcp, ssh, https, tls); status in scope is computed over
the narrowed checks
- chart "successful checks per target (egress) / site (ingress)" when both
direction and protocol are chosen; a row opens the list of addresses
(dialog, CSV)
- API: direction/protocol parameters and run in GET /admin/registry,
GET /admin/registry/breakdown and /breakdown/list
- subnet filter passes ids as one JSON parameter (SQLite variable limit)
Analytics (/analytics):
- subnet filter recomputes the whole page over the addresses of the run
inside the subnet; only their checks are read; cache per run and subnet
- direction and protocol focus the page; with both set the registry chart
is shown
- subnet parameter in GET /admin/analytics/runs/{id} and lists (JSON, CSV)
Docs: plans and summaries in docs/changes, README, API, USAGE.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
068c10ea1c
commit
ded196ec8d
40 files changed
+2545
-188
No files matched your search
@@ -22,6 +22,9 @@ type Input struct {
|
||||
Subnets []db.Subnet
|
||||
SiteNames map[int]string // site index -> site id
|
||||
Rechecked int // addresses with more than one cycle in the run
|
||||
// Subnet narrows the report to the addresses of the run inside it, nested
|
||||
// subnets included; the zero prefix means the whole run.
|
||||
Subnet netip.Prefix
|
||||
// Each feeds every check of the run's result cycles to fn.
|
||||
Each func(fn func(db.RunCheck)) error
|
||||
}
|
||||
@@ -29,6 +32,7 @@ type Input struct {
|
||||
// Report is the data of the analytics page for one run.
|
||||
type Report struct {
|
||||
Run RunInfo `json:"run"`
|
||||
Scope *Scope `json:"scope,omitempty"` // set when the report is narrowed to a subnet
|
||||
Summary Summary `json:"summary"`
|
||||
Reasons []Reason `json:"reasons"`
|
||||
Quality Quality `json:"quality"`
|
||||
@@ -50,6 +54,14 @@ type RunInfo struct {
|
||||
Rechecked int `json:"rechecked"`
|
||||
}
|
||||
|
||||
// Scope says which part of the run a narrowed report covers: Addresses of
|
||||
// Summary are those inside Subnet, RunAddresses all of the run (cancelled
|
||||
// ones excluded in both).
|
||||
type Scope struct {
|
||||
Subnet string `json:"subnet"`
|
||||
RunAddresses int `json:"run_addresses"`
|
||||
}
|
||||
|
||||
type Summary struct {
|
||||
Addresses int `json:"addresses"`
|
||||
Pass int `json:"pass"`
|
||||
@@ -176,7 +188,17 @@ func Compute(in Input) (*Analysis, error) {
|
||||
byReg := make(map[int64]*addr, len(in.Results))
|
||||
var addrs []*addr
|
||||
subnetOf := newSubnetMatcher(in.Subnets)
|
||||
runAddrs := 0
|
||||
for _, r := range in.Results {
|
||||
if r.Verdict != db.ResultCancelled {
|
||||
runAddrs++
|
||||
}
|
||||
if in.Subnet.IsValid() {
|
||||
// checks of an address outside the subnet find no entry in byReg below
|
||||
if ip, err := netip.ParseAddr(r.IPAddress); err != nil || !in.Subnet.Contains(ip) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
a := &addr{res: r, subnet: subnetOf(r.IPAddress), failedTargets: map[string]bool{}}
|
||||
a.ssh.errs = map[string]bool{}
|
||||
byReg[r.RegistryID] = a
|
||||
@@ -279,6 +301,9 @@ func Compute(in Input) (*Analysis, error) {
|
||||
rep := Report{Matrix: map[string][]MatrixRow{}}
|
||||
rep.Run = RunInfo{ID: in.Run.ID, Kind: in.Run.Kind, State: in.Run.State, StartedAt: in.Run.StartedAt,
|
||||
FinalizedAt: in.Run.FinalizedAt, Rechecked: in.Rechecked}
|
||||
if in.Subnet.IsValid() {
|
||||
rep.Scope = &Scope{Subnet: in.Subnet.Masked().String(), RunAddresses: runAddrs}
|
||||
}
|
||||
if in.Run.FinalizedAt != nil {
|
||||
rep.Run.DurationSec = int(in.Run.FinalizedAt.Sub(in.Run.StartedAt).Seconds())
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -30,12 +31,18 @@ func (f *fixture) check(reg int64, source, typ, target string, ok bool, validato
|
||||
}
|
||||
|
||||
func (f *fixture) compute(t *testing.T, subnets []db.Subnet) *Analysis {
|
||||
t.Helper()
|
||||
return f.computeIn(t, subnets, netip.Prefix{})
|
||||
}
|
||||
|
||||
func (f *fixture) computeIn(t *testing.T, subnets []db.Subnet, subnet netip.Prefix) *Analysis {
|
||||
t.Helper()
|
||||
end := t0.Add(10 * time.Minute)
|
||||
an, err := Compute(Input{
|
||||
Run: db.CheckRun{ID: 7, Kind: db.RunManual, State: db.RunFinalized, StartedAt: t0, FinalizedAt: &end},
|
||||
Results: f.results,
|
||||
Subnets: subnets,
|
||||
Subnet: subnet,
|
||||
SiteNames: map[int]string{1: "rxmsk", 2: "rxyc"},
|
||||
Each: func(fn func(db.RunCheck)) error {
|
||||
for _, c := range f.checks {
|
||||
@@ -153,6 +160,57 @@ func TestComputeCountsFactsPerAddress(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeNarrowedToSubnet: with Input.Subnet every block and every list is
|
||||
// built from the addresses inside it (nested subnets included) and nothing else.
|
||||
func TestComputeNarrowedToSubnet(t *testing.T) {
|
||||
f := &fixture{}
|
||||
f.addr(1, "10.0.0.1", db.ResultPass, 3)
|
||||
f.addr(2, "10.0.0.2", db.ResultPartial, 3)
|
||||
f.addr(3, "10.0.1.1", db.ResultPartial, 3)
|
||||
f.addr(4, "10.0.1.200", db.ResultCancelled, 3)
|
||||
for reg := int64(1); reg <= 4; reg++ {
|
||||
f.check(reg, eg, "https", "https://a.test", reg != 2, "vkiplab-v1", "", false)
|
||||
f.check(reg, s1, "icmp", "ip", true, "vkiplab-v1", "", false)
|
||||
f.check(reg, s1, "ssh", "ip", reg != 3, "vkiplab-v1", "dial tcp: i/o timeout", false)
|
||||
}
|
||||
subnets := []db.Subnet{{CIDR: "10.0.0.0/24"}, {CIDR: "10.0.1.0/24"}}
|
||||
|
||||
all := f.compute(t, subnets).Report
|
||||
if all.Scope != nil || all.Summary.Addresses != 3 || len(all.Subnets) != 2 {
|
||||
t.Fatalf("without a subnet the whole run is reported: %+v %+v", all.Scope, all.Summary)
|
||||
}
|
||||
|
||||
// 10.0.0.0/23 holds both configured subnets: the same numbers as the whole run.
|
||||
if wide := f.computeIn(t, subnets, netip.MustParsePrefix("10.0.0.0/23")).Report; wide.Summary != all.Summary || len(wide.Subnets) != 2 {
|
||||
t.Errorf("a wider subnet changes nothing: %+v", wide.Summary)
|
||||
}
|
||||
|
||||
an := f.computeIn(t, subnets, netip.MustParsePrefix("10.0.1.0/24"))
|
||||
r := an.Report
|
||||
if r.Scope == nil || r.Scope.Subnet != "10.0.1.0/24" || r.Scope.RunAddresses != 3 {
|
||||
t.Errorf("scope = %+v", r.Scope)
|
||||
}
|
||||
if r.Summary.Addresses != 1 || r.Summary.Partial != 1 || r.Summary.Pass != 0 || r.Summary.Cancelled != 1 || r.Summary.IngressSSHAny != 1 || r.Summary.EgressHTTPSAny != 0 {
|
||||
t.Errorf("summary = %+v", r.Summary)
|
||||
}
|
||||
if len(r.Subnets) != 1 || r.Subnets[0].CIDR != "10.0.1.0/24" || r.Subnets[0].Addresses != 1 {
|
||||
t.Errorf("subnets = %+v", r.Subnets)
|
||||
}
|
||||
if !reflect.DeepEqual(r.Targets.Failed["https"], []int{0}) || len(r.Sites.Rows) != 1 || r.Sites.Rows[0].Stats[1] != (SiteStat{Total: 1, OK: 0}) {
|
||||
t.Errorf("targets %+v, sites %+v", r.Targets, r.Sites)
|
||||
}
|
||||
// the lists are cut from the same subset, so they agree with the tiles
|
||||
for kind, want := range map[string]int{ListVerdictPartial: r.Summary.Partial, ListVerdictPass: 0, ListIngressSSHAny: r.Summary.IngressSSHAny, ListEgressHTTPSAny: 0} {
|
||||
l, err := an.List(kind, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(l.Rows) != want {
|
||||
t.Errorf("%s: %d rows, want %d", kind, len(l.Rows), want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubnetMatching(t *testing.T) {
|
||||
in := []db.Subnet{{CIDR: "10.0.0.0/8"}, {CIDR: "10.1.0.0/16"}}
|
||||
m := newSubnetMatcher(in)
|
||||
|
||||
@@ -2,12 +2,14 @@ package analytics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/netip"
|
||||
|
||||
"cloudipvalidator/internal/db"
|
||||
)
|
||||
|
||||
// Load reads a finished run from the database and computes its analysis.
|
||||
func Load(ctx context.Context, d *db.DB, runID int64) (*Analysis, error) {
|
||||
// Load reads a finished run from the database and computes its analysis,
|
||||
// narrowed to subnet unless that is the zero prefix.
|
||||
func Load(ctx context.Context, d *db.DB, runID int64, subnet netip.Prefix) (*Analysis, error) {
|
||||
run, err := d.GetRun(ctx, runID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -32,8 +34,19 @@ func Load(ctx context.Context, d *db.DB, runID int64) (*Analysis, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// With a subnet only the checks of its addresses are read; Compute narrows
|
||||
// the results by the same rule.
|
||||
var ids []int64
|
||||
if subnet.IsValid() {
|
||||
ids = []int64{}
|
||||
for _, r := range results {
|
||||
if ip, err := netip.ParseAddr(r.IPAddress); err == nil && subnet.Contains(ip) {
|
||||
ids = append(ids, r.RegistryID)
|
||||
}
|
||||
}
|
||||
}
|
||||
return Compute(Input{
|
||||
Run: *run, Results: results, Subnets: subnets, SiteNames: names, Rechecked: rechecked,
|
||||
Each: func(fn func(db.RunCheck)) error { return d.EachRunCheck(ctx, runID, fn) },
|
||||
Run: *run, Results: results, Subnets: subnets, SiteNames: names, Rechecked: rechecked, Subnet: subnet,
|
||||
Each: func(fn func(db.RunCheck)) error { return d.EachRunCheck(ctx, runID, ids, fn) },
|
||||
})
|
||||
}
|
||||
@@ -244,8 +244,10 @@ func (c *client) ScanStatus(ctx context.Context) (scanStatusDTO, error) {
|
||||
type registryQuery struct {
|
||||
Q string
|
||||
LastResult string
|
||||
Run int64 // only addresses with a result in this run
|
||||
Run int64 // only addresses with a result in this run; the results shown are the run's
|
||||
Subnet string // only addresses inside this CIDR
|
||||
Direction string // egress|ingress — only checks of this direction
|
||||
Protocol string // icmp|tcp|ssh|https|tls — only checks of this protocol
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
@@ -255,11 +257,20 @@ type registryQuery struct {
|
||||
// history — survives an address being deleted from the queue and later
|
||||
// re-added) plus the total number of rows matching the filter.
|
||||
func (c *client) ListRegistryPage(ctx context.Context, q registryQuery) (registryPage, error) {
|
||||
v := url.Values{}
|
||||
v := q.filter()
|
||||
v.Set("limit", strconv.Itoa(clampLimit(q.Limit)))
|
||||
if q.Offset > 0 {
|
||||
v.Set("offset", strconv.Itoa(q.Offset))
|
||||
}
|
||||
var out registryPage
|
||||
err := c.do(ctx, http.MethodGet, "/api/v1/admin/registry?"+v.Encode(), nil, &out)
|
||||
return out, err
|
||||
}
|
||||
|
||||
// filter is the filter part of the query (everything but the page) as the
|
||||
// parameters GET /admin/registry and its breakdown endpoints take.
|
||||
func (q registryQuery) filter() url.Values {
|
||||
v := url.Values{}
|
||||
if q.Q != "" {
|
||||
v.Set("q", q.Q)
|
||||
}
|
||||
@@ -272,11 +283,52 @@ func (c *client) ListRegistryPage(ctx context.Context, q registryQuery) (registr
|
||||
if q.Subnet != "" {
|
||||
v.Set("subnet", q.Subnet)
|
||||
}
|
||||
var out registryPage
|
||||
err := c.do(ctx, http.MethodGet, "/api/v1/admin/registry?"+v.Encode(), nil, &out)
|
||||
if q.Direction != "" {
|
||||
v.Set("direction", q.Direction)
|
||||
}
|
||||
if q.Protocol != "" {
|
||||
v.Set("protocol", q.Protocol)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// registryBreakdownPath is the path of GET /admin/registry/breakdown, or of its
|
||||
// list of one row (key) when list is set, as a CSV file when csv is.
|
||||
func registryBreakdownPath(q registryQuery, key string, list, csv bool) string {
|
||||
v := q.filter()
|
||||
p := "/api/v1/admin/registry/breakdown"
|
||||
if list {
|
||||
p += "/list"
|
||||
v.Set("key", key)
|
||||
if csv {
|
||||
v.Set("format", "csv")
|
||||
}
|
||||
}
|
||||
return p + "?" + v.Encode()
|
||||
}
|
||||
|
||||
// GetRegistryBreakdown returns the checks of the query's direction and protocol
|
||||
// per target or site, over the addresses the query selects.
|
||||
func (c *client) GetRegistryBreakdown(ctx context.Context, q registryQuery) (registryBreakdown, error) {
|
||||
var out registryBreakdown
|
||||
err := c.do(ctx, http.MethodGet, registryBreakdownPath(q, "", false, false), nil, &out)
|
||||
return out, err
|
||||
}
|
||||
|
||||
// GetRegistryBreakdownList returns the table (JSON) of the checks behind one
|
||||
// row of the breakdown.
|
||||
func (c *client) GetRegistryBreakdownList(ctx context.Context, q registryQuery, key string) (json.RawMessage, error) {
|
||||
var out json.RawMessage
|
||||
err := c.do(ctx, http.MethodGet, registryBreakdownPath(q, key, true, false), nil, &out)
|
||||
return out, err
|
||||
}
|
||||
|
||||
// GetRegistryBreakdownCSV returns the CSV file of that table, with the file
|
||||
// name control-api proposed.
|
||||
func (c *client) GetRegistryBreakdownCSV(ctx context.Context, q registryQuery, key string) ([]byte, string, error) {
|
||||
return c.getCSV(ctx, registryBreakdownPath(q, key, true, true))
|
||||
}
|
||||
|
||||
// GetRegistryHistory returns one address's registry record plus its full
|
||||
// retained check history across every cycle still kept.
|
||||
func (c *client) GetRegistryHistory(ctx context.Context, ip string) (registryHistoryResponse, error) {
|
||||
@@ -423,19 +475,27 @@ func (c *client) ListAnalyticsRuns(ctx context.Context) ([]analyticsRun, error)
|
||||
return out, err
|
||||
}
|
||||
|
||||
// GetAnalyticsReport returns the analytics of one finished run as the raw
|
||||
// JSON control-api computed; the page's script reads it as it is.
|
||||
func (c *client) GetAnalyticsReport(ctx context.Context, runID int64) (json.RawMessage, error) {
|
||||
// GetAnalyticsReport returns the analytics of one finished run (of the
|
||||
// addresses inside subnet, unless it is empty) as the raw JSON control-api
|
||||
// computed; the page's script reads it as it is.
|
||||
func (c *client) GetAnalyticsReport(ctx context.Context, runID int64, subnet string) (json.RawMessage, error) {
|
||||
var out json.RawMessage
|
||||
err := c.do(ctx, http.MethodGet, "/api/v1/admin/analytics/runs/"+strconv.FormatInt(runID, 10), nil, &out)
|
||||
p := "/api/v1/admin/analytics/runs/" + strconv.FormatInt(runID, 10)
|
||||
if subnet != "" {
|
||||
p += "?" + url.Values{"subnet": {subnet}}.Encode()
|
||||
}
|
||||
err := c.do(ctx, http.MethodGet, p, nil, &out)
|
||||
return out, err
|
||||
}
|
||||
|
||||
func analyticsListPath(runID int64, kind, class string, csv bool) string {
|
||||
func analyticsListPath(runID int64, kind, class, subnet string, csv bool) string {
|
||||
v := url.Values{}
|
||||
if class != "" {
|
||||
v.Set("class", class)
|
||||
}
|
||||
if subnet != "" {
|
||||
v.Set("subnet", subnet)
|
||||
}
|
||||
if csv {
|
||||
v.Set("format", "csv")
|
||||
}
|
||||
@@ -446,17 +506,18 @@ func analyticsListPath(runID int64, kind, class string, csv bool) string {
|
||||
return p
|
||||
}
|
||||
|
||||
// GetAnalyticsList returns one address table (JSON) of a run.
|
||||
func (c *client) GetAnalyticsList(ctx context.Context, runID int64, kind, class string) (json.RawMessage, error) {
|
||||
// GetAnalyticsList returns one address table (JSON) of a run, narrowed to
|
||||
// subnet unless it is empty.
|
||||
func (c *client) GetAnalyticsList(ctx context.Context, runID int64, kind, class, subnet string) (json.RawMessage, error) {
|
||||
var out json.RawMessage
|
||||
err := c.do(ctx, http.MethodGet, analyticsListPath(runID, kind, class, false), nil, &out)
|
||||
err := c.do(ctx, http.MethodGet, analyticsListPath(runID, kind, class, subnet, false), nil, &out)
|
||||
return out, err
|
||||
}
|
||||
|
||||
// GetAnalyticsListCSV returns the CSV file of one address table, with the
|
||||
// file name control-api proposed.
|
||||
func (c *client) GetAnalyticsListCSV(ctx context.Context, runID int64, kind, class string) ([]byte, string, error) {
|
||||
return c.getCSV(ctx, analyticsListPath(runID, kind, class, true))
|
||||
func (c *client) GetAnalyticsListCSV(ctx context.Context, runID int64, kind, class, subnet string) ([]byte, string, error) {
|
||||
return c.getCSV(ctx, analyticsListPath(runID, kind, class, subnet, true))
|
||||
}
|
||||
|
||||
func (c *client) getCSV(ctx context.Context, path string) ([]byte, string, error) {
|
||||
|
||||
@@ -60,6 +60,13 @@ type fakeControlAPI struct {
|
||||
scanFreeAddresses []string
|
||||
registry map[string]registryItem
|
||||
registryChecks map[string][]check
|
||||
// The chart of /registry: the breakdown served (breakdownStatus != 0 makes
|
||||
// it fail with that status), the table behind a row and the raw queries
|
||||
// of both endpoints.
|
||||
breakdown registryBreakdown
|
||||
breakdownStatus int
|
||||
breakdownList string
|
||||
breakdownReqs []string
|
||||
|
||||
// Scan job state machine (see the scan handlers): POST starts a job that
|
||||
// stays "running" for scanRunPolls GET polls (0 = finishes at once), then
|
||||
@@ -483,6 +490,29 @@ func (f *fakeControlAPI) handler() http.Handler {
|
||||
}
|
||||
writeJSON(w, http.StatusOK, registryPage{Items: page, Total: len(matched), Limit: limit, Offset: offset})
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/admin/registry/breakdown", func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.breakdownReqs = append(f.breakdownReqs, r.URL.RequestURI())
|
||||
if f.breakdownStatus != 0 {
|
||||
writeAPIErr(w, f.breakdownStatus, "breakdown unavailable")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, f.breakdown)
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/admin/registry/breakdown/list", func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.breakdownReqs = append(f.breakdownReqs, r.URL.RequestURI())
|
||||
if r.URL.Query().Get("format") == "csv" {
|
||||
w.Header().Set("Content-Type", "text/csv; charset=utf-8")
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="registry_ingress_tls_rxmsk.csv"`)
|
||||
_, _ = w.Write([]byte("Адрес\r\n1.2.3.4\r\n"))
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(f.breakdownList))
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/admin/registry/{ip}", func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
@@ -142,6 +142,25 @@ type registryPage struct {
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
Run int64 `json:"run"` // the run the results are read from, 0 = newest cycles
|
||||
}
|
||||
|
||||
// registryBreakdown is GET /admin/registry/breakdown: the checks of one
|
||||
// direction and protocol per target (Group "target") or site (Group "site").
|
||||
type registryBreakdown struct {
|
||||
Group string `json:"group"`
|
||||
Direction string `json:"direction"`
|
||||
Protocol string `json:"protocol"`
|
||||
Run int64 `json:"run"`
|
||||
Addresses int `json:"addresses"`
|
||||
Rows []breakdownRow `json:"rows"`
|
||||
}
|
||||
|
||||
type breakdownRow struct {
|
||||
Key string `json:"key"`
|
||||
Label string `json:"label"`
|
||||
Total int `json:"total"`
|
||||
OK int `json:"ok"`
|
||||
}
|
||||
|
||||
// scanStatusDTO is the state of control-api's background floating-IP scan job
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -24,6 +25,17 @@ type analyticsPageData struct {
|
||||
RunID int64
|
||||
PrevURL string // older run, "" when there is none
|
||||
NextURL string // newer run
|
||||
// CompareURL is the comparison page with this run as the target.
|
||||
CompareURL string
|
||||
// The filter of the page: a subnet recomputes every block, a direction
|
||||
// and a protocol focus the page on them (and give the chart).
|
||||
Subnet string
|
||||
Direction string
|
||||
Protocol string
|
||||
Protocols []string
|
||||
SubnetOptions []subnetOption
|
||||
Filtered bool // a subnet, direction or protocol is chosen
|
||||
Breakdown *breakdownView // chart of the direction and protocol; nil without both
|
||||
// DataJSON is the page's data for analytics.js (run meta, labels, report),
|
||||
// HTML-safe JSON.
|
||||
DataJSON template.JS
|
||||
@@ -40,12 +52,25 @@ type analyticsMeta struct {
|
||||
Rechecked int `json:"rechecked"`
|
||||
ListURL string `json:"list_url"`
|
||||
CSVURL string `json:"csv_url"`
|
||||
Registry string `json:"registry_url"`
|
||||
Registry string `json:"registry_url"` // registry of the run with the page's direction and protocol; a link adds the subnet
|
||||
Subnet string `json:"subnet"`
|
||||
Direction string `json:"direction"`
|
||||
Protocol string `json:"protocol"`
|
||||
}
|
||||
|
||||
// analyticsURL is the analytics page of a run with the slice filter f (subnet,
|
||||
// direction, protocol) of the page that links to it.
|
||||
func analyticsURL(run int64, f registryQuery) string {
|
||||
v := f.filter()
|
||||
v.Set("run", strconv.FormatInt(run, 10))
|
||||
return "/analytics?" + v.Encode()
|
||||
}
|
||||
|
||||
// handleAnalyticsPage renders the analytics of one finished run: ?run=ID, by
|
||||
// default the newest finished run. The run selector lists every run, the open
|
||||
// one disabled; nothing of any other run is on the page.
|
||||
// one disabled; nothing of any other run is on the page. ?subnet= narrows every
|
||||
// block to the addresses of that subnet; ?direction=&protocol= focus the page
|
||||
// (analytics.js) and, both given, add the chart of the registry.
|
||||
func (s *Server) handleAnalyticsPage(w http.ResponseWriter, r *http.Request) {
|
||||
data := analyticsPageData{}
|
||||
data.ActiveNav = "analytics"
|
||||
@@ -56,7 +81,16 @@ func (s *Server) handleAnalyticsPage(w http.ResponseWriter, r *http.Request) {
|
||||
s.renderPage(w, r, "analytics_page", data)
|
||||
return
|
||||
}
|
||||
want, _ := strconv.ParseInt(r.URL.Query().Get("run"), 10, 64)
|
||||
f := parseSliceFilter(r.URL.Query())
|
||||
want := f.Run
|
||||
f.Run = 0 // the links below name the run themselves
|
||||
// the subnet choices come from the configured list; without it the filter
|
||||
// still works, just without the choices
|
||||
subnets, subnetsErr := s.CA.GetSubnets(r.Context())
|
||||
data.Subnet, data.Direction, data.Protocol = f.Subnet, f.Direction, f.Protocol
|
||||
data.Protocols = registryProtocols
|
||||
data.SubnetOptions = registrySubnetOptions(subnets.Subnets, f.Subnet)
|
||||
data.Filtered = f.Subnet != "" || f.Direction != "" || f.Protocol != ""
|
||||
var chosen *analyticsRun
|
||||
for i := range runs { // newest first
|
||||
if runs[i].State != "finalized" || runs[i].Addresses == 0 {
|
||||
@@ -94,15 +128,16 @@ func (s *Server) handleAnalyticsPage(w http.ResponseWriter, r *http.Request) {
|
||||
for i, x := range finished {
|
||||
if x.ID == chosen.ID {
|
||||
if i+1 < len(finished) {
|
||||
data.PrevURL = "/analytics?run=" + strconv.FormatInt(finished[i+1].ID, 10)
|
||||
data.PrevURL = analyticsURL(finished[i+1].ID, f)
|
||||
}
|
||||
if i > 0 {
|
||||
data.NextURL = "/analytics?run=" + strconv.FormatInt(finished[i-1].ID, 10)
|
||||
data.NextURL = analyticsURL(finished[i-1].ID, f)
|
||||
}
|
||||
}
|
||||
}
|
||||
data.CompareURL = compareURL(chosen.ID, f)
|
||||
|
||||
report, err := s.CA.GetAnalyticsReport(r.Context(), chosen.ID)
|
||||
report, err := s.CA.GetAnalyticsReport(r.Context(), chosen.ID, f.Subnet)
|
||||
if err != nil {
|
||||
data.Banner = bannerFor(err)
|
||||
s.renderPage(w, r, "analytics_page", data)
|
||||
@@ -120,7 +155,12 @@ func (s *Server) handleAnalyticsPage(w http.ResponseWriter, r *http.Request) {
|
||||
Start: fmtShort(chosen.StartedAt), Duration: "—", Rechecked: info.Run.Rechecked,
|
||||
ListURL: "/analytics/lists/",
|
||||
CSVURL: "/analytics/csv/",
|
||||
Registry: "/registry?run=" + id,
|
||||
Registry: "/registry?run=" + id, Subnet: f.Subnet, Direction: f.Direction, Protocol: f.Protocol,
|
||||
}
|
||||
for _, p := range [][2]string{{"direction", f.Direction}, {"protocol", f.Protocol}} {
|
||||
if p[1] != "" {
|
||||
meta.Registry += "&" + p[0] + "=" + url.QueryEscape(p[1])
|
||||
}
|
||||
}
|
||||
if chosen.FinalizedAt != nil {
|
||||
meta.End = fmtShort(*chosen.FinalizedAt)
|
||||
@@ -137,14 +177,33 @@ func (s *Server) handleAnalyticsPage(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
data.HasRun = true
|
||||
data.DataJSON = template.JS(payload)
|
||||
if f.Direction != "" && f.Protocol != "" {
|
||||
f.Run = chosen.ID
|
||||
data.Breakdown = s.registryBreakdown(r, f, f.filter())
|
||||
}
|
||||
data.Banner = bannerFor(subnetsErr)
|
||||
s.renderPage(w, r, "analytics_page", data)
|
||||
}
|
||||
|
||||
// compareURL is the comparison page with a run as the target; it carries the
|
||||
// slice filter along, as the other links of the page do.
|
||||
func compareURL(target int64, f registryQuery) string {
|
||||
v := f.filter()
|
||||
v.Set("target", strconv.FormatInt(target, 10))
|
||||
return "/analytics/compare?" + v.Encode()
|
||||
}
|
||||
|
||||
func parseRunParam(r *http.Request) (int64, bool) {
|
||||
id, err := strconv.ParseInt(r.URL.Query().Get("run"), 10, 64)
|
||||
return id, err == nil && id > 0
|
||||
}
|
||||
|
||||
// analyticsSubnetParam is the ?subnet= of a list request as it came: control-api
|
||||
// validates it, so a malformed one is an error and never a silently wider list.
|
||||
func analyticsSubnetParam(r *http.Request) string {
|
||||
return strings.TrimSpace(r.URL.Query().Get("subnet"))
|
||||
}
|
||||
|
||||
// handleAnalyticsList proxies one address table of a run as JSON.
|
||||
func (s *Server) handleAnalyticsList(w http.ResponseWriter, r *http.Request) {
|
||||
id, ok := parseRunParam(r)
|
||||
@@ -152,7 +211,7 @@ func (s *Server) handleAnalyticsList(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "run is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
out, err := s.CA.GetAnalyticsList(r.Context(), id, r.PathValue("kind"), r.URL.Query().Get("class"))
|
||||
out, err := s.CA.GetAnalyticsList(r.Context(), id, r.PathValue("kind"), r.URL.Query().Get("class"), analyticsSubnetParam(r))
|
||||
if err != nil {
|
||||
writeProxyError(w, err)
|
||||
return
|
||||
@@ -169,7 +228,7 @@ func (s *Server) handleAnalyticsCSV(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "run is required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
body, disposition, err := s.CA.GetAnalyticsListCSV(r.Context(), id, r.PathValue("kind"), r.URL.Query().Get("class"))
|
||||
body, disposition, err := s.CA.GetAnalyticsListCSV(r.Context(), id, r.PathValue("kind"), r.URL.Query().Get("class"), analyticsSubnetParam(r))
|
||||
if err != nil {
|
||||
writeProxyError(w, err)
|
||||
return
|
||||
|
||||
@@ -80,6 +80,72 @@ func TestAnalyticsPageShowsOneRun(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The analytics filters: the subnet goes to control-api with the report and the
|
||||
// lists, direction and protocol focus the page and, both given, add the chart of
|
||||
// the registry for the run and subnet; every link of the page keeps the filter.
|
||||
func TestAnalyticsPageFilters(t *testing.T) {
|
||||
fake, ts := analyticsFake(t)
|
||||
fake.subnets = subnetList{Subnets: []subnetEntry{{CIDR: "9.9.9.0/24", Label: "Офис"}, {CIDR: "10.0.0.0/8"}}}
|
||||
fake.breakdown = registryBreakdown{Group: "target", Direction: "egress", Protocol: "https", Addresses: 5, Rows: []breakdownRow{{Key: "a.test", Label: "a.test", Total: 5, OK: 4}}}
|
||||
lastReq := func(reqs []string) string { fake.mu.Lock(); defer fake.mu.Unlock(); return reqs[len(reqs)-1] }
|
||||
|
||||
page := get(t, ts, "/analytics?run=1&subnet=9.9.9.0/24&direction=egress&protocol=https")
|
||||
for _, want := range []string{
|
||||
`<option value="9.9.9.0/24" selected>9.9.9.0/24 — Офис</option>`, `<option value="egress" selected>`, `<option value="https" selected>`,
|
||||
`"subnet":"9.9.9.0/24"`, `"direction":"egress"`, `"protocol":"https"`, "сбросить фильтры",
|
||||
`id="registry-breakdown"`, "Успешные проверки по целям · Egress https", `data-slice="запуск 1 · подсеть 9.9.9.0/24"`,
|
||||
`data-qs="direction=egress&protocol=https&run=1&subnet=9.9.9.0%2F24"`,
|
||||
// the newer run is one step forward, the comparison is opened for this run; both keep the filter
|
||||
`href="/analytics?direction=egress&protocol=https&run=2&subnet=9.9.9.0%2F24"`,
|
||||
`href="/analytics/compare?direction=egress&protocol=https&subnet=9.9.9.0%2F24&target=1"`,
|
||||
} {
|
||||
if !strings.Contains(page, want) {
|
||||
t.Fatalf("expected %q in the page, got:\n%s", want, page)
|
||||
}
|
||||
}
|
||||
if got := lastReq(fake.analyticsReqs); !strings.HasSuffix(got, "/runs/1?subnet=9.9.9.0%2F24") {
|
||||
t.Errorf("the report must be requested for the subnet, got %q", got)
|
||||
}
|
||||
req := lastReq(fake.breakdownReqs)
|
||||
for _, want := range []string{"run=1", "subnet=9.9.9.0%2F24", "direction=egress", "protocol=https"} {
|
||||
if !strings.Contains(req, want) {
|
||||
t.Errorf("chart request %q lacks %s", req, want)
|
||||
}
|
||||
}
|
||||
|
||||
// No chart without both direction and protocol (and no request for it); a
|
||||
// malformed subnet is dropped and the whole run is shown.
|
||||
fake.mu.Lock()
|
||||
fake.breakdownReqs = nil
|
||||
fake.mu.Unlock()
|
||||
for _, q := range []string{"run=1", "run=1&direction=ingress", "run=1&protocol=tls&subnet=garbage"} {
|
||||
page = get(t, ts, "/analytics?"+q)
|
||||
if strings.Contains(page, `id="registry-breakdown"`) || strings.Contains(page, "Выберите") {
|
||||
t.Errorf("%s: no chart and no hint expected:\n%s", q, page)
|
||||
}
|
||||
}
|
||||
if len(fake.breakdownReqs) != 0 {
|
||||
t.Errorf("the chart was requested without direction and protocol: %v", fake.breakdownReqs)
|
||||
}
|
||||
if got := lastReq(fake.analyticsReqs); strings.Contains(got, "subnet") {
|
||||
t.Errorf("a malformed subnet must be dropped, got %q", got)
|
||||
}
|
||||
|
||||
// The lists keep the subnet, a malformed one is passed on for control-api to refuse.
|
||||
for _, path := range []string{"/analytics/lists/egress_https_any?run=1&subnet=9.9.9.0/24", "/analytics/csv/egress_https_any?run=1&subnet=9.9.9.0/24"} {
|
||||
if page = get(t, ts, path); page == "" {
|
||||
t.Fatalf("%s: empty answer", path)
|
||||
}
|
||||
if got := lastReq(fake.analyticsReqs); !strings.Contains(got, "subnet=9.9.9.0%2F24") {
|
||||
t.Errorf("%s: control-api request %q lacks the subnet", path, got)
|
||||
}
|
||||
}
|
||||
get(t, ts, "/analytics/lists/egress_https_any?run=1&subnet=bad")
|
||||
if got := lastReq(fake.analyticsReqs); !strings.Contains(got, "subnet=bad") {
|
||||
t.Errorf("a malformed subnet must reach control-api, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyticsPageWithoutRunsAndWithUnknownRun(t *testing.T) {
|
||||
_, caURL := newFakeControlAPI(t)
|
||||
ts := newTestServer(t, caURL)
|
||||
@@ -389,7 +455,7 @@ func TestSettingsSubnetsForm(t *testing.T) {
|
||||
}
|
||||
|
||||
// The drill-down from the analytics page: run and subnet go to control-api,
|
||||
// come back as hidden fields and a visible chip; a malformed subnet is dropped.
|
||||
// come back in the filter fields and the reset link; a malformed subnet is dropped.
|
||||
func TestRegistryDrillDownFromAnalytics(t *testing.T) {
|
||||
fake, caURL := newFakeControlAPI(t)
|
||||
ts := newTestServer(t, caURL)
|
||||
@@ -402,7 +468,8 @@ func TestRegistryDrillDownFromAnalytics(t *testing.T) {
|
||||
if !strings.Contains(last, "run=2") || !strings.Contains(last, "subnet=10.0.0.0%2F24") {
|
||||
t.Fatalf("control-api request %q lacks the run or subnet", last)
|
||||
}
|
||||
for _, want := range []string{`name="run" value="2"`, `name="subnet" value="10.0.0.0/24"`, "запуск 2", "подсеть 10.0.0.0/24", "сбросить фильтр"} {
|
||||
for _, want := range []string{`<option value="2" selected>`, `<option value="10.0.0.0/24" selected>10.0.0.0/24 (нет в списке)</option>`,
|
||||
`href="/analytics?run=2&subnet=10.0.0.0%2F24"`, "сбросить фильтры"} {
|
||||
if !strings.Contains(page, want) {
|
||||
t.Fatalf("expected %q in:\n%s", want, page)
|
||||
}
|
||||
@@ -410,7 +477,7 @@ func TestRegistryDrillDownFromAnalytics(t *testing.T) {
|
||||
|
||||
page = get(t, ts, "/registry?subnet=garbage")
|
||||
last = fake.registryQueries[len(fake.registryQueries)-1]
|
||||
if strings.Contains(last, "subnet=") || strings.Contains(page, "сбросить фильтр") {
|
||||
if strings.Contains(last, "subnet=") || strings.Contains(page, `garbage`) {
|
||||
t.Fatalf("a malformed subnet must be ignored: %q", last)
|
||||
}
|
||||
}
|
||||
@@ -426,3 +493,122 @@ func TestAnalyticsCompareListPath(t *testing.T) {
|
||||
t.Errorf("path = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The "Подсеть" selector: configured subnets as "CIDR — label", disabled with a
|
||||
// link to /settings when there are none.
|
||||
func TestRegistrySubnetSelect(t *testing.T) {
|
||||
fake, caURL := newFakeControlAPI(t)
|
||||
ts := newTestServer(t, caURL)
|
||||
|
||||
page := get(t, ts, "/registry")
|
||||
for _, want := range []string{`<select id="registry-subnet" name="subnet" disabled`, `<option value="">Все подсети</option>`, `href="/settings">добавить в настройках`} {
|
||||
if !strings.Contains(page, want) {
|
||||
t.Fatalf("no subnets configured: expected %q in:\n%s", want, page)
|
||||
}
|
||||
}
|
||||
|
||||
fake.subnets = subnetList{Subnets: []subnetEntry{{CIDR: "9.9.9.0/24", Label: "Офис"}, {CIDR: "10.0.0.0/8"}}}
|
||||
page = get(t, ts, "/registry?subnet=10.0.0.0/8")
|
||||
for _, want := range []string{`<option value="9.9.9.0/24" >9.9.9.0/24 — Офис</option>`, `<option value="10.0.0.0/8" selected>10.0.0.0/8</option>`} {
|
||||
if !strings.Contains(page, want) {
|
||||
t.Fatalf("expected %q in:\n%s", want, page)
|
||||
}
|
||||
}
|
||||
if strings.Contains(page, "disabled") || strings.Contains(page, "нет в списке") {
|
||||
t.Fatalf("a configured subnet must neither disable the selector nor be added again:\n%s", page)
|
||||
}
|
||||
}
|
||||
|
||||
// The chart above the registry table: only with a direction and a protocol (a
|
||||
// hint for the missing one), rows as control-api sorted them, an error inside
|
||||
// the block that leaves the table in place; the list proxies forward the filter.
|
||||
func TestRegistryBreakdownChartAndProxy(t *testing.T) {
|
||||
fake, caURL := newFakeControlAPI(t)
|
||||
now := time.Now()
|
||||
fake.registry["9.9.9.9"] = registryItem{IPAddress: "9.9.9.9", FirstSeenAt: now, LastSeenAt: now}
|
||||
fake.breakdown = registryBreakdown{Group: "site", Direction: "ingress", Protocol: "tls", Addresses: 1, Rows: []breakdownRow{
|
||||
{Key: "inbound-site-2", Label: "rxspb", Total: 1250, OK: 1234}, {Key: "inbound-site-1", Label: "rxmsk", Total: 617, OK: 617},
|
||||
}}
|
||||
ts := newTestServer(t, caURL)
|
||||
|
||||
page := get(t, ts, "/registry")
|
||||
if strings.Contains(page, `id="registry-breakdown"`) || strings.Contains(page, "Выберите") {
|
||||
t.Fatalf("no chart and no hint without filters:\n%s", page)
|
||||
}
|
||||
page = get(t, ts, "/registry?direction=ingress")
|
||||
if !strings.Contains(page, "Выберите протокол, чтобы увидеть распределение по площадкам") || strings.Contains(page, `id="registry-breakdown"`) {
|
||||
t.Fatalf("a direction alone gives a hint:\n%s", page)
|
||||
}
|
||||
|
||||
page = get(t, ts, "/registry?direction=ingress&protocol=tls&run=3&subnet=9.9.9.0/24")
|
||||
for _, want := range []string{
|
||||
"Успешные проверки по площадкам · Ingress tls", `data-bd-key="inbound-site-2"`, `data-bd-label="rxspb"`,
|
||||
"1\u00a0234 из 1\u00a0250 · 98,7%", "617 из 617 · 100%", `style="width:100.0%"`, `style="width:50.0%"`,
|
||||
`data-slice="запуск 3 · подсеть 9.9.9.0/24"`, `data-qs="direction=ingress&protocol=tls&run=3&subnet=9.9.9.0%2F24"`, "Адресов под фильтром: 1",
|
||||
"9.9.9.9", // the table is still there
|
||||
} {
|
||||
if !strings.Contains(page, want) {
|
||||
t.Fatalf("expected %q in:\n%s", want, page)
|
||||
}
|
||||
}
|
||||
if strings.Index(page, "rxspb") > strings.Index(page, "rxmsk") {
|
||||
t.Fatalf("rows must keep the order control-api gave:\n%s", page)
|
||||
}
|
||||
fake.mu.Lock()
|
||||
req := fake.breakdownReqs[len(fake.breakdownReqs)-1]
|
||||
fake.mu.Unlock()
|
||||
for _, want := range []string{"direction=ingress", "protocol=tls", "run=3", "subnet=9.9.9.0%2F24"} {
|
||||
if !strings.Contains(req, want) {
|
||||
t.Fatalf("breakdown request %q lacks %s", req, want)
|
||||
}
|
||||
}
|
||||
|
||||
fake.breakdown.Rows = nil
|
||||
if page = get(t, ts, "/registry?direction=egress&protocol=tls"); !strings.Contains(page, "Для этого сочетания проверок нет") {
|
||||
t.Fatalf("an empty chart says so:\n%s", page)
|
||||
}
|
||||
fake.breakdownStatus = http.StatusInternalServerError
|
||||
page = get(t, ts, "/registry?direction=egress&protocol=https")
|
||||
if !strings.Contains(page, "Не удалось получить распределение") || !strings.Contains(page, "9.9.9.9") {
|
||||
t.Fatalf("a chart error is shown in its block and the table stays:\n%s", page)
|
||||
}
|
||||
|
||||
// The list proxies: filter and key reach control-api; the filter is checked.
|
||||
fake.breakdownList = `{"columns":["Адрес"],"rows":[["1.2.3.4"]]}`
|
||||
for path, want := range map[string]int{
|
||||
"/registry/breakdown/list?direction=ingress&protocol=tls&key=inbound-site-1&run=3&status=fail": http.StatusOK,
|
||||
"/registry/breakdown/csv?direction=ingress&protocol=tls&key=inbound-site-1": http.StatusOK,
|
||||
"/registry/breakdown/list?direction=ingress&key=inbound-site-1": http.StatusBadRequest, // no protocol
|
||||
"/registry/breakdown/csv?direction=ingress&protocol=tls": http.StatusBadRequest, // no key
|
||||
} {
|
||||
resp, err := http.Get(ts.URL + path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != want {
|
||||
t.Fatalf("%s: %d, want %d", path, resp.StatusCode, want)
|
||||
}
|
||||
if strings.Contains(path, "/csv") && want == http.StatusOK &&
|
||||
(!strings.HasPrefix(resp.Header.Get("Content-Type"), "text/csv") || !strings.Contains(resp.Header.Get("Content-Disposition"), "registry_ingress_tls_rxmsk.csv")) {
|
||||
t.Fatalf("csv: %v %s", resp.Header, body)
|
||||
}
|
||||
if strings.Contains(path, "/list") && want == http.StatusOK && !strings.Contains(string(body), `"1.2.3.4"`) {
|
||||
t.Fatalf("list: %s", body)
|
||||
}
|
||||
}
|
||||
fake.mu.Lock()
|
||||
var list string
|
||||
for _, r := range fake.breakdownReqs {
|
||||
if strings.Contains(r, "/breakdown/list") && !strings.Contains(r, "format=csv") {
|
||||
list = r
|
||||
}
|
||||
}
|
||||
fake.mu.Unlock()
|
||||
for _, want := range []string{"key=inbound-site-1", "run=3", "last_result=fail", "direction=ingress", "protocol=tls"} {
|
||||
if !strings.Contains(list, want) {
|
||||
t.Fatalf("list request %q lacks %s", list, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
@@ -8,13 +9,28 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Values of the registry's direction and protocol filters; same as
|
||||
// db.RegistryFilter's Level and Family.
|
||||
var (
|
||||
registryDirections = []string{"egress", "ingress"}
|
||||
registryProtocols = []string{"icmp", "tcp", "ssh", "https", "tls"}
|
||||
)
|
||||
|
||||
type registryPageData struct {
|
||||
PageData
|
||||
Items []registryItem
|
||||
Query string
|
||||
StatusFilter string
|
||||
Run int64 // drill-down from the analytics page
|
||||
Run int64 // data slice: the address's cycle in this run (also the drill-down from analytics)
|
||||
Subnet string
|
||||
Direction string
|
||||
Protocol string
|
||||
Protocols []string
|
||||
Scoped bool // direction or protocol is set
|
||||
Runs []runOption
|
||||
SubnetOptions []subnetOption
|
||||
Breakdown *breakdownView // nil unless a direction or protocol is chosen
|
||||
AnalyticsURL string // the analytics page of Run with the same subnet, direction and protocol
|
||||
Page, PerPage int
|
||||
Total int
|
||||
Pager pagerData
|
||||
@@ -31,25 +47,18 @@ type registryDetailData struct {
|
||||
// record that survives an address being deleted from /ips and later
|
||||
// re-added. See internal/db/migrations/0007_ip_registry.sql. Optional
|
||||
// ?q=&status= query params narrow the list by address substring and by
|
||||
// LastResult, and ?page=&per_page= select a page — all applied server-side
|
||||
// (control-api's ListRegistryPage), so only the visible rows are transferred.
|
||||
// LastResult, ?run=&subnet=&direction=&protocol= by run, subnet and the
|
||||
// direction/protocol of the checks, and ?page=&per_page= select a page — all
|
||||
// applied server-side (control-api's ListRegistryPage), so only the visible
|
||||
// rows are transferred. The run and subnet choices come from the analytics
|
||||
// run list and the configured subnets; if either list is unavailable the
|
||||
// filters still work, just without those choices.
|
||||
func (s *Server) handleRegistryPage(w http.ResponseWriter, r *http.Request) {
|
||||
q := strings.TrimSpace(r.URL.Query().Get("q"))
|
||||
status := r.URL.Query().Get("status")
|
||||
if !containsStr(ipResults, status) {
|
||||
status = ""
|
||||
}
|
||||
query := parseRegistryQuery(r)
|
||||
q, status, run, subnet, direction, protocol := query.Q, query.LastResult, query.Run, query.Subnet, query.Direction, query.Protocol
|
||||
perPage := parsePerPage(r.URL.Query().Get("per_page"))
|
||||
page := parsePage(r.URL.Query().Get("page"))
|
||||
run, _ := strconv.ParseInt(r.URL.Query().Get("run"), 10, 64)
|
||||
if run < 0 {
|
||||
run = 0
|
||||
}
|
||||
subnet := strings.TrimSpace(r.URL.Query().Get("subnet"))
|
||||
if _, err := netip.ParsePrefix(subnet); err != nil {
|
||||
subnet = ""
|
||||
}
|
||||
query := registryQuery{Q: q, LastResult: status, Run: run, Subnet: subnet, Limit: perPage, Offset: (page - 1) * perPage}
|
||||
query.Limit, query.Offset = perPage, (page-1)*perPage
|
||||
|
||||
res, err := s.CA.ListRegistryPage(r.Context(), query)
|
||||
if err == nil {
|
||||
@@ -72,9 +81,31 @@ func (s *Server) handleRegistryPage(w http.ResponseWriter, r *http.Request) {
|
||||
if subnet != "" {
|
||||
params.Set("subnet", subnet)
|
||||
}
|
||||
if direction != "" {
|
||||
params.Set("direction", direction)
|
||||
}
|
||||
if protocol != "" {
|
||||
params.Set("protocol", protocol)
|
||||
}
|
||||
// A filter/pager request from htmx swaps only #registry-table-wrap
|
||||
// (hx-select), so the run and subnet choices of the form are not needed.
|
||||
// A history-restore fetch needs the full page.
|
||||
var runs []analyticsRun
|
||||
var subnets subnetList
|
||||
var runsErr, subnetsErr error
|
||||
if r.Header.Get("HX-Request") != "true" || r.Header.Get("HX-History-Restore-Request") == "true" {
|
||||
runs, runsErr = s.CA.ListAnalyticsRuns(r.Context())
|
||||
subnets, subnetsErr = s.CA.GetSubnets(r.Context())
|
||||
}
|
||||
data := registryPageData{
|
||||
Run: run,
|
||||
Subnet: subnet,
|
||||
Direction: direction,
|
||||
Protocol: protocol,
|
||||
Protocols: registryProtocols,
|
||||
Scoped: direction != "" || protocol != "",
|
||||
Runs: registryRunOptions(runs, run),
|
||||
SubnetOptions: registrySubnetOptions(subnets.Subnets, subnet),
|
||||
Items: res.Items,
|
||||
Query: q,
|
||||
StatusFilter: status,
|
||||
@@ -85,10 +116,223 @@ func (s *Server) handleRegistryPage(w http.ResponseWriter, r *http.Request) {
|
||||
PerPageOptions: perPageOptions,
|
||||
}
|
||||
data.ActiveNav = "registry"
|
||||
if run > 0 {
|
||||
data.AnalyticsURL = analyticsURL(run, query)
|
||||
}
|
||||
if err == nil {
|
||||
data.Breakdown = s.registryBreakdown(r, query, params)
|
||||
err = errors.Join(runsErr, subnetsErr)
|
||||
}
|
||||
data.Banner = bannerFor(err)
|
||||
s.renderPage(w, r, "registry_page", data)
|
||||
}
|
||||
|
||||
// parseRegistryQuery reads the filter of the registry page and of its chart
|
||||
// requests (?q=&status=&run=&subnet=&direction=&protocol=); a value that is
|
||||
// not valid is dropped.
|
||||
func parseRegistryQuery(r *http.Request) registryQuery {
|
||||
v := r.URL.Query()
|
||||
q := parseSliceFilter(v)
|
||||
q.Q, q.LastResult = strings.TrimSpace(v.Get("q")), v.Get("status")
|
||||
if !containsStr(ipResults, q.LastResult) {
|
||||
q.LastResult = ""
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
// parseSliceFilter reads the part of the filter that the registry and the
|
||||
// analytics page share (?run=&subnet=&direction=&protocol=): the data slice
|
||||
// of a run, a subnet, and the direction and protocol of the checks. A value
|
||||
// that is not valid is dropped.
|
||||
func parseSliceFilter(v url.Values) registryQuery {
|
||||
var q registryQuery
|
||||
if q.Run, _ = strconv.ParseInt(v.Get("run"), 10, 64); q.Run < 0 {
|
||||
q.Run = 0
|
||||
}
|
||||
if q.Subnet = strings.TrimSpace(v.Get("subnet")); q.Subnet != "" {
|
||||
if _, err := netip.ParsePrefix(q.Subnet); err != nil {
|
||||
q.Subnet = ""
|
||||
}
|
||||
}
|
||||
if q.Direction = v.Get("direction"); !containsStr(registryDirections, q.Direction) {
|
||||
q.Direction = ""
|
||||
}
|
||||
if q.Protocol = v.Get("protocol"); !containsStr(registryProtocols, q.Protocol) {
|
||||
q.Protocol = ""
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
// subnetOption is one entry of the "Подсеть" selector.
|
||||
type subnetOption struct {
|
||||
CIDR, Text string
|
||||
Selected bool
|
||||
}
|
||||
|
||||
// registrySubnetOptions lists the configured subnets for the selector as
|
||||
// "CIDR — label". The chosen subnet is always present and selected: one that is
|
||||
// not configured (a link from analytics, or the list is unavailable) is added
|
||||
// as a separate entry.
|
||||
func registrySubnetOptions(subnets []subnetEntry, chosen string) []subnetOption {
|
||||
var out []subnetOption
|
||||
found := false
|
||||
for _, x := range subnets {
|
||||
text := x.CIDR
|
||||
if x.Label != "" {
|
||||
text += " — " + x.Label
|
||||
}
|
||||
out = append(out, subnetOption{CIDR: x.CIDR, Text: text, Selected: x.CIDR == chosen})
|
||||
found = found || x.CIDR == chosen
|
||||
}
|
||||
if chosen != "" && !found {
|
||||
out = append(out, subnetOption{CIDR: chosen, Text: chosen + " (нет в списке)", Selected: true})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// breakdownView is the chart "successful checks per target / site" above the
|
||||
// registry table. With Hint set, it is only a prompt to choose the missing
|
||||
// filter; with Err set, the chart could not be loaded.
|
||||
type breakdownView struct {
|
||||
Title, Scope, Slice, QS string
|
||||
Hint, Err string
|
||||
Addresses int
|
||||
Rows []breakdownRowView
|
||||
}
|
||||
|
||||
type breakdownRowView struct {
|
||||
Key, Label, Width, Text, Tip string
|
||||
}
|
||||
|
||||
// registryBreakdown builds the chart for the page's filter: nothing without a
|
||||
// direction and a protocol, then a hint for the missing one. params is the
|
||||
// filter as the page's links carry it; the chart's dialog requests its lists
|
||||
// with the same query string.
|
||||
func (s *Server) registryBreakdown(r *http.Request, q registryQuery, params url.Values) *breakdownView {
|
||||
switch {
|
||||
case q.Direction == "" && q.Protocol == "":
|
||||
return nil
|
||||
case q.Protocol == "":
|
||||
return &breakdownView{Hint: "Выберите протокол, чтобы увидеть распределение по " + breakdownGroupName(q.Direction) + "."}
|
||||
case q.Direction == "":
|
||||
return &breakdownView{Hint: "Выберите направление, чтобы увидеть распределение по целям или площадкам."}
|
||||
}
|
||||
v := &breakdownView{Scope: strings.ToUpper(q.Direction[:1]) + q.Direction[1:] + " " + q.Protocol, QS: params.Encode(), Slice: "последний цикл адреса"}
|
||||
if q.Run > 0 {
|
||||
v.Slice = "запуск " + strconv.FormatInt(q.Run, 10)
|
||||
}
|
||||
if q.Subnet != "" {
|
||||
v.Slice += " · подсеть " + q.Subnet
|
||||
}
|
||||
v.Title = "Успешные проверки по " + breakdownGroupName(q.Direction)
|
||||
b, err := s.CA.GetRegistryBreakdown(r.Context(), q)
|
||||
if err != nil {
|
||||
v.Err = "Не удалось получить распределение: " + err.Error()
|
||||
return v
|
||||
}
|
||||
v.Addresses = b.Addresses
|
||||
most := 0
|
||||
for _, x := range b.Rows {
|
||||
most = max(most, x.OK)
|
||||
}
|
||||
for _, x := range b.Rows { // control-api sorts them, most successful first
|
||||
width := 0.0
|
||||
if most > 0 {
|
||||
width = float64(x.OK) * 100 / float64(most)
|
||||
}
|
||||
ok, total := groupThousands(x.OK), groupThousands(x.Total)
|
||||
v.Rows = append(v.Rows, breakdownRowView{
|
||||
Key: x.Key, Label: x.Label, Width: strconv.FormatFloat(width, 'f', 1, 64),
|
||||
Text: ok + " из " + total + " · " + pct1(x.OK, x.Total) + "%",
|
||||
Tip: x.Label + ": успешно " + ok + " из " + total + " проверок. Нажмите, чтобы открыть список адресов",
|
||||
})
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// breakdownGroupName is what the chart groups the checks of a direction by.
|
||||
func breakdownGroupName(direction string) string {
|
||||
if direction == "ingress" {
|
||||
return "площадкам"
|
||||
}
|
||||
return "целям"
|
||||
}
|
||||
|
||||
// pct1 is a/b in percent with at most one decimal and a decimal comma: 98,7.
|
||||
func pct1(a, b int) string {
|
||||
if b == 0 {
|
||||
return "0"
|
||||
}
|
||||
s := strconv.FormatFloat(float64(a)*100/float64(b), 'f', 1, 64)
|
||||
return strings.Replace(strings.TrimSuffix(s, ".0"), ".", ",", 1)
|
||||
}
|
||||
|
||||
// breakdownQuery reads the request of the chart's list proxies: the page's
|
||||
// filter, with direction and protocol and the row's key required.
|
||||
func breakdownQuery(w http.ResponseWriter, r *http.Request) (q registryQuery, key string, ok bool) {
|
||||
q, key = parseRegistryQuery(r), r.URL.Query().Get("key")
|
||||
if q.Direction == "" || q.Protocol == "" || key == "" {
|
||||
http.Error(w, "direction, protocol and key are required", http.StatusBadRequest)
|
||||
return q, key, false
|
||||
}
|
||||
return q, key, true
|
||||
}
|
||||
|
||||
// handleRegistryBreakdownList proxies the checks behind one row of the chart as JSON.
|
||||
func (s *Server) handleRegistryBreakdownList(w http.ResponseWriter, r *http.Request) {
|
||||
q, key, ok := breakdownQuery(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
out, err := s.CA.GetRegistryBreakdownList(r.Context(), q, key)
|
||||
if err != nil {
|
||||
writeProxyError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
_, _ = w.Write(out)
|
||||
}
|
||||
|
||||
// handleRegistryBreakdownCSV proxies the same table as a CSV download.
|
||||
func (s *Server) handleRegistryBreakdownCSV(w http.ResponseWriter, r *http.Request) {
|
||||
q, key, ok := breakdownQuery(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
body, disposition, err := s.CA.GetRegistryBreakdownCSV(r.Context(), q, key)
|
||||
if err != nil {
|
||||
writeProxyError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/csv; charset=utf-8")
|
||||
if disposition != "" {
|
||||
w.Header().Set("Content-Disposition", disposition)
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
_, _ = w.Write(body)
|
||||
}
|
||||
|
||||
// registryRunOptions lists the runs for the "Запуск" selector, newest first as
|
||||
// control-api returns them; a finished run without addresses is hidden. The
|
||||
// chosen run is always present and selected, even if the list lacks it (the
|
||||
// history was cleaned up, or the list is unavailable).
|
||||
func registryRunOptions(runs []analyticsRun, chosen int64) []runOption {
|
||||
var out []runOption
|
||||
found := false
|
||||
for _, x := range runs {
|
||||
if x.State == "finalized" && x.Addresses == 0 && x.ID != chosen {
|
||||
continue
|
||||
}
|
||||
out = append(out, runOption{ID: x.ID, Label: runLabel(x), Selected: x.ID == chosen})
|
||||
found = found || x.ID == chosen
|
||||
}
|
||||
if chosen > 0 && !found {
|
||||
out = append(out, runOption{ID: chosen, Label: "Запуск " + strconv.FormatInt(chosen, 10), Selected: true})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// handleRegistryDetail shows one address's full retained check history
|
||||
// across every cycle it has ever run, not just the current attempt — see
|
||||
// ip_detail_content in ip_detail.html for the attempt-scoped equivalent.
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -632,6 +633,53 @@ func TestRegistryPageAndDetail(t *testing.T) {
|
||||
if !strings.Contains(notFound, "alert-warning") {
|
||||
t.Fatalf("expected client error banner for unknown registry address, got:\n%s", notFound)
|
||||
}
|
||||
|
||||
// The four filters: the run and subnet choices come from control-api, the
|
||||
// values reach it and the pager, unknown direction/protocol are dropped, and
|
||||
// with a direction/protocol only the levels that have checks are shown.
|
||||
fake.runs = []analyticsRun{fakeRun(2, "open", 120, 40), fakeRun(1, "finalized", 900, 300)}
|
||||
fake.subnets = subnetList{Subnets: []subnetEntry{{CIDR: "9.9.9.0/24", Label: "Офис"}}}
|
||||
fake.registry["9.9.9.8"] = registryItem{
|
||||
IPAddress: "9.9.9.8", FirstSeenAt: now, LastSeenAt: now, TotalCycles: 1, LastResult: "pass",
|
||||
LastCycleID: 1, Ingress: levelResult{Total: 1, OK: 1, ByType: []typeStat{{"tls", 1, 1}}},
|
||||
}
|
||||
for i := 0; i < 30; i++ { // a second page for the pager
|
||||
ip := fmt.Sprintf("9.9.9.%d", 100+i)
|
||||
fake.registry[ip] = registryItem{IPAddress: ip, FirstSeenAt: now, LastSeenAt: now, TotalCycles: 1, LastResult: "pass",
|
||||
LastCycleID: 1, Ingress: levelResult{Total: 1, OK: 1, ByType: []typeStat{{"tls", 1, 1}}}}
|
||||
}
|
||||
page = get(t, ts, "/registry?run=1&subnet=9.9.9.0/24&direction=ingress&protocol=tls&per_page=25")
|
||||
fake.mu.Lock()
|
||||
last := fake.registryQueries[len(fake.registryQueries)-1]
|
||||
fake.mu.Unlock()
|
||||
for _, want := range []string{"run=1", "subnet=9.9.9.0%2F24", "direction=ingress", "protocol=tls"} {
|
||||
if !strings.Contains(last, want) {
|
||||
t.Fatalf("control-api request %q lacks %s", last, want)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{
|
||||
`<option value="1" selected>`, `<option value="2" >`, `<option value="9.9.9.0/24" selected>9.9.9.0/24 — Офис</option>`,
|
||||
`value="ingress" selected`, `value="tls" selected`, "Результат в запуске 1", `level-name">Ingress`,
|
||||
} {
|
||||
if !strings.Contains(page, want) {
|
||||
t.Fatalf("expected %q in the filtered registry page, got:\n%s", want, page)
|
||||
}
|
||||
}
|
||||
if strings.Contains(page, `level-name">Egress`) {
|
||||
t.Fatalf("the Egress level must be hidden when only ingress checks are selected, got:\n%s", page)
|
||||
}
|
||||
next := pagerLink(t, page, "next")
|
||||
if next == nil || next.Query().Get("run") != "1" || next.Query().Get("subnet") != "9.9.9.0/24" ||
|
||||
next.Query().Get("direction") != "ingress" || next.Query().Get("protocol") != "tls" {
|
||||
t.Fatalf("pager link lost the filters: %v", next)
|
||||
}
|
||||
get(t, ts, "/registry?direction=sideways&protocol=udp")
|
||||
fake.mu.Lock()
|
||||
last = fake.registryQueries[len(fake.registryQueries)-1]
|
||||
fake.mu.Unlock()
|
||||
if strings.Contains(last, "direction=") || strings.Contains(last, "protocol=") {
|
||||
t.Fatalf("unknown direction/protocol must be dropped: %q", last)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRegistryPageShowsEgressIngressLevels proves the list shows, under the
|
||||
|
||||
@@ -28,6 +28,8 @@ func (s *Server) routes(mux *http.ServeMux) {
|
||||
|
||||
mux.HandleFunc("GET /registry", s.handleRegistryPage)
|
||||
mux.HandleFunc("GET /registry/{ip}", s.handleRegistryDetail)
|
||||
mux.HandleFunc("GET /registry/breakdown/list", s.handleRegistryBreakdownList)
|
||||
mux.HandleFunc("GET /registry/breakdown/csv", s.handleRegistryBreakdownCSV)
|
||||
|
||||
mux.HandleFunc("GET /analytics", s.handleAnalyticsPage)
|
||||
mux.HandleFunc("GET /analytics/lists/{kind}", s.handleAnalyticsList)
|
||||
|
||||
@@ -36,6 +36,16 @@
|
||||
.an select, .an .an-btn { font: 500 13px var(--font-mono); background: var(--surface); color: var(--text); border: 1px solid var(--border); border-radius: var(--an-radius); padding: 7px 10px; }
|
||||
.an select { min-width: 0; max-width: 100%; flex: 1 1 160px; }
|
||||
.an-btn { cursor: pointer; } .an-btn:hover { background: var(--surface-alt); }
|
||||
.an [hidden] { display: none !important; }
|
||||
.an-body { display: grid; gap: 16px; min-width: 0; align-content: start; }
|
||||
#an-filter { display: grid; gap: 12px; }
|
||||
/* filter fields: label above the select, wrapping as whole pairs; in a column the fields keep their own height */
|
||||
.an-fields { display: flex; flex-wrap: wrap; gap: 10px 14px; align-items: flex-end; }
|
||||
.an-field { display: flex; flex-direction: column; gap: 5px; flex: 1 1 160px; min-width: 0; }
|
||||
.an-field label { font: 700 12px var(--font-mono); color: var(--text-muted); text-transform: uppercase; letter-spacing: .06em; }
|
||||
.an-field select { flex: 0 0 auto; }
|
||||
.an-reset { text-decoration: none; }
|
||||
.an .bd-wrap { margin-bottom: 0; } /* the chart of the registry brings its own space; here the grid gap is enough */
|
||||
.an-tabs { display: inline-flex; gap: 0; }
|
||||
.an-tabs button { font: 500 12px var(--font-mono); background: var(--surface); color: var(--text-muted); border: 1px solid var(--border); padding: 5px 12px; cursor: pointer; }
|
||||
.an-tabs button + button { border-left: 0; }
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
/* Analytics page: renders the report of one finished run (embedded as JSON in
|
||||
#analytics-data) and opens the address lists behind the indicators and the
|
||||
error classes in the shared dialog (analytics-dialog.js). No other run's data
|
||||
is on the page. */
|
||||
is on the page. The report is already narrowed to the chosen subnet; the
|
||||
direction and protocol only focus the page (blocks of the other level are
|
||||
hidden, the type tab and the sites column follow the protocol). */
|
||||
(function () {
|
||||
'use strict';
|
||||
var D = JSON.parse(document.getElementById('analytics-data').textContent);
|
||||
@@ -14,6 +16,7 @@
|
||||
function vshort(id) { var n = vnum(id); return n === null ? id : 'v' + n; }
|
||||
|
||||
var state = { sort: 'worst', all: false, type: R.targets.types.indexOf('https') >= 0 ? 'https' : (R.targets.types[0] || '') };
|
||||
if (R.targets.types.indexOf(M.protocol) >= 0) state.type = M.protocol;
|
||||
|
||||
/* ---- indicators ---- */
|
||||
function renderKpis() {
|
||||
@@ -124,11 +127,12 @@
|
||||
}
|
||||
|
||||
function renderSites() {
|
||||
var T = R.sites;
|
||||
$('an-sites').innerHTML = '<thead><tr><th>Площадка</th>' + T.types.map(function (t) { return '<th class="an-r">' + esc(t) + '</th>'; }).join('') + '</tr></thead><tbody>' +
|
||||
var T = R.sites, only = T.types.indexOf(M.protocol); // with a protocol only its column stays
|
||||
function shown(x, i) { return only < 0 || i === only; }
|
||||
$('an-sites').innerHTML = '<thead><tr><th>Площадка</th>' + T.types.filter(shown).map(function (t) { return '<th class="an-r">' + esc(t) + '</th>'; }).join('') + '</tr></thead><tbody>' +
|
||||
T.rows.map(function (row) {
|
||||
return '<tr><td class="an-a">' + esc(row.site) + '</td>' + row.stats.map(function (st, i) {
|
||||
return '<td class="an-r" ' + tipAttr(row.site + ' · ' + T.types[i] + ': успешно ' + fmt(st.ok) + ' из ' + fmt(st.total)) + '><span class="an-num">' + pct1(st.total - st.ok, st.total) + '%</span> <span class="an-note">провал</span></td>';
|
||||
return shown(st, i) ? '<td class="an-r" ' + tipAttr(row.site + ' · ' + T.types[i] + ': успешно ' + fmt(st.ok) + ' из ' + fmt(st.total)) + '><span class="an-num">' + pct1(st.total - st.ok, st.total) + '%</span> <span class="an-note">провал</span></td>' : '';
|
||||
}).join('') + '</tr>';
|
||||
}).join('') + '</tbody>';
|
||||
}
|
||||
@@ -206,14 +210,14 @@
|
||||
};
|
||||
|
||||
function listURL(base, kind, cls) {
|
||||
return base + encodeURIComponent(kind) + '?run=' + M.run_id + (cls ? '&class=' + encodeURIComponent(cls) : '');
|
||||
return base + encodeURIComponent(kind) + '?run=' + M.run_id + (cls ? '&class=' + encodeURIComponent(cls) : '') + (M.subnet ? '&subnet=' + encodeURIComponent(M.subnet) : '');
|
||||
}
|
||||
|
||||
function loadList(kind, cls, meta) { return A.load(listURL(M.list_url, kind, cls), meta.title); }
|
||||
|
||||
function runLabel() {
|
||||
var o = document.getElementById('an-run');
|
||||
return o && o.selectedOptions[0] ? o.selectedOptions[0].textContent : 'запуск ' + M.run_id;
|
||||
return (o && o.selectedOptions[0] ? o.selectedOptions[0].textContent : 'запуск ' + M.run_id) + (M.subnet ? ' · подсеть ' + M.subnet : '');
|
||||
}
|
||||
|
||||
function openIndicator(kind) {
|
||||
@@ -250,6 +254,12 @@
|
||||
});
|
||||
}
|
||||
|
||||
/* ---- focus: direction and protocol ---- */
|
||||
function applyFocus() {
|
||||
$('an-sec-in').hidden = M.direction === 'egress';
|
||||
$('an-sec-eg').hidden = $('an-sec-val').hidden = M.direction === 'ingress';
|
||||
}
|
||||
|
||||
/* ---- wiring ---- */
|
||||
function renderAll() {
|
||||
renderSubnets();
|
||||
@@ -258,8 +268,15 @@
|
||||
}
|
||||
|
||||
$('an-runnote').textContent = 'Тип: ' + M.kind + '. Начало ' + M.start + ', завершён ' + M.end + ', длительность ' + M.duration + '.' +
|
||||
(M.rechecked ? ' Перепроверено внутри запуска: ' + M.rechecked + ' адр. (берётся последний цикл).' : '');
|
||||
renderKpis(); renderReasons(); renderQuality(); renderErrors(); renderSites(); renderValidators(); renderAll();
|
||||
(M.rechecked ? ' Перепроверено внутри запуска: ' + M.rechecked + ' адр. (берётся последний цикл).' : '') +
|
||||
(R.scope ? ' Подсеть ' + R.scope.subnet + ': ' + fmt(S.addresses) + ' адр. из ' + fmt(R.scope.run_addresses) + ' в запуске.' : '');
|
||||
if (R.scope && !S.addresses) {
|
||||
$('an-empty').textContent = 'В запуске нет адресов этой подсети.';
|
||||
$('an-empty').hidden = false;
|
||||
$('an-body').hidden = true;
|
||||
return;
|
||||
}
|
||||
renderKpis(); renderReasons(); renderQuality(); renderErrors(); renderSites(); renderValidators(); renderAll(); applyFocus();
|
||||
|
||||
$('an-kpis').addEventListener('click', function (e) { var b = e.target.closest('[data-list]'); if (b) openIndicator(b.dataset.list); });
|
||||
$('an-errs').addEventListener('click', function (e) { var b = e.target.closest('[data-cls]'); if (b) openError(b.dataset.cls); });
|
||||
|
||||
@@ -507,8 +507,16 @@ code.inline { font-family: var(--font-mono); background: var(--surface-alt); bor
|
||||
.main { padding: 16px 14px 50px; }
|
||||
}
|
||||
|
||||
.bd-wrap { margin-bottom: 16px; }
|
||||
/* Chart of the registry: the figures column has one width in every row, so the bars line up. */
|
||||
#registry-breakdown .an-bar-row { grid-template-columns: minmax(110px, 30%) minmax(0, 1fr) 15em; }
|
||||
|
||||
@media (max-width: 640px) {
|
||||
.field-row { flex-direction: column; align-items: stretch; }
|
||||
/* the registry fields carry an inline flex basis (for rows); in a column it would become a height */
|
||||
#registry-filter .field { flex: 0 0 auto !important; }
|
||||
#registry-breakdown .an-bar-row { grid-template-columns: minmax(0, 1fr) auto; }
|
||||
#registry-breakdown .an-bar-row .an-track { grid-column: 1 / -1; grid-row: 2; }
|
||||
thead { display: none; }
|
||||
table, tbody, tr, td { display: block; width: 100%; }
|
||||
tbody tr { border-bottom: 1px solid var(--border-soft); padding: 9px 16px; }
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
/* The chart of the registry ("registry_breakdown" in registry.html): a click on
|
||||
a row opens the checks behind it in the analytics dialog (analytics-dialog.js).
|
||||
The listener sits on the document, so it keeps working after htmx replaces the
|
||||
table; the filter of the list is the one the chart was built with (data-qs). */
|
||||
(function () {
|
||||
'use strict';
|
||||
if (window.registryBreakdownBound) return;
|
||||
window.registryBreakdownBound = true;
|
||||
|
||||
var HINTS = {
|
||||
'Результат': 'Итог именно этой проверки. Список начинается с проваленных.',
|
||||
'Тип проверки': 'https, icmp, ssh, tcp-22, tls-443 и т. д.',
|
||||
'Валидатор': 'Валидатор, с которого шла egress-проверка.',
|
||||
'Задержка, мс': 'Сколько заняла проверка.',
|
||||
'Детали': 'Ошибка проверки, если она провалена.'
|
||||
};
|
||||
|
||||
function open(btn) {
|
||||
var A = window.AnalyticsDialog, box = btn.closest('#registry-breakdown');
|
||||
if (!A || !box) return;
|
||||
var tail = (box.dataset.qs ? box.dataset.qs + '&' : '') + 'key=' + encodeURIComponent(btn.dataset.bdKey);
|
||||
var title = box.dataset.scope + ' · ' + btn.dataset.bdLabel;
|
||||
A.load('/registry/breakdown/list?' + tail, title).then(function (l) {
|
||||
if (!l) return;
|
||||
var ok = l.rows.filter(function (r) { return r[1] === 'успешно'; }).length;
|
||||
var hints = {};
|
||||
l.columns.forEach(function (c, i) { if (HINTS[c]) hints[i] = HINTS[c]; });
|
||||
A.fill({
|
||||
title: title,
|
||||
scope: 'Срез',
|
||||
runLabel: box.dataset.slice,
|
||||
note: 'Проверки этого типа в цикле адреса среза, у адресов под текущим фильтром. Провалы сверху.',
|
||||
cols: l.columns, rows: l.rows, hints: hints,
|
||||
dist: '<div class="an-chips"><span class="an-tag an-t-ok">успешно ' + A.fmt(ok) + '</span><span class="an-tag an-t-bad">провал ' + A.fmt(l.rows.length - ok) + '</span></div>',
|
||||
csvURL: '/registry/breakdown/csv?' + tail
|
||||
});
|
||||
// the error text of a failed check is long: give its column room instead of a narrow wrapped strip
|
||||
var d = l.columns.indexOf('Детали');
|
||||
if (d >= 0) document.querySelectorAll('#an-dlg-tbl tr').forEach(function (tr) { if (tr.children[d]) tr.children[d].style.minWidth = '280px'; });
|
||||
});
|
||||
}
|
||||
|
||||
document.addEventListener('click', function (e) {
|
||||
var b = e.target.closest('[data-bd-key]');
|
||||
if (b) open(b);
|
||||
});
|
||||
})();
|
||||
@@ -20,6 +20,7 @@
|
||||
{{if .HasRun}}
|
||||
<script type="application/json" id="analytics-data">{{.DataJSON}}</script>
|
||||
<script src="/static/analytics-dialog.js"></script>
|
||||
<script src="/static/registry.js"></script>
|
||||
<script src="/static/analytics.js"></script>
|
||||
{{end}}
|
||||
</body>
|
||||
@@ -38,22 +39,62 @@
|
||||
<p class="an-note">Запусков проверки пока нет. Они появляются, когда адреса ставятся в очередь на странице <a href="/ips">«Очередь IP»</a> или запускается автоматический цикл.</p>
|
||||
</section>
|
||||
{{else}}
|
||||
<section class="an-panel" aria-label="Выбор запуска">
|
||||
<section class="an-panel" aria-label="Выбор запуска и фильтры">
|
||||
<form id="an-filter" action="/analytics" method="get" onsubmit="return false">
|
||||
{{/* a change goes to the page of the new filter; empty values stay out of the address */}}
|
||||
<script>function anGo(f) { var q = new URLSearchParams(); new FormData(f).forEach(function (v, k) { if (v) q.set(k, v); }); location.href = '/analytics?' + q; }</script>
|
||||
<div class="an-runbar">
|
||||
<label for="an-run">Запуск</label>
|
||||
{{if .PrevURL}}<a class="an-btn" href="{{.PrevURL}}" aria-label="Предыдущий запуск">◀</a>{{else}}<span class="an-btn" aria-disabled="true" style="opacity:.4">◀</span>{{end}}
|
||||
<select id="an-run" onchange="if (this.value) location.href = '/analytics?run=' + encodeURIComponent(this.value)">
|
||||
<select id="an-run" name="run" onchange="if (this.value) anGo(this.form)">
|
||||
{{range .Runs}}<option value="{{.ID}}"{{if .Selected}} selected{{end}}{{if .Disabled}} disabled{{end}}>{{.Label}}</option>
|
||||
{{end}}
|
||||
</select>
|
||||
{{if .NextURL}}<a class="an-btn" href="{{.NextURL}}" aria-label="Следующий запуск">▶</a>{{else}}<span class="an-btn" aria-disabled="true" style="opacity:.4">▶</span>{{end}}
|
||||
{{if .HasRun}}<a class="an-btn" href="/analytics/compare?target={{.RunID}}">Сравнить с другим запуском</a>{{end}}
|
||||
{{if .HasRun}}<a class="an-btn" href="{{.CompareURL}}">Сравнить с другим запуском</a>{{end}}
|
||||
</div>
|
||||
{{if .HasRun}}<p class="an-note" id="an-runnote"></p>{{else}}<p class="an-note">Завершённых запусков пока нет: данные появятся, когда все адреса запуска получат итог.</p>{{end}}
|
||||
{{if .HasRun}}
|
||||
<div class="an-fields">
|
||||
<div class="an-field">
|
||||
<label for="an-subnet">Подсеть</label>
|
||||
<select id="an-subnet" name="subnet" onchange="anGo(this.form)"{{if not .SubnetOptions}} disabled{{end}}>
|
||||
<option value="">Все подсети</option>
|
||||
{{range .SubnetOptions}}<option value="{{.CIDR}}"{{if .Selected}} selected{{end}}>{{.Text}}</option>
|
||||
{{end}}
|
||||
</select>
|
||||
{{if not .SubnetOptions}}<span class="an-note">Подсети не настроены — <a href="/settings">добавить в настройках</a></span>{{end}}
|
||||
</div>
|
||||
<div class="an-field">
|
||||
<label for="an-direction">Направление</label>
|
||||
<select id="an-direction" name="direction" onchange="anGo(this.form)">
|
||||
<option value="">Все</option>
|
||||
<option value="egress"{{if eq .Direction "egress"}} selected{{end}}>Egress</option>
|
||||
<option value="ingress"{{if eq .Direction "ingress"}} selected{{end}}>Ingress</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="an-field">
|
||||
<label for="an-protocol">Протокол</label>
|
||||
<select id="an-protocol" name="protocol" onchange="anGo(this.form)">
|
||||
<option value="">Все</option>
|
||||
{{$pr := .Protocol}}
|
||||
{{range $p := .Protocols}}<option value="{{$p}}"{{if eq $p $pr}} selected{{end}}>{{$p}}</option>
|
||||
{{end}}
|
||||
</select>
|
||||
</div>
|
||||
{{if .Filtered}}<a class="an-btn an-reset" href="/analytics?run={{.RunID}}">сбросить фильтры</a>{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
</form>
|
||||
{{if .HasRun}}<p class="an-note" id="an-runnote"></p>
|
||||
<p class="an-note">Подсеть пересчитывает все блоки страницы по её адресам. Направление и протокол задают фокус: скрывают блоки другого уровня и выбирают тип проверки, а вердикты и «Качество данных» остаются по всем проверкам запуска. С направлением и протоколом сразу показывается чарт.{{if .Breakdown}} Чарт считает проверки цикла запуска, в том числе пришедшие после вердикта, а плитки ниже берут вердикты запуска, поэтому числа могут расходиться.{{end}}</p>
|
||||
{{else}}<p class="an-note">Завершённых запусков пока нет: данные появятся, когда все адреса запуска получат итог.</p>{{end}}
|
||||
</section>
|
||||
{{end}}
|
||||
|
||||
{{if .HasRun}}
|
||||
{{with .Breakdown}}{{template "registry_breakdown" .}}{{end}}
|
||||
<p class="an-note" id="an-empty" hidden></p>
|
||||
<div class="an-body" id="an-body">
|
||||
<div class="an-kpis" id="an-kpis"></div>
|
||||
|
||||
<div class="an-cols">
|
||||
@@ -84,7 +125,7 @@
|
||||
<p class="an-note">Строка ведёт в «Реестр» с фильтром по запуску и подсети.</p>
|
||||
</section>
|
||||
|
||||
<section class="an-panel" aria-labelledby="an-h-eg">
|
||||
<section class="an-panel" id="an-sec-eg" aria-labelledby="an-h-eg">
|
||||
<div class="an-head">
|
||||
<h2 id="an-h-eg">Egress по целям</h2>
|
||||
<div class="an-tabs" role="group" aria-label="Тип проверки" id="an-types"></div>
|
||||
@@ -98,7 +139,7 @@
|
||||
<p class="an-note" id="an-matrixnote"></p>
|
||||
</section>
|
||||
|
||||
<div class="an-cols">
|
||||
<div class="an-cols" id="an-sec-in">
|
||||
<section class="an-panel" aria-labelledby="an-h-in">
|
||||
<h2 id="an-h-in">Ingress по площадкам</h2>
|
||||
<div class="an-scroll"><table id="an-sites"></table></div>
|
||||
@@ -109,12 +150,13 @@
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<section class="an-panel" aria-labelledby="an-h-val">
|
||||
<section class="an-panel" id="an-sec-val" aria-labelledby="an-h-val">
|
||||
<h2 id="an-h-val">Валидаторы: доля провалов egress https</h2>
|
||||
<div class="an-vals" id="an-vals" role="img" aria-label="Доля проваленных https-проверок по валидаторам"></div>
|
||||
<div class="an-vlab"><span id="an-vfirst"></span><span id="an-vavg"></span><span id="an-vlast"></span></div>
|
||||
<p class="an-note">Ровная полоса значит: проблема зависит от подсети адреса, а не от валидатора.</p>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
{{template "analytics_dialog"}}
|
||||
{{end}}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
{{define "registry_page"}}
|
||||
<!doctype html>
|
||||
<html lang="ru">
|
||||
<head>{{template "html_head" .}}</head>
|
||||
<head>{{template "html_head" .}}
|
||||
<link rel="stylesheet" href="/static/analytics.css">
|
||||
</head>
|
||||
<body>
|
||||
<div class="bg-grid"></div>
|
||||
<input type="checkbox" id="nav-toggle" class="nav-toggle">
|
||||
@@ -15,6 +17,10 @@
|
||||
</main>
|
||||
</div>
|
||||
</div>
|
||||
{{/* The list behind a row of the chart opens in the analytics dialog. */}}
|
||||
<div class="an">{{template "analytics_dialog"}}</div>
|
||||
<script src="/static/analytics-dialog.js"></script>
|
||||
<script src="/static/registry.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
{{end}}
|
||||
@@ -26,13 +32,6 @@
|
||||
Глубина хранимой истории на адрес настраивается на <a href="/settings">странице настроек</a>.</p>
|
||||
|
||||
<form id="registry-filter" class="panel" onsubmit="return false" style="margin-bottom:16px">
|
||||
{{if .Run}}<input type="hidden" name="run" value="{{.Run}}">{{end}}
|
||||
{{if .Subnet}}<input type="hidden" name="subnet" value="{{.Subnet}}">{{end}}
|
||||
{{if or .Run .Subnet}}<div class="panel-body" style="padding-bottom:0">
|
||||
<span class="pill pill-info">Из аналитики:{{if .Run}} запуск {{.Run}}{{end}}{{if .Subnet}} · подсеть {{.Subnet}}{{end}}</span>
|
||||
<a href="/registry" style="margin-left:10px">сбросить фильтр</a>
|
||||
{{if .Run}}<a href="/analytics?run={{.Run}}" style="margin-left:10px">к аналитике</a>{{end}}
|
||||
</div>{{end}}
|
||||
<div class="panel-body field-row">
|
||||
<div class="field" style="flex:1 1 260px">
|
||||
<label for="registry-q">Поиск по IP</label>
|
||||
@@ -41,6 +40,52 @@
|
||||
hx-include="#registry-filter" hx-trigger="input changed delay:300ms"
|
||||
hx-replace-url="true" hx-sync="#registry-table-wrap:queue last">
|
||||
</div>
|
||||
<div class="field" style="flex:1 1 300px">
|
||||
<label for="registry-run">Запуск</label>
|
||||
<select id="registry-run" name="run"
|
||||
hx-get="/registry" hx-select="#registry-table-wrap" hx-target="#registry-table-wrap" hx-swap="outerHTML"
|
||||
hx-include="#registry-filter" hx-trigger="change"
|
||||
hx-replace-url="true" hx-sync="#registry-table-wrap:queue last">
|
||||
<option value="">Последний цикл (по умолчанию)</option>
|
||||
{{range .Runs}}<option value="{{.ID}}" {{if .Selected}}selected{{end}}>{{.Label}}</option>
|
||||
{{end}}
|
||||
</select>
|
||||
</div>
|
||||
<div class="field" style="flex:1 1 200px">
|
||||
<label for="registry-subnet">Подсеть</label>
|
||||
<select id="registry-subnet" name="subnet" {{if not .SubnetOptions}}disabled{{end}}
|
||||
hx-get="/registry" hx-select="#registry-table-wrap" hx-target="#registry-table-wrap" hx-swap="outerHTML"
|
||||
hx-include="#registry-filter" hx-trigger="change"
|
||||
hx-replace-url="true" hx-sync="#registry-table-wrap:queue last">
|
||||
<option value="">Все подсети</option>
|
||||
{{range .SubnetOptions}}<option value="{{.CIDR}}" {{if .Selected}}selected{{end}}>{{.Text}}</option>
|
||||
{{end}}
|
||||
</select>
|
||||
{{if not .SubnetOptions}}<span class="muted" style="font-size:12px">Подсети не настроены — <a href="/settings">добавить в настройках</a></span>{{end}}
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="registry-direction">Направление</label>
|
||||
<select id="registry-direction" name="direction"
|
||||
hx-get="/registry" hx-select="#registry-table-wrap" hx-target="#registry-table-wrap" hx-swap="outerHTML"
|
||||
hx-include="#registry-filter" hx-trigger="change"
|
||||
hx-replace-url="true" hx-sync="#registry-table-wrap:queue last">
|
||||
<option value="">Все</option>
|
||||
<option value="egress" {{if eq .Direction "egress"}}selected{{end}}>Egress</option>
|
||||
<option value="ingress" {{if eq .Direction "ingress"}}selected{{end}}>Ingress</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="registry-protocol">Протокол</label>
|
||||
<select id="registry-protocol" name="protocol"
|
||||
hx-get="/registry" hx-select="#registry-table-wrap" hx-target="#registry-table-wrap" hx-swap="outerHTML"
|
||||
hx-include="#registry-filter" hx-trigger="change"
|
||||
hx-replace-url="true" hx-sync="#registry-table-wrap:queue last">
|
||||
<option value="">Все</option>
|
||||
{{$pr := .Protocol}}
|
||||
{{range $p := .Protocols}}<option value="{{$p}}" {{if eq $p $pr}}selected{{end}}>{{$p}}</option>
|
||||
{{end}}
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="registry-status">Статус</label>
|
||||
<select id="registry-status" name="status"
|
||||
@@ -66,6 +111,12 @@
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel-body muted" style="padding-top:0;font-size:12.5px">
|
||||
Запуск задаёт срез: результат берётся по циклу адреса в этом запуске, а статус — по вердикту запуска.
|
||||
Направление и протокол оставляют только такие проверки, и статус тогда считается по ним, а не по общему вердикту.
|
||||
tls проверяется только на входе (Ingress).
|
||||
<a href="/registry" style="margin-left:10px">сбросить фильтры</a>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<div id="registry-table-wrap">
|
||||
@@ -82,12 +133,34 @@
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
{{define "registry_breakdown"}}
|
||||
{{if .Hint}}<p class="muted" style="margin-bottom:12px">{{.Hint}}</p>
|
||||
{{else}}
|
||||
<div class="an bd-wrap">
|
||||
<section class="an-panel" id="registry-breakdown" aria-labelledby="registry-breakdown-h" data-qs="{{.QS}}" data-scope="{{.Scope}}" data-slice="{{.Slice}}">
|
||||
<h2 id="registry-breakdown-h">{{.Title}} · {{.Scope}}</h2>
|
||||
{{if .Err}}<p class="an-note">{{.Err}}</p>
|
||||
{{else if not .Rows}}<p class="an-note">Для этого сочетания проверок нет.</p>
|
||||
{{else}}
|
||||
<div class="an-rows">
|
||||
{{range .Rows}}<button type="button" class="an-bar-row" data-bd-key="{{.Key}}" data-bd-label="{{.Label}}" aria-haspopup="dialog" data-tip="{{.Tip}}" aria-label="{{.Tip}}"><span class="an-n">{{.Label}}</span><div class="an-track"><div class="an-fill" style="width:{{.Width}}%"></div></div><span class="an-num">{{.Text}}</span></button>
|
||||
{{end}}
|
||||
</div>
|
||||
<p class="an-note">Адресов под фильтром: {{.Addresses}}. Срез: {{.Slice}}. Строки идут от большего числа успешных проверок к меньшему. Считаются проверки, а не адреса: у tcp и tls на ingress у адреса может быть по проверке на каждую площадку и порт (22, 443). Строка открывает список адресов.</p>
|
||||
{{end}}
|
||||
</section>
|
||||
</div>
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
{{define "registry_table"}}
|
||||
{{with .Breakdown}}{{template "registry_breakdown" .}}{{end}}
|
||||
{{if or .Items .Run}}<p class="muted" style="margin-bottom:8px">Найдено адресов: {{.Total}}{{if .Run}} · <a href="{{.AnalyticsURL}}">к аналитике запуска {{.Run}}</a>{{end}}</p>{{end}}
|
||||
{{if .Items}}
|
||||
<div class="panel">
|
||||
<div class="table-scroll">
|
||||
<table>
|
||||
<thead><tr><th>Адрес</th><th>Впервые замечен</th><th>Последний раз замечен</th><th>Циклов</th><th>Последний результат</th><th>Сейчас в очереди</th></tr></thead>
|
||||
<thead><tr><th>Адрес</th><th>Впервые замечен</th><th>Последний раз замечен</th><th>Циклов</th><th>{{if .Run}}Результат в запуске {{.Run}}{{else}}Последний результат{{end}}</th><th>Сейчас в очереди</th></tr></thead>
|
||||
<tbody>
|
||||
{{range .Items}}
|
||||
<tr>
|
||||
@@ -95,15 +168,15 @@
|
||||
<td data-label="Впервые замечен">{{fmtTime .FirstSeenAt}}</td>
|
||||
<td data-label="Последний раз замечен">{{fmtTime .LastSeenAt}}</td>
|
||||
<td class="num" data-label="Циклов">{{.TotalCycles}}</td>
|
||||
<td data-label="Последний результат">
|
||||
<td data-label="{{if $.Run}}Результат в запуске {{$.Run}}{{else}}Последний результат{{end}}">
|
||||
{{if eq .LastResult "pass"}}<span class="pill pill-success">pass</span>
|
||||
{{else if eq .LastResult "partial"}}<span class="pill pill-warning">partial</span>
|
||||
{{else if eq .LastResult "fail"}}<span class="pill pill-danger">fail</span>
|
||||
{{else if eq .LastResult "cancelled"}}<span class="pill pill-cancel">cancelled</span>
|
||||
{{else}}<span class="pill pill-neutral">—</span>{{end}}
|
||||
{{if .LastCycleID}}<div class="levels" title="Считаются записанные проверки цикла {{.LastCycleID}}; вердикт учитывает ещё и недостающие результаты.">
|
||||
{{template "registry_level" dict "Name" "Egress" "L" .Egress}}
|
||||
{{template "registry_level" dict "Name" "Ingress" "L" .Ingress}}
|
||||
{{if or (not $.Scoped) .Egress.Total}}{{template "registry_level" dict "Name" "Egress" "L" .Egress}}{{end}}
|
||||
{{if or (not $.Scoped) .Ingress.Total}}{{template "registry_level" dict "Name" "Ingress" "L" .Ingress}}{{end}}
|
||||
</div>{{end}}
|
||||
</td>
|
||||
<td data-label="Сейчас в очереди">
|
||||
@@ -116,6 +189,6 @@
|
||||
</div>
|
||||
{{template "pager" .Pager}}
|
||||
</div>
|
||||
{{else if or .Query .StatusFilter}}<p class="muted">Ничего не найдено по текущему фильтру.</p>
|
||||
{{else if or .Query .StatusFilter .Run .Subnet .Direction .Protocol}}<p class="muted">Ничего не найдено по текущему фильтру.</p>
|
||||
{{else}}<p class="muted">Реестр пуст — ни один адрес ещё не ставился на проверку.</p>{{end}}
|
||||
{{end}}
|
||||
+188
-39
@@ -3,8 +3,10 @@ package db
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -106,28 +108,97 @@ func (d *DB) ListRegistry(ctx context.Context) ([]RegistrySummary, error) {
|
||||
out := make([]RegistrySummary, len(items))
|
||||
for i, item := range items {
|
||||
s := RegistrySummary{RegistryItem: item}
|
||||
if err := d.fillRegistrySummary(ctx, &s); err != nil {
|
||||
if err := d.fillRegistrySummary(ctx, &s, registrySlice{}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[i] = s
|
||||
}
|
||||
if err := d.fillRegistryLevels(ctx, out); err != nil {
|
||||
if err := d.fillRegistryLevels(ctx, out, registrySlice{}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// RegistryFamilies lists the check families RegistryFilter.Family accepts.
|
||||
// tls is an ingress-only family (the prober's TLS handshake on 443).
|
||||
var RegistryFamilies = []string{"icmp", "tcp", "ssh", "https", "tls"}
|
||||
|
||||
// IsValidRegistryLevel reports whether s is a RegistryFilter.Level value.
|
||||
func IsValidRegistryLevel(s string) bool { return s == LevelEgress || s == LevelIngress }
|
||||
|
||||
// IsValidRegistryFamily reports whether s is one of RegistryFamilies.
|
||||
func IsValidRegistryFamily(s string) bool { return slices.Contains(RegistryFamilies, s) }
|
||||
|
||||
// RegistryFilter narrows ListRegistryPage. The zero value matches everything.
|
||||
//
|
||||
// RunID, Level and Family also choose which data of an address is read (the
|
||||
// "slice"): with RunID, its cycle in that run (run_results.cycle_id) and the
|
||||
// run's verdict; without, its newest cycle. Level and Family narrow the checks
|
||||
// of that cycle: the address must have such checks, and LastResult then
|
||||
// classifies just them (all succeeded = pass, none = fail, else partial).
|
||||
type RegistryFilter struct {
|
||||
Query string // substring of ip_address
|
||||
LastResult string // pass|partial|fail|cancelled — same meaning as RegistrySummary.LastResult
|
||||
RunID int64 // only addresses that have a result in this run
|
||||
Subnet string // only addresses inside this CIDR
|
||||
Level string // egress|ingress — only checks of this level (see CheckLevel)
|
||||
Family string // one of RegistryFamilies — only checks of this family (see CheckFamily)
|
||||
}
|
||||
|
||||
// registrySlice is the part of RegistryFilter that selects what is read of an
|
||||
// address; the zero value is "newest cycle, all checks".
|
||||
type registrySlice struct {
|
||||
RunID int64
|
||||
Level, Family string
|
||||
}
|
||||
|
||||
func (sl registrySlice) scoped() bool { return sl.Level != "" || sl.Family != "" }
|
||||
|
||||
// validate rejects a Level or Family outside the accepted values.
|
||||
func (sl registrySlice) validate() error {
|
||||
if sl.Level != "" && !IsValidRegistryLevel(sl.Level) {
|
||||
return fmt.Errorf("level %q: %w", sl.Level, ErrValidation)
|
||||
}
|
||||
if sl.Family != "" && !IsValidRegistryFamily(sl.Family) {
|
||||
return fmt.Errorf("family %q: %w", sl.Family, ErrValidation)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// scopeCond is the SQL condition of Level and Family over `checks c`, "1 = 1"
|
||||
// without them. Level and Family are validated, so the LIKE patterns are safe.
|
||||
func (sl registrySlice) scopeCond() (string, []any) {
|
||||
conds := []string{"1 = 1"}
|
||||
var args []any
|
||||
switch sl.Level {
|
||||
case LevelEgress:
|
||||
conds = append(conds, "c.source = 'egress'")
|
||||
case LevelIngress:
|
||||
conds = append(conds, "c.source LIKE 'inbound-site-%'")
|
||||
}
|
||||
if sl.Family != "" {
|
||||
conds = append(conds, "(c.check_type = ? OR c.check_type LIKE ? || '-%')")
|
||||
args = append(args, sl.Family, sl.Family)
|
||||
}
|
||||
return strings.Join(conds, " AND "), args
|
||||
}
|
||||
|
||||
// scopeFrom is `FROM checks c WHERE …` over the checks of the slice of
|
||||
// registry row r (with the run's run_results row as rr when RunID is set):
|
||||
// the run's cycle or the newest one, narrowed by Level and Family.
|
||||
func (sl registrySlice) scopeFrom() (string, []any) {
|
||||
sc, args := sl.scopeCond()
|
||||
if sl.RunID > 0 {
|
||||
return `FROM checks c WHERE c.run_id = ? AND c.registry_id = r.id AND c.cycle_id = rr.cycle_id AND ` + sc,
|
||||
append([]any{sl.RunID}, args...)
|
||||
}
|
||||
return `FROM checks c WHERE c.registry_id = r.id
|
||||
AND c.cycle_id = (SELECT MAX(c2.cycle_id) FROM checks c2 WHERE c2.registry_id = r.id) AND ` + sc, args
|
||||
}
|
||||
|
||||
// subnetIDs returns the registry ids of the addresses inside prefix. SQLite
|
||||
// has no CIDR operators, so the registry's addresses are filtered here.
|
||||
func (d *DB) subnetIDs(ctx context.Context, cidr string) ([]any, error) {
|
||||
func (d *DB) subnetIDs(ctx context.Context, cidr string) ([]int64, error) {
|
||||
p, err := netip.ParsePrefix(cidr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("subnet %q: %v: %w", cidr, err, ErrValidation)
|
||||
@@ -137,7 +208,7 @@ func (d *DB) subnetIDs(ctx context.Context, cidr string) ([]any, error) {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var ids []any
|
||||
var ids []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var ip string
|
||||
@@ -169,42 +240,88 @@ const lastResultCond = `(
|
||||
) = ?)
|
||||
)`
|
||||
|
||||
// ListRegistryPage returns one page (limit/offset) of the registry in the same
|
||||
// order as ListRegistry, filtered by f, plus the total number of matching
|
||||
// rows. LIMIT/OFFSET are applied in SQL before the per-row summary queries,
|
||||
// so only the rows of the page pay for them. limit <= 0 means no limit.
|
||||
func (d *DB) ListRegistryPage(ctx context.Context, f RegistryFilter, limit, offset int) ([]RegistrySummary, int, error) {
|
||||
// registryFilterSQL is the FROM and the WHERE condition (without the keyword;
|
||||
// "1 = 1" for no filter) of the addresses f selects, over `ip_registry r LEFT
|
||||
// JOIN ip_queue q`, with the run's run_results row as rr when sl.RunID is set,
|
||||
// and its arguments in order. ListRegistryPage and RegistryBreakdown share it, so
|
||||
// the list and the chart over it always cover the same addresses.
|
||||
func (d *DB) registryFilterSQL(ctx context.Context, f RegistryFilter, sl registrySlice) (from, cond string, args []any, err error) {
|
||||
var conds []string
|
||||
var args []any
|
||||
if f.Query != "" {
|
||||
conds = append(conds, "instr(r.ip_address, ?) > 0")
|
||||
args = append(args, f.Query)
|
||||
}
|
||||
if f.LastResult != "" {
|
||||
switch {
|
||||
case f.LastResult != "" && sl.scoped():
|
||||
// The verdict covers all checks, so a narrowed scope has none; cancelled
|
||||
// leaves no checks at all.
|
||||
if f.LastResult == ResultCancelled {
|
||||
conds = append(conds, "0 = 1")
|
||||
break
|
||||
}
|
||||
sfrom, fargs := sl.scopeFrom()
|
||||
conds = append(conds, `(SELECT CASE WHEN SUM(c.success) = 0 THEN 'fail'
|
||||
WHEN SUM(c.success) = COUNT(*) THEN 'pass' ELSE 'partial' END `+sfrom+` HAVING COUNT(*) > 0) = ?`)
|
||||
args = append(args, fargs...)
|
||||
args = append(args, f.LastResult)
|
||||
case f.LastResult != "" && sl.RunID > 0:
|
||||
conds = append(conds, "rr.verdict = ?")
|
||||
args = append(args, f.LastResult)
|
||||
case f.LastResult != "":
|
||||
conds = append(conds, lastResultCond)
|
||||
args = append(args, f.LastResult, f.LastResult)
|
||||
}
|
||||
if f.RunID > 0 {
|
||||
conds = append(conds, "r.id IN (SELECT registry_id FROM run_results WHERE run_id = ?)")
|
||||
args = append(args, f.RunID)
|
||||
case sl.scoped():
|
||||
sfrom, fargs := sl.scopeFrom()
|
||||
conds = append(conds, "EXISTS (SELECT 1 "+sfrom+")")
|
||||
args = append(args, fargs...)
|
||||
}
|
||||
if f.Subnet != "" {
|
||||
ids, err := d.subnetIDs(ctx, f.Subnet)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
return "", "", nil, err
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
conds = append(conds, "0 = 1")
|
||||
} else {
|
||||
conds = append(conds, "r.id IN ("+strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",")+")")
|
||||
args = append(args, ids...)
|
||||
// One JSON parameter instead of an id per "?", which would hit
|
||||
// SQLite's bound-variable limit on a large subnet.
|
||||
b, err := json.Marshal(ids)
|
||||
if err != nil {
|
||||
return "", "", nil, err
|
||||
}
|
||||
conds = append(conds, "r.id IN (SELECT value FROM json_each(?))")
|
||||
args = append(args, string(b))
|
||||
}
|
||||
}
|
||||
from := ` FROM ip_registry r LEFT JOIN ip_queue q ON q.registry_id = r.id `
|
||||
where := ""
|
||||
if len(conds) > 0 {
|
||||
where = "WHERE " + strings.Join(conds, " AND ") + " "
|
||||
from = ` FROM ip_registry r LEFT JOIN ip_queue q ON q.registry_id = r.id `
|
||||
if sl.RunID > 0 {
|
||||
from += `JOIN run_results rr ON rr.registry_id = r.id AND rr.run_id = ? `
|
||||
args = append([]any{sl.RunID}, args...)
|
||||
}
|
||||
cond = "1 = 1"
|
||||
if len(conds) > 0 {
|
||||
cond = strings.Join(conds, " AND ")
|
||||
}
|
||||
return from, cond, args, nil
|
||||
}
|
||||
|
||||
// ListRegistryPage returns one page (limit/offset) of the registry in the same
|
||||
// order as ListRegistry, filtered by f, plus the total number of matching
|
||||
// rows. LIMIT/OFFSET are applied in SQL before the per-row summary queries,
|
||||
// so only the rows of the page pay for them. limit <= 0 means no limit. With
|
||||
// f.RunID the rows are the run's addresses and LastResult, LastCycleID, Egress
|
||||
// and Ingress describe the address in that run; an unknown run is an empty
|
||||
// list. Level and Family narrow Egress and Ingress to the matching checks.
|
||||
func (d *DB) ListRegistryPage(ctx context.Context, f RegistryFilter, limit, offset int) ([]RegistrySummary, int, error) {
|
||||
sl := registrySlice{RunID: f.RunID, Level: f.Level, Family: f.Family}
|
||||
if err := sl.validate(); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
from, cond, args, err := d.registryFilterSQL(ctx, f, sl)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
where := "WHERE " + cond + " "
|
||||
|
||||
var total int
|
||||
if err := d.QueryRowContext(ctx, `SELECT COUNT(*)`+from+where, args...).Scan(&total); err != nil {
|
||||
@@ -231,12 +348,12 @@ func (d *DB) ListRegistryPage(ctx context.Context, f RegistryFilter, limit, offs
|
||||
out := make([]RegistrySummary, len(items))
|
||||
for i, item := range items {
|
||||
s := RegistrySummary{RegistryItem: item}
|
||||
if err := d.fillRegistrySummary(ctx, &s); err != nil {
|
||||
if err := d.fillRegistrySummary(ctx, &s, sl); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
out[i] = s
|
||||
}
|
||||
if err := d.fillRegistryLevels(ctx, out); err != nil {
|
||||
if err := d.fillRegistryLevels(ctx, out, sl); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return out, total, nil
|
||||
@@ -257,11 +374,11 @@ func (d *DB) GetRegistryByAddress(ctx context.Context, address string) (*Registr
|
||||
return nil, err
|
||||
}
|
||||
s := &RegistrySummary{RegistryItem: *item}
|
||||
if err := d.fillRegistrySummary(ctx, s); err != nil {
|
||||
if err := d.fillRegistrySummary(ctx, s, registrySlice{}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
one := []RegistrySummary{*s}
|
||||
if err := d.fillRegistryLevels(ctx, one); err != nil {
|
||||
if err := d.fillRegistryLevels(ctx, one, registrySlice{}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
*s = one[0]
|
||||
@@ -275,7 +392,10 @@ func (d *DB) GetRegistryByAddress(ctx context.Context, address string) (*Registr
|
||||
// checked_at — a cycle with a mix of passing and failing checks (e.g. one
|
||||
// egress target timed out while the rest succeeded) is "partial", even
|
||||
// though the chronologically-last check to report in might have passed.
|
||||
func (d *DB) fillRegistrySummary(ctx context.Context, s *RegistrySummary) error {
|
||||
//
|
||||
// With sl.RunID, LastResult and LastCycleID are instead the address's verdict
|
||||
// and cycle in that run (from run_results); the rest is unchanged.
|
||||
func (d *DB) fillRegistrySummary(ctx context.Context, s *RegistrySummary, sl registrySlice) error {
|
||||
if err := d.QueryRowContext(ctx, `
|
||||
SELECT COUNT(DISTINCT cycle_id) FROM checks WHERE registry_id=?
|
||||
`, s.ID).Scan(&s.TotalCycles); err != nil {
|
||||
@@ -305,6 +425,18 @@ func (d *DB) fillRegistrySummary(ctx context.Context, s *RegistrySummary) error
|
||||
s.CurrentState = state.String
|
||||
}
|
||||
|
||||
if sl.RunID > 0 {
|
||||
var cycle sql.NullInt64
|
||||
var verdict sql.NullString
|
||||
err := d.QueryRowContext(ctx, `SELECT cycle_id, verdict FROM run_results WHERE run_id=? AND registry_id=?`, sl.RunID, s.ID).
|
||||
Scan(&cycle, &verdict)
|
||||
if err != nil && err != sql.ErrNoRows {
|
||||
return err
|
||||
}
|
||||
s.LastResult, s.LastCycleID = verdict.String, int(cycle.Int64)
|
||||
return nil
|
||||
}
|
||||
|
||||
switch {
|
||||
case overallResult.Valid && overallResult.String != "":
|
||||
// The address has a live ip_queue row with a finished cycle
|
||||
@@ -336,30 +468,46 @@ func (d *DB) fillRegistrySummary(ctx context.Context, s *RegistrySummary) error
|
||||
// SQLite's bound-variable limit.
|
||||
const registryLevelsChunk = 500
|
||||
|
||||
// registryLevelsQuery is the grouped query of fillRegistryLevels for n
|
||||
// registry ids: per address, the counts of its newest cycle by source and
|
||||
// check type.
|
||||
func registryLevelsQuery(n int) string {
|
||||
// registryLevelsQuery is the grouped query of fillRegistryLevels for the given
|
||||
// registry ids and its arguments: per address, the counts of its cycle by
|
||||
// source and check type — the newest one, or the run's with sl.RunID — over
|
||||
// the checks that match sl.Level and sl.Family.
|
||||
func registryLevelsQuery(ids []any, sl registrySlice) (string, []any) {
|
||||
in := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",")
|
||||
sc, scArgs := sl.scopeCond()
|
||||
if sl.RunID > 0 {
|
||||
args := append([]any{sl.RunID, sl.RunID}, ids...)
|
||||
return `
|
||||
SELECT c.registry_id, rr.cycle_id, c.source, c.check_type, COUNT(*), COALESCE(SUM(c.success), 0)
|
||||
FROM checks c
|
||||
JOIN run_results rr ON rr.run_id = ? AND rr.registry_id = c.registry_id AND rr.cycle_id = c.cycle_id
|
||||
WHERE c.run_id = ? AND c.registry_id IN (` + in + `) AND ` + sc + `
|
||||
GROUP BY c.registry_id, rr.cycle_id, c.source, c.check_type
|
||||
`, append(args, scArgs...)
|
||||
}
|
||||
return `
|
||||
SELECT c.registry_id, m.cid, c.source, c.check_type, COUNT(*), COALESCE(SUM(c.success), 0)
|
||||
FROM checks c
|
||||
JOIN (SELECT registry_id, MAX(cycle_id) AS cid FROM checks
|
||||
WHERE registry_id IN (` + strings.TrimSuffix(strings.Repeat("?,", n), ",") + `)
|
||||
WHERE registry_id IN (` + in + `)
|
||||
GROUP BY registry_id) m
|
||||
ON m.registry_id = c.registry_id AND m.cid = c.cycle_id
|
||||
WHERE ` + sc + `
|
||||
GROUP BY c.registry_id, m.cid, c.source, c.check_type
|
||||
`
|
||||
`, append(ids, scArgs...)
|
||||
}
|
||||
|
||||
// fillRegistryLevels sets LastCycleID, Egress and Ingress on every summary in
|
||||
// sums: the recorded checks of each address's newest cycle (the same cycle
|
||||
// whose time is LastCheckedAt), counted per level and per check family. It
|
||||
// runs one grouped query per chunk of addresses, not one per address, over
|
||||
// idx_checks_registry_cycle. Checks whose source is neither egress nor an
|
||||
// inbound site are not counted. The counts follow the recorded rows only, so
|
||||
// they can differ from LastResult, which also treats missing results as
|
||||
// failures.
|
||||
func (d *DB) fillRegistryLevels(ctx context.Context, sums []RegistrySummary) error {
|
||||
// idx_checks_registry_cycle. With sl.RunID the cycle is the address's one in
|
||||
// that run (over idx_checks_run) and sl.Level and sl.Family leave only the
|
||||
// matching checks, so the levels and types outside them stay empty. Checks
|
||||
// whose source is neither egress nor an inbound site are not counted. The
|
||||
// counts follow the recorded rows only, so they can differ from LastResult,
|
||||
// which also treats missing results as failures.
|
||||
func (d *DB) fillRegistryLevels(ctx context.Context, sums []RegistrySummary, sl registrySlice) error {
|
||||
pos := make(map[int64]int, len(sums))
|
||||
for i := range sums {
|
||||
pos[sums[i].ID] = i
|
||||
@@ -374,7 +522,8 @@ func (d *DB) fillRegistryLevels(ctx context.Context, sums []RegistrySummary) err
|
||||
for _, s := range sums[start:end] {
|
||||
args = append(args, s.ID)
|
||||
}
|
||||
rows, err := d.QueryContext(ctx, registryLevelsQuery(len(args)), args...)
|
||||
q, qargs := registryLevelsQuery(args, sl)
|
||||
rows, err := d.QueryContext(ctx, q, qargs...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Groups of a RegistryBreakdown: egress checks are grouped by target, ingress
|
||||
// ones by prober site.
|
||||
const (
|
||||
BreakdownTarget = "target"
|
||||
BreakdownSite = "site"
|
||||
)
|
||||
|
||||
// BreakdownRow is one group of a RegistryBreakdown: Key is the checks.target
|
||||
// (egress) or the checks.source (ingress, "inbound-site-N"), Total the checks
|
||||
// recorded for it and OK the successful ones.
|
||||
type BreakdownRow struct {
|
||||
Key string
|
||||
Total int
|
||||
OK int
|
||||
}
|
||||
|
||||
// Breakdown counts the checks of one direction and protocol per target or site.
|
||||
type Breakdown struct {
|
||||
Group string // BreakdownTarget or BreakdownSite
|
||||
Addresses int // addresses of the filter, the same as ListRegistryPage's total
|
||||
Rows []BreakdownRow
|
||||
}
|
||||
|
||||
// breakdownChecks is `FROM … JOIN checks c … WHERE …` over the checks of the
|
||||
// addresses f selects (registryFilterSQL), in the slice of f: the address's cycle
|
||||
// in the run or its newest one, narrowed by Level and Family, which must be
|
||||
// set. It also returns the column the checks are grouped by. CROSS JOIN keeps
|
||||
// the checks as the inner table, so the addresses drive the lookups over the
|
||||
// checks indexes whatever the table statistics say.
|
||||
func (d *DB) breakdownChecks(ctx context.Context, f RegistryFilter) (q, group string, args []any, err error) {
|
||||
sl := registrySlice{RunID: f.RunID, Level: f.Level, Family: f.Family}
|
||||
if err := sl.validate(); err != nil {
|
||||
return "", "", nil, err
|
||||
}
|
||||
if sl.Level == "" || sl.Family == "" {
|
||||
return "", "", nil, fmt.Errorf("direction and protocol are required: %w", ErrValidation)
|
||||
}
|
||||
from, cond, args, err := d.registryFilterSQL(ctx, f, sl)
|
||||
if err != nil {
|
||||
return "", "", nil, err
|
||||
}
|
||||
group = "c.target"
|
||||
if sl.Level == LevelIngress {
|
||||
group = "c.source"
|
||||
}
|
||||
sc, scArgs := sl.scopeCond()
|
||||
q = from + `CROSS JOIN checks c ON c.registry_id = r.id AND c.cycle_id = `
|
||||
if sl.RunID > 0 {
|
||||
q += `rr.cycle_id WHERE ` + cond + ` AND c.run_id = ? AND ` + sc
|
||||
args = append(args, sl.RunID)
|
||||
} else {
|
||||
q += `(SELECT MAX(c2.cycle_id) FROM checks c2 WHERE c2.registry_id = r.id) WHERE ` + cond + ` AND ` + sc
|
||||
}
|
||||
return q, group, append(args, scArgs...), nil
|
||||
}
|
||||
|
||||
// RegistryBreakdown counts the recorded checks of f.Level and f.Family (both
|
||||
// required, else ErrValidation) per target (egress) or site (ingress), over
|
||||
// the same slice and the same addresses — all under the filter, not one page —
|
||||
// as ListRegistryPage. Rows are sorted by successful checks, most first, then by
|
||||
// key. It counts checks, not addresses: an address can have several checks per
|
||||
// site (tcp-22 and tcp-443).
|
||||
func (d *DB) RegistryBreakdown(ctx context.Context, f RegistryFilter) (*Breakdown, error) {
|
||||
from, group, args, err := d.breakdownChecks(ctx, f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b := &Breakdown{Group: BreakdownTarget}
|
||||
if f.Level == LevelIngress {
|
||||
b.Group = BreakdownSite
|
||||
}
|
||||
rows, err := d.QueryContext(ctx, `SELECT `+group+`, COUNT(*), COALESCE(SUM(c.success), 0) `+from+` GROUP BY `+group, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var r BreakdownRow
|
||||
if err := rows.Scan(&r.Key, &r.Total, &r.OK); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b.Rows = append(b.Rows, r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Slice(b.Rows, func(i, j int) bool {
|
||||
if b.Rows[i].OK != b.Rows[j].OK {
|
||||
return b.Rows[i].OK > b.Rows[j].OK
|
||||
}
|
||||
return b.Rows[i].Key < b.Rows[j].Key
|
||||
})
|
||||
|
||||
// The addresses of the filter, as ListRegistryPage counts them.
|
||||
sl := registrySlice{RunID: f.RunID, Level: f.Level, Family: f.Family}
|
||||
afrom, cond, aargs, err := d.registryFilterSQL(ctx, f, sl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := d.QueryRowContext(ctx, `SELECT COUNT(*)`+afrom+`WHERE `+cond, aargs...).Scan(&b.Addresses); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// RegistryBreakdownList returns the checks behind one row of RegistryBreakdown:
|
||||
// those whose target (egress) or source (ingress) is key, with the address
|
||||
// (IPAddress), type, validator, latency, detail and time, failures first, then
|
||||
// by registry order. A key without checks is ErrNotFound.
|
||||
func (d *DB) RegistryBreakdownList(ctx context.Context, f RegistryFilter, key string) ([]Check, error) {
|
||||
from, group, args, err := d.breakdownChecks(ctx, f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := d.QueryContext(ctx, `
|
||||
SELECT r.ip_address, c.validator_id, c.source, c.check_type, c.target, c.success, c.latency_ms, c.detail, c.checked_at `+
|
||||
from+` AND `+group+` = ? ORDER BY c.success, r.first_seen_at, r.id, c.check_type, c.source, c.target`, append(args, key)...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Check
|
||||
for rows.Next() {
|
||||
var c Check
|
||||
var checkedAt string
|
||||
if err := rows.Scan(&c.IPAddress, &c.ValidatorID, &c.Source, &c.CheckType, &c.Target, &c.Success, &c.LatencyMS, &c.Detail, &checkedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.CheckedAt, err = dbToTime(checkedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, fmt.Errorf("breakdown key %q: %w", key, ErrNotFound)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// RegistryBreakdown and RegistryBreakdownList over the slice of the registry
|
||||
// filter. Run 1 holds three addresses (cycle 1), run 2 a re-check of .1 (cycle 2).
|
||||
func TestRegistryBreakdown(t *testing.T) {
|
||||
d, ctx := newTestDB(t)
|
||||
const a1, a2, a3 = "10.0.0.1", "10.0.0.2", "10.0.1.1"
|
||||
submit(t, d, RunManual, a1, a2, a3)
|
||||
s1, s2 := InboundSource(1), InboundSource(2)
|
||||
for _, c := range []struct {
|
||||
addr, source, typ, target string
|
||||
ok bool
|
||||
}{
|
||||
{a1, SourceEgress, "https", "T1", true}, {a1, SourceEgress, "https", "T2", false},
|
||||
{a1, s1, "tcp-22", a1, true}, {a1, s1, "tcp-443", a1, false}, {a1, s2, "tcp-443", a1, true},
|
||||
{a2, SourceEgress, "https", "T1", false}, {a2, SourceEgress, "https", "T2", false}, {a2, s1, "tcp-443", a2, true},
|
||||
{a3, SourceEgress, "https", "T1", true}, {a3, SourceEgress, "https", "T3", true},
|
||||
} {
|
||||
addCheck(t, d, c.addr, c.source, c.typ, c.target, c.ok)
|
||||
}
|
||||
finish(t, d, a1, ResultPartial, -1)
|
||||
finish(t, d, a2, ResultFail, -1)
|
||||
finish(t, d, a3, ResultPass, -1)
|
||||
run1 := runs(t, d)[0].ID
|
||||
|
||||
submit(t, d, RunManual, a1)
|
||||
addCheck(t, d, a1, SourceEgress, "https", "T1", false)
|
||||
addCheck(t, d, a1, SourceEgress, "https", "T2", true)
|
||||
finish(t, d, a1, ResultPartial, -1)
|
||||
|
||||
eg := RegistryFilter{Level: LevelEgress, Family: "https"}
|
||||
in := RegistryFilter{Level: LevelIngress, Family: "tcp"}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
f RegistryFilter
|
||||
group string
|
||||
addrs int
|
||||
want []BreakdownRow
|
||||
}{
|
||||
{"egress, run 1: most successful first", withRun(eg, run1), BreakdownTarget, 3,
|
||||
[]BreakdownRow{{"T1", 3, 2}, {"T3", 1, 1}, {"T2", 2, 0}}},
|
||||
{"egress, newest cycle; equal counts by key", eg, BreakdownTarget, 3,
|
||||
[]BreakdownRow{{"T1", 3, 1}, {"T2", 2, 1}, {"T3", 1, 1}}},
|
||||
{"egress, subnet", RegistryFilter{RunID: run1, Level: LevelEgress, Family: "https", Subnet: "10.0.0.0/24"}, BreakdownTarget, 2,
|
||||
[]BreakdownRow{{"T1", 2, 1}, {"T2", 2, 0}}},
|
||||
{"egress, status in scope", RegistryFilter{RunID: run1, Level: LevelEgress, Family: "https", LastResult: ResultFail}, BreakdownTarget, 1,
|
||||
[]BreakdownRow{{"T1", 1, 0}, {"T2", 1, 0}}},
|
||||
{"ingress by site, checks not addresses", withRun(in, run1), BreakdownSite, 2,
|
||||
[]BreakdownRow{{s1, 3, 2}, {s2, 1, 1}}},
|
||||
{"egress tls does not exist", RegistryFilter{Level: LevelEgress, Family: "tls"}, BreakdownTarget, 0, nil},
|
||||
} {
|
||||
b, err := d.RegistryBreakdown(ctx, tc.f)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", tc.name, err)
|
||||
}
|
||||
if b.Group != tc.group || b.Addresses != tc.addrs || !reflect.DeepEqual(b.Rows, tc.want) {
|
||||
t.Errorf("%s: group=%s addresses=%d rows=%v, want %s %d %v", tc.name, b.Group, b.Addresses, b.Rows, tc.group, tc.addrs, tc.want)
|
||||
}
|
||||
// The chart covers the addresses of the list.
|
||||
if _, total, err := d.ListRegistryPage(ctx, tc.f, 10, 0); err != nil || total != b.Addresses {
|
||||
t.Errorf("%s: list total=%d err=%v, chart addresses=%d", tc.name, total, err, b.Addresses)
|
||||
}
|
||||
}
|
||||
|
||||
// The list: failures first, then registry order; the key picks the group.
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
f RegistryFilter
|
||||
key string
|
||||
want [][3]string // address, type, success
|
||||
}{
|
||||
{"egress T1", withRun(eg, run1), "T1", [][3]string{{a2, "https", "0"}, {a1, "https", "1"}, {a3, "https", "1"}}},
|
||||
{"ingress site 1", withRun(in, run1), s1, [][3]string{{a1, "tcp-443", "0"}, {a1, "tcp-22", "1"}, {a2, "tcp-443", "1"}}},
|
||||
} {
|
||||
got, err := d.RegistryBreakdownList(ctx, tc.f, tc.key)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", tc.name, err)
|
||||
}
|
||||
var rows [][3]string
|
||||
for _, c := range got {
|
||||
ok := "0"
|
||||
if c.Success {
|
||||
ok = "1"
|
||||
}
|
||||
rows = append(rows, [3]string{c.IPAddress, c.CheckType, ok})
|
||||
}
|
||||
if !reflect.DeepEqual(rows, tc.want) {
|
||||
t.Errorf("%s: %v, want %v", tc.name, rows, tc.want)
|
||||
}
|
||||
}
|
||||
if _, err := d.RegistryBreakdownList(ctx, withRun(eg, run1), "nope"); !errors.Is(err, ErrNotFound) {
|
||||
t.Errorf("unknown key: %v", err)
|
||||
}
|
||||
for _, f := range []RegistryFilter{{}, {Level: LevelEgress}, {Family: "https"}, {Level: "sideways", Family: "https"}} {
|
||||
if _, err := d.RegistryBreakdown(ctx, f); !errors.Is(err, ErrValidation) {
|
||||
t.Errorf("%+v: %v", f, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func withRun(f RegistryFilter, run int64) RegistryFilter {
|
||||
f.RunID = run
|
||||
return f
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -167,3 +169,102 @@ func TestRegistryLevelsLatestCycleAndManyAddresses(t *testing.T) {
|
||||
t.Errorf("after delete: %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRegistryPageSlice covers the run / level / protocol filters of
|
||||
// ListRegistryPage together with the status and the subnet. Run 1 holds three
|
||||
// addresses (cycle 1), run 2 only a re-check of .1 (cycle 2) whose result
|
||||
// differs from run 1.
|
||||
func TestRegistryPageSlice(t *testing.T) {
|
||||
d, ctx := newTestDB(t)
|
||||
const a1, a2, a3 = "10.0.0.1", "10.0.0.2", "10.0.1.1"
|
||||
submit(t, d, RunManual, a1, a2, a3)
|
||||
for _, c := range []struct {
|
||||
addr, source, typ string
|
||||
ok bool
|
||||
}{
|
||||
{a1, SourceEgress, "https", true}, {a1, SourceEgress, "icmp", true},
|
||||
{a1, InboundSource(1), "tcp-22", true}, {a1, InboundSource(1), "tcp-443", false}, {a1, InboundSource(1), "tls-443", false},
|
||||
{a2, SourceEgress, "https", false},
|
||||
{a2, InboundSource(1), "tcp-443", true}, {a2, InboundSource(1), "tls-443", true}, {a2, InboundSource(1), "ssh", true},
|
||||
{a3, InboundSource(1), "icmp", true},
|
||||
} {
|
||||
addCheck(t, d, c.addr, c.source, c.typ, c.addr, c.ok)
|
||||
}
|
||||
finish(t, d, a1, ResultPartial, -1)
|
||||
finish(t, d, a2, ResultPartial, -1)
|
||||
finish(t, d, a3, ResultPass, -1)
|
||||
run1 := runs(t, d)[0].ID
|
||||
|
||||
submit(t, d, RunManual, a1)
|
||||
addCheck(t, d, a1, SourceEgress, "https", a1, false)
|
||||
addCheck(t, d, a1, InboundSource(1), "tcp-443", a1, true)
|
||||
addCheck(t, d, a1, InboundSource(1), "tls-443", a1, true)
|
||||
finish(t, d, a1, ResultPartial, -1)
|
||||
run2 := runs(t, d)[0].ID
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
f RegistryFilter
|
||||
want []string
|
||||
}{
|
||||
{"no filter", RegistryFilter{}, []string{a1, a2, a3}},
|
||||
{"run 1", RegistryFilter{RunID: run1}, []string{a1, a2, a3}},
|
||||
{"run 2", RegistryFilter{RunID: run2}, []string{a1}},
|
||||
{"unknown run", RegistryFilter{RunID: 999}, nil},
|
||||
{"egress, newest cycle", RegistryFilter{Level: LevelEgress}, []string{a1, a2}},
|
||||
{"ingress, newest cycle", RegistryFilter{Level: LevelIngress}, []string{a1, a2, a3}},
|
||||
{"verdict of the run", RegistryFilter{RunID: run1, LastResult: ResultPartial}, []string{a1, a2}},
|
||||
{"cancelled in the run", RegistryFilter{RunID: run1, LastResult: ResultCancelled}, nil},
|
||||
{"tcp-22 and tcp-443 are tcp", RegistryFilter{RunID: run1, Family: "tcp"}, []string{a1, a2}},
|
||||
{"tcp partial", RegistryFilter{RunID: run1, Family: "tcp", LastResult: ResultPartial}, []string{a1}},
|
||||
{"tcp pass", RegistryFilter{RunID: run1, Family: "tcp", LastResult: ResultPass}, []string{a2}},
|
||||
{"tcp fail", RegistryFilter{RunID: run1, Family: "tcp", LastResult: ResultFail}, nil},
|
||||
{"tls fail in run 1", RegistryFilter{RunID: run1, Family: "tls", LastResult: ResultFail}, []string{a1}},
|
||||
{"tls pass in run 2", RegistryFilter{RunID: run2, Family: "tls", LastResult: ResultPass}, []string{a1}},
|
||||
{"tls on egress is empty", RegistryFilter{RunID: run1, Level: LevelEgress, Family: "tls"}, nil},
|
||||
{"egress status is of egress checks", RegistryFilter{RunID: run1, Level: LevelEgress, LastResult: ResultPass}, []string{a1}},
|
||||
{"egress fail", RegistryFilter{RunID: run1, Level: LevelEgress, LastResult: ResultFail}, []string{a2}},
|
||||
{"cancelled has no scope", RegistryFilter{Level: LevelEgress, LastResult: ResultCancelled}, nil},
|
||||
{"icmp on ingress", RegistryFilter{RunID: run1, Level: LevelIngress, Family: "icmp"}, []string{a3}},
|
||||
{"subnet, run and level", RegistryFilter{RunID: run1, Level: LevelIngress, Subnet: "10.0.0.0/24"}, []string{a1, a2}},
|
||||
{"subnet with no checks of the level", RegistryFilter{RunID: run1, Level: LevelEgress, Subnet: "10.0.1.0/24"}, nil},
|
||||
{"query and level", RegistryFilter{Query: ".2", Level: LevelEgress}, []string{a2}},
|
||||
} {
|
||||
page, total, err := d.ListRegistryPage(ctx, tc.f, 50, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", tc.name, err)
|
||||
}
|
||||
var got []string
|
||||
for _, s := range page {
|
||||
got = append(got, s.IPAddress)
|
||||
}
|
||||
sort.Strings(got)
|
||||
if total != len(tc.want) || !reflect.DeepEqual(got, tc.want) {
|
||||
t.Errorf("%s: total=%d got=%v, want %v", tc.name, total, got, tc.want)
|
||||
}
|
||||
}
|
||||
|
||||
// The same address shows its own result in each run, narrowed to the scope.
|
||||
for run, want := range map[int64]struct {
|
||||
cycle int
|
||||
result LevelResult
|
||||
}{
|
||||
run1: {1, LevelResult{Total: 1, OK: 0, ByType: []TypeStat{{"tls", 1, 0}}}},
|
||||
run2: {2, LevelResult{Total: 1, OK: 1, ByType: []TypeStat{{"tls", 1, 1}}}},
|
||||
} {
|
||||
page, _, err := d.ListRegistryPage(ctx, RegistryFilter{RunID: run, Query: a1, Family: "tls"}, 10, 0)
|
||||
if err != nil || len(page) != 1 {
|
||||
t.Fatalf("run %d: %v %+v", run, err, page)
|
||||
}
|
||||
s := page[0]
|
||||
if s.LastCycleID != want.cycle || s.LastResult != ResultPartial || s.Egress.Total != 0 || !reflect.DeepEqual(s.Ingress, want.result) {
|
||||
t.Errorf("run %d: cycle=%d result=%q egress=%+v ingress=%+v", run, s.LastCycleID, s.LastResult, s.Egress, s.Ingress)
|
||||
}
|
||||
}
|
||||
|
||||
for _, f := range []RegistryFilter{{Level: "sideways"}, {Family: "dns"}} {
|
||||
if _, _, err := d.ListRegistryPage(ctx, f, 10, 0); !errors.Is(err, ErrValidation) {
|
||||
t.Errorf("%+v: %v", f, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package db
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"sort"
|
||||
@@ -273,13 +274,25 @@ type RunCheck struct {
|
||||
|
||||
// EachRunCheck calls fn for every check of the cycles that make up the run's
|
||||
// results (the latest cycle of each address in the run), in one pass over the
|
||||
// run_id index. fn must not call back into the DB (one connection).
|
||||
func (d *DB) EachRunCheck(ctx context.Context, runID int64, fn func(RunCheck)) error {
|
||||
rows, err := d.QueryContext(ctx, `
|
||||
// run_id index. With registryIDs (not nil) only the checks of those addresses
|
||||
// are read. fn must not call back into the DB (one connection).
|
||||
func (d *DB) EachRunCheck(ctx context.Context, runID int64, registryIDs []int64, fn func(RunCheck)) error {
|
||||
q := `
|
||||
SELECT c.registry_id, c.source, c.check_type, c.target, c.success, c.validator_id, c.detail,
|
||||
COALESCE(c.recorded_at, c.created_at), c.after_verdict
|
||||
FROM checks c JOIN run_results r ON r.run_id=c.run_id AND r.registry_id=c.registry_id AND r.cycle_id=c.cycle_id
|
||||
WHERE c.run_id=?`, runID)
|
||||
WHERE c.run_id=?`
|
||||
args := []any{runID}
|
||||
if registryIDs != nil {
|
||||
ids, err := json.Marshal(registryIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// One JSON parameter, not an id per "?" (SQLite's bound-variable limit).
|
||||
q += ` AND c.registry_id IN (SELECT value FROM json_each(?))`
|
||||
args = append(args, string(ids))
|
||||
}
|
||||
rows, err := d.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -120,9 +120,17 @@ func TestRecheckAfterFinalizeOpensNewRun(t *testing.T) {
|
||||
}
|
||||
// The checks of a run are the ones of its result cycle, nothing else.
|
||||
var got []RunCheck
|
||||
if err := d.EachRunCheck(ctx, first.ID, func(c RunCheck) { got = append(got, c) }); err != nil || len(got) != 1 || !got[0].Success {
|
||||
if err := d.EachRunCheck(ctx, first.ID, nil, func(c RunCheck) { got = append(got, c) }); err != nil || len(got) != 1 || !got[0].Success {
|
||||
t.Fatalf("run 1 checks: %+v %v", got, err)
|
||||
}
|
||||
// With a list of addresses only theirs are read; an empty list reads none.
|
||||
got = nil
|
||||
if err := d.EachRunCheck(ctx, first.ID, []int64{}, func(c RunCheck) { got = append(got, c) }); err != nil || len(got) != 0 {
|
||||
t.Fatalf("empty address list must read no checks: %+v %v", got, err)
|
||||
}
|
||||
if err := d.EachRunCheck(ctx, first.ID, []int64{old[0].RegistryID}, func(c RunCheck) { got = append(got, c) }); err != nil || len(got) != 1 {
|
||||
t.Fatalf("listed address: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A re-check while the run is still open joins it and replaces the address's
|
||||
|
||||
@@ -351,30 +351,36 @@ func TestMigration0009Indexes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRegistryLevelsQueryUsesIndex guards against a full scan of checks: both
|
||||
// the per-address MAX(cycle_id) and the join back must go through
|
||||
// idx_checks_registry_cycle.
|
||||
// TestRegistryLevelsQueryUsesIndex guards against a full scan of checks: the
|
||||
// per-address MAX(cycle_id) and the join back must go through
|
||||
// idx_checks_registry_cycle, the run's slice through idx_checks_run.
|
||||
func TestRegistryLevelsQueryUsesIndex(t *testing.T) {
|
||||
d, ctx := newTestDB(t)
|
||||
rows, err := d.QueryContext(ctx, "EXPLAIN QUERY PLAN "+registryLevelsQuery(3), 1, 2, 3)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var plan string
|
||||
for rows.Next() {
|
||||
var id, parent, unused int
|
||||
var detail string
|
||||
if err := rows.Scan(&id, &parent, &unused, &detail); err != nil {
|
||||
for sl, index := range map[registrySlice]string{
|
||||
{}: "idx_checks_registry_cycle",
|
||||
{RunID: 1, Level: LevelIngress, Family: "tls"}: "idx_checks_run",
|
||||
} {
|
||||
q, args := registryLevelsQuery([]any{1, 2, 3}, sl)
|
||||
rows, err := d.QueryContext(ctx, "EXPLAIN QUERY PLAN "+q, args...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan += detail + "\n"
|
||||
if strings.HasPrefix(detail, "SCAN") && strings.Contains(detail, "checks") {
|
||||
t.Errorf("full scan of checks in plan:\n%s", plan)
|
||||
var plan string
|
||||
for rows.Next() {
|
||||
var id, parent, unused int
|
||||
var detail string
|
||||
if err := rows.Scan(&id, &parent, &unused, &detail); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plan += detail + "\n"
|
||||
if strings.HasPrefix(detail, "SCAN") && strings.Contains(detail, "checks") {
|
||||
t.Errorf("%+v: full scan of checks in plan:\n%s", sl, plan)
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
if !strings.Contains(plan, index) {
|
||||
t.Errorf("%+v: expected %s in plan:\n%s", sl, index, plan)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(plan, "idx_checks_registry_cycle") {
|
||||
t.Errorf("expected idx_checks_registry_cycle in plan:\n%s", plan)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -417,6 +423,17 @@ func TestScaleSmoke6440(t *testing.T) {
|
||||
t.Fatalf("upsert check: %v", err)
|
||||
}
|
||||
}
|
||||
// The run's result of the address (as aggregation would record it).
|
||||
if _, err := d.ExecContext(ctx, `
|
||||
INSERT INTO run_results (run_id, registry_id, ip_address, cycle_id, verdict, aggregated_at)
|
||||
SELECT run_id, registry_id, ip_address, cycle_id, 'partial', ? FROM ip_queue WHERE id=?
|
||||
`, timeToDB(Now()), ip.ID); err != nil {
|
||||
t.Fatalf("run result: %v", err)
|
||||
}
|
||||
}
|
||||
var runID int64
|
||||
if err := d.QueryRowContext(ctx, `SELECT run_id FROM run_results LIMIT 1`).Scan(&runID); err != nil {
|
||||
t.Fatalf("run id: %v", err)
|
||||
}
|
||||
|
||||
start = time.Now()
|
||||
@@ -430,6 +447,25 @@ func TestScaleSmoke6440(t *testing.T) {
|
||||
if _, total, err = d.ListRegistryPage(ctx, RegistryFilter{LastResult: ResultPass}, 100, 0); err != nil || total != 0 {
|
||||
t.Fatalf("registry last_result filter: total=%d err=%v", total, err)
|
||||
}
|
||||
// All filters at once, the subnet covering the whole registry.
|
||||
page, total, err = d.ListRegistryPage(ctx, RegistryFilter{
|
||||
RunID: runID, Subnet: "10.0.0.0/8", Level: LevelIngress, Family: "tcp", LastResult: ResultPartial,
|
||||
}, 50, 0)
|
||||
if err != nil || total != 100 || len(page) != 50 || page[0].Egress.Total != 0 || page[0].Ingress.Total != 18 {
|
||||
t.Fatalf("registry slice: total=%d len=%d err=%v", total, len(page), err)
|
||||
}
|
||||
// The chart and the list behind one of its rows: 100 addresses x 6 checks per site.
|
||||
bf := RegistryFilter{RunID: runID, Subnet: "10.0.0.0/8", Level: LevelIngress, Family: "tcp", LastResult: ResultPartial}
|
||||
bd, err := d.RegistryBreakdown(ctx, bf)
|
||||
if err != nil || bd.Addresses != 100 || len(bd.Rows) != 3 || bd.Rows[0].Total != 600 || bd.Rows[0].OK != 500 {
|
||||
t.Fatalf("breakdown: %+v err=%v", bd, err)
|
||||
}
|
||||
if l, err := d.RegistryBreakdownList(ctx, bf, bd.Rows[0].Key); err != nil || len(l) != 600 {
|
||||
t.Fatalf("breakdown list: len=%d err=%v", len(l), err)
|
||||
}
|
||||
if bd, err = d.RegistryBreakdown(ctx, RegistryFilter{Level: LevelEgress, Family: "https"}); err != nil || bd.Addresses != 100 || len(bd.Rows) != 6 {
|
||||
t.Fatalf("breakdown, newest cycle: %+v err=%v", bd, err)
|
||||
}
|
||||
registryDur := time.Since(start)
|
||||
|
||||
start = time.Now()
|
||||
|
||||
@@ -97,8 +97,8 @@ func TestRouteTableIsClassified(t *testing.T) {
|
||||
t.Fatalf("admin route %q is %s, want admin", rt.Pattern, rt.Access)
|
||||
}
|
||||
}
|
||||
if counts["admin"] != 41 || counts["agent"] != 5 || counts["open"] != 8 {
|
||||
t.Fatalf("access counts = %v, want admin=41 agent=5 open=8", counts)
|
||||
if counts["admin"] != 43 || counts["agent"] != 5 || counts["open"] != 8 {
|
||||
t.Fatalf("access counts = %v, want admin=43 agent=5 open=8", counts)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -80,6 +80,28 @@ type registryPageResponse struct {
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
Run int64 `json:"run"` // the run the results are read from, 0 = each address's newest cycle
|
||||
}
|
||||
|
||||
// registryBreakdownDTO is GET /admin/registry/breakdown: the checks of one
|
||||
// direction and protocol per target (group "target", egress) or site (group
|
||||
// "site", ingress). Key is what ".../breakdown/list?key=" takes, Label the
|
||||
// name to show; Total counts checks, OK the successful ones. Run is 0 for each
|
||||
// address's newest cycle.
|
||||
type registryBreakdownDTO struct {
|
||||
Group string `json:"group"`
|
||||
Direction string `json:"direction"`
|
||||
Protocol string `json:"protocol"`
|
||||
Run int64 `json:"run"`
|
||||
Addresses int `json:"addresses"`
|
||||
Rows []breakdownRowDTO `json:"rows"`
|
||||
}
|
||||
|
||||
type breakdownRowDTO struct {
|
||||
Key string `json:"key"`
|
||||
Label string `json:"label"`
|
||||
Total int `json:"total"`
|
||||
OK int `json:"ok"`
|
||||
}
|
||||
|
||||
// registryDTO is one row of the durable per-address registry — see
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -42,17 +43,29 @@ type subnetsDTO struct {
|
||||
Subnets []subnetDTO `json:"subnets"`
|
||||
}
|
||||
|
||||
// analyticsCache keeps the computed analysis of finalized runs. An entry is
|
||||
// analyticsCache keeps the computed analysis of finalized runs, per run and
|
||||
// subnet (the subnet narrows the report; "" is the whole run). An entry is
|
||||
// valid while the run's data version (checks written, results) is unchanged;
|
||||
// the subnet list is part of the key because it changes the grouping.
|
||||
// the subnet list is part of the version because it changes the grouping. At
|
||||
// most analyticsCacheMax entries are kept, the least recently used one goes
|
||||
// first, so trying many subnets does not grow the memory without limit.
|
||||
type analyticsCache struct {
|
||||
mu sync.Mutex
|
||||
entries map[int64]analyticsEntry
|
||||
entries map[analyticsKey]analyticsEntry
|
||||
clock uint64
|
||||
}
|
||||
|
||||
const analyticsCacheMax = 16
|
||||
|
||||
type analyticsKey struct {
|
||||
run int64
|
||||
subnet string
|
||||
}
|
||||
|
||||
type analyticsEntry struct {
|
||||
version string
|
||||
an *analytics.Analysis
|
||||
used uint64
|
||||
}
|
||||
|
||||
func (s *Server) handleAnalyticsRuns(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -72,21 +85,30 @@ func (s *Server) handleAnalyticsRuns(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// analysisFor returns the analysis of the run named by the {id} of the path;
|
||||
// see analysisByID.
|
||||
// analysisFor returns the analysis of the run named by the {id} of the path,
|
||||
// narrowed to the ?subnet= (a CIDR) when given; see analysisByID.
|
||||
func (s *Server) analysisFor(w http.ResponseWriter, r *http.Request) *analytics.Analysis {
|
||||
id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
|
||||
if err != nil || id <= 0 {
|
||||
writeError(w, http.StatusBadRequest, "invalid run id")
|
||||
return nil
|
||||
}
|
||||
return s.analysisByID(w, r, id)
|
||||
var subnet netip.Prefix
|
||||
if v := strings.TrimSpace(r.URL.Query().Get("subnet")); v != "" {
|
||||
if subnet, err = netip.ParsePrefix(v); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid subnet "+strconv.Quote(v)+" (a CIDR such as 203.0.113.0/24 is expected)")
|
||||
return nil
|
||||
}
|
||||
subnet = subnet.Masked()
|
||||
}
|
||||
return s.analysisByID(w, r, id, subnet)
|
||||
}
|
||||
|
||||
// analysisByID returns the analysis of a finalized run, from the cache when
|
||||
// the run's data has not changed since it was computed. It writes the error
|
||||
// response itself and returns nil when it cannot.
|
||||
func (s *Server) analysisByID(w http.ResponseWriter, r *http.Request, id int64) *analytics.Analysis {
|
||||
// analysisByID returns the analysis of a finalized run (of the addresses
|
||||
// inside subnet, unless it is the zero prefix), from the cache when the run's
|
||||
// data has not changed since it was computed. It writes the error response
|
||||
// itself and returns nil when it cannot.
|
||||
func (s *Server) analysisByID(w http.ResponseWriter, r *http.Request, id int64, subnet netip.Prefix) *analytics.Analysis {
|
||||
ctx := r.Context()
|
||||
run, err := s.DB.GetRun(ctx, id)
|
||||
if err != nil {
|
||||
@@ -113,23 +135,42 @@ func (s *Server) analysisByID(w http.ResponseWriter, r *http.Request, id int64)
|
||||
}
|
||||
version := data + "#" + sb.String()
|
||||
|
||||
key := analyticsKey{run: id}
|
||||
if subnet.IsValid() {
|
||||
key.subnet = subnet.String()
|
||||
}
|
||||
s.analytics.mu.Lock()
|
||||
defer s.analytics.mu.Unlock()
|
||||
if e, ok := s.analytics.entries[id]; ok && e.version == version {
|
||||
s.analytics.clock++
|
||||
if e, ok := s.analytics.entries[key]; ok && e.version == version {
|
||||
e.used = s.analytics.clock
|
||||
s.analytics.entries[key] = e
|
||||
return e.an
|
||||
}
|
||||
an, err := analytics.Load(ctx, s.DB, id)
|
||||
an, err := analytics.Load(ctx, s.DB, id, subnet)
|
||||
if err != nil {
|
||||
writeDBError(w, err)
|
||||
return nil
|
||||
}
|
||||
if s.analytics.entries == nil {
|
||||
s.analytics.entries = map[int64]analyticsEntry{}
|
||||
s.analytics.entries = map[analyticsKey]analyticsEntry{}
|
||||
}
|
||||
s.analytics.entries[id] = analyticsEntry{version: version, an: an}
|
||||
if _, ok := s.analytics.entries[key]; !ok && len(s.analytics.entries) >= analyticsCacheMax {
|
||||
var oldest analyticsKey
|
||||
least := ^uint64(0)
|
||||
for k, e := range s.analytics.entries {
|
||||
if e.used < least {
|
||||
oldest, least = k, e.used
|
||||
}
|
||||
}
|
||||
delete(s.analytics.entries, oldest)
|
||||
}
|
||||
s.analytics.entries[key] = analyticsEntry{version: version, an: an, used: s.analytics.clock}
|
||||
return an
|
||||
}
|
||||
|
||||
// handleAnalyticsRun serves the report of a finished run, narrowed to
|
||||
// ?subnet= (a CIDR) when given.
|
||||
func (s *Server) handleAnalyticsRun(w http.ResponseWriter, r *http.Request) {
|
||||
if an := s.analysisFor(w, r); an != nil {
|
||||
writeJSON(w, http.StatusOK, an.Report)
|
||||
@@ -138,7 +179,7 @@ func (s *Server) handleAnalyticsRun(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
var nonSlug = regexp.MustCompile(`[^a-z0-9]+`)
|
||||
|
||||
// handleAnalyticsList serves the address table behind one indicator
|
||||
// handleAnalyticsList serves the address table (of the ?subnet= when given) behind one indicator
|
||||
// (kind = egress_https_any|egress_https_all|ingress_ssh_any|ingress_ssh_all),
|
||||
// the addresses of one verdict (kind = verdict_pass|verdict_partial|verdict_fail)
|
||||
// or one ingress error class (kind = error, ?class=...), as JSON or, with
|
||||
@@ -201,11 +242,11 @@ func (s *Server) compareFor(w http.ResponseWriter, r *http.Request) (c *analytic
|
||||
writeError(w, http.StatusBadRequest, "base and target must be different runs")
|
||||
return nil, 0, 0
|
||||
}
|
||||
a := s.analysisByID(w, r, ids[0])
|
||||
a := s.analysisByID(w, r, ids[0], netip.Prefix{})
|
||||
if a == nil {
|
||||
return nil, 0, 0
|
||||
}
|
||||
b := s.analysisByID(w, r, ids[1])
|
||||
b := s.analysisByID(w, r, ids[1], netip.Prefix{})
|
||||
if b == nil {
|
||||
return nil, 0, 0
|
||||
}
|
||||
|
||||
@@ -3,8 +3,12 @@ package httpapi
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -355,6 +359,82 @@ func TestAnalyticsCacheFollowsData(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// ?subnet= narrows the report and the lists to the addresses inside it, a
|
||||
// malformed CIDR is a 400, and the cache keeps the subnets apart and bounded.
|
||||
func TestAnalyticsSubnetFilter(t *testing.T) {
|
||||
fc, d, _, _ := newConfigTestHarness(t)
|
||||
id := finishedRun(t, d) // 9.9.9.1 passes, 9.9.9.2 is partial
|
||||
base := "/api/v1/admin/analytics/runs/" + itoa64(id)
|
||||
report := func(subnet string) (addresses, partial int, scope string) {
|
||||
resp, body := fc.do(http.MethodGet, base+"?subnet="+url.QueryEscape(subnet), nil)
|
||||
var rep struct {
|
||||
Summary struct{ Addresses, Partial int } `json:"summary"`
|
||||
Scope *struct {
|
||||
Subnet string `json:"subnet"`
|
||||
RunAddresses int `json:"run_addresses"`
|
||||
} `json:"scope"`
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &rep) != nil {
|
||||
t.Fatalf("report %q: %d %s", subnet, resp.StatusCode, body)
|
||||
}
|
||||
if rep.Scope != nil {
|
||||
scope = fmt.Sprintf("%s/%d", rep.Scope.Subnet, rep.Scope.RunAddresses)
|
||||
}
|
||||
return rep.Summary.Addresses, rep.Summary.Partial, scope
|
||||
}
|
||||
for _, c := range []struct {
|
||||
subnet string
|
||||
addrs, partial int
|
||||
scope string
|
||||
}{
|
||||
{"9.9.9.2/32", 1, 1, "9.9.9.2/32/2"},
|
||||
{"9.9.9.1/32", 1, 0, "9.9.9.1/32/2"},
|
||||
{"9.9.9.0/24", 2, 1, "9.9.9.0/24/2"},
|
||||
{"9.9.9.77/24", 2, 1, "9.9.9.0/24/2"}, // host bits are masked
|
||||
{"10.0.0.0/8", 0, 0, "10.0.0.0/8/2"},
|
||||
{"", 2, 1, ""},
|
||||
{"9.9.9.2/32", 1, 1, "9.9.9.2/32/2"}, // the first subnet again: not mixed up with the others
|
||||
} {
|
||||
if a, p, sc := report(c.subnet); a != c.addrs || p != c.partial || sc != c.scope {
|
||||
t.Errorf("subnet %q: addresses %d, partial %d, scope %q; want %d, %d, %q", c.subnet, a, p, sc, c.addrs, c.partial, c.scope)
|
||||
}
|
||||
}
|
||||
|
||||
var l struct {
|
||||
Rows [][]string `json:"rows"`
|
||||
}
|
||||
resp, body := fc.do(http.MethodGet, base+"/lists/verdict_partial?subnet=9.9.9.2/32", nil)
|
||||
if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &l) != nil || len(l.Rows) != 1 || l.Rows[0][0] != "9.9.9.2" {
|
||||
t.Fatalf("list in the subnet: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
resp, body = fc.do(http.MethodGet, base+"/lists/verdict_partial?subnet=9.9.9.1/32", nil)
|
||||
if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &l) != nil || len(l.Rows) != 0 {
|
||||
t.Fatalf("list outside the subnet: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
resp, body = fc.do(http.MethodGet, base+"/lists/verdict_partial?format=csv&subnet=9.9.9.1/32", nil)
|
||||
if resp.StatusCode != http.StatusOK || strings.Contains(string(body), "9.9.9.2") {
|
||||
t.Fatalf("csv outside the subnet: %d %q", resp.StatusCode, body)
|
||||
}
|
||||
for _, path := range []string{base + "?subnet=nonsense", base + "/lists/verdict_pass?subnet=9.9.9.0", base + "/lists/verdict_pass?subnet=9.9.9.0/33"} {
|
||||
if resp, body := fc.do(http.MethodGet, path, nil); resp.StatusCode != http.StatusBadRequest {
|
||||
t.Errorf("%s: %d %s, want 400", path, resp.StatusCode, body)
|
||||
}
|
||||
}
|
||||
|
||||
// The cache holds a bounded number of entries.
|
||||
s := &Server{DB: d}
|
||||
for i := 0; i < analyticsCacheMax+5; i++ {
|
||||
rec := httptest.NewRecorder()
|
||||
subnet := netip.PrefixFrom(netip.AddrFrom4([4]byte{9, 9, byte(i), 0}), 24)
|
||||
if an := s.analysisByID(rec, httptest.NewRequest(http.MethodGet, "/", nil), id, subnet); an == nil {
|
||||
t.Fatalf("analysis of %s: %d %s", subnet, rec.Code, rec.Body)
|
||||
}
|
||||
}
|
||||
if n := len(s.analytics.entries); n != analyticsCacheMax {
|
||||
t.Errorf("cache entries = %d, want %d", n, analyticsCacheMax)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubnetsConfigEndpoints(t *testing.T) {
|
||||
fc, _, _, _ := newConfigTestHarness(t)
|
||||
resp, body := fc.do(http.MethodPut, "/api/v1/admin/config/subnets", subnetsDTO{Subnets: []subnetDTO{{CIDR: " 10.1.2.3/24 ", Label: "a"}, {CIDR: "10.0.0.0/8"}}})
|
||||
@@ -407,3 +487,63 @@ func TestRegistryRunAndSubnetFilters(t *testing.T) {
|
||||
}
|
||||
|
||||
func itoa64(n int64) string { return strconv.FormatInt(n, 10) }
|
||||
|
||||
// The breakdown endpoints: 400 without direction and protocol, the shape of the
|
||||
// chart rows (site names, run slice), the list behind a row as JSON and CSV, 404
|
||||
// for an unknown key.
|
||||
func TestRegistryBreakdownEndpoints(t *testing.T) {
|
||||
fc, d, _, _ := newConfigTestHarness(t)
|
||||
fc.do(http.MethodPut, "/api/v1/admin/config/sites/1", putSiteRequest{SiteID: "rxmsk"})
|
||||
id := finishedRun(t, d)
|
||||
const base = "/api/v1/admin/registry/breakdown"
|
||||
|
||||
for _, bad := range []string{"", "?direction=egress", "?protocol=ssh", "?direction=up&protocol=ssh", "?direction=egress&protocol=dns"} {
|
||||
if resp, _ := fc.do(http.MethodGet, base+bad, nil); resp.StatusCode != http.StatusBadRequest {
|
||||
t.Errorf("%q: %d, want 400", bad, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
if resp, _ := fc.do(http.MethodGet, base+"/list?direction=ingress&protocol=ssh", nil); resp.StatusCode != http.StatusBadRequest {
|
||||
t.Errorf("list without key: %d, want 400", resp.StatusCode)
|
||||
}
|
||||
|
||||
resp, body := fc.do(http.MethodGet, base+"?direction=ingress&protocol=ssh&run="+itoa64(id), nil)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("breakdown: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
var b registryBreakdownDTO
|
||||
if err := json.Unmarshal(body, &b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := registryBreakdownDTO{Group: "site", Direction: "ingress", Protocol: "ssh", Run: id, Addresses: 2,
|
||||
Rows: []breakdownRowDTO{{Key: "inbound-site-1", Label: "rxmsk", Total: 2, OK: 1}}}
|
||||
if !reflect.DeepEqual(b, want) {
|
||||
t.Errorf("breakdown = %+v, want %+v", b, want)
|
||||
}
|
||||
|
||||
resp, body = fc.do(http.MethodGet, base+"?direction=egress&protocol=https", nil)
|
||||
if err := json.Unmarshal(body, &b); err != nil || resp.StatusCode != http.StatusOK || b.Group != "target" ||
|
||||
len(b.Rows) != 1 || b.Rows[0].Key != "https://a.test" || b.Rows[0].Label != "a.test" || b.Rows[0].OK != 2 {
|
||||
t.Errorf("egress breakdown: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
listURL := base + "/list?direction=ingress&protocol=ssh&run=" + itoa64(id) + "&key=inbound-site-1"
|
||||
resp, body = fc.do(http.MethodGet, listURL, nil)
|
||||
var l struct {
|
||||
Columns []string `json:"columns"`
|
||||
Rows [][]string `json:"rows"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &l); err != nil || resp.StatusCode != http.StatusOK || len(l.Rows) != 2 || len(l.Rows[0]) != len(l.Columns) {
|
||||
t.Fatalf("list: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
if r := l.Rows[0]; r[0] != "9.9.9.2" || r[1] != "провал" || r[3] != "rxmsk" || r[5] != "dial tcp: i/o timeout" {
|
||||
t.Errorf("failure must come first: %v", r)
|
||||
}
|
||||
resp, body = fc.do(http.MethodGet, listURL+"&format=csv", nil)
|
||||
if resp.StatusCode != http.StatusOK || !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/csv") ||
|
||||
!strings.Contains(resp.Header.Get("Content-Disposition"), "registry_ingress_ssh_rxmsk.csv") || !strings.Contains(string(body), "9.9.9.2") {
|
||||
t.Errorf("csv: %d %v %s", resp.StatusCode, resp.Header, body)
|
||||
}
|
||||
if resp, _ := fc.do(http.MethodGet, base+"/list?direction=ingress&protocol=ssh&key=inbound-site-9", nil); resp.StatusCode != http.StatusNotFound {
|
||||
t.Errorf("unknown key: %d, want 404", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
@@ -1,21 +1,58 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"cloudipvalidator/internal/analytics"
|
||||
"cloudipvalidator/internal/db"
|
||||
)
|
||||
|
||||
// parseRegistryFilter reads the filter parameters of the registry endpoints
|
||||
// (q, last_result, run, subnet, direction, protocol). A non-empty message is
|
||||
// the reason a value is invalid.
|
||||
func parseRegistryFilter(q url.Values) (f db.RegistryFilter, msg string) {
|
||||
f = db.RegistryFilter{Query: strings.TrimSpace(q.Get("q")), LastResult: q.Get("last_result"), Subnet: strings.TrimSpace(q.Get("subnet"))}
|
||||
if f.Subnet != "" {
|
||||
if _, err := netip.ParsePrefix(f.Subnet); err != nil {
|
||||
return f, "invalid subnet " + strconv.Quote(f.Subnet) + " (a CIDR such as 203.0.113.0/24 is expected)"
|
||||
}
|
||||
}
|
||||
if v := q.Get("run"); v != "" {
|
||||
id, err := strconv.ParseInt(v, 10, 64)
|
||||
if err != nil || id <= 0 {
|
||||
return f, "invalid run " + strconv.Quote(v)
|
||||
}
|
||||
f.RunID = id
|
||||
}
|
||||
if f.LastResult != "" && !db.IsValidResult(f.LastResult) {
|
||||
return f, "invalid last_result " + strconv.Quote(f.LastResult) + " (valid: pass, partial, fail, cancelled)"
|
||||
}
|
||||
f.Level, f.Family = q.Get("direction"), q.Get("protocol")
|
||||
if f.Level != "" && !db.IsValidRegistryLevel(f.Level) {
|
||||
return f, "invalid direction " + strconv.Quote(f.Level) + " (valid: egress, ingress)"
|
||||
}
|
||||
if f.Family != "" && !db.IsValidRegistryFamily(f.Family) {
|
||||
return f, "invalid protocol " + strconv.Quote(f.Family) + " (valid: " + strings.Join(db.RegistryFamilies, ", ") + ")"
|
||||
}
|
||||
return f, ""
|
||||
}
|
||||
|
||||
// handleAdminRegistry lists every address ever submitted to the check
|
||||
// queue, each with a summary of its accumulated check history — the
|
||||
// durable record that survives an address being deleted from ip_queue and
|
||||
// later re-added. See migrations/0007_ip_registry.sql. Without `limit` it is
|
||||
// the bare array of every row; with `limit` (1..1000) it returns the envelope
|
||||
// {items,total,limit,offset} (filters: offset, q = substring of the address,
|
||||
// last_result = pass|partial|fail|cancelled).
|
||||
// {items,total,limit,offset,run} (filters: offset, q = substring of the
|
||||
// address, last_result = pass|partial|fail|cancelled, run, subnet, direction =
|
||||
// egress|ingress, protocol = icmp|tcp|ssh|https|tls). With `run` the result
|
||||
// fields are those of the address in that run, narrowed by direction/protocol;
|
||||
// see db.ListRegistryPage.
|
||||
func (s *Server) handleAdminRegistry(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
limit, offset, paged, err := parsePaging(q)
|
||||
@@ -23,23 +60,9 @@ func (s *Server) handleAdminRegistry(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
filter := db.RegistryFilter{Query: strings.TrimSpace(q.Get("q")), LastResult: q.Get("last_result"), Subnet: strings.TrimSpace(q.Get("subnet"))}
|
||||
if filter.Subnet != "" {
|
||||
if _, err := netip.ParsePrefix(filter.Subnet); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid subnet "+strconv.Quote(filter.Subnet)+" (a CIDR such as 203.0.113.0/24 is expected)")
|
||||
return
|
||||
}
|
||||
}
|
||||
if v := q.Get("run"); v != "" {
|
||||
id, err := strconv.ParseInt(v, 10, 64)
|
||||
if err != nil || id <= 0 {
|
||||
writeError(w, http.StatusBadRequest, "invalid run "+strconv.Quote(v))
|
||||
return
|
||||
}
|
||||
filter.RunID = id
|
||||
}
|
||||
if filter.LastResult != "" && !db.IsValidResult(filter.LastResult) {
|
||||
writeError(w, http.StatusBadRequest, "invalid last_result "+strconv.Quote(filter.LastResult)+" (valid: pass, partial, fail, cancelled)")
|
||||
filter, msg := parseRegistryFilter(q)
|
||||
if msg != "" {
|
||||
writeError(w, http.StatusBadRequest, msg)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -62,7 +85,7 @@ func (s *Server) handleAdminRegistry(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, registryPageResponse{Items: out, Total: total, Limit: limit, Offset: offset})
|
||||
writeJSON(w, http.StatusOK, registryPageResponse{Items: out, Total: total, Limit: limit, Offset: offset, Run: filter.RunID})
|
||||
}
|
||||
|
||||
// handleAdminRegistryHistory returns one address's registry record plus its
|
||||
@@ -117,3 +140,130 @@ func levelResultToDTO(l db.LevelResult) levelResultDTO {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// breakdownFor reads the filter of the breakdown endpoints, which need both a
|
||||
// direction and a protocol, and writes the 400 response itself when it cannot.
|
||||
func breakdownFor(w http.ResponseWriter, r *http.Request) (db.RegistryFilter, bool) {
|
||||
f, msg := parseRegistryFilter(r.URL.Query())
|
||||
if msg == "" && (f.Level == "" || f.Family == "") {
|
||||
msg = "direction and protocol are required"
|
||||
}
|
||||
if msg != "" {
|
||||
writeError(w, http.StatusBadRequest, msg)
|
||||
return f, false
|
||||
}
|
||||
return f, true
|
||||
}
|
||||
|
||||
// breakdownLabel is the name of a breakdown group: the target without the
|
||||
// scheme and the trailing "/" (egress), or the site name, "site-N" when the
|
||||
// site is no longer configured (ingress; key is the checks.source).
|
||||
func breakdownLabel(group, key string, sites map[int]string) string {
|
||||
if group == db.BreakdownTarget {
|
||||
if i := strings.Index(key, "://"); i >= 0 {
|
||||
key = key[i+3:]
|
||||
}
|
||||
return strings.TrimRight(key, "/")
|
||||
}
|
||||
idx, _ := strconv.Atoi(strings.TrimPrefix(key, "inbound-site-"))
|
||||
if n := sites[idx]; n != "" {
|
||||
return n
|
||||
}
|
||||
return "site-" + strconv.Itoa(idx)
|
||||
}
|
||||
|
||||
func (s *Server) siteNames(r *http.Request) (map[int]string, error) {
|
||||
sites, err := s.DB.ListSites(r.Context())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := make(map[int]string, len(sites))
|
||||
for _, x := range sites {
|
||||
names[x.Index] = x.SiteID
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// handleAdminRegistryBreakdown counts the checks of one direction and protocol
|
||||
// per target (egress) or site (ingress) over the addresses the registry filter
|
||||
// selects (same parameters as GET /admin/registry, direction and protocol
|
||||
// required), most successful first; see db.RegistryBreakdown.
|
||||
func (s *Server) handleAdminRegistryBreakdown(w http.ResponseWriter, r *http.Request) {
|
||||
f, ok := breakdownFor(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
b, err := s.DB.RegistryBreakdown(r.Context(), f)
|
||||
if err != nil {
|
||||
writeDBError(w, err)
|
||||
return
|
||||
}
|
||||
names, err := s.siteNames(r)
|
||||
if err != nil {
|
||||
writeDBError(w, err)
|
||||
return
|
||||
}
|
||||
out := registryBreakdownDTO{Group: b.Group, Direction: f.Level, Protocol: f.Family, Run: f.RunID, Addresses: b.Addresses,
|
||||
Rows: make([]breakdownRowDTO, len(b.Rows))}
|
||||
for i, x := range b.Rows {
|
||||
out.Rows[i] = breakdownRowDTO{Key: x.Key, Label: breakdownLabel(b.Group, x.Key, names), Total: x.Total, OK: x.OK}
|
||||
}
|
||||
sort.SliceStable(out.Rows, func(i, j int) bool {
|
||||
if out.Rows[i].OK != out.Rows[j].OK {
|
||||
return out.Rows[i].OK > out.Rows[j].OK
|
||||
}
|
||||
return out.Rows[i].Label < out.Rows[j].Label
|
||||
})
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// handleAdminRegistryBreakdownList serves the checks behind one row of the
|
||||
// breakdown (?key=, as in its rows) as a table, failures first, or with
|
||||
// ?format=csv as a downloadable CSV file. An unknown key is 404.
|
||||
func (s *Server) handleAdminRegistryBreakdownList(w http.ResponseWriter, r *http.Request) {
|
||||
f, ok := breakdownFor(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
key := r.URL.Query().Get("key")
|
||||
if key == "" {
|
||||
writeError(w, http.StatusBadRequest, "key is required")
|
||||
return
|
||||
}
|
||||
checks, err := s.DB.RegistryBreakdownList(r.Context(), f, key)
|
||||
if err != nil {
|
||||
writeDBError(w, err)
|
||||
return
|
||||
}
|
||||
names, err := s.siteNames(r)
|
||||
if err != nil {
|
||||
writeDBError(w, err)
|
||||
return
|
||||
}
|
||||
group := db.BreakdownTarget
|
||||
columns := []string{"Адрес", "Результат", "Тип проверки", "Цель", "Валидатор", "Задержка, мс", "Детали", "Проверено (UTC)"}
|
||||
if f.Level == db.LevelIngress {
|
||||
group = db.BreakdownSite
|
||||
columns = []string{"Адрес", "Результат", "Тип проверки", "Площадка", "Задержка, мс", "Детали", "Проверено (UTC)"}
|
||||
}
|
||||
rows := make([][]string, len(checks))
|
||||
for i, c := range checks {
|
||||
result := "провал"
|
||||
if c.Success {
|
||||
result = "успешно"
|
||||
}
|
||||
row := []string{c.IPAddress, result, c.CheckType, breakdownLabel(group, key, names)}
|
||||
if group == db.BreakdownTarget {
|
||||
row = append(row, analytics.ShortValidator(c.ValidatorID))
|
||||
}
|
||||
rows[i] = append(row, strconv.FormatInt(c.LatencyMS, 10), c.Detail, c.CheckedAt.UTC().Format("2006-01-02 15:04:05"))
|
||||
}
|
||||
if r.URL.Query().Get("format") == "csv" {
|
||||
writeCSV(w, columns, rows, fmt.Sprintf("registry_%s_%s_%s.csv", f.Level, f.Family, strings.Trim(nonSlug.ReplaceAllString(strings.ToLower(breakdownLabel(group, key, names)), "-"), "-")))
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, struct {
|
||||
Columns []string `json:"columns"`
|
||||
Rows [][]string `json:"rows"`
|
||||
}{columns, rows})
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -303,7 +304,21 @@ func TestAdminRegistryPaginationFiltersAndCompat(t *testing.T) {
|
||||
if p = page("limit=50&q=0.0.1"); p.Total != 4 { // 10.0.0.1, .10, .11, .12
|
||||
t.Fatalf("q: %+v", p)
|
||||
}
|
||||
for _, bad := range []string{"limit=0", "limit=1001", "offset=1", "limit=5&last_result=bogus", "limit=5&offset=-3"} {
|
||||
// run, subnet, direction and protocol together with the older filters; the
|
||||
// status is then that of the narrowed checks, and the run is echoed.
|
||||
runs, err := d.ListRuns(context.Background())
|
||||
if err != nil || len(runs) != 1 {
|
||||
t.Fatalf("runs: %+v %v", runs, err)
|
||||
}
|
||||
runQ := "limit=50&run=" + strconv.FormatInt(runs[0].ID, 10) + "&direction=egress&protocol="
|
||||
if p = page(runQ + "https&subnet=10.0.0.0/28&last_result=pass&q=10.0.0."); p.Total != 2 || p.Run != runs[0].ID || p.Items[0].Egress.Total != 1 || p.Items[0].Ingress.Total != 0 {
|
||||
t.Fatalf("run+subnet+direction+protocol+status: %+v", p)
|
||||
}
|
||||
if p = page(runQ + "tls"); p.Total != 0 {
|
||||
t.Fatalf("tls on egress: %+v", p)
|
||||
}
|
||||
for _, bad := range []string{"limit=0", "limit=1001", "offset=1", "limit=5&last_result=bogus", "limit=5&offset=-3",
|
||||
"limit=5&direction=sideways", "limit=5&protocol=udp", "limit=5&protocol=TCP"} {
|
||||
if resp, body := fc.do(http.MethodGet, "/api/v1/admin/registry?"+bad, nil); resp.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("%s: expected 400, got %d %s", bad, resp.StatusCode, body)
|
||||
}
|
||||
|
||||
@@ -64,6 +64,8 @@ func (s *Server) routeTable() []route {
|
||||
{"POST /api/v1/admin/auto-cycle/start", s.handleAdminStartAutoCycle, accessAdmin},
|
||||
{"POST /api/v1/admin/auto-cycle/stop", s.handleAdminStopAutoCycle, accessAdmin},
|
||||
{"GET /api/v1/admin/registry", s.handleAdminRegistry, accessAdmin},
|
||||
{"GET /api/v1/admin/registry/breakdown", s.handleAdminRegistryBreakdown, accessAdmin},
|
||||
{"GET /api/v1/admin/registry/breakdown/list", s.handleAdminRegistryBreakdownList, accessAdmin},
|
||||
{"GET /api/v1/admin/registry/{ip}", s.handleAdminRegistryHistory, accessAdmin},
|
||||
{"GET /api/v1/admin/analytics/runs", s.handleAnalyticsRuns, accessAdmin},
|
||||
{"GET /api/v1/admin/analytics/runs/{id}", s.handleAnalyticsRun, accessAdmin},
|
||||
|
||||
Reference in new issue
Block a user