Analytics: compare two finished runs
New page /analytics/compare and API GET /admin/analytics/compare (+ /lists/{group}):
the administrator picks an old (A) and a new (B) run; the report shows the new
addresses (only in B), the ones that left (only in A) and the common ones whose
membership in the seven indicators (pass, partial, fail, egress https any/all,
ingress ssh any/all) differs, with a "what changed" summary per address; the
dynamics of each indicator (delta = new - left + entered - exited) and a verdict
transition matrix. Every number opens a list with CSV. Cancelled addresses are not
part of a run. The list dialog moved to a shared analytics-dialog.js and template;
/analytics got a "compare with another run" button.
Docs, plan and summary in docs/changes/.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ayurishchevandClaude Sonnet 5.5 committed
1 parent
2f038f8362
commit
068c10ea1c
28 files changed
+2057
-138
No files matched your search
@@ -136,6 +136,178 @@ func TestAnalyticsEndpoints(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// secondRun builds the run after finishedRun's and returns its id: 9.9.9.1 is
|
||||
// not in it, 9.9.9.2 is checked again and passes now, 9.9.9.3 is new and has a
|
||||
// failed ssh. Against the first run: one new, one left, one changed address.
|
||||
func secondRun(t *testing.T, d *db.DB) int64 {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
if _, err := d.SubmitIPsAs(ctx, []string{"9.9.9.2", "9.9.9.3"}, db.RunManual); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, addr := range []string{"9.9.9.2", "9.9.9.3"} {
|
||||
ip, err := d.GetIPByAddress(ctx, addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.SetChecking(ctx, ip.ID, time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ip, _ = d.GetIP(ctx, ip.ID)
|
||||
put := func(src, typ, target string, ok bool) {
|
||||
if _, err := d.UpsertCheckIfOpen(ctx, db.Check{IPID: ip.ID, IPAddress: addr, AttemptNumber: ip.AttemptNumber,
|
||||
ValidatorID: "vkiplab-v1", Source: src, CheckType: typ, Target: target, Success: ok, Detail: "dial tcp: i/o timeout", CheckedAt: db.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put(db.SourceEgress, "https", "https://a.test", true)
|
||||
put(db.InboundSource(1), "icmp", addr, true)
|
||||
sshOK := addr == "9.9.9.2"
|
||||
put(db.InboundSource(1), "ssh", addr, sshOK)
|
||||
verdict := db.ResultPass
|
||||
if !sshOK {
|
||||
verdict = db.ResultPartial
|
||||
}
|
||||
if err := d.FinishIPExpected(ctx, ip.ID, verdict, 3); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
rs, err := d.ListRuns(ctx)
|
||||
if err != nil || len(rs) != 2 || rs[0].State != db.RunFinalized {
|
||||
t.Fatalf("expected two finalized runs: %+v %v", rs, err)
|
||||
}
|
||||
return rs[0].ID
|
||||
}
|
||||
|
||||
func TestAnalyticsCompareEndpoints(t *testing.T) {
|
||||
fc, d, _, _ := newConfigTestHarness(t)
|
||||
fc.do(http.MethodPut, "/api/v1/admin/config/sites/1", putSiteRequest{SiteID: "rxmsk"})
|
||||
a := finishedRun(t, d)
|
||||
b := secondRun(t, d)
|
||||
cmp := "/api/v1/admin/analytics/compare"
|
||||
q := "?base=" + itoa64(a) + "&target=" + itoa64(b)
|
||||
|
||||
resp, body := fc.do(http.MethodGet, cmp+q, nil)
|
||||
var rep struct {
|
||||
Runs struct {
|
||||
Base struct{ ID, Addresses int64 } `json:"base"`
|
||||
Target struct{ ID, Addresses int64 } `json:"target"`
|
||||
} `json:"runs"`
|
||||
Groups struct {
|
||||
New, Left, Common, Changed, Same int
|
||||
} `json:"groups"`
|
||||
Indicators []struct {
|
||||
Key string
|
||||
|
||||
Base, Target, Delta, New, Left, Entered, Exited int
|
||||
} `json:"indicators"`
|
||||
Transitions struct {
|
||||
Matrix [][]int `json:"matrix"`
|
||||
} `json:"transitions"`
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &rep) != nil || rep.Runs.Base.ID != a || rep.Runs.Target.ID != b ||
|
||||
rep.Runs.Base.Addresses != 2 || rep.Runs.Target.Addresses != 2 ||
|
||||
rep.Groups.New != 1 || rep.Groups.Left != 1 || rep.Groups.Common != 1 || rep.Groups.Changed != 1 || rep.Groups.Same != 0 ||
|
||||
len(rep.Indicators) != 7 || len(rep.Transitions.Matrix) != 3 || rep.Transitions.Matrix[1][0] != 1 {
|
||||
t.Fatalf("compare: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
for _, ind := range rep.Indicators {
|
||||
if ind.Delta != ind.New-ind.Left+ind.Entered-ind.Exited || ind.Delta != ind.Target-ind.Base {
|
||||
t.Errorf("indicator %s: %+v", ind.Key, ind)
|
||||
}
|
||||
if ind.Key == "ingress_ssh_any" && (ind.Base != 1 || ind.Target != 1 || ind.New != 1 || ind.Exited != 1) {
|
||||
t.Errorf("ssh any: %+v", ind)
|
||||
}
|
||||
}
|
||||
|
||||
// Lists: JSON, then CSV with BOM and the name of the file.
|
||||
var l struct {
|
||||
Group string `json:"group"`
|
||||
Columns []string `json:"columns"`
|
||||
Rows [][]string `json:"rows"`
|
||||
}
|
||||
resp, body = fc.do(http.MethodGet, cmp+"/lists/changed"+q, nil)
|
||||
if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &l) != nil || l.Group != "changed" || len(l.Rows) != 1 ||
|
||||
l.Rows[0][0] != "9.9.9.2" || !strings.HasPrefix(l.Rows[0][2], "partial → pass") {
|
||||
t.Fatalf("changed list: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
resp, body = fc.do(http.MethodGet, cmp+"/lists/new"+q+"&indicator=verdict_partial", nil)
|
||||
if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &l) != nil || len(l.Rows) != 1 || l.Rows[0][0] != "9.9.9.3" {
|
||||
t.Fatalf("new list: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
resp, body = fc.do(http.MethodGet, cmp+"/lists/left"+q+"&indicator=verdict_pass", nil)
|
||||
if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &l) != nil || len(l.Rows) != 1 || l.Rows[0][0] != "9.9.9.1" {
|
||||
t.Fatalf("left list: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
resp, body = fc.do(http.MethodGet, cmp+"/lists/common"+q+"&from=partial&to=pass", nil)
|
||||
if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &l) != nil || len(l.Rows) != 1 || l.Rows[0][0] != "9.9.9.2" {
|
||||
t.Fatalf("matrix cell list: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
resp, body = fc.do(http.MethodGet, cmp+"/lists/exited"+q+"&indicator=ingress_ssh_any", nil)
|
||||
if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &l) != nil || len(l.Rows) != 1 || l.Rows[0][0] != "9.9.9.2" {
|
||||
t.Fatalf("exited list: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
resp, body = fc.do(http.MethodGet, cmp+"/lists/changed"+q+"&format=csv", nil)
|
||||
if resp.StatusCode != http.StatusOK || !strings.HasPrefix(string(body), "\xef\xbb\xbf") || !strings.Contains(string(body), "9.9.9.2") ||
|
||||
!strings.HasPrefix(resp.Header.Get("Content-Type"), "text/csv") ||
|
||||
!strings.Contains(resp.Header.Get("Content-Disposition"), `attachment; filename="compare_changed_run`+itoa64(a)+"-"+itoa64(b)+`.csv"`) {
|
||||
t.Fatalf("csv: %d %v %q", resp.StatusCode, resp.Header, body)
|
||||
}
|
||||
resp, body = fc.do(http.MethodGet, cmp+"/lists/new"+q+"&indicator=verdict_partial&format=csv", nil)
|
||||
if !strings.Contains(resp.Header.Get("Content-Disposition"), `filename="compare_new_verdict_partial_run`+itoa64(a)+"-"+itoa64(b)+`.csv"`) {
|
||||
t.Fatalf("csv with an indicator: %d %v %q", resp.StatusCode, resp.Header, body)
|
||||
}
|
||||
resp, body = fc.do(http.MethodGet, cmp+"/lists/common"+q+"&from=partial&to=pass&format=csv", nil)
|
||||
if !strings.Contains(resp.Header.Get("Content-Disposition"), `filename="compare_common_partial-pass_run`+itoa64(a)+"-"+itoa64(b)+`.csv"`) {
|
||||
t.Fatalf("csv with verdicts: %d %v %q", resp.StatusCode, resp.Header, body)
|
||||
}
|
||||
|
||||
// The order is the caller's: swapped, the new address is the one that left.
|
||||
swapped := "?base=" + itoa64(b) + "&target=" + itoa64(a)
|
||||
resp, body = fc.do(http.MethodGet, cmp+"/lists/new"+swapped, nil)
|
||||
if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &l) != nil || len(l.Rows) != 1 || l.Rows[0][0] != "9.9.9.1" {
|
||||
t.Fatalf("swapped: %d %s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
// An open run cannot be compared.
|
||||
if _, err := d.SubmitIPs(context.Background(), []string{"9.9.9.9"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rs, _ := d.ListRuns(context.Background())
|
||||
open := itoa64(rs[0].ID)
|
||||
if rs[0].State == db.RunFinalized {
|
||||
t.Fatalf("expected an open run: %+v", rs[0])
|
||||
}
|
||||
|
||||
ida, idb := itoa64(a), itoa64(b)
|
||||
for _, c := range []struct {
|
||||
path string
|
||||
want int
|
||||
}{
|
||||
{cmp, http.StatusBadRequest}, // no ids
|
||||
{cmp + "?base=" + ida, http.StatusBadRequest},
|
||||
{cmp + "?target=" + ida, http.StatusBadRequest},
|
||||
{cmp + "?base=abc&target=" + ida, http.StatusBadRequest},
|
||||
{cmp + "?base=0&target=" + ida, http.StatusBadRequest},
|
||||
{cmp + "?base=" + ida + "&target=" + ida, http.StatusBadRequest}, // the same run twice
|
||||
{cmp + "/lists/changed?base=" + ida + "&target=" + ida, http.StatusBadRequest},
|
||||
{cmp + "/lists/changed", http.StatusBadRequest},
|
||||
{cmp + "?base=9999&target=" + ida, http.StatusNotFound},
|
||||
{cmp + "?base=" + ida + "&target=9999", http.StatusNotFound},
|
||||
{cmp + "?base=" + ida + "&target=" + open, http.StatusConflict},
|
||||
{cmp + "/lists/changed?base=" + open + "&target=" + idb, http.StatusConflict},
|
||||
{cmp + "/lists/nonsense" + q, http.StatusNotFound},
|
||||
{cmp + "/lists/new" + q + "&indicator=nonsense", http.StatusNotFound},
|
||||
{cmp + "/lists/entered" + q, http.StatusNotFound}, // an indicator is required
|
||||
{cmp + "/lists/common" + q + "&from=pass", http.StatusNotFound},
|
||||
{cmp + "/lists/common" + q + "&from=pass&to=cancelled", http.StatusNotFound},
|
||||
} {
|
||||
if resp, body := fc.do(http.MethodGet, c.path, nil); resp.StatusCode != c.want {
|
||||
t.Errorf("%s: %d %s, want %d", c.path, resp.StatusCode, body, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An open run has no analytics yet.
|
||||
func TestAnalyticsOfOpenRunIsRefused(t *testing.T) {
|
||||
fc, d, _, _ := newConfigTestHarness(t)
|
||||
|
||||
Reference in new issue
Block a user