Analytics: compare two finished runs

New page /analytics/compare and API GET /admin/analytics/compare (+ /lists/{group}):
the administrator picks an old (A) and a new (B) run; the report shows the new
addresses (only in B), the ones that left (only in A) and the common ones whose
membership in the seven indicators (pass, partial, fail, egress https any/all,
ingress ssh any/all) differs, with a "what changed" summary per address; the
dynamics of each indicator (delta = new - left + entered - exited) and a verdict
transition matrix. Every number opens a list with CSV. Cancelled addresses are not
part of a run. The list dialog moved to a shared analytics-dialog.js and template;
/analytics got a "compare with another run" button.

Docs, plan and summary in docs/changes/.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

internal/httpapi/auth_test.go

@@ -97,8 +97,8 @@ func TestRouteTableIsClassified(t *testing.T) {
 			t.Fatalf("admin route %q is %s, want admin", rt.Pattern, rt.Access)
 		}
 	}
-	if counts["admin"] != 39 || counts["agent"] != 5 || counts["open"] != 8 {
-		t.Fatalf("access counts = %v, want admin=39 agent=5 open=8", counts)
+	if counts["admin"] != 41 || counts["agent"] != 5 || counts["open"] != 8 {
+		t.Fatalf("access counts = %v, want admin=41 agent=5 open=8", counts)
 	}
 }
 

internal/httpapi/handlers_analytics.go

@@ -72,15 +72,21 @@ func (s *Server) handleAnalyticsRuns(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, out)
 }
 
-// analysisFor returns the analysis of a finalized run, from the cache when the
-// run's data has not changed since it was computed. It writes the error
-// response itself and returns nil when it cannot.
+// analysisFor returns the analysis of the run named by the {id} of the path;
+// see analysisByID.
 func (s *Server) analysisFor(w http.ResponseWriter, r *http.Request) *analytics.Analysis {
 	id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
 	if err != nil || id <= 0 {
 		writeError(w, http.StatusBadRequest, "invalid run id")
 		return nil
 	}
+	return s.analysisByID(w, r, id)
+}
+
+// analysisByID returns the analysis of a finalized run, from the cache when
+// the run's data has not changed since it was computed. It writes the error
+// response itself and returns nil when it cannot.
+func (s *Server) analysisByID(w http.ResponseWriter, r *http.Request, id int64) *analytics.Analysis {
 	ctx := r.Context()
 	run, err := s.DB.GetRun(ctx, id)
 	if err != nil {
@@ -152,12 +158,6 @@ func (s *Server) handleAnalyticsList(w http.ResponseWriter, r *http.Request) {
 		writeJSON(w, http.StatusOK, list)
 		return
 	}
-	var buf bytes.Buffer
-	buf.WriteString("\xef\xbb\xbf") // UTF-8 BOM, so Excel opens the file as UTF-8
-	cw := csv.NewWriter(&buf)
-	cw.UseCRLF = true
-	_ = cw.Write(list.Columns)
-	_ = cw.WriteAll(list.Rows)
 	name := kind
 	if kind == analytics.ListError {
 		if slug := strings.Trim(nonSlug.ReplaceAllString(strings.ToLower(class), "-"), "-"); slug != "" {
@@ -166,12 +166,91 @@ func (s *Server) handleAnalyticsList(w http.ResponseWriter, r *http.Request) {
 			name += "-class"
 		}
 	}
+	writeCSV(w, list.Columns, list.Rows, fmt.Sprintf("%s_run%s.csv", name, r.PathValue("id")))
+}
+
+// writeCSV sends a table as a downloadable CSV file.
+func writeCSV(w http.ResponseWriter, columns []string, rows [][]string, filename string) {
+	var buf bytes.Buffer
+	buf.WriteString("\xef\xbb\xbf") // UTF-8 BOM, so Excel opens the file as UTF-8
+	cw := csv.NewWriter(&buf)
+	cw.UseCRLF = true
+	_ = cw.Write(columns)
+	_ = cw.WriteAll(rows)
 	w.Header().Set("Content-Type", "text/csv; charset=utf-8")
-	w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s_run%s.csv"`, name, r.PathValue("id")))
+	w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename))
 	w.WriteHeader(http.StatusOK)
 	_, _ = w.Write(buf.Bytes())
 }
 
+// compareFor loads the two runs named by ?base=A&target=B (the older and the
+// newer one) and compares them. It writes the error response itself and
+// returns nil when it cannot: 400 for a missing or malformed id or the same
+// run twice, 404 for an unknown run, 409 for one that is still open.
+func (s *Server) compareFor(w http.ResponseWriter, r *http.Request) (c *analytics.Comparison, base, target int64) {
+	ids := [2]int64{}
+	for i, name := range []string{"base", "target"} {
+		id, err := strconv.ParseInt(r.URL.Query().Get(name), 10, 64)
+		if err != nil || id <= 0 {
+			writeError(w, http.StatusBadRequest, "invalid or missing "+name+" run id")
+			return nil, 0, 0
+		}
+		ids[i] = id
+	}
+	if ids[0] == ids[1] {
+		writeError(w, http.StatusBadRequest, "base and target must be different runs")
+		return nil, 0, 0
+	}
+	a := s.analysisByID(w, r, ids[0])
+	if a == nil {
+		return nil, 0, 0
+	}
+	b := s.analysisByID(w, r, ids[1])
+	if b == nil {
+		return nil, 0, 0
+	}
+	return analytics.Compare(a, b), ids[0], ids[1]
+}
+
+// handleAnalyticsCompare serves the comparison of two finished runs:
+// ?base=A (older) &target=B (newer).
+func (s *Server) handleAnalyticsCompare(w http.ResponseWriter, r *http.Request) {
+	if c, _, _ := s.compareFor(w, r); c != nil {
+		writeJSON(w, http.StatusOK, c)
+	}
+}
+
+// handleAnalyticsCompareList serves the address table of one group of the
+// comparison (new|left|common|changed|same|entered|exited), narrowed by
+// ?indicator=... and ?from=...&to=... (verdicts), as JSON or, with
+// ?format=csv, as a downloadable CSV file.
+func (s *Server) handleAnalyticsCompareList(w http.ResponseWriter, r *http.Request) {
+	c, base, target := s.compareFor(w, r)
+	if c == nil {
+		return
+	}
+	q := r.URL.Query()
+	group := r.PathValue("group")
+	f := analytics.CompareFilter{Indicator: q.Get("indicator"), From: q.Get("from"), To: q.Get("to")}
+	list, err := c.List(group, f)
+	if err != nil {
+		writeError(w, http.StatusNotFound, err.Error())
+		return
+	}
+	if q.Get("format") != "csv" {
+		writeJSON(w, http.StatusOK, list)
+		return
+	}
+	name := "compare_" + group
+	if f.Indicator != "" {
+		name += "_" + f.Indicator
+	}
+	if f.From != "" {
+		name += "_" + f.From + "-" + f.To
+	}
+	writeCSV(w, list.Columns, list.Rows, fmt.Sprintf("%s_run%d-%d.csv", name, base, target))
+}
+
 func (s *Server) handleConfigGetSubnets(w http.ResponseWriter, r *http.Request) {
 	list, err := s.DB.ListSubnets(r.Context())
 	if err != nil {

internal/httpapi/handlers_analytics_test.go

@@ -136,6 +136,178 @@ func TestAnalyticsEndpoints(t *testing.T) {
 	}
 }
 
+// secondRun builds the run after finishedRun's and returns its id: 9.9.9.1 is
+// not in it, 9.9.9.2 is checked again and passes now, 9.9.9.3 is new and has a
+// failed ssh. Against the first run: one new, one left, one changed address.
+func secondRun(t *testing.T, d *db.DB) int64 {
+	t.Helper()
+	ctx := context.Background()
+	if _, err := d.SubmitIPsAs(ctx, []string{"9.9.9.2", "9.9.9.3"}, db.RunManual); err != nil {
+		t.Fatal(err)
+	}
+	for _, addr := range []string{"9.9.9.2", "9.9.9.3"} {
+		ip, err := d.GetIPByAddress(ctx, addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := d.SetChecking(ctx, ip.ID, time.Minute); err != nil {
+			t.Fatal(err)
+		}
+		ip, _ = d.GetIP(ctx, ip.ID)
+		put := func(src, typ, target string, ok bool) {
+			if _, err := d.UpsertCheckIfOpen(ctx, db.Check{IPID: ip.ID, IPAddress: addr, AttemptNumber: ip.AttemptNumber,
+				ValidatorID: "vkiplab-v1", Source: src, CheckType: typ, Target: target, Success: ok, Detail: "dial tcp: i/o timeout", CheckedAt: db.Now()}); err != nil {
+				t.Fatal(err)
+			}
+		}
+		put(db.SourceEgress, "https", "https://a.test", true)
+		put(db.InboundSource(1), "icmp", addr, true)
+		sshOK := addr == "9.9.9.2"
+		put(db.InboundSource(1), "ssh", addr, sshOK)
+		verdict := db.ResultPass
+		if !sshOK {
+			verdict = db.ResultPartial
+		}
+		if err := d.FinishIPExpected(ctx, ip.ID, verdict, 3); err != nil {
+			t.Fatal(err)
+		}
+	}
+	rs, err := d.ListRuns(ctx)
+	if err != nil || len(rs) != 2 || rs[0].State != db.RunFinalized {
+		t.Fatalf("expected two finalized runs: %+v %v", rs, err)
+	}
+	return rs[0].ID
+}
+
+func TestAnalyticsCompareEndpoints(t *testing.T) {
+	fc, d, _, _ := newConfigTestHarness(t)
+	fc.do(http.MethodPut, "/api/v1/admin/config/sites/1", putSiteRequest{SiteID: "rxmsk"})
+	a := finishedRun(t, d)
+	b := secondRun(t, d)
+	cmp := "/api/v1/admin/analytics/compare"
+	q := "?base=" + itoa64(a) + "&target=" + itoa64(b)
+
+	resp, body := fc.do(http.MethodGet, cmp+q, nil)
+	var rep struct {
+		Runs struct {
+			Base   struct{ ID, Addresses int64 } `json:"base"`
+			Target struct{ ID, Addresses int64 } `json:"target"`
+		} `json:"runs"`
+		Groups struct {
+			New, Left, Common, Changed, Same int
+		} `json:"groups"`
+		Indicators []struct {
+			Key string
+
+			Base, Target, Delta, New, Left, Entered, Exited int
+		} `json:"indicators"`
+		Transitions struct {
+			Matrix [][]int `json:"matrix"`
+		} `json:"transitions"`
+	}
+	if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &rep) != nil || rep.Runs.Base.ID != a || rep.Runs.Target.ID != b ||
+		rep.Runs.Base.Addresses != 2 || rep.Runs.Target.Addresses != 2 ||
+		rep.Groups.New != 1 || rep.Groups.Left != 1 || rep.Groups.Common != 1 || rep.Groups.Changed != 1 || rep.Groups.Same != 0 ||
+		len(rep.Indicators) != 7 || len(rep.Transitions.Matrix) != 3 || rep.Transitions.Matrix[1][0] != 1 {
+		t.Fatalf("compare: %d %s", resp.StatusCode, body)
+	}
+	for _, ind := range rep.Indicators {
+		if ind.Delta != ind.New-ind.Left+ind.Entered-ind.Exited || ind.Delta != ind.Target-ind.Base {
+			t.Errorf("indicator %s: %+v", ind.Key, ind)
+		}
+		if ind.Key == "ingress_ssh_any" && (ind.Base != 1 || ind.Target != 1 || ind.New != 1 || ind.Exited != 1) {
+			t.Errorf("ssh any: %+v", ind)
+		}
+	}
+
+	// Lists: JSON, then CSV with BOM and the name of the file.
+	var l struct {
+		Group   string     `json:"group"`
+		Columns []string   `json:"columns"`
+		Rows    [][]string `json:"rows"`
+	}
+	resp, body = fc.do(http.MethodGet, cmp+"/lists/changed"+q, nil)
+	if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &l) != nil || l.Group != "changed" || len(l.Rows) != 1 ||
+		l.Rows[0][0] != "9.9.9.2" || !strings.HasPrefix(l.Rows[0][2], "partial → pass") {
+		t.Fatalf("changed list: %d %s", resp.StatusCode, body)
+	}
+	resp, body = fc.do(http.MethodGet, cmp+"/lists/new"+q+"&indicator=verdict_partial", nil)
+	if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &l) != nil || len(l.Rows) != 1 || l.Rows[0][0] != "9.9.9.3" {
+		t.Fatalf("new list: %d %s", resp.StatusCode, body)
+	}
+	resp, body = fc.do(http.MethodGet, cmp+"/lists/left"+q+"&indicator=verdict_pass", nil)
+	if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &l) != nil || len(l.Rows) != 1 || l.Rows[0][0] != "9.9.9.1" {
+		t.Fatalf("left list: %d %s", resp.StatusCode, body)
+	}
+	resp, body = fc.do(http.MethodGet, cmp+"/lists/common"+q+"&from=partial&to=pass", nil)
+	if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &l) != nil || len(l.Rows) != 1 || l.Rows[0][0] != "9.9.9.2" {
+		t.Fatalf("matrix cell list: %d %s", resp.StatusCode, body)
+	}
+	resp, body = fc.do(http.MethodGet, cmp+"/lists/exited"+q+"&indicator=ingress_ssh_any", nil)
+	if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &l) != nil || len(l.Rows) != 1 || l.Rows[0][0] != "9.9.9.2" {
+		t.Fatalf("exited list: %d %s", resp.StatusCode, body)
+	}
+	resp, body = fc.do(http.MethodGet, cmp+"/lists/changed"+q+"&format=csv", nil)
+	if resp.StatusCode != http.StatusOK || !strings.HasPrefix(string(body), "\xef\xbb\xbf") || !strings.Contains(string(body), "9.9.9.2") ||
+		!strings.HasPrefix(resp.Header.Get("Content-Type"), "text/csv") ||
+		!strings.Contains(resp.Header.Get("Content-Disposition"), `attachment; filename="compare_changed_run`+itoa64(a)+"-"+itoa64(b)+`.csv"`) {
+		t.Fatalf("csv: %d %v %q", resp.StatusCode, resp.Header, body)
+	}
+	resp, body = fc.do(http.MethodGet, cmp+"/lists/new"+q+"&indicator=verdict_partial&format=csv", nil)
+	if !strings.Contains(resp.Header.Get("Content-Disposition"), `filename="compare_new_verdict_partial_run`+itoa64(a)+"-"+itoa64(b)+`.csv"`) {
+		t.Fatalf("csv with an indicator: %d %v %q", resp.StatusCode, resp.Header, body)
+	}
+	resp, body = fc.do(http.MethodGet, cmp+"/lists/common"+q+"&from=partial&to=pass&format=csv", nil)
+	if !strings.Contains(resp.Header.Get("Content-Disposition"), `filename="compare_common_partial-pass_run`+itoa64(a)+"-"+itoa64(b)+`.csv"`) {
+		t.Fatalf("csv with verdicts: %d %v %q", resp.StatusCode, resp.Header, body)
+	}
+
+	// The order is the caller's: swapped, the new address is the one that left.
+	swapped := "?base=" + itoa64(b) + "&target=" + itoa64(a)
+	resp, body = fc.do(http.MethodGet, cmp+"/lists/new"+swapped, nil)
+	if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &l) != nil || len(l.Rows) != 1 || l.Rows[0][0] != "9.9.9.1" {
+		t.Fatalf("swapped: %d %s", resp.StatusCode, body)
+	}
+
+	// An open run cannot be compared.
+	if _, err := d.SubmitIPs(context.Background(), []string{"9.9.9.9"}); err != nil {
+		t.Fatal(err)
+	}
+	rs, _ := d.ListRuns(context.Background())
+	open := itoa64(rs[0].ID)
+	if rs[0].State == db.RunFinalized {
+		t.Fatalf("expected an open run: %+v", rs[0])
+	}
+
+	ida, idb := itoa64(a), itoa64(b)
+	for _, c := range []struct {
+		path string
+		want int
+	}{
+		{cmp, http.StatusBadRequest}, // no ids
+		{cmp + "?base=" + ida, http.StatusBadRequest},
+		{cmp + "?target=" + ida, http.StatusBadRequest},
+		{cmp + "?base=abc&target=" + ida, http.StatusBadRequest},
+		{cmp + "?base=0&target=" + ida, http.StatusBadRequest},
+		{cmp + "?base=" + ida + "&target=" + ida, http.StatusBadRequest}, // the same run twice
+		{cmp + "/lists/changed?base=" + ida + "&target=" + ida, http.StatusBadRequest},
+		{cmp + "/lists/changed", http.StatusBadRequest},
+		{cmp + "?base=9999&target=" + ida, http.StatusNotFound},
+		{cmp + "?base=" + ida + "&target=9999", http.StatusNotFound},
+		{cmp + "?base=" + ida + "&target=" + open, http.StatusConflict},
+		{cmp + "/lists/changed?base=" + open + "&target=" + idb, http.StatusConflict},
+		{cmp + "/lists/nonsense" + q, http.StatusNotFound},
+		{cmp + "/lists/new" + q + "&indicator=nonsense", http.StatusNotFound},
+		{cmp + "/lists/entered" + q, http.StatusNotFound}, // an indicator is required
+		{cmp + "/lists/common" + q + "&from=pass", http.StatusNotFound},
+		{cmp + "/lists/common" + q + "&from=pass&to=cancelled", http.StatusNotFound},
+	} {
+		if resp, body := fc.do(http.MethodGet, c.path, nil); resp.StatusCode != c.want {
+			t.Errorf("%s: %d %s, want %d", c.path, resp.StatusCode, body, c.want)
+		}
+	}
+}
+
 // An open run has no analytics yet.
 func TestAnalyticsOfOpenRunIsRefused(t *testing.T) {
 	fc, d, _, _ := newConfigTestHarness(t)

internal/httpapi/routes.go

@@ -68,6 +68,8 @@ func (s *Server) routeTable() []route {
 		{"GET /api/v1/admin/analytics/runs", s.handleAnalyticsRuns, accessAdmin},
 		{"GET /api/v1/admin/analytics/runs/{id}", s.handleAnalyticsRun, accessAdmin},
 		{"GET /api/v1/admin/analytics/runs/{id}/lists/{kind}", s.handleAnalyticsList, accessAdmin},
+		{"GET /api/v1/admin/analytics/compare", s.handleAnalyticsCompare, accessAdmin},
+		{"GET /api/v1/admin/analytics/compare/lists/{group}", s.handleAnalyticsCompareList, accessAdmin},
 		{"GET /api/v1/admin/config/subnets", s.handleConfigGetSubnets, accessAdmin},
 		{"PUT /api/v1/admin/config/subnets", s.handleConfigPutSubnets, accessAdmin},
 		{"GET /api/v1/admin/config/validators", s.handleConfigListValidators, accessAdmin},