Analytics: compare two finished runs

New page /analytics/compare and API GET /admin/analytics/compare (+ /lists/{group}):
the administrator picks an old (A) and a new (B) run; the report shows the new
addresses (only in B), the ones that left (only in A) and the common ones whose
membership in the seven indicators (pass, partial, fail, egress https any/all,
ingress ssh any/all) differs, with a "what changed" summary per address; the
dynamics of each indicator (delta = new - left + entered - exited) and a verdict
transition matrix. Every number opens a list with CSV. Cancelled addresses are not
part of a run. The list dialog moved to a shared analytics-dialog.js and template;
/analytics got a "compare with another run" button.

Docs, plan and summary in docs/changes/.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

internal/analytics/compare.go

@@ -0,0 +1,432 @@
+package analytics
+
+import (
+	"sort"
+	"strings"
+
+	"cloudipvalidator/internal/db"
+)
+
+// indicator is one of the seven counters of the analytics page. Its key is the
+// name of the list behind the counter; has repeats the condition Compute counts
+// it by (an address with a cancelled result is never asked).
+type indicator struct {
+	key, name string
+	has       func(a *addr) bool
+}
+
+var indicators = []indicator{
+	{ListVerdictPass, "pass", func(a *addr) bool { return a.res.Verdict == db.ResultPass }},
+	{ListVerdictPartial, "partial", func(a *addr) bool { return a.res.Verdict == db.ResultPartial }},
+	{ListVerdictFail, "fail", func(a *addr) bool { return a.res.Verdict == db.ResultFail }},
+	{ListEgressHTTPSAny, "Egress https: есть провалы", func(a *addr) bool { return a.https.n > 0 && a.https.ok < a.https.n }},
+	{ListEgressHTTPSAll, "Egress https: все провалены", func(a *addr) bool { return a.https.n > 0 && a.https.ok == 0 }},
+	{ListIngressSSHAny, "Ingress ssh: есть провалы", func(a *addr) bool { return a.ssh.n > 0 && a.ssh.ok < a.ssh.n }},
+	{ListIngressSSHAll, "Ingress ssh: все провалены", func(a *addr) bool { return a.ssh.n > 0 && a.ssh.ok == 0 }},
+}
+
+// indicatorIndex is the position of an indicator key in the table, -1 if unknown.
+func indicatorIndex(key string) int {
+	for i, ind := range indicators {
+		if ind.key == key {
+			return i
+		}
+	}
+	return -1
+}
+
+// verdicts are the rows and columns of the transition matrix.
+var verdicts = []string{db.ResultPass, db.ResultPartial, db.ResultFail}
+
+func verdictIndex(v string) int {
+	for i, x := range verdicts {
+		if x == v {
+			return i
+		}
+	}
+	return -1
+}
+
+// membership is the set of indicators an address belongs to, one bit per
+// entry of the indicators table.
+func membership(a *addr) uint8 {
+	var m uint8
+	for i, ind := range indicators {
+		if ind.has(a) {
+			m |= 1 << i
+		}
+	}
+	return m
+}
+
+// Comparison is the difference between two finished runs: the base (older) and
+// the target (newer) one. An address is its IP; one with a cancelled result is
+// not in its run.
+type Comparison struct {
+	Runs        CompareRuns     `json:"runs"`
+	Groups      CompareGroups   `json:"groups"`
+	Indicators  []IndicatorDiff `json:"indicators"`
+	Transitions Transitions     `json:"transitions"`
+	Cancelled   CompareCancel   `json:"cancelled"`
+
+	rows   []*cmpRow // numeric address order
+	target *Analysis
+}
+
+type CompareRuns struct {
+	Base   CompareRun `json:"base"`
+	Target CompareRun `json:"target"`
+}
+
+type CompareRun struct {
+	RunInfo
+	Addresses int `json:"addresses"`
+}
+
+// CompareGroups counts the addresses by how they relate to the two runs:
+// New are only in the target, Left only in the base, Common in both, and
+// Common = Changed + Same.
+type CompareGroups struct {
+	New     int `json:"new"`
+	Left    int `json:"left"`
+	Common  int `json:"common"`
+	Changed int `json:"changed"`
+	Same    int `json:"same"`
+}
+
+// IndicatorDiff is one indicator in both runs. Delta = Target - Base =
+// New - Left + Entered - Exited.
+type IndicatorDiff struct {
+	Key     string `json:"key"`
+	Name    string `json:"name"`
+	Base    int    `json:"base"`
+	Target  int    `json:"target"`
+	Delta   int    `json:"delta"`
+	New     int    `json:"new"`     // new addresses that are in the indicator
+	Left    int    `json:"left"`    // left addresses that were in it
+	Entered int    `json:"entered"` // common addresses that entered it
+	Exited  int    `json:"exited"`  // common addresses that left it
+}
+
+// Transitions is the verdict of the common addresses: Matrix[from][to] with
+// the verdicts of the base on the rows and of the target on the columns. New
+// holds the new addresses by their verdict in the target, Left the addresses
+// that left by their verdict in the base.
+type Transitions struct {
+	Verdicts []string `json:"verdicts"`
+	Matrix   [][]int  `json:"matrix"`
+	New      []int    `json:"new"`
+	Left     []int    `json:"left"`
+}
+
+type CompareCancel struct {
+	Base   int `json:"base"`
+	Target int `json:"target"`
+}
+
+// cmpRow is one address with its state in each run (nil when absent) and the
+// indicators it belongs to there.
+type cmpRow struct {
+	ip     string
+	a, b   *addr
+	ma, mb uint8
+}
+
+func (r *cmpRow) common() bool  { return r.a != nil && r.b != nil }
+func (r *cmpRow) changed() bool { return r.common() && r.ma != r.mb }
+
+// Compare puts two analyses side by side; base is the older run, target the newer.
+func Compare(base, target *Analysis) *Comparison {
+	byIP := map[string]*cmpRow{}
+	var rows []*cmpRow
+	add := func(list []*addr, isBase bool) {
+		for _, x := range list {
+			r := byIP[x.res.IPAddress]
+			if r == nil {
+				r = &cmpRow{ip: x.res.IPAddress}
+				byIP[r.ip] = r
+				rows = append(rows, r)
+			}
+			if isBase {
+				r.a, r.ma = x, membership(x)
+			} else {
+				r.b, r.mb = x, membership(x)
+			}
+		}
+	}
+	add(base.sorted(), true)
+	add(target.sorted(), false)
+	sort.Slice(rows, func(i, j int) bool { return lessIP(rows[i].ip, rows[j].ip) })
+
+	c := &Comparison{rows: rows, target: target}
+	c.Runs = CompareRuns{
+		Base:   CompareRun{RunInfo: base.Report.Run, Addresses: base.Report.Summary.Addresses},
+		Target: CompareRun{RunInfo: target.Report.Run, Addresses: target.Report.Summary.Addresses},
+	}
+	c.Cancelled = CompareCancel{Base: base.Report.Summary.Cancelled, Target: target.Report.Summary.Cancelled}
+	c.Indicators = make([]IndicatorDiff, len(indicators))
+	for i, ind := range indicators {
+		c.Indicators[i] = IndicatorDiff{Key: ind.key, Name: ind.name}
+	}
+	tr := &c.Transitions
+	tr.Verdicts = verdicts
+	tr.New, tr.Left = make([]int, len(verdicts)), make([]int, len(verdicts))
+	tr.Matrix = make([][]int, len(verdicts))
+	for i := range tr.Matrix {
+		tr.Matrix[i] = make([]int, len(verdicts))
+	}
+
+	for _, r := range rows {
+		switch {
+		case r.a == nil:
+			c.Groups.New++
+			if v := verdictIndex(r.b.res.Verdict); v >= 0 {
+				tr.New[v]++
+			}
+		case r.b == nil:
+			c.Groups.Left++
+			if v := verdictIndex(r.a.res.Verdict); v >= 0 {
+				tr.Left[v]++
+			}
+		default:
+			c.Groups.Common++
+			if r.changed() {
+				c.Groups.Changed++
+			} else {
+				c.Groups.Same++
+			}
+			if from, to := verdictIndex(r.a.res.Verdict), verdictIndex(r.b.res.Verdict); from >= 0 && to >= 0 {
+				tr.Matrix[from][to]++
+			}
+		}
+		for i := range indicators {
+			bit := uint8(1) << i
+			inA, inB := r.ma&bit != 0, r.mb&bit != 0
+			d := &c.Indicators[i]
+			if inA {
+				d.Base++
+			}
+			if inB {
+				d.Target++
+			}
+			switch {
+			case r.a == nil && inB:
+				d.New++
+			case r.b == nil && inA:
+				d.Left++
+			case r.common() && !inA && inB:
+				d.Entered++
+			case r.common() && inA && !inB:
+				d.Exited++
+			}
+		}
+	}
+	for i := range c.Indicators {
+		c.Indicators[i].Delta = c.Indicators[i].Target - c.Indicators[i].Base
+	}
+	return c
+}
+
+// Groups served by Comparison.List.
+const (
+	GroupNew     = "new"
+	GroupLeft    = "left"
+	GroupCommon  = "common"
+	GroupChanged = "changed"
+	GroupSame    = "same"
+	GroupEntered = "entered"
+	GroupExited  = "exited"
+)
+
+// CompareFilter narrows a group. Indicator is a list key of the indicator
+// table: for new and left the address is in it in its own run, for common,
+// changed and same in either run, for entered and exited it is required.
+// From and To, only together, keep the common addresses whose verdict was From
+// in the base and is To in the target.
+type CompareFilter struct {
+	Indicator string
+	From, To  string
+}
+
+// CompareList is a table of addresses of one group.
+type CompareList struct {
+	Group     string     `json:"group"`
+	Indicator string     `json:"indicator,omitempty"`
+	Columns   []string   `json:"columns"`
+	Rows      [][]string `json:"rows"`
+}
+
+// List builds the table of a group; an unknown group, indicator or verdict, or
+// a filter that does not fit the group, is an ErrUnknownList.
+func (c *Comparison) List(group string, f CompareFilter) (*CompareList, error) {
+	switch group {
+	case GroupNew, GroupLeft, GroupCommon, GroupChanged, GroupSame, GroupEntered, GroupExited:
+	default:
+		return nil, ErrUnknownList(group)
+	}
+	var bit uint8
+	if f.Indicator != "" {
+		i := indicatorIndex(f.Indicator)
+		if i < 0 {
+			return nil, ErrUnknownList("indicator " + f.Indicator)
+		}
+		bit = 1 << i
+	}
+	if bit == 0 && (group == GroupEntered || group == GroupExited) {
+		return nil, ErrUnknownList(group + " without indicator")
+	}
+	if (f.From != "") != (f.To != "") {
+		return nil, ErrUnknownList("from without to")
+	}
+	if f.From != "" {
+		if verdictIndex(f.From) < 0 || verdictIndex(f.To) < 0 {
+			return nil, ErrUnknownList("verdict " + f.From + " → " + f.To)
+		}
+		if group == GroupNew || group == GroupLeft {
+			return nil, ErrUnknownList("from and to for " + group)
+		}
+	}
+
+	l := &CompareList{Group: group, Indicator: f.Indicator, Rows: [][]string{}}
+	if group == GroupNew || group == GroupLeft {
+		l.Columns = []string{"Адрес", "Подсеть", "Вердикт", "Egress", "Ingress", "Индикаторы"}
+	} else {
+		l.Columns = []string{"Адрес", "Подсеть", "Вердикт (A → B)", "Egress (A → B)", "Ingress (A → B)", "Что изменилось"}
+	}
+	for _, r := range c.rows {
+		var ok bool
+		switch group {
+		case GroupNew:
+			ok = r.a == nil && (bit == 0 || r.mb&bit != 0)
+		case GroupLeft:
+			ok = r.b == nil && (bit == 0 || r.ma&bit != 0)
+		case GroupCommon:
+			ok = r.common()
+		case GroupChanged:
+			ok = r.changed()
+		case GroupSame:
+			ok = r.common() && !r.changed()
+		case GroupEntered:
+			ok = r.common() && r.ma&bit == 0 && r.mb&bit != 0
+		case GroupExited:
+			ok = r.common() && r.ma&bit != 0 && r.mb&bit == 0
+		}
+		if !ok {
+			continue
+		}
+		if r.common() && bit != 0 && group != GroupEntered && group != GroupExited && (r.ma|r.mb)&bit == 0 {
+			continue
+		}
+		if f.From != "" && (r.a.res.Verdict != f.From || r.b.res.Verdict != f.To) {
+			continue
+		}
+		switch {
+		case r.a == nil:
+			l.Rows = append(l.Rows, []string{r.ip, r.b.subnet, r.b.res.Verdict, okOf(r.b.egress), okOf(r.b.ingress), indicatorsCell(r.mb)})
+		case r.b == nil:
+			l.Rows = append(l.Rows, []string{r.ip, r.a.subnet, r.a.res.Verdict, okOf(r.a.egress), okOf(r.a.ingress), indicatorsCell(r.ma)})
+		default:
+			l.Rows = append(l.Rows, []string{r.ip, r.b.subnet, arrow(r.a.res.Verdict, r.b.res.Verdict),
+				arrow(okOf(r.a.egress), okOf(r.b.egress)), arrow(okOf(r.a.ingress), okOf(r.b.ingress)), c.changeText(r)})
+		}
+	}
+	return l, nil
+}
+
+func arrow(from, to string) string { return from + " → " + to }
+
+// indicatorsCell names the indicators of a membership set, "—" for none.
+func indicatorsCell(m uint8) string {
+	var names []string
+	for i, ind := range indicators {
+		if m&(1<<i) != 0 {
+			names = append(names, ind.name)
+		}
+	}
+	if len(names) == 0 {
+		return "—"
+	}
+	return strings.Join(names, ", ")
+}
+
+// changeText says step by step what differs between the two runs for a common
+// address. An address with the same indicators is "без изменений" even if its
+// failed targets or sites differ.
+func (c *Comparison) changeText(r *cmpRow) string {
+	if r.ma == r.mb {
+		return "без изменений"
+	}
+	var steps []string
+	if r.a.res.Verdict != r.b.res.Verdict {
+		steps = append(steps, "вердикт "+arrow(r.a.res.Verdict, r.b.res.Verdict))
+	}
+	var in, out []string
+	for i, ind := range indicators {
+		if strings.HasPrefix(ind.key, "verdict_") {
+			continue // the verdict step says it
+		}
+		bit := uint8(1) << i
+		switch {
+		case r.ma&bit == 0 && r.mb&bit != 0:
+			in = append(in, ind.name)
+		case r.ma&bit != 0 && r.mb&bit == 0:
+			out = append(out, ind.name)
+		}
+	}
+	if len(in) > 0 {
+		steps = append(steps, "вошёл в: "+strings.Join(in, ", "))
+	}
+	if len(out) > 0 {
+		steps = append(steps, "вышел из: "+strings.Join(out, ", "))
+	}
+	if added, removed := setDiff(r.a.https.failedTargets, r.b.https.failedTargets, sortedStrings); len(added)+len(removed) > 0 {
+		steps = append(steps, "https: провалены цели "+signed(added, removed))
+	}
+	if added, removed := setDiff(r.a.ssh.sites, r.b.ssh.sites, c.target.sortSites); len(added)+len(removed) > 0 {
+		steps = append(steps, "ssh: площадки "+signed(added, removed))
+	}
+	if was, now := r.a.https.validator, r.b.https.validator; was != "" && now != "" && was != now {
+		steps = append(steps, "валидатор "+arrow(ShortValidator(was), ShortValidator(now)))
+	}
+	return strings.Join(steps, "; ")
+}
+
+// setDiff is what is in now and not in was, and the reverse, each ordered by order.
+func setDiff(was, now []string, order func([]string) []string) (added, removed []string) {
+	in := func(list []string) map[string]bool {
+		m := make(map[string]bool, len(list))
+		for _, s := range list {
+			m[s] = true
+		}
+		return m
+	}
+	w, n := in(was), in(now)
+	for s := range n {
+		if !w[s] {
+			added = append(added, s)
+		}
+	}
+	for s := range w {
+		if !n[s] {
+			removed = append(removed, s)
+		}
+	}
+	return order(added), order(removed)
+}
+
+func sortedStrings(s []string) []string {
+	sort.Strings(s)
+	return s
+}
+
+// signed writes a set difference as "+new1 +new2 −gone1".
+func signed(added, removed []string) string {
+	parts := make([]string, 0, len(added)+len(removed))
+	for _, s := range added {
+		parts = append(parts, "+"+s)
+	}
+	for _, s := range removed {
+		parts = append(parts, "−"+s)
+	}
+	return strings.Join(parts, " ")
+}

internal/analytics/compare_test.go

@@ -0,0 +1,335 @@
+package analytics
+
+import (
+	"reflect"
+	"testing"
+
+	"cloudipvalidator/internal/db"
+)
+
+// set stores the four checks of an address: https to a.test and b.test, ssh
+// from the two sites.
+func (f *fixture) set(reg int64, validator string, a, b, ssh1, ssh2 bool) {
+	f.check(reg, eg, "https", "https://a.test", a, validator, "", false)
+	f.check(reg, eg, "https", "https://b.test", b, validator, "", false)
+	f.check(reg, s1, "ssh", "ip", ssh1, validator, "dial tcp: i/o timeout", false)
+	f.check(reg, s2, "ssh", "ip", ssh2, validator, "dial tcp: i/o timeout", false)
+}
+
+// comparedRuns builds the older and the newer run:
+//
+//	10.0.0.1   pass -> pass, nothing changed
+//	10.0.0.2   partial -> pass: https recovered on both targets, another validator
+//	10.0.0.3   pass -> partial: ssh fails from rxmsk
+//	10.0.0.20  partial -> partial: another https target fails, same indicators
+//	10.0.0.10  only in the old run (fail)
+//	10.0.0.14  only in the old run (pass), cancelled in the new one
+//	10.0.0.11  only in the new run (partial, https fails everywhere)
+//	10.0.0.12  only in the new run (pass)
+//	10.0.0.13  only in the new run (pass), cancelled in the old one
+func comparedRuns(t *testing.T) (base, target *Analysis) {
+	t.Helper()
+	fa, fb := &fixture{}, &fixture{}
+	fa.addr(1, "10.0.0.1", db.ResultPass, 4)
+	fa.set(1, "vkiplab-v1", true, true, true, true)
+	fa.addr(2, "10.0.0.2", db.ResultPartial, 4)
+	fa.set(2, "vkiplab-v2", false, false, true, true)
+	fa.addr(3, "10.0.0.3", db.ResultPass, 4)
+	fa.set(3, "vkiplab-v3", true, true, true, true)
+	fa.addr(4, "10.0.0.20", db.ResultPartial, 4)
+	fa.set(4, "vkiplab-v4", false, true, true, true)
+	fa.addr(5, "10.0.0.10", db.ResultFail, 4)
+	fa.set(5, "vkiplab-v5", false, false, false, false)
+	fa.addr(6, "10.0.0.13", db.ResultCancelled, 4)
+	fa.set(6, "vkiplab-v6", true, true, true, true)
+	fa.addr(7, "10.0.0.14", db.ResultPass, 4)
+	fa.set(7, "vkiplab-v7", true, true, true, true)
+
+	fb.addr(1, "10.0.0.1", db.ResultPass, 4)
+	fb.set(1, "vkiplab-v1", true, true, true, true)
+	fb.addr(2, "10.0.0.2", db.ResultPass, 4)
+	fb.set(2, "vkiplab-v5", true, true, true, true)
+	fb.addr(3, "10.0.0.3", db.ResultPartial, 4)
+	fb.set(3, "vkiplab-v3", true, true, false, true)
+	fb.addr(4, "10.0.0.20", db.ResultPartial, 4)
+	fb.set(4, "vkiplab-v4", true, false, true, true)
+	fb.addr(8, "10.0.0.11", db.ResultPartial, 4)
+	fb.set(8, "vkiplab-v6", false, false, true, true)
+	fb.addr(9, "10.0.0.12", db.ResultPass, 4)
+	fb.set(9, "vkiplab-v6", true, true, true, true)
+	fb.addr(10, "10.0.0.13", db.ResultPass, 4)
+	fb.set(10, "vkiplab-v6", true, true, true, true)
+	fb.addr(11, "10.0.0.14", db.ResultCancelled, 4)
+	fb.set(11, "vkiplab-v7", true, true, true, true)
+	return fa.compute(t, nil), fb.compute(t, nil)
+}
+
+func ips(l *CompareList) []string {
+	out := []string{}
+	for _, r := range l.Rows {
+		out = append(out, r[0])
+	}
+	return out
+}
+
+func mustList(t *testing.T, c *Comparison, group string, f CompareFilter) *CompareList {
+	t.Helper()
+	l, err := c.List(group, f)
+	if err != nil {
+		t.Fatalf("%s %+v: %v", group, f, err)
+	}
+	for _, r := range l.Rows {
+		if len(r) != len(l.Columns) {
+			t.Errorf("%s: row %v for columns %v", group, r, l.Columns)
+		}
+	}
+	return l
+}
+
+func TestCompareGroupsAndIndicators(t *testing.T) {
+	base, target := comparedRuns(t)
+	c := Compare(base, target)
+
+	if want := (CompareGroups{New: 3, Left: 2, Common: 4, Changed: 2, Same: 2}); c.Groups != want {
+		t.Errorf("groups = %+v, want %+v", c.Groups, want)
+	}
+	// Every address is in exactly one of new, left, common; common = changed + same.
+	if c.Groups.New+c.Groups.Common != target.Report.Summary.Addresses || c.Groups.Left+c.Groups.Common != base.Report.Summary.Addresses ||
+		c.Groups.Changed+c.Groups.Same != c.Groups.Common {
+		t.Errorf("groups do not add up: %+v, runs %d and %d addresses", c.Groups, base.Report.Summary.Addresses, target.Report.Summary.Addresses)
+	}
+	if c.Cancelled != (CompareCancel{Base: 1, Target: 1}) {
+		t.Errorf("cancelled = %+v", c.Cancelled)
+	}
+	if c.Runs.Base.Addresses != 6 || c.Runs.Target.Addresses != 7 || c.Runs.Base.ID != 7 {
+		t.Errorf("runs = %+v", c.Runs)
+	}
+
+	// The count of each indicator in each run is the number of its card.
+	sa, sb := base.Report.Summary, target.Report.Summary
+	cards := map[string][2]int{
+		ListVerdictPass: {sa.Pass, sb.Pass}, ListVerdictPartial: {sa.Partial, sb.Partial}, ListVerdictFail: {sa.Fail, sb.Fail},
+		ListEgressHTTPSAny: {sa.EgressHTTPSAny, sb.EgressHTTPSAny}, ListEgressHTTPSAll: {sa.EgressHTTPSAll, sb.EgressHTTPSAll},
+		ListIngressSSHAny: {sa.IngressSSHAny, sb.IngressSSHAny}, ListIngressSSHAll: {sa.IngressSSHAll, sb.IngressSSHAll},
+	}
+	if len(c.Indicators) != len(cards) {
+		t.Fatalf("%d indicators", len(c.Indicators))
+	}
+	for _, d := range c.Indicators {
+		if want := cards[d.Key]; d.Base != want[0] || d.Target != want[1] {
+			t.Errorf("%s: %d -> %d, summaries say %v", d.Key, d.Base, d.Target, want)
+		}
+		if d.Delta != d.Target-d.Base || d.Delta != d.New-d.Left+d.Entered-d.Exited {
+			t.Errorf("%s: delta %d, new %d left %d entered %d exited %d", d.Key, d.Delta, d.New, d.Left, d.Entered, d.Exited)
+		}
+	}
+
+	want := map[string]IndicatorDiff{
+		ListVerdictPass:    {Base: 3, Target: 4, Delta: 1, New: 2, Left: 1, Entered: 1, Exited: 1},
+		ListVerdictPartial: {Base: 2, Target: 3, Delta: 1, New: 1, Entered: 1, Exited: 1},
+		ListVerdictFail:    {Base: 1, Target: 0, Delta: -1, Left: 1},
+		ListEgressHTTPSAny: {Base: 3, Target: 2, Delta: -1, New: 1, Left: 1, Exited: 1},
+		ListEgressHTTPSAll: {Base: 2, Target: 1, Delta: -1, New: 1, Left: 1, Exited: 1},
+		ListIngressSSHAny:  {Base: 1, Target: 1, Left: 1, Entered: 1},
+		ListIngressSSHAll:  {Base: 1, Target: 0, Delta: -1, Left: 1},
+	}
+	for _, d := range c.Indicators {
+		w := want[d.Key]
+		w.Key, w.Name = d.Key, d.Name
+		if d != w || d.Name == "" {
+			t.Errorf("%s = %+v, want %+v", d.Key, d, w)
+		}
+	}
+
+	// Verdicts of the common addresses, and of the new and of the left ones.
+	tr := c.Transitions
+	if !reflect.DeepEqual(tr.Verdicts, []string{"pass", "partial", "fail"}) ||
+		!reflect.DeepEqual(tr.Matrix, [][]int{{1, 1, 0}, {1, 1, 0}, {0, 0, 0}}) ||
+		!reflect.DeepEqual(tr.New, []int{2, 1, 0}) || !reflect.DeepEqual(tr.Left, []int{1, 0, 1}) {
+		t.Errorf("transitions = %+v", tr)
+	}
+}
+
+func TestCompareLists(t *testing.T) {
+	base, target := comparedRuns(t)
+	c := Compare(base, target)
+
+	// New and left: the state in the run the address is in; numeric order.
+	l := mustList(t, c, GroupNew, CompareFilter{})
+	if want := []string{"Адрес", "Подсеть", "Вердикт", "Egress", "Ingress", "Индикаторы"}; !reflect.DeepEqual(l.Columns, want) {
+		t.Errorf("new columns: %v", l.Columns)
+	}
+	if got := ips(l); !reflect.DeepEqual(got, []string{"10.0.0.11", "10.0.0.12", "10.0.0.13"}) {
+		t.Errorf("new: %v", got)
+	}
+	if want := []string{"10.0.0.11", "10.0.0.0/24", "partial", "0 из 2", "2 из 2", "partial, Egress https: есть провалы, Egress https: все провалены"}; !reflect.DeepEqual(l.Rows[0], want) {
+		t.Errorf("new row: %v", l.Rows[0])
+	}
+	if got := ips(mustList(t, c, GroupNew, CompareFilter{Indicator: ListVerdictPass})); !reflect.DeepEqual(got, []string{"10.0.0.12", "10.0.0.13"}) {
+		t.Errorf("new, pass: %v", got)
+	}
+	if got := ips(mustList(t, c, GroupNew, CompareFilter{Indicator: ListEgressHTTPSAll})); !reflect.DeepEqual(got, []string{"10.0.0.11"}) {
+		t.Errorf("new, https all: %v", got)
+	}
+	l = mustList(t, c, GroupLeft, CompareFilter{})
+	if got := ips(l); !reflect.DeepEqual(got, []string{"10.0.0.10", "10.0.0.14"}) {
+		t.Errorf("left: %v", got)
+	}
+	if want := "fail"; l.Rows[0][2] != want || l.Rows[0][5] != "fail, Egress https: есть провалы, Egress https: все провалены, Ingress ssh: есть провалы, Ingress ssh: все провалены" {
+		t.Errorf("left row: %v", l.Rows[0])
+	}
+	if got := ips(mustList(t, c, GroupLeft, CompareFilter{Indicator: ListVerdictPass})); !reflect.DeepEqual(got, []string{"10.0.0.14"}) {
+		t.Errorf("left, pass: %v", got)
+	}
+
+	// Changed: what changed, step by step.
+	l = mustList(t, c, GroupChanged, CompareFilter{})
+	if got := ips(l); !reflect.DeepEqual(got, []string{"10.0.0.2", "10.0.0.3"}) {
+		t.Fatalf("changed: %v", got)
+	}
+	if l.Columns[2] != "Вердикт (A → B)" || l.Columns[5] != "Что изменилось" {
+		t.Errorf("changed columns: %v", l.Columns)
+	}
+	if want := []string{"10.0.0.2", "10.0.0.0/24", "partial → pass", "0 из 2 → 2 из 2", "2 из 2 → 2 из 2",
+		"вердикт partial → pass; вышел из: Egress https: есть провалы, Egress https: все провалены; https: провалены цели −a.test −b.test; валидатор v2 → v5"}; !reflect.DeepEqual(l.Rows[0], want) {
+		t.Errorf("changed row 1: %v", l.Rows[0])
+	}
+	if want := []string{"10.0.0.3", "10.0.0.0/24", "pass → partial", "2 из 2 → 2 из 2", "2 из 2 → 1 из 2",
+		"вердикт pass → partial; вошёл в: Ingress ssh: есть провалы; ssh: площадки +rxmsk"}; !reflect.DeepEqual(l.Rows[1], want) {
+		t.Errorf("changed row 2: %v", l.Rows[1])
+	}
+	// A filter by indicator keeps the addresses that are in it in either run.
+	if got := ips(mustList(t, c, GroupChanged, CompareFilter{Indicator: ListIngressSSHAny})); !reflect.DeepEqual(got, []string{"10.0.0.3"}) {
+		t.Errorf("changed, ssh any: %v", got)
+	}
+	if got := ips(mustList(t, c, GroupChanged, CompareFilter{Indicator: ListEgressHTTPSAll})); !reflect.DeepEqual(got, []string{"10.0.0.2"}) {
+		t.Errorf("changed, https all: %v", got)
+	}
+
+	// Same: another failed target does not change the indicators; numeric order (20 after 1).
+	l = mustList(t, c, GroupSame, CompareFilter{})
+	if got := ips(l); !reflect.DeepEqual(got, []string{"10.0.0.1", "10.0.0.20"}) {
+		t.Fatalf("same: %v", got)
+	}
+	if want := []string{"10.0.0.20", "10.0.0.0/24", "partial → partial", "1 из 2 → 1 из 2", "2 из 2 → 2 из 2", "без изменений"}; !reflect.DeepEqual(l.Rows[1], want) {
+		t.Errorf("same row: %v", l.Rows[1])
+	}
+	if got := ips(mustList(t, c, GroupSame, CompareFilter{Indicator: ListEgressHTTPSAny})); !reflect.DeepEqual(got, []string{"10.0.0.20"}) {
+		t.Errorf("same, https any: %v", got)
+	}
+
+	// Common is changed + same, in address order.
+	l = mustList(t, c, GroupCommon, CompareFilter{})
+	if got := ips(l); !reflect.DeepEqual(got, []string{"10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.20"}) || l.Rows[3][5] != "без изменений" {
+		t.Errorf("common: %v", l.Rows)
+	}
+
+	// Entered and exited need an indicator.
+	for _, x := range []struct {
+		group, ind string
+		want       []string
+	}{
+		{GroupEntered, ListVerdictPass, []string{"10.0.0.2"}},
+		{GroupExited, ListVerdictPass, []string{"10.0.0.3"}},
+		{GroupEntered, ListVerdictPartial, []string{"10.0.0.3"}},
+		{GroupEntered, ListIngressSSHAny, []string{"10.0.0.3"}},
+		{GroupExited, ListEgressHTTPSAll, []string{"10.0.0.2"}},
+		{GroupExited, ListEgressHTTPSAny, []string{"10.0.0.2"}},
+		{GroupEntered, ListEgressHTTPSAny, []string{}},
+		{GroupEntered, ListVerdictFail, []string{}},
+	} {
+		got := ips(mustList(t, c, x.group, CompareFilter{Indicator: x.ind}))
+		if !reflect.DeepEqual(got, x.want) {
+			t.Errorf("%s %s: %v, want %v", x.group, x.ind, got, x.want)
+		}
+	}
+
+	// A cell of the transition matrix: the verdict in the old and in the new run.
+	for _, x := range []struct {
+		group, from, to string
+		want            []string
+	}{
+		{GroupCommon, "partial", "pass", []string{"10.0.0.2"}},
+		{GroupCommon, "pass", "partial", []string{"10.0.0.3"}},
+		{GroupCommon, "pass", "pass", []string{"10.0.0.1"}},
+		{GroupCommon, "partial", "partial", []string{"10.0.0.20"}},
+		{GroupSame, "partial", "partial", []string{"10.0.0.20"}},
+		{GroupChanged, "pass", "pass", []string{}},
+		{GroupCommon, "fail", "pass", []string{}},
+	} {
+		got := ips(mustList(t, c, x.group, CompareFilter{From: x.from, To: x.to}))
+		if !reflect.DeepEqual(got, x.want) {
+			t.Errorf("%s %s -> %s: %v, want %v", x.group, x.from, x.to, got, x.want)
+		}
+	}
+	// The cells of the matrix and the lists add up.
+	n := 0
+	for _, from := range verdicts {
+		for _, to := range verdicts {
+			n += len(mustList(t, c, GroupCommon, CompareFilter{From: from, To: to}).Rows)
+		}
+	}
+	if n != c.Groups.Common {
+		t.Errorf("matrix cells hold %d addresses, %d are common", n, c.Groups.Common)
+	}
+	// The size of every list equals its number in the report.
+	for _, d := range c.Indicators {
+		for group, want := range map[string]int{GroupNew: d.New, GroupLeft: d.Left, GroupEntered: d.Entered, GroupExited: d.Exited} {
+			if got := len(mustList(t, c, group, CompareFilter{Indicator: d.Key}).Rows); got != want {
+				t.Errorf("%s %s: %d rows, report says %d", group, d.Key, got, want)
+			}
+		}
+	}
+	for group, want := range map[string]int{GroupNew: c.Groups.New, GroupLeft: c.Groups.Left, GroupCommon: c.Groups.Common, GroupChanged: c.Groups.Changed, GroupSame: c.Groups.Same} {
+		if got := len(mustList(t, c, group, CompareFilter{}).Rows); got != want {
+			t.Errorf("%s: %d rows, report says %d", group, got, want)
+		}
+	}
+}
+
+func TestCompareListErrors(t *testing.T) {
+	base, target := comparedRuns(t)
+	c := Compare(base, target)
+	for name, x := range map[string]struct {
+		group string
+		f     CompareFilter
+	}{
+		"unknown group":          {"nonsense", CompareFilter{}},
+		"unknown indicator":      {GroupNew, CompareFilter{Indicator: "nonsense"}},
+		"entered needs one":      {GroupEntered, CompareFilter{}},
+		"exited needs one":       {GroupExited, CompareFilter{}},
+		"from without to":        {GroupCommon, CompareFilter{From: "pass"}},
+		"to without from":        {GroupCommon, CompareFilter{To: "pass"}},
+		"unknown verdict":        {GroupCommon, CompareFilter{From: "pass", To: "cancelled"}},
+		"verdicts of new addrs":  {GroupNew, CompareFilter{From: "pass", To: "pass"}},
+		"verdicts of left addrs": {GroupLeft, CompareFilter{From: "pass", To: "pass"}},
+	} {
+		l, err := c.List(x.group, x.f)
+		if _, ok := err.(ErrUnknownList); !ok || l != nil {
+			t.Errorf("%s: %v %v", name, l, err)
+		}
+	}
+}
+
+// A run compared with itself: nothing new, nothing left, nothing changed, and
+// the empty lists are empty tables, not nil.
+func TestCompareWithItself(t *testing.T) {
+	base, _ := comparedRuns(t)
+	c := Compare(base, base)
+	if want := (CompareGroups{Common: 6, Same: 6}); c.Groups != want {
+		t.Errorf("groups = %+v", c.Groups)
+	}
+	for _, d := range c.Indicators {
+		if d.Delta != 0 || d.New != 0 || d.Left != 0 || d.Entered != 0 || d.Exited != 0 || d.Base != d.Target {
+			t.Errorf("%+v", d)
+		}
+	}
+	for _, group := range []string{GroupNew, GroupLeft, GroupChanged} {
+		if l := mustList(t, c, group, CompareFilter{}); l.Rows == nil || len(l.Rows) != 0 {
+			t.Errorf("%s: %#v", group, l.Rows)
+		}
+	}
+	if !reflect.DeepEqual(c.Transitions.Matrix, [][]int{{3, 0, 0}, {0, 2, 0}, {0, 0, 1}}) {
+		t.Errorf("matrix = %v", c.Transitions.Matrix)
+	}
+}

internal/analytics/lists.go

@@ -149,17 +149,20 @@ func (an *Analysis) sorted() []*addr {
 			out = append(out, a)
 		}
 	}
-	sort.Slice(out, func(i, j int) bool {
-		x, errX := netip.ParseAddr(out[i].res.IPAddress)
-		y, errY := netip.ParseAddr(out[j].res.IPAddress)
-		if errX != nil || errY != nil {
-			return out[i].res.IPAddress < out[j].res.IPAddress
-		}
-		return x.Less(y)
-	})
+	sort.Slice(out, func(i, j int) bool { return lessIP(out[i].res.IPAddress, out[j].res.IPAddress) })
 	return out
 }
 
+// lessIP orders addresses numerically; text that is not an address by its text.
+func lessIP(a, b string) bool {
+	x, errX := netip.ParseAddr(a)
+	y, errY := netip.ParseAddr(b)
+	if errX != nil || errY != nil {
+		return a < b
+	}
+	return x.Less(y)
+}
+
 func (an *Analysis) siteIndex(site string) int { return siteIndexOf(an.siteNames, site) }
 
 func (an *Analysis) sortSites(sites []string) []string {