Analytics: compare two finished runs
New page /analytics/compare and API GET /admin/analytics/compare (+ /lists/{group}):
the administrator picks an old (A) and a new (B) run; the report shows the new
addresses (only in B), the ones that left (only in A) and the common ones whose
membership in the seven indicators (pass, partial, fail, egress https any/all,
ingress ssh any/all) differs, with a "what changed" summary per address; the
dynamics of each indicator (delta = new - left + entered - exited) and a verdict
transition matrix. Every number opens a list with CSV. Cancelled addresses are not
part of a run. The list dialog moved to a shared analytics-dialog.js and template;
/analytics got a "compare with another run" button.
Docs, plan and summary in docs/changes/.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ayurishchevandClaude Sonnet 5.5 committed
1 parent
2f038f8362
commit
068c10ea1c
28 files changed
+2057
-138
No files matched your search
@@ -0,0 +1,335 @@
|
||||
package analytics
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"cloudipvalidator/internal/db"
|
||||
)
|
||||
|
||||
// set stores the four checks of an address: https to a.test and b.test, ssh
|
||||
// from the two sites.
|
||||
func (f *fixture) set(reg int64, validator string, a, b, ssh1, ssh2 bool) {
|
||||
f.check(reg, eg, "https", "https://a.test", a, validator, "", false)
|
||||
f.check(reg, eg, "https", "https://b.test", b, validator, "", false)
|
||||
f.check(reg, s1, "ssh", "ip", ssh1, validator, "dial tcp: i/o timeout", false)
|
||||
f.check(reg, s2, "ssh", "ip", ssh2, validator, "dial tcp: i/o timeout", false)
|
||||
}
|
||||
|
||||
// comparedRuns builds the older and the newer run:
|
||||
//
|
||||
// 10.0.0.1 pass -> pass, nothing changed
|
||||
// 10.0.0.2 partial -> pass: https recovered on both targets, another validator
|
||||
// 10.0.0.3 pass -> partial: ssh fails from rxmsk
|
||||
// 10.0.0.20 partial -> partial: another https target fails, same indicators
|
||||
// 10.0.0.10 only in the old run (fail)
|
||||
// 10.0.0.14 only in the old run (pass), cancelled in the new one
|
||||
// 10.0.0.11 only in the new run (partial, https fails everywhere)
|
||||
// 10.0.0.12 only in the new run (pass)
|
||||
// 10.0.0.13 only in the new run (pass), cancelled in the old one
|
||||
func comparedRuns(t *testing.T) (base, target *Analysis) {
|
||||
t.Helper()
|
||||
fa, fb := &fixture{}, &fixture{}
|
||||
fa.addr(1, "10.0.0.1", db.ResultPass, 4)
|
||||
fa.set(1, "vkiplab-v1", true, true, true, true)
|
||||
fa.addr(2, "10.0.0.2", db.ResultPartial, 4)
|
||||
fa.set(2, "vkiplab-v2", false, false, true, true)
|
||||
fa.addr(3, "10.0.0.3", db.ResultPass, 4)
|
||||
fa.set(3, "vkiplab-v3", true, true, true, true)
|
||||
fa.addr(4, "10.0.0.20", db.ResultPartial, 4)
|
||||
fa.set(4, "vkiplab-v4", false, true, true, true)
|
||||
fa.addr(5, "10.0.0.10", db.ResultFail, 4)
|
||||
fa.set(5, "vkiplab-v5", false, false, false, false)
|
||||
fa.addr(6, "10.0.0.13", db.ResultCancelled, 4)
|
||||
fa.set(6, "vkiplab-v6", true, true, true, true)
|
||||
fa.addr(7, "10.0.0.14", db.ResultPass, 4)
|
||||
fa.set(7, "vkiplab-v7", true, true, true, true)
|
||||
|
||||
fb.addr(1, "10.0.0.1", db.ResultPass, 4)
|
||||
fb.set(1, "vkiplab-v1", true, true, true, true)
|
||||
fb.addr(2, "10.0.0.2", db.ResultPass, 4)
|
||||
fb.set(2, "vkiplab-v5", true, true, true, true)
|
||||
fb.addr(3, "10.0.0.3", db.ResultPartial, 4)
|
||||
fb.set(3, "vkiplab-v3", true, true, false, true)
|
||||
fb.addr(4, "10.0.0.20", db.ResultPartial, 4)
|
||||
fb.set(4, "vkiplab-v4", true, false, true, true)
|
||||
fb.addr(8, "10.0.0.11", db.ResultPartial, 4)
|
||||
fb.set(8, "vkiplab-v6", false, false, true, true)
|
||||
fb.addr(9, "10.0.0.12", db.ResultPass, 4)
|
||||
fb.set(9, "vkiplab-v6", true, true, true, true)
|
||||
fb.addr(10, "10.0.0.13", db.ResultPass, 4)
|
||||
fb.set(10, "vkiplab-v6", true, true, true, true)
|
||||
fb.addr(11, "10.0.0.14", db.ResultCancelled, 4)
|
||||
fb.set(11, "vkiplab-v7", true, true, true, true)
|
||||
return fa.compute(t, nil), fb.compute(t, nil)
|
||||
}
|
||||
|
||||
func ips(l *CompareList) []string {
|
||||
out := []string{}
|
||||
for _, r := range l.Rows {
|
||||
out = append(out, r[0])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mustList(t *testing.T, c *Comparison, group string, f CompareFilter) *CompareList {
|
||||
t.Helper()
|
||||
l, err := c.List(group, f)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %+v: %v", group, f, err)
|
||||
}
|
||||
for _, r := range l.Rows {
|
||||
if len(r) != len(l.Columns) {
|
||||
t.Errorf("%s: row %v for columns %v", group, r, l.Columns)
|
||||
}
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
func TestCompareGroupsAndIndicators(t *testing.T) {
|
||||
base, target := comparedRuns(t)
|
||||
c := Compare(base, target)
|
||||
|
||||
if want := (CompareGroups{New: 3, Left: 2, Common: 4, Changed: 2, Same: 2}); c.Groups != want {
|
||||
t.Errorf("groups = %+v, want %+v", c.Groups, want)
|
||||
}
|
||||
// Every address is in exactly one of new, left, common; common = changed + same.
|
||||
if c.Groups.New+c.Groups.Common != target.Report.Summary.Addresses || c.Groups.Left+c.Groups.Common != base.Report.Summary.Addresses ||
|
||||
c.Groups.Changed+c.Groups.Same != c.Groups.Common {
|
||||
t.Errorf("groups do not add up: %+v, runs %d and %d addresses", c.Groups, base.Report.Summary.Addresses, target.Report.Summary.Addresses)
|
||||
}
|
||||
if c.Cancelled != (CompareCancel{Base: 1, Target: 1}) {
|
||||
t.Errorf("cancelled = %+v", c.Cancelled)
|
||||
}
|
||||
if c.Runs.Base.Addresses != 6 || c.Runs.Target.Addresses != 7 || c.Runs.Base.ID != 7 {
|
||||
t.Errorf("runs = %+v", c.Runs)
|
||||
}
|
||||
|
||||
// The count of each indicator in each run is the number of its card.
|
||||
sa, sb := base.Report.Summary, target.Report.Summary
|
||||
cards := map[string][2]int{
|
||||
ListVerdictPass: {sa.Pass, sb.Pass}, ListVerdictPartial: {sa.Partial, sb.Partial}, ListVerdictFail: {sa.Fail, sb.Fail},
|
||||
ListEgressHTTPSAny: {sa.EgressHTTPSAny, sb.EgressHTTPSAny}, ListEgressHTTPSAll: {sa.EgressHTTPSAll, sb.EgressHTTPSAll},
|
||||
ListIngressSSHAny: {sa.IngressSSHAny, sb.IngressSSHAny}, ListIngressSSHAll: {sa.IngressSSHAll, sb.IngressSSHAll},
|
||||
}
|
||||
if len(c.Indicators) != len(cards) {
|
||||
t.Fatalf("%d indicators", len(c.Indicators))
|
||||
}
|
||||
for _, d := range c.Indicators {
|
||||
if want := cards[d.Key]; d.Base != want[0] || d.Target != want[1] {
|
||||
t.Errorf("%s: %d -> %d, summaries say %v", d.Key, d.Base, d.Target, want)
|
||||
}
|
||||
if d.Delta != d.Target-d.Base || d.Delta != d.New-d.Left+d.Entered-d.Exited {
|
||||
t.Errorf("%s: delta %d, new %d left %d entered %d exited %d", d.Key, d.Delta, d.New, d.Left, d.Entered, d.Exited)
|
||||
}
|
||||
}
|
||||
|
||||
want := map[string]IndicatorDiff{
|
||||
ListVerdictPass: {Base: 3, Target: 4, Delta: 1, New: 2, Left: 1, Entered: 1, Exited: 1},
|
||||
ListVerdictPartial: {Base: 2, Target: 3, Delta: 1, New: 1, Entered: 1, Exited: 1},
|
||||
ListVerdictFail: {Base: 1, Target: 0, Delta: -1, Left: 1},
|
||||
ListEgressHTTPSAny: {Base: 3, Target: 2, Delta: -1, New: 1, Left: 1, Exited: 1},
|
||||
ListEgressHTTPSAll: {Base: 2, Target: 1, Delta: -1, New: 1, Left: 1, Exited: 1},
|
||||
ListIngressSSHAny: {Base: 1, Target: 1, Left: 1, Entered: 1},
|
||||
ListIngressSSHAll: {Base: 1, Target: 0, Delta: -1, Left: 1},
|
||||
}
|
||||
for _, d := range c.Indicators {
|
||||
w := want[d.Key]
|
||||
w.Key, w.Name = d.Key, d.Name
|
||||
if d != w || d.Name == "" {
|
||||
t.Errorf("%s = %+v, want %+v", d.Key, d, w)
|
||||
}
|
||||
}
|
||||
|
||||
// Verdicts of the common addresses, and of the new and of the left ones.
|
||||
tr := c.Transitions
|
||||
if !reflect.DeepEqual(tr.Verdicts, []string{"pass", "partial", "fail"}) ||
|
||||
!reflect.DeepEqual(tr.Matrix, [][]int{{1, 1, 0}, {1, 1, 0}, {0, 0, 0}}) ||
|
||||
!reflect.DeepEqual(tr.New, []int{2, 1, 0}) || !reflect.DeepEqual(tr.Left, []int{1, 0, 1}) {
|
||||
t.Errorf("transitions = %+v", tr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareLists(t *testing.T) {
|
||||
base, target := comparedRuns(t)
|
||||
c := Compare(base, target)
|
||||
|
||||
// New and left: the state in the run the address is in; numeric order.
|
||||
l := mustList(t, c, GroupNew, CompareFilter{})
|
||||
if want := []string{"Адрес", "Подсеть", "Вердикт", "Egress", "Ingress", "Индикаторы"}; !reflect.DeepEqual(l.Columns, want) {
|
||||
t.Errorf("new columns: %v", l.Columns)
|
||||
}
|
||||
if got := ips(l); !reflect.DeepEqual(got, []string{"10.0.0.11", "10.0.0.12", "10.0.0.13"}) {
|
||||
t.Errorf("new: %v", got)
|
||||
}
|
||||
if want := []string{"10.0.0.11", "10.0.0.0/24", "partial", "0 из 2", "2 из 2", "partial, Egress https: есть провалы, Egress https: все провалены"}; !reflect.DeepEqual(l.Rows[0], want) {
|
||||
t.Errorf("new row: %v", l.Rows[0])
|
||||
}
|
||||
if got := ips(mustList(t, c, GroupNew, CompareFilter{Indicator: ListVerdictPass})); !reflect.DeepEqual(got, []string{"10.0.0.12", "10.0.0.13"}) {
|
||||
t.Errorf("new, pass: %v", got)
|
||||
}
|
||||
if got := ips(mustList(t, c, GroupNew, CompareFilter{Indicator: ListEgressHTTPSAll})); !reflect.DeepEqual(got, []string{"10.0.0.11"}) {
|
||||
t.Errorf("new, https all: %v", got)
|
||||
}
|
||||
l = mustList(t, c, GroupLeft, CompareFilter{})
|
||||
if got := ips(l); !reflect.DeepEqual(got, []string{"10.0.0.10", "10.0.0.14"}) {
|
||||
t.Errorf("left: %v", got)
|
||||
}
|
||||
if want := "fail"; l.Rows[0][2] != want || l.Rows[0][5] != "fail, Egress https: есть провалы, Egress https: все провалены, Ingress ssh: есть провалы, Ingress ssh: все провалены" {
|
||||
t.Errorf("left row: %v", l.Rows[0])
|
||||
}
|
||||
if got := ips(mustList(t, c, GroupLeft, CompareFilter{Indicator: ListVerdictPass})); !reflect.DeepEqual(got, []string{"10.0.0.14"}) {
|
||||
t.Errorf("left, pass: %v", got)
|
||||
}
|
||||
|
||||
// Changed: what changed, step by step.
|
||||
l = mustList(t, c, GroupChanged, CompareFilter{})
|
||||
if got := ips(l); !reflect.DeepEqual(got, []string{"10.0.0.2", "10.0.0.3"}) {
|
||||
t.Fatalf("changed: %v", got)
|
||||
}
|
||||
if l.Columns[2] != "Вердикт (A → B)" || l.Columns[5] != "Что изменилось" {
|
||||
t.Errorf("changed columns: %v", l.Columns)
|
||||
}
|
||||
if want := []string{"10.0.0.2", "10.0.0.0/24", "partial → pass", "0 из 2 → 2 из 2", "2 из 2 → 2 из 2",
|
||||
"вердикт partial → pass; вышел из: Egress https: есть провалы, Egress https: все провалены; https: провалены цели −a.test −b.test; валидатор v2 → v5"}; !reflect.DeepEqual(l.Rows[0], want) {
|
||||
t.Errorf("changed row 1: %v", l.Rows[0])
|
||||
}
|
||||
if want := []string{"10.0.0.3", "10.0.0.0/24", "pass → partial", "2 из 2 → 2 из 2", "2 из 2 → 1 из 2",
|
||||
"вердикт pass → partial; вошёл в: Ingress ssh: есть провалы; ssh: площадки +rxmsk"}; !reflect.DeepEqual(l.Rows[1], want) {
|
||||
t.Errorf("changed row 2: %v", l.Rows[1])
|
||||
}
|
||||
// A filter by indicator keeps the addresses that are in it in either run.
|
||||
if got := ips(mustList(t, c, GroupChanged, CompareFilter{Indicator: ListIngressSSHAny})); !reflect.DeepEqual(got, []string{"10.0.0.3"}) {
|
||||
t.Errorf("changed, ssh any: %v", got)
|
||||
}
|
||||
if got := ips(mustList(t, c, GroupChanged, CompareFilter{Indicator: ListEgressHTTPSAll})); !reflect.DeepEqual(got, []string{"10.0.0.2"}) {
|
||||
t.Errorf("changed, https all: %v", got)
|
||||
}
|
||||
|
||||
// Same: another failed target does not change the indicators; numeric order (20 after 1).
|
||||
l = mustList(t, c, GroupSame, CompareFilter{})
|
||||
if got := ips(l); !reflect.DeepEqual(got, []string{"10.0.0.1", "10.0.0.20"}) {
|
||||
t.Fatalf("same: %v", got)
|
||||
}
|
||||
if want := []string{"10.0.0.20", "10.0.0.0/24", "partial → partial", "1 из 2 → 1 из 2", "2 из 2 → 2 из 2", "без изменений"}; !reflect.DeepEqual(l.Rows[1], want) {
|
||||
t.Errorf("same row: %v", l.Rows[1])
|
||||
}
|
||||
if got := ips(mustList(t, c, GroupSame, CompareFilter{Indicator: ListEgressHTTPSAny})); !reflect.DeepEqual(got, []string{"10.0.0.20"}) {
|
||||
t.Errorf("same, https any: %v", got)
|
||||
}
|
||||
|
||||
// Common is changed + same, in address order.
|
||||
l = mustList(t, c, GroupCommon, CompareFilter{})
|
||||
if got := ips(l); !reflect.DeepEqual(got, []string{"10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.20"}) || l.Rows[3][5] != "без изменений" {
|
||||
t.Errorf("common: %v", l.Rows)
|
||||
}
|
||||
|
||||
// Entered and exited need an indicator.
|
||||
for _, x := range []struct {
|
||||
group, ind string
|
||||
want []string
|
||||
}{
|
||||
{GroupEntered, ListVerdictPass, []string{"10.0.0.2"}},
|
||||
{GroupExited, ListVerdictPass, []string{"10.0.0.3"}},
|
||||
{GroupEntered, ListVerdictPartial, []string{"10.0.0.3"}},
|
||||
{GroupEntered, ListIngressSSHAny, []string{"10.0.0.3"}},
|
||||
{GroupExited, ListEgressHTTPSAll, []string{"10.0.0.2"}},
|
||||
{GroupExited, ListEgressHTTPSAny, []string{"10.0.0.2"}},
|
||||
{GroupEntered, ListEgressHTTPSAny, []string{}},
|
||||
{GroupEntered, ListVerdictFail, []string{}},
|
||||
} {
|
||||
got := ips(mustList(t, c, x.group, CompareFilter{Indicator: x.ind}))
|
||||
if !reflect.DeepEqual(got, x.want) {
|
||||
t.Errorf("%s %s: %v, want %v", x.group, x.ind, got, x.want)
|
||||
}
|
||||
}
|
||||
|
||||
// A cell of the transition matrix: the verdict in the old and in the new run.
|
||||
for _, x := range []struct {
|
||||
group, from, to string
|
||||
want []string
|
||||
}{
|
||||
{GroupCommon, "partial", "pass", []string{"10.0.0.2"}},
|
||||
{GroupCommon, "pass", "partial", []string{"10.0.0.3"}},
|
||||
{GroupCommon, "pass", "pass", []string{"10.0.0.1"}},
|
||||
{GroupCommon, "partial", "partial", []string{"10.0.0.20"}},
|
||||
{GroupSame, "partial", "partial", []string{"10.0.0.20"}},
|
||||
{GroupChanged, "pass", "pass", []string{}},
|
||||
{GroupCommon, "fail", "pass", []string{}},
|
||||
} {
|
||||
got := ips(mustList(t, c, x.group, CompareFilter{From: x.from, To: x.to}))
|
||||
if !reflect.DeepEqual(got, x.want) {
|
||||
t.Errorf("%s %s -> %s: %v, want %v", x.group, x.from, x.to, got, x.want)
|
||||
}
|
||||
}
|
||||
// The cells of the matrix and the lists add up.
|
||||
n := 0
|
||||
for _, from := range verdicts {
|
||||
for _, to := range verdicts {
|
||||
n += len(mustList(t, c, GroupCommon, CompareFilter{From: from, To: to}).Rows)
|
||||
}
|
||||
}
|
||||
if n != c.Groups.Common {
|
||||
t.Errorf("matrix cells hold %d addresses, %d are common", n, c.Groups.Common)
|
||||
}
|
||||
// The size of every list equals its number in the report.
|
||||
for _, d := range c.Indicators {
|
||||
for group, want := range map[string]int{GroupNew: d.New, GroupLeft: d.Left, GroupEntered: d.Entered, GroupExited: d.Exited} {
|
||||
if got := len(mustList(t, c, group, CompareFilter{Indicator: d.Key}).Rows); got != want {
|
||||
t.Errorf("%s %s: %d rows, report says %d", group, d.Key, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
for group, want := range map[string]int{GroupNew: c.Groups.New, GroupLeft: c.Groups.Left, GroupCommon: c.Groups.Common, GroupChanged: c.Groups.Changed, GroupSame: c.Groups.Same} {
|
||||
if got := len(mustList(t, c, group, CompareFilter{}).Rows); got != want {
|
||||
t.Errorf("%s: %d rows, report says %d", group, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareListErrors(t *testing.T) {
|
||||
base, target := comparedRuns(t)
|
||||
c := Compare(base, target)
|
||||
for name, x := range map[string]struct {
|
||||
group string
|
||||
f CompareFilter
|
||||
}{
|
||||
"unknown group": {"nonsense", CompareFilter{}},
|
||||
"unknown indicator": {GroupNew, CompareFilter{Indicator: "nonsense"}},
|
||||
"entered needs one": {GroupEntered, CompareFilter{}},
|
||||
"exited needs one": {GroupExited, CompareFilter{}},
|
||||
"from without to": {GroupCommon, CompareFilter{From: "pass"}},
|
||||
"to without from": {GroupCommon, CompareFilter{To: "pass"}},
|
||||
"unknown verdict": {GroupCommon, CompareFilter{From: "pass", To: "cancelled"}},
|
||||
"verdicts of new addrs": {GroupNew, CompareFilter{From: "pass", To: "pass"}},
|
||||
"verdicts of left addrs": {GroupLeft, CompareFilter{From: "pass", To: "pass"}},
|
||||
} {
|
||||
l, err := c.List(x.group, x.f)
|
||||
if _, ok := err.(ErrUnknownList); !ok || l != nil {
|
||||
t.Errorf("%s: %v %v", name, l, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A run compared with itself: nothing new, nothing left, nothing changed, and
|
||||
// the empty lists are empty tables, not nil.
|
||||
func TestCompareWithItself(t *testing.T) {
|
||||
base, _ := comparedRuns(t)
|
||||
c := Compare(base, base)
|
||||
if want := (CompareGroups{Common: 6, Same: 6}); c.Groups != want {
|
||||
t.Errorf("groups = %+v", c.Groups)
|
||||
}
|
||||
for _, d := range c.Indicators {
|
||||
if d.Delta != 0 || d.New != 0 || d.Left != 0 || d.Entered != 0 || d.Exited != 0 || d.Base != d.Target {
|
||||
t.Errorf("%+v", d)
|
||||
}
|
||||
}
|
||||
for _, group := range []string{GroupNew, GroupLeft, GroupChanged} {
|
||||
if l := mustList(t, c, group, CompareFilter{}); l.Rows == nil || len(l.Rows) != 0 {
|
||||
t.Errorf("%s: %#v", group, l.Rows)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(c.Transitions.Matrix, [][]int{{3, 0, 0}, {0, 2, 0}, {0, 0, 1}}) {
|
||||
t.Errorf("matrix = %v", c.Transitions.Matrix)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user