Show egress/ingress levels in the registry; freeze checks at the verdict

Registry: the "last result" column now also shows, per level (egress,
ingress), how many of the recorded checks of the latest cycle succeeded, split
by check family (tcp-22 and tcp-443 are both "tcp"). One grouped query per
chunk of addresses; new fields last_cycle_id, egress, ingress in
GET /admin/registry; the dashboard renders them under the verdict.

Verdict integrity (migration 0010):
- the prober is handed an address once per site and attempt, not on every
  poll, so results are no longer overwritten by later probe rounds;
- UpsertCheckIfOpen refuses writes once the address is aggregating or has its
  verdict, or for an older attempt; senders get {"ok":true,"ignored":N} and a
  result_dropped event is recorded;
- the checking window counts from checking_started_at, not from assigned_at;
- checks.recorded_at (server clock) and checks.after_verdict (flag for rows
  written after the verdict in existing data);
- the verdict rule is a pure function (computeVerdict) and the aggregated
  event carries the egress/ingress check counts.

Rebuilt bin/control-api and bin/admin-dashboard to match. Plans and summaries
are in docs/changes; README, API, USAGE, DASHBOARD and DIAGRAMS are updated.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

internal/orchestrator/verdict_integrity_test.go

@@ -0,0 +1,127 @@
+package orchestrator
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"cloudipvalidator/internal/db"
+)
+
+func chk(src, ct string, ok bool) db.Check {
+	return db.Check{Source: src, CheckType: ct, Success: ok}
+}
+
+func TestComputeVerdict(t *testing.T) {
+	in1 := db.InboundSource(1)
+	cases := []struct {
+		name     string
+		checks   []db.Check
+		expected int
+		want     string
+		missing  int
+	}{
+		{"all pass, complete", []db.Check{chk(db.SourceEgress, "https", true), chk(in1, "icmp", true)}, 2, db.ResultPass, 0},
+		{"egress failure", []db.Check{chk(db.SourceEgress, "https", false), chk(in1, "icmp", true)}, 2, db.ResultPartial, 0},
+		{"ingress failure", []db.Check{chk(db.SourceEgress, "https", true), chk(in1, "ssh", false)}, 2, db.ResultPartial, 0},
+		{"all recorded pass but one missing", []db.Check{chk(db.SourceEgress, "https", true)}, 2, db.ResultPartial, 1},
+		{"all fail", []db.Check{chk(db.SourceEgress, "https", false), chk(in1, "icmp", false)}, 2, db.ResultFail, 0},
+		{"nothing recorded", nil, 2, db.ResultFail, 2},
+		{"more recorded than expected", []db.Check{chk(db.SourceEgress, "https", true), chk(in1, "icmp", true)}, 1, db.ResultPass, 0},
+	}
+	for _, c := range cases {
+		got, _, missing := computeVerdict(c.checks, c.expected, true)
+		if got != c.want || missing != c.missing {
+			t.Errorf("%s: got %s missing=%d, want %s missing=%d", c.name, got, missing, c.want, c.missing)
+		}
+	}
+}
+
+// The window starts when checking starts: an address handed out long ago but
+// that only just began checking must not be cut off.
+func TestAggregationWindowCountsFromCheckingStart(t *testing.T) {
+	ctx := context.Background()
+	o, _, _ := newTestOrchestrator(t, 180)
+	deadline := time.Now().Add(-120 * time.Second)
+	long, recent := time.Now().Add(-10*time.Minute), time.Now()
+
+	item := db.IPQueueItem{AssignedAt: &long, CheckingStartedAt: &recent}
+	ready, err := o.isReadyToAggregate(ctx, item, deadline, nil)
+	if err != nil || ready {
+		t.Fatalf("recent checking start must not be cut off by an old assignment: ready=%v err=%v", ready, err)
+	}
+
+	started := time.Now().Add(-3 * time.Minute)
+	item.CheckingStartedAt = &started
+	if ready, err := o.isReadyToAggregate(ctx, item, deadline, nil); err != nil || !ready {
+		t.Fatalf("window elapsed since checking start: ready=%v err=%v", ready, err)
+	}
+
+	// Rows from before checking_started_at existed fall back to assigned_at.
+	item.CheckingStartedAt = nil
+	if ready, err := o.isReadyToAggregate(ctx, item, deadline, nil); err != nil || !ready {
+		t.Fatalf("fallback to assigned_at: ready=%v err=%v", ready, err)
+	}
+}
+
+// After the verdict nothing can change the stored checks, and the verdict is
+// exactly what the stored checks give when computed again.
+func TestVerdictMatchesStoredChecksAndLateResultIsDropped(t *testing.T) {
+	ctx := context.Background()
+	o, d, mock := newTestOrchestrator(t, 180)
+	mock.Seed("fip-1", "1.2.3.4", "svc-project")
+	_ = d.RegisterValidator(ctx, "validator-1", "host-1", "port-1", "v0.1")
+	_ = d.SeedQueue(ctx, []string{"1.2.3.4"})
+
+	o.Tick(ctx)
+	ip, _ := d.GetIPByAddress(ctx, "1.2.3.4")
+	_ = o.SelfCheckResult(ctx, "validator-1", ip.ID, true, "ok")
+	ip, _ = d.GetIP(ctx, ip.ID)
+
+	rec := func(src, ct string, ok bool) bool {
+		t.Helper()
+		written, err := o.RecordCheckIfOpen(ctx, db.Check{
+			IPID: ip.ID, IPAddress: ip.IPAddress, AttemptNumber: ip.AttemptNumber, ValidatorID: "validator-1",
+			Source: src, CheckType: ct, Target: "t", Success: ok, CheckedAt: db.Now(),
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		return written
+	}
+	if !rec(db.SourceEgress, "https", true) || !rec(db.InboundSource(1), "tcp-22", false) {
+		t.Fatal("results while checking must be stored")
+	}
+	_ = o.MarkEgressComplete(ctx, ip.ID)
+	o.Cfg.CheckingWindowSeconds = 0
+	time.Sleep(5 * time.Millisecond)
+	o.Tick(ctx)
+
+	ip, _ = d.GetIP(ctx, ip.ID)
+	if ip.State != db.IPDone {
+		t.Fatalf("expected done, got %s", ip.State)
+	}
+
+	// A late result: a new check, and an attempt to flip a stored one.
+	if rec(db.InboundSource(2), "icmp", true) {
+		t.Fatal("new check after the verdict was stored")
+	}
+	if rec(db.InboundSource(1), "tcp-22", true) {
+		t.Fatal("overwrite after the verdict was accepted")
+	}
+
+	stored, err := d.ListChecksForAttempt(ctx, ip.ID, ip.AttemptNumber)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(stored) != 2 {
+		t.Fatalf("stored checks changed after the verdict: %d rows", len(stored))
+	}
+	expected, err := o.expectedCheckCount(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got, _, _ := computeVerdict(stored, expected, o.Agg.MissingCountsAsFail); got != ip.OverallResult {
+		t.Fatalf("recomputed verdict %s differs from stored %s", got, ip.OverallResult)
+	}
+}