Show egress/ingress levels in the registry; freeze checks at the verdict

Registry: the "last result" column now also shows, per level (egress,
ingress), how many of the recorded checks of the latest cycle succeeded, split
by check family (tcp-22 and tcp-443 are both "tcp"). One grouped query per
chunk of addresses; new fields last_cycle_id, egress, ingress in
GET /admin/registry; the dashboard renders them under the verdict.

Verdict integrity (migration 0010):
- the prober is handed an address once per site and attempt, not on every
  poll, so results are no longer overwritten by later probe rounds;
- UpsertCheckIfOpen refuses writes once the address is aggregating or has its
  verdict, or for an older attempt; senders get {"ok":true,"ignored":N} and a
  result_dropped event is recorded;
- the checking window counts from checking_started_at, not from assigned_at;
- checks.recorded_at (server clock) and checks.after_verdict (flag for rows
  written after the verdict in existing data);
- the verdict rule is a pure function (computeVerdict) and the aggregated
  event carries the egress/ingress check counts.

Rebuilt bin/control-api and bin/admin-dashboard to match. Plans and summaries
are in docs/changes; README, API, USAGE, DASHBOARD and DIAGRAMS are updated.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

internal/httpapi/dto.go

@@ -23,6 +23,15 @@ type okResponse struct {
 	OK bool `json:"ok"`
 }
 
+// resultsResponse answers the result-submission endpoints. Ignored counts the
+// submitted checks that were dropped because the address already has its
+// verdict (or the result belongs to an earlier attempt); the sender must not
+// retry them.
+type resultsResponse struct {
+	OK      bool `json:"ok"`
+	Ignored int  `json:"ignored,omitempty"`
+}
+
 type observedIPResponse struct {
 	IP     string `json:"ip"`
 	Source string `json:"source"`

internal/httpapi/dto_admin.go

@@ -93,6 +93,24 @@ type registryDTO struct {
 	LastCheckedAt *time.Time `json:"last_checked_at"`
 	InQueue       bool       `json:"in_queue"`
 	CurrentState  string     `json:"current_state"`
+	// LastCycleID is the cycle counted by Egress and Ingress (0 = no checks).
+	LastCycleID int            `json:"last_cycle_id"`
+	Egress      levelResultDTO `json:"egress"`
+	Ingress     levelResultDTO `json:"ingress"`
+}
+
+// levelResultDTO is "ok of total" recorded checks of one level in the last
+// cycle, split by check family (tcp-22 and tcp-443 are both "tcp").
+type levelResultDTO struct {
+	Total  int           `json:"total"`
+	OK     int           `json:"ok"`
+	ByType []typeStatDTO `json:"by_type"`
+}
+
+type typeStatDTO struct {
+	Type  string `json:"type"`
+	Total int    `json:"total"`
+	OK    int    `json:"ok"`
 }
 
 type validatorDTO struct {

internal/httpapi/handlers_agent.go

@@ -149,6 +149,7 @@ func (s *Server) handleAgentResults(w http.ResponseWriter, r *http.Request) {
 		writeError(w, http.StatusBadRequest, "invalid body: "+err.Error())
 		return
 	}
+	dropped := map[int64]int{}
 	for _, res := range req.Results {
 		item, err := s.DB.GetIP(r.Context(), res.IPID)
 		if err != nil {
@@ -159,7 +160,7 @@ func (s *Server) handleAgentResults(w http.ResponseWriter, r *http.Request) {
 		if err != nil {
 			checkedAt = db.Now()
 		}
-		err = s.Orch.RecordCheck(r.Context(), db.Check{
+		written, err := s.Orch.RecordCheckIfOpen(r.Context(), db.Check{
 			IPID: item.ID, IPAddress: item.IPAddress, AttemptNumber: item.AttemptNumber,
 			ValidatorID: id, Source: db.SourceEgress, CheckType: res.CheckType, Target: res.Target,
 			Success: res.Success, LatencyMS: res.LatencyMS, Detail: res.Detail, CheckedAt: checkedAt,
@@ -168,8 +169,11 @@ func (s *Server) handleAgentResults(w http.ResponseWriter, r *http.Request) {
 			writeError(w, http.StatusInternalServerError, err.Error())
 			return
 		}
+		if !written {
+			dropped[item.ID]++
+		}
 	}
-	writeJSON(w, http.StatusOK, okResponse{OK: true})
+	writeJSON(w, http.StatusOK, s.reportDropped(r.Context(), "validator-agent", id, db.SourceEgress, dropped))
 }
 
 func (s *Server) handleAgentComplete(w http.ResponseWriter, r *http.Request) {

internal/httpapi/handlers_prober.go

@@ -1,6 +1,8 @@
 package httpapi
 
 import (
+	"context"
+	"fmt"
 	"net/http"
 	"time"
 
@@ -41,9 +43,12 @@ func (s *Server) handleProberHeartbeat(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, okResponse{OK: true})
 }
 
-// handleProberAssignments returns every IP currently in the checking
-// state — probers work the whole active set each poll, not one IP at a
-// time, since multiple validators run in parallel.
+// handleProberAssignments returns every IP currently in the checking state
+// that this site has not yet finished probing in the current attempt —
+// probers work the whole active set each poll, not one IP at a time, since
+// multiple validators run in parallel. An address is handed out until the
+// site reports it complete, then no more: one probe round per site per
+// attempt, so results are written once and never overwritten.
 func (s *Server) handleProberAssignments(w http.ResponseWriter, r *http.Request) {
 	siteID := r.PathValue("site_id")
 	idx, err := s.Orch.SiteIndexForID(r.Context(), siteID)
@@ -55,7 +60,7 @@ func (s *Server) handleProberAssignments(w http.ResponseWriter, r *http.Request)
 		writeError(w, http.StatusNotFound, "unknown site_id: "+siteID)
 		return
 	}
-	items, err := s.DB.ListChecking(r.Context())
+	items, err := s.DB.ListCheckingForSite(r.Context(), idx)
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
 		return
@@ -93,6 +98,7 @@ func (s *Server) handleProberResults(w http.ResponseWriter, r *http.Request) {
 	}
 
 	completed := map[int64]bool{}
+	dropped := map[int64]int{}
 	for _, res := range req.Results {
 		item, err := s.DB.GetIP(r.Context(), res.IPID)
 		if err != nil {
@@ -103,7 +109,7 @@ func (s *Server) handleProberResults(w http.ResponseWriter, r *http.Request) {
 		if err != nil {
 			checkedAt = db.Now()
 		}
-		err = s.Orch.RecordCheck(r.Context(), db.Check{
+		written, err := s.Orch.RecordCheckIfOpen(r.Context(), db.Check{
 			IPID: item.ID, IPAddress: item.IPAddress, AttemptNumber: item.AttemptNumber,
 			Source: db.InboundSource(siteIndex), CheckType: res.CheckType, Target: res.IPAddress,
 			Success: res.Success, LatencyMS: res.LatencyMS, Detail: res.Detail, CheckedAt: checkedAt,
@@ -112,6 +118,10 @@ func (s *Server) handleProberResults(w http.ResponseWriter, r *http.Request) {
 			writeError(w, http.StatusInternalServerError, err.Error())
 			return
 		}
+		if !written {
+			dropped[item.ID]++
+			continue
+		}
 		if res.Complete {
 			completed[res.IPID] = true
 		}
@@ -122,5 +132,20 @@ func (s *Server) handleProberResults(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
-	writeJSON(w, http.StatusOK, okResponse{OK: true})
+	writeJSON(w, http.StatusOK, s.reportDropped(r.Context(), "prober", siteID, db.InboundSource(siteIndex), dropped))
+}
+
+// reportDropped records one result_dropped event per address whose submitted
+// checks were refused (the verdict already exists, or the attempt is old) and
+// builds the response. The events are the audit trail of how often senders
+// are late; in a healthy run there are none.
+func (s *Server) reportDropped(ctx context.Context, sourceType, sourceID, checkSource string, dropped map[int64]int) resultsResponse {
+	total := 0
+	for ipID, n := range dropped {
+		total += n
+		id := ipID
+		s.Orch.RecordEvent(ctx, sourceType, sourceID, &id, "result_dropped",
+			fmt.Sprintf(`{"source":%q,"dropped":%d}`, checkSource, n))
+	}
+	return resultsResponse{OK: true, Ignored: total}
 }

internal/httpapi/handlers_registry.go

@@ -82,5 +82,16 @@ func registrySummaryToDTO(s db.RegistrySummary) registryDTO {
 		LastCheckedAt: s.LastCheckedAt,
 		InQueue:       s.InQueue,
 		CurrentState:  s.CurrentState,
+		LastCycleID:   s.LastCycleID,
+		Egress:        levelResultToDTO(s.Egress),
+		Ingress:       levelResultToDTO(s.Ingress),
 	}
 }
+
+func levelResultToDTO(l db.LevelResult) levelResultDTO {
+	out := levelResultDTO{Total: l.Total, OK: l.OK, ByType: make([]typeStatDTO, len(l.ByType))}
+	for i, t := range l.ByType {
+		out.ByType[i] = typeStatDTO{Type: t.Type, Total: t.Total, OK: t.OK}
+	}
+	return out
+}

internal/httpapi/handlers_registry_test.go

@@ -1,9 +1,11 @@
 package httpapi
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"net/http"
+	"reflect"
 	"testing"
 	"time"
 
@@ -149,3 +151,71 @@ func TestRegistryHistoryOutlivesIPDeletion(t *testing.T) {
 		t.Fatalf("expected a single registry entry for the address, got %+v", all)
 	}
 }
+
+// TestAdminRegistryLevels proves the registry endpoints report the last
+// cycle's recorded checks as "ok of total" per level (egress / ingress) and
+// per check family, and that by_type is an empty list, not null, when a
+// level has no checks.
+func TestAdminRegistryLevels(t *testing.T) {
+	fc, d, _, _ := newConfigTestHarness(t)
+	ctx := context.Background()
+	if _, err := d.SubmitIPs(ctx, []string{"7.7.7.7", "8.8.8.8"}); err != nil {
+		t.Fatal(err)
+	}
+	ip, err := d.GetIPByAddress(ctx, "7.7.7.7")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, c := range []struct {
+		source, typ, target string
+		ok                  bool
+	}{
+		{db.SourceEgress, "https", "https://a.test", true},
+		{db.SourceEgress, "https", "https://b.test", false},
+		{db.SourceEgress, "icmp", "a.test", true},
+		{db.InboundSource(1), "tcp-22", "7.7.7.7", true},
+		{db.InboundSource(1), "tcp-443", "7.7.7.7", true},
+		{db.InboundSource(2), "tcp-22", "7.7.7.7", false},
+	} {
+		if err := d.UpsertCheck(ctx, db.Check{
+			IPID: ip.ID, IPAddress: "7.7.7.7", AttemptNumber: ip.AttemptNumber,
+			Source: c.source, CheckType: c.typ, Target: c.target, Success: c.ok, CheckedAt: db.Now(),
+		}); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	resp, body := fc.do(http.MethodGet, "/api/v1/admin/registry/7.7.7.7", nil)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("detail: %d %s", resp.StatusCode, body)
+	}
+	var detail struct {
+		Registry registryDTO `json:"registry"`
+	}
+	if err := json.Unmarshal(body, &detail); err != nil {
+		t.Fatal(err)
+	}
+	r := detail.Registry
+	if r.LastCycleID != 1 {
+		t.Errorf("last_cycle_id = %d", r.LastCycleID)
+	}
+	wantEgress := levelResultDTO{Total: 3, OK: 2, ByType: []typeStatDTO{{"https", 2, 1}, {"icmp", 1, 1}}}
+	wantIngress := levelResultDTO{Total: 3, OK: 2, ByType: []typeStatDTO{{"tcp", 3, 2}}}
+	if !reflect.DeepEqual(r.Egress, wantEgress) || !reflect.DeepEqual(r.Ingress, wantIngress) {
+		t.Errorf("egress=%+v ingress=%+v", r.Egress, r.Ingress)
+	}
+
+	// The paginated list carries the same fields; an address without checks
+	// serialises empty levels as {"total":0,"ok":0,"by_type":[]}.
+	_, body = fc.do(http.MethodGet, "/api/v1/admin/registry?limit=10", nil)
+	var page registryPageResponse
+	if err := json.Unmarshal(body, &page); err != nil || len(page.Items) != 2 {
+		t.Fatalf("list: %v %s", err, body)
+	}
+	if !reflect.DeepEqual(page.Items[0].Egress, wantEgress) {
+		t.Errorf("list egress = %+v", page.Items[0].Egress)
+	}
+	if !bytes.Contains(body, []byte(`"egress":{"total":0,"ok":0,"by_type":[]}`)) {
+		t.Errorf("empty level must serialise by_type as [], got %s", body)
+	}
+}

internal/httpapi/handlers_verdict_integrity_test.go

@@ -0,0 +1,133 @@
+package httpapi
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"testing"
+	"time"
+
+	"cloudipvalidator/internal/db"
+)
+
+// setupCheckingIP stands up two sites and one validator and leaves 9.9.9.9 in
+// the checking state.
+func setupCheckingIP(t *testing.T) (*fakeClient, *db.DB, *db.IPQueueItem) {
+	t.Helper()
+	fc, d, orch, mock := newConfigTestHarness(t)
+	ctx := context.Background()
+	mock.Seed("fip-1", "9.9.9.9", "svc-project")
+	fc.do(http.MethodPut, "/api/v1/admin/config/sites/1", putSiteRequest{SiteID: "site-1"})
+	fc.do(http.MethodPut, "/api/v1/admin/config/sites/2", putSiteRequest{SiteID: "site-2"})
+	fc.do(http.MethodPost, "/api/v1/admin/config/validators", createValidatorRequest{ValidatorID: "validator-1", OSPortID: "port-1"})
+	fc.do(http.MethodPost, "/api/v1/agents/register", registerAgentRequest{ValidatorID: "validator-1"})
+	fc.do(http.MethodPost, "/api/v1/admin/ips", submitIPsRequest{Addresses: []string{"9.9.9.9"}})
+	orch.Tick(ctx)
+	_, body := fc.do(http.MethodGet, "/api/v1/agents/validator-1/assignment", nil)
+	var a assignmentResponse
+	if err := json.Unmarshal(body, &a); err != nil {
+		t.Fatal(err)
+	}
+	fc.do(http.MethodPost, "/api/v1/agents/validator-1/self-check", selfCheckRequest{IPID: a.IPID, DetectedEgress: "9.9.9.9", Success: true, Detail: "matched"})
+	fc.do(http.MethodPost, "/api/v1/probers/register", registerProberRequest{SiteID: "site-1"})
+	fc.do(http.MethodPost, "/api/v1/probers/register", registerProberRequest{SiteID: "site-2"})
+	ip, err := d.GetIP(ctx, a.IPID)
+	if err != nil || ip.State != db.IPChecking {
+		t.Fatalf("expected checking: %v %+v", err, ip)
+	}
+	return fc, d, ip
+}
+
+func proberAssignmentsFor(t *testing.T, fc *fakeClient, site string) []proberAssignment {
+	t.Helper()
+	resp, body := fc.do(http.MethodGet, "/api/v1/probers/"+site+"/assignments", nil)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("%s assignments: %d %s", site, resp.StatusCode, body)
+	}
+	var out []proberAssignment
+	if err := json.Unmarshal(body, &out); err != nil {
+		t.Fatal(err)
+	}
+	return out
+}
+
+func proberResult(ip *db.IPQueueItem, ct string, ok, complete bool) proberResultDTO {
+	return proberResultDTO{IPID: ip.ID, IPAddress: ip.IPAddress, CheckType: ct, Success: ok,
+		CheckedAt: time.Now().UTC().Format(time.RFC3339Nano), Complete: complete}
+}
+
+// A prober site gets an address until it has reported it complete, then not
+// again; the other site still gets it.
+func TestProberAssignmentsOncePerSite(t *testing.T) {
+	fc, _, ip := setupCheckingIP(t)
+
+	if len(proberAssignmentsFor(t, fc, "site-1")) != 1 || len(proberAssignmentsFor(t, fc, "site-2")) != 1 {
+		t.Fatal("both sites must get the address before reporting")
+	}
+	resp, body := fc.do(http.MethodPost, "/api/v1/probers/site-1/results", proberResultsRequest{
+		Results: []proberResultDTO{proberResult(ip, "tcp-22", true, false), proberResult(ip, "icmp", true, true)},
+	})
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("results: %d %s", resp.StatusCode, body)
+	}
+	if got := proberAssignmentsFor(t, fc, "site-1"); len(got) != 0 {
+		t.Fatalf("site-1 must not get the address again after completing it: %+v", got)
+	}
+	if got := proberAssignmentsFor(t, fc, "site-2"); len(got) != 1 {
+		t.Fatalf("site-2 still has to probe the address: %+v", got)
+	}
+}
+
+// Results that arrive after the verdict are refused with ignored>0, leave the
+// stored checks untouched and leave a result_dropped event behind; the same
+// for the validator agent's results.
+func TestResultsAfterVerdictAreIgnored(t *testing.T) {
+	fc, d, ip := setupCheckingIP(t)
+	ctx := context.Background()
+
+	fc.do(http.MethodPost, "/api/v1/probers/site-1/results", proberResultsRequest{
+		Results: []proberResultDTO{proberResult(ip, "tcp-22", true, true)},
+	})
+	fc.do(http.MethodPost, "/api/v1/agents/validator-1/results", agentResultsRequest{
+		Results: []checkResultDTO{{IPID: ip.ID, CheckType: "https", Target: "https://example.test", Success: true, CheckedAt: time.Now().UTC().Format(time.RFC3339Nano)}},
+	})
+	if err := d.SetAggregating(ctx, ip.ID); err != nil {
+		t.Fatal(err)
+	}
+	if err := d.FinishIP(ctx, ip.ID, db.ResultPartial); err != nil {
+		t.Fatal(err)
+	}
+
+	// prober: one new check and one that would flip a stored success
+	resp, body := fc.do(http.MethodPost, "/api/v1/probers/site-1/results", proberResultsRequest{
+		Results: []proberResultDTO{proberResult(ip, "tcp-22", false, false), proberResult(ip, "ssh", false, true)},
+	})
+	var rr resultsResponse
+	if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &rr) != nil || !rr.OK || rr.Ignored != 2 {
+		t.Fatalf("prober late results: %d %s", resp.StatusCode, body)
+	}
+	// agent
+	resp, body = fc.do(http.MethodPost, "/api/v1/agents/validator-1/results", agentResultsRequest{
+		Results: []checkResultDTO{{IPID: ip.ID, CheckType: "https", Target: "https://example.test", Success: false, CheckedAt: time.Now().UTC().Format(time.RFC3339Nano)}},
+	})
+	if resp.StatusCode != http.StatusOK || json.Unmarshal(body, &rr) != nil || rr.Ignored != 1 {
+		t.Fatalf("agent late results: %d %s", resp.StatusCode, body)
+	}
+
+	rows, err := d.ListChecksForAttempt(ctx, ip.ID, ip.AttemptNumber)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rows) != 2 {
+		t.Fatalf("expected the 2 checks written before the verdict, got %d", len(rows))
+	}
+	for _, c := range rows {
+		if !c.Success {
+			t.Errorf("%s/%s changed after the verdict", c.Source, c.CheckType)
+		}
+	}
+	var dropped int
+	if err := d.QueryRowContext(ctx, `SELECT COUNT(*) FROM events WHERE event_type='result_dropped' AND ip_id=?`, ip.ID).Scan(&dropped); err != nil || dropped != 2 {
+		t.Fatalf("expected 2 result_dropped events (prober, agent), got %d err=%v", dropped, err)
+	}
+}