Registry and Analytics: run, subnet, direction and protocol filters, successes-by-target chart

Registry (/registry):
- filters by run (slice by the address's cycle in that run), subnet
  (drop-down of configured subnets), direction (egress/ingress) and
  protocol (icmp, tcp, ssh, https, tls); status in scope is computed over
  the narrowed checks
- chart "successful checks per target (egress) / site (ingress)" when both
  direction and protocol are chosen; a row opens the list of addresses
  (dialog, CSV)
- API: direction/protocol parameters and run in GET /admin/registry,
  GET /admin/registry/breakdown and /breakdown/list
- subnet filter passes ids as one JSON parameter (SQLite variable limit)

Analytics (/analytics):
- subnet filter recomputes the whole page over the addresses of the run
  inside the subnet; only their checks are read; cache per run and subnet
- direction and protocol focus the page; with both set the registry chart
  is shown
- subnet parameter in GET /admin/analytics/runs/{id} and lists (JSON, CSV)

Docs: plans and summaries in docs/changes, README, API, USAGE.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

internal/db/queries_registry.go

@@ -3,8 +3,10 @@ package db
 import (
 	"context"
 	"database/sql"
+	"encoding/json"
 	"fmt"
 	"net/netip"
+	"slices"
 	"sort"
 	"strings"
 	"time"
@@ -106,28 +108,97 @@ func (d *DB) ListRegistry(ctx context.Context) ([]RegistrySummary, error) {
 	out := make([]RegistrySummary, len(items))
 	for i, item := range items {
 		s := RegistrySummary{RegistryItem: item}
-		if err := d.fillRegistrySummary(ctx, &s); err != nil {
+		if err := d.fillRegistrySummary(ctx, &s, registrySlice{}); err != nil {
 			return nil, err
 		}
 		out[i] = s
 	}
-	if err := d.fillRegistryLevels(ctx, out); err != nil {
+	if err := d.fillRegistryLevels(ctx, out, registrySlice{}); err != nil {
 		return nil, err
 	}
 	return out, nil
 }
 
+// RegistryFamilies lists the check families RegistryFilter.Family accepts.
+// tls is an ingress-only family (the prober's TLS handshake on 443).
+var RegistryFamilies = []string{"icmp", "tcp", "ssh", "https", "tls"}
+
+// IsValidRegistryLevel reports whether s is a RegistryFilter.Level value.
+func IsValidRegistryLevel(s string) bool { return s == LevelEgress || s == LevelIngress }
+
+// IsValidRegistryFamily reports whether s is one of RegistryFamilies.
+func IsValidRegistryFamily(s string) bool { return slices.Contains(RegistryFamilies, s) }
+
 // RegistryFilter narrows ListRegistryPage. The zero value matches everything.
+//
+// RunID, Level and Family also choose which data of an address is read (the
+// "slice"): with RunID, its cycle in that run (run_results.cycle_id) and the
+// run's verdict; without, its newest cycle. Level and Family narrow the checks
+// of that cycle: the address must have such checks, and LastResult then
+// classifies just them (all succeeded = pass, none = fail, else partial).
 type RegistryFilter struct {
 	Query      string // substring of ip_address
 	LastResult string // pass|partial|fail|cancelled — same meaning as RegistrySummary.LastResult
 	RunID      int64  // only addresses that have a result in this run
 	Subnet     string // only addresses inside this CIDR
+	Level      string // egress|ingress — only checks of this level (see CheckLevel)
+	Family     string // one of RegistryFamilies — only checks of this family (see CheckFamily)
+}
+
+// registrySlice is the part of RegistryFilter that selects what is read of an
+// address; the zero value is "newest cycle, all checks".
+type registrySlice struct {
+	RunID         int64
+	Level, Family string
+}
+
+func (sl registrySlice) scoped() bool { return sl.Level != "" || sl.Family != "" }
+
+// validate rejects a Level or Family outside the accepted values.
+func (sl registrySlice) validate() error {
+	if sl.Level != "" && !IsValidRegistryLevel(sl.Level) {
+		return fmt.Errorf("level %q: %w", sl.Level, ErrValidation)
+	}
+	if sl.Family != "" && !IsValidRegistryFamily(sl.Family) {
+		return fmt.Errorf("family %q: %w", sl.Family, ErrValidation)
+	}
+	return nil
+}
+
+// scopeCond is the SQL condition of Level and Family over `checks c`, "1 = 1"
+// without them. Level and Family are validated, so the LIKE patterns are safe.
+func (sl registrySlice) scopeCond() (string, []any) {
+	conds := []string{"1 = 1"}
+	var args []any
+	switch sl.Level {
+	case LevelEgress:
+		conds = append(conds, "c.source = 'egress'")
+	case LevelIngress:
+		conds = append(conds, "c.source LIKE 'inbound-site-%'")
+	}
+	if sl.Family != "" {
+		conds = append(conds, "(c.check_type = ? OR c.check_type LIKE ? || '-%')")
+		args = append(args, sl.Family, sl.Family)
+	}
+	return strings.Join(conds, " AND "), args
+}
+
+// scopeFrom is `FROM checks c WHERE …` over the checks of the slice of
+// registry row r (with the run's run_results row as rr when RunID is set):
+// the run's cycle or the newest one, narrowed by Level and Family.
+func (sl registrySlice) scopeFrom() (string, []any) {
+	sc, args := sl.scopeCond()
+	if sl.RunID > 0 {
+		return `FROM checks c WHERE c.run_id = ? AND c.registry_id = r.id AND c.cycle_id = rr.cycle_id AND ` + sc,
+			append([]any{sl.RunID}, args...)
+	}
+	return `FROM checks c WHERE c.registry_id = r.id
+		  AND c.cycle_id = (SELECT MAX(c2.cycle_id) FROM checks c2 WHERE c2.registry_id = r.id) AND ` + sc, args
 }
 
 // subnetIDs returns the registry ids of the addresses inside prefix. SQLite
 // has no CIDR operators, so the registry's addresses are filtered here.
-func (d *DB) subnetIDs(ctx context.Context, cidr string) ([]any, error) {
+func (d *DB) subnetIDs(ctx context.Context, cidr string) ([]int64, error) {
 	p, err := netip.ParsePrefix(cidr)
 	if err != nil {
 		return nil, fmt.Errorf("subnet %q: %v: %w", cidr, err, ErrValidation)
@@ -137,7 +208,7 @@ func (d *DB) subnetIDs(ctx context.Context, cidr string) ([]any, error) {
 		return nil, err
 	}
 	defer rows.Close()
-	var ids []any
+	var ids []int64
 	for rows.Next() {
 		var id int64
 		var ip string
@@ -169,42 +240,88 @@ const lastResultCond = `(
 	) = ?)
 )`
 
-// ListRegistryPage returns one page (limit/offset) of the registry in the same
-// order as ListRegistry, filtered by f, plus the total number of matching
-// rows. LIMIT/OFFSET are applied in SQL before the per-row summary queries,
-// so only the rows of the page pay for them. limit <= 0 means no limit.
-func (d *DB) ListRegistryPage(ctx context.Context, f RegistryFilter, limit, offset int) ([]RegistrySummary, int, error) {
+// registryFilterSQL is the FROM and the WHERE condition (without the keyword;
+// "1 = 1" for no filter) of the addresses f selects, over `ip_registry r LEFT
+// JOIN ip_queue q`, with the run's run_results row as rr when sl.RunID is set,
+// and its arguments in order. ListRegistryPage and RegistryBreakdown share it, so
+// the list and the chart over it always cover the same addresses.
+func (d *DB) registryFilterSQL(ctx context.Context, f RegistryFilter, sl registrySlice) (from, cond string, args []any, err error) {
 	var conds []string
-	var args []any
 	if f.Query != "" {
 		conds = append(conds, "instr(r.ip_address, ?) > 0")
 		args = append(args, f.Query)
 	}
-	if f.LastResult != "" {
+	switch {
+	case f.LastResult != "" && sl.scoped():
+		// The verdict covers all checks, so a narrowed scope has none; cancelled
+		// leaves no checks at all.
+		if f.LastResult == ResultCancelled {
+			conds = append(conds, "0 = 1")
+			break
+		}
+		sfrom, fargs := sl.scopeFrom()
+		conds = append(conds, `(SELECT CASE WHEN SUM(c.success) = 0 THEN 'fail'
+			WHEN SUM(c.success) = COUNT(*) THEN 'pass' ELSE 'partial' END `+sfrom+` HAVING COUNT(*) > 0) = ?`)
+		args = append(args, fargs...)
+		args = append(args, f.LastResult)
+	case f.LastResult != "" && sl.RunID > 0:
+		conds = append(conds, "rr.verdict = ?")
+		args = append(args, f.LastResult)
+	case f.LastResult != "":
 		conds = append(conds, lastResultCond)
 		args = append(args, f.LastResult, f.LastResult)
-	}
-	if f.RunID > 0 {
-		conds = append(conds, "r.id IN (SELECT registry_id FROM run_results WHERE run_id = ?)")
-		args = append(args, f.RunID)
+	case sl.scoped():
+		sfrom, fargs := sl.scopeFrom()
+		conds = append(conds, "EXISTS (SELECT 1 "+sfrom+")")
+		args = append(args, fargs...)
 	}
 	if f.Subnet != "" {
 		ids, err := d.subnetIDs(ctx, f.Subnet)
 		if err != nil {
-			return nil, 0, err
+			return "", "", nil, err
 		}
 		if len(ids) == 0 {
 			conds = append(conds, "0 = 1")
 		} else {
-			conds = append(conds, "r.id IN ("+strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",")+")")
-			args = append(args, ids...)
+			// One JSON parameter instead of an id per "?", which would hit
+			// SQLite's bound-variable limit on a large subnet.
+			b, err := json.Marshal(ids)
+			if err != nil {
+				return "", "", nil, err
+			}
+			conds = append(conds, "r.id IN (SELECT value FROM json_each(?))")
+			args = append(args, string(b))
 		}
 	}
-	from := ` FROM ip_registry r LEFT JOIN ip_queue q ON q.registry_id = r.id `
-	where := ""
-	if len(conds) > 0 {
-		where = "WHERE " + strings.Join(conds, " AND ") + " "
+	from = ` FROM ip_registry r LEFT JOIN ip_queue q ON q.registry_id = r.id `
+	if sl.RunID > 0 {
+		from += `JOIN run_results rr ON rr.registry_id = r.id AND rr.run_id = ? `
+		args = append([]any{sl.RunID}, args...)
 	}
+	cond = "1 = 1"
+	if len(conds) > 0 {
+		cond = strings.Join(conds, " AND ")
+	}
+	return from, cond, args, nil
+}
+
+// ListRegistryPage returns one page (limit/offset) of the registry in the same
+// order as ListRegistry, filtered by f, plus the total number of matching
+// rows. LIMIT/OFFSET are applied in SQL before the per-row summary queries,
+// so only the rows of the page pay for them. limit <= 0 means no limit. With
+// f.RunID the rows are the run's addresses and LastResult, LastCycleID, Egress
+// and Ingress describe the address in that run; an unknown run is an empty
+// list. Level and Family narrow Egress and Ingress to the matching checks.
+func (d *DB) ListRegistryPage(ctx context.Context, f RegistryFilter, limit, offset int) ([]RegistrySummary, int, error) {
+	sl := registrySlice{RunID: f.RunID, Level: f.Level, Family: f.Family}
+	if err := sl.validate(); err != nil {
+		return nil, 0, err
+	}
+	from, cond, args, err := d.registryFilterSQL(ctx, f, sl)
+	if err != nil {
+		return nil, 0, err
+	}
+	where := "WHERE " + cond + " "
 
 	var total int
 	if err := d.QueryRowContext(ctx, `SELECT COUNT(*)`+from+where, args...).Scan(&total); err != nil {
@@ -231,12 +348,12 @@ func (d *DB) ListRegistryPage(ctx context.Context, f RegistryFilter, limit, offs
 	out := make([]RegistrySummary, len(items))
 	for i, item := range items {
 		s := RegistrySummary{RegistryItem: item}
-		if err := d.fillRegistrySummary(ctx, &s); err != nil {
+		if err := d.fillRegistrySummary(ctx, &s, sl); err != nil {
 			return nil, 0, err
 		}
 		out[i] = s
 	}
-	if err := d.fillRegistryLevels(ctx, out); err != nil {
+	if err := d.fillRegistryLevels(ctx, out, sl); err != nil {
 		return nil, 0, err
 	}
 	return out, total, nil
@@ -257,11 +374,11 @@ func (d *DB) GetRegistryByAddress(ctx context.Context, address string) (*Registr
 		return nil, err
 	}
 	s := &RegistrySummary{RegistryItem: *item}
-	if err := d.fillRegistrySummary(ctx, s); err != nil {
+	if err := d.fillRegistrySummary(ctx, s, registrySlice{}); err != nil {
 		return nil, err
 	}
 	one := []RegistrySummary{*s}
-	if err := d.fillRegistryLevels(ctx, one); err != nil {
+	if err := d.fillRegistryLevels(ctx, one, registrySlice{}); err != nil {
 		return nil, err
 	}
 	*s = one[0]
@@ -275,7 +392,10 @@ func (d *DB) GetRegistryByAddress(ctx context.Context, address string) (*Registr
 // checked_at — a cycle with a mix of passing and failing checks (e.g. one
 // egress target timed out while the rest succeeded) is "partial", even
 // though the chronologically-last check to report in might have passed.
-func (d *DB) fillRegistrySummary(ctx context.Context, s *RegistrySummary) error {
+//
+// With sl.RunID, LastResult and LastCycleID are instead the address's verdict
+// and cycle in that run (from run_results); the rest is unchanged.
+func (d *DB) fillRegistrySummary(ctx context.Context, s *RegistrySummary, sl registrySlice) error {
 	if err := d.QueryRowContext(ctx, `
 		SELECT COUNT(DISTINCT cycle_id) FROM checks WHERE registry_id=?
 	`, s.ID).Scan(&s.TotalCycles); err != nil {
@@ -305,6 +425,18 @@ func (d *DB) fillRegistrySummary(ctx context.Context, s *RegistrySummary) error
 		s.CurrentState = state.String
 	}
 
+	if sl.RunID > 0 {
+		var cycle sql.NullInt64
+		var verdict sql.NullString
+		err := d.QueryRowContext(ctx, `SELECT cycle_id, verdict FROM run_results WHERE run_id=? AND registry_id=?`, sl.RunID, s.ID).
+			Scan(&cycle, &verdict)
+		if err != nil && err != sql.ErrNoRows {
+			return err
+		}
+		s.LastResult, s.LastCycleID = verdict.String, int(cycle.Int64)
+		return nil
+	}
+
 	switch {
 	case overallResult.Valid && overallResult.String != "":
 		// The address has a live ip_queue row with a finished cycle
@@ -336,30 +468,46 @@ func (d *DB) fillRegistrySummary(ctx context.Context, s *RegistrySummary) error
 // SQLite's bound-variable limit.
 const registryLevelsChunk = 500
 
-// registryLevelsQuery is the grouped query of fillRegistryLevels for n
-// registry ids: per address, the counts of its newest cycle by source and
-// check type.
-func registryLevelsQuery(n int) string {
+// registryLevelsQuery is the grouped query of fillRegistryLevels for the given
+// registry ids and its arguments: per address, the counts of its cycle by
+// source and check type — the newest one, or the run's with sl.RunID — over
+// the checks that match sl.Level and sl.Family.
+func registryLevelsQuery(ids []any, sl registrySlice) (string, []any) {
+	in := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",")
+	sc, scArgs := sl.scopeCond()
+	if sl.RunID > 0 {
+		args := append([]any{sl.RunID, sl.RunID}, ids...)
+		return `
+			SELECT c.registry_id, rr.cycle_id, c.source, c.check_type, COUNT(*), COALESCE(SUM(c.success), 0)
+			FROM checks c
+			JOIN run_results rr ON rr.run_id = ? AND rr.registry_id = c.registry_id AND rr.cycle_id = c.cycle_id
+			WHERE c.run_id = ? AND c.registry_id IN (` + in + `) AND ` + sc + `
+			GROUP BY c.registry_id, rr.cycle_id, c.source, c.check_type
+		`, append(args, scArgs...)
+	}
 	return `
 		SELECT c.registry_id, m.cid, c.source, c.check_type, COUNT(*), COALESCE(SUM(c.success), 0)
 		FROM checks c
 		JOIN (SELECT registry_id, MAX(cycle_id) AS cid FROM checks
-		      WHERE registry_id IN (` + strings.TrimSuffix(strings.Repeat("?,", n), ",") + `)
+		      WHERE registry_id IN (` + in + `)
 		      GROUP BY registry_id) m
 		  ON m.registry_id = c.registry_id AND m.cid = c.cycle_id
+		WHERE ` + sc + `
 		GROUP BY c.registry_id, m.cid, c.source, c.check_type
-	`
+	`, append(ids, scArgs...)
 }
 
 // fillRegistryLevels sets LastCycleID, Egress and Ingress on every summary in
 // sums: the recorded checks of each address's newest cycle (the same cycle
 // whose time is LastCheckedAt), counted per level and per check family. It
 // runs one grouped query per chunk of addresses, not one per address, over
-// idx_checks_registry_cycle. Checks whose source is neither egress nor an
-// inbound site are not counted. The counts follow the recorded rows only, so
-// they can differ from LastResult, which also treats missing results as
-// failures.
-func (d *DB) fillRegistryLevels(ctx context.Context, sums []RegistrySummary) error {
+// idx_checks_registry_cycle. With sl.RunID the cycle is the address's one in
+// that run (over idx_checks_run) and sl.Level and sl.Family leave only the
+// matching checks, so the levels and types outside them stay empty. Checks
+// whose source is neither egress nor an inbound site are not counted. The
+// counts follow the recorded rows only, so they can differ from LastResult,
+// which also treats missing results as failures.
+func (d *DB) fillRegistryLevels(ctx context.Context, sums []RegistrySummary, sl registrySlice) error {
 	pos := make(map[int64]int, len(sums))
 	for i := range sums {
 		pos[sums[i].ID] = i
@@ -374,7 +522,8 @@ func (d *DB) fillRegistryLevels(ctx context.Context, sums []RegistrySummary) err
 		for _, s := range sums[start:end] {
 			args = append(args, s.ID)
 		}
-		rows, err := d.QueryContext(ctx, registryLevelsQuery(len(args)), args...)
+		q, qargs := registryLevelsQuery(args, sl)
+		rows, err := d.QueryContext(ctx, q, qargs...)
 		if err != nil {
 			return err
 		}

internal/db/queries_registry_breakdown.go

@@ -0,0 +1,149 @@
+package db
+
+import (
+	"context"
+	"fmt"
+	"sort"
+)
+
+// Groups of a RegistryBreakdown: egress checks are grouped by target, ingress
+// ones by prober site.
+const (
+	BreakdownTarget = "target"
+	BreakdownSite   = "site"
+)
+
+// BreakdownRow is one group of a RegistryBreakdown: Key is the checks.target
+// (egress) or the checks.source (ingress, "inbound-site-N"), Total the checks
+// recorded for it and OK the successful ones.
+type BreakdownRow struct {
+	Key   string
+	Total int
+	OK    int
+}
+
+// Breakdown counts the checks of one direction and protocol per target or site.
+type Breakdown struct {
+	Group     string // BreakdownTarget or BreakdownSite
+	Addresses int    // addresses of the filter, the same as ListRegistryPage's total
+	Rows      []BreakdownRow
+}
+
+// breakdownChecks is `FROM … JOIN checks c … WHERE …` over the checks of the
+// addresses f selects (registryFilterSQL), in the slice of f: the address's cycle
+// in the run or its newest one, narrowed by Level and Family, which must be
+// set. It also returns the column the checks are grouped by. CROSS JOIN keeps
+// the checks as the inner table, so the addresses drive the lookups over the
+// checks indexes whatever the table statistics say.
+func (d *DB) breakdownChecks(ctx context.Context, f RegistryFilter) (q, group string, args []any, err error) {
+	sl := registrySlice{RunID: f.RunID, Level: f.Level, Family: f.Family}
+	if err := sl.validate(); err != nil {
+		return "", "", nil, err
+	}
+	if sl.Level == "" || sl.Family == "" {
+		return "", "", nil, fmt.Errorf("direction and protocol are required: %w", ErrValidation)
+	}
+	from, cond, args, err := d.registryFilterSQL(ctx, f, sl)
+	if err != nil {
+		return "", "", nil, err
+	}
+	group = "c.target"
+	if sl.Level == LevelIngress {
+		group = "c.source"
+	}
+	sc, scArgs := sl.scopeCond()
+	q = from + `CROSS JOIN checks c ON c.registry_id = r.id AND c.cycle_id = `
+	if sl.RunID > 0 {
+		q += `rr.cycle_id WHERE ` + cond + ` AND c.run_id = ? AND ` + sc
+		args = append(args, sl.RunID)
+	} else {
+		q += `(SELECT MAX(c2.cycle_id) FROM checks c2 WHERE c2.registry_id = r.id) WHERE ` + cond + ` AND ` + sc
+	}
+	return q, group, append(args, scArgs...), nil
+}
+
+// RegistryBreakdown counts the recorded checks of f.Level and f.Family (both
+// required, else ErrValidation) per target (egress) or site (ingress), over
+// the same slice and the same addresses — all under the filter, not one page —
+// as ListRegistryPage. Rows are sorted by successful checks, most first, then by
+// key. It counts checks, not addresses: an address can have several checks per
+// site (tcp-22 and tcp-443).
+func (d *DB) RegistryBreakdown(ctx context.Context, f RegistryFilter) (*Breakdown, error) {
+	from, group, args, err := d.breakdownChecks(ctx, f)
+	if err != nil {
+		return nil, err
+	}
+	b := &Breakdown{Group: BreakdownTarget}
+	if f.Level == LevelIngress {
+		b.Group = BreakdownSite
+	}
+	rows, err := d.QueryContext(ctx, `SELECT `+group+`, COUNT(*), COALESCE(SUM(c.success), 0) `+from+` GROUP BY `+group, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var r BreakdownRow
+		if err := rows.Scan(&r.Key, &r.Total, &r.OK); err != nil {
+			return nil, err
+		}
+		b.Rows = append(b.Rows, r)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	sort.Slice(b.Rows, func(i, j int) bool {
+		if b.Rows[i].OK != b.Rows[j].OK {
+			return b.Rows[i].OK > b.Rows[j].OK
+		}
+		return b.Rows[i].Key < b.Rows[j].Key
+	})
+
+	// The addresses of the filter, as ListRegistryPage counts them.
+	sl := registrySlice{RunID: f.RunID, Level: f.Level, Family: f.Family}
+	afrom, cond, aargs, err := d.registryFilterSQL(ctx, f, sl)
+	if err != nil {
+		return nil, err
+	}
+	if err := d.QueryRowContext(ctx, `SELECT COUNT(*)`+afrom+`WHERE `+cond, aargs...).Scan(&b.Addresses); err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+// RegistryBreakdownList returns the checks behind one row of RegistryBreakdown:
+// those whose target (egress) or source (ingress) is key, with the address
+// (IPAddress), type, validator, latency, detail and time, failures first, then
+// by registry order. A key without checks is ErrNotFound.
+func (d *DB) RegistryBreakdownList(ctx context.Context, f RegistryFilter, key string) ([]Check, error) {
+	from, group, args, err := d.breakdownChecks(ctx, f)
+	if err != nil {
+		return nil, err
+	}
+	rows, err := d.QueryContext(ctx, `
+		SELECT r.ip_address, c.validator_id, c.source, c.check_type, c.target, c.success, c.latency_ms, c.detail, c.checked_at `+
+		from+` AND `+group+` = ? ORDER BY c.success, r.first_seen_at, r.id, c.check_type, c.source, c.target`, append(args, key)...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []Check
+	for rows.Next() {
+		var c Check
+		var checkedAt string
+		if err := rows.Scan(&c.IPAddress, &c.ValidatorID, &c.Source, &c.CheckType, &c.Target, &c.Success, &c.LatencyMS, &c.Detail, &checkedAt); err != nil {
+			return nil, err
+		}
+		if c.CheckedAt, err = dbToTime(checkedAt); err != nil {
+			return nil, err
+		}
+		out = append(out, c)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	if len(out) == 0 {
+		return nil, fmt.Errorf("breakdown key %q: %w", key, ErrNotFound)
+	}
+	return out, nil
+}

internal/db/queries_registry_breakdown_test.go

@@ -0,0 +1,110 @@
+package db
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+)
+
+// RegistryBreakdown and RegistryBreakdownList over the slice of the registry
+// filter. Run 1 holds three addresses (cycle 1), run 2 a re-check of .1 (cycle 2).
+func TestRegistryBreakdown(t *testing.T) {
+	d, ctx := newTestDB(t)
+	const a1, a2, a3 = "10.0.0.1", "10.0.0.2", "10.0.1.1"
+	submit(t, d, RunManual, a1, a2, a3)
+	s1, s2 := InboundSource(1), InboundSource(2)
+	for _, c := range []struct {
+		addr, source, typ, target string
+		ok                        bool
+	}{
+		{a1, SourceEgress, "https", "T1", true}, {a1, SourceEgress, "https", "T2", false},
+		{a1, s1, "tcp-22", a1, true}, {a1, s1, "tcp-443", a1, false}, {a1, s2, "tcp-443", a1, true},
+		{a2, SourceEgress, "https", "T1", false}, {a2, SourceEgress, "https", "T2", false}, {a2, s1, "tcp-443", a2, true},
+		{a3, SourceEgress, "https", "T1", true}, {a3, SourceEgress, "https", "T3", true},
+	} {
+		addCheck(t, d, c.addr, c.source, c.typ, c.target, c.ok)
+	}
+	finish(t, d, a1, ResultPartial, -1)
+	finish(t, d, a2, ResultFail, -1)
+	finish(t, d, a3, ResultPass, -1)
+	run1 := runs(t, d)[0].ID
+
+	submit(t, d, RunManual, a1)
+	addCheck(t, d, a1, SourceEgress, "https", "T1", false)
+	addCheck(t, d, a1, SourceEgress, "https", "T2", true)
+	finish(t, d, a1, ResultPartial, -1)
+
+	eg := RegistryFilter{Level: LevelEgress, Family: "https"}
+	in := RegistryFilter{Level: LevelIngress, Family: "tcp"}
+	for _, tc := range []struct {
+		name  string
+		f     RegistryFilter
+		group string
+		addrs int
+		want  []BreakdownRow
+	}{
+		{"egress, run 1: most successful first", withRun(eg, run1), BreakdownTarget, 3,
+			[]BreakdownRow{{"T1", 3, 2}, {"T3", 1, 1}, {"T2", 2, 0}}},
+		{"egress, newest cycle; equal counts by key", eg, BreakdownTarget, 3,
+			[]BreakdownRow{{"T1", 3, 1}, {"T2", 2, 1}, {"T3", 1, 1}}},
+		{"egress, subnet", RegistryFilter{RunID: run1, Level: LevelEgress, Family: "https", Subnet: "10.0.0.0/24"}, BreakdownTarget, 2,
+			[]BreakdownRow{{"T1", 2, 1}, {"T2", 2, 0}}},
+		{"egress, status in scope", RegistryFilter{RunID: run1, Level: LevelEgress, Family: "https", LastResult: ResultFail}, BreakdownTarget, 1,
+			[]BreakdownRow{{"T1", 1, 0}, {"T2", 1, 0}}},
+		{"ingress by site, checks not addresses", withRun(in, run1), BreakdownSite, 2,
+			[]BreakdownRow{{s1, 3, 2}, {s2, 1, 1}}},
+		{"egress tls does not exist", RegistryFilter{Level: LevelEgress, Family: "tls"}, BreakdownTarget, 0, nil},
+	} {
+		b, err := d.RegistryBreakdown(ctx, tc.f)
+		if err != nil {
+			t.Fatalf("%s: %v", tc.name, err)
+		}
+		if b.Group != tc.group || b.Addresses != tc.addrs || !reflect.DeepEqual(b.Rows, tc.want) {
+			t.Errorf("%s: group=%s addresses=%d rows=%v, want %s %d %v", tc.name, b.Group, b.Addresses, b.Rows, tc.group, tc.addrs, tc.want)
+		}
+		// The chart covers the addresses of the list.
+		if _, total, err := d.ListRegistryPage(ctx, tc.f, 10, 0); err != nil || total != b.Addresses {
+			t.Errorf("%s: list total=%d err=%v, chart addresses=%d", tc.name, total, err, b.Addresses)
+		}
+	}
+
+	// The list: failures first, then registry order; the key picks the group.
+	for _, tc := range []struct {
+		name string
+		f    RegistryFilter
+		key  string
+		want [][3]string // address, type, success
+	}{
+		{"egress T1", withRun(eg, run1), "T1", [][3]string{{a2, "https", "0"}, {a1, "https", "1"}, {a3, "https", "1"}}},
+		{"ingress site 1", withRun(in, run1), s1, [][3]string{{a1, "tcp-443", "0"}, {a1, "tcp-22", "1"}, {a2, "tcp-443", "1"}}},
+	} {
+		got, err := d.RegistryBreakdownList(ctx, tc.f, tc.key)
+		if err != nil {
+			t.Fatalf("%s: %v", tc.name, err)
+		}
+		var rows [][3]string
+		for _, c := range got {
+			ok := "0"
+			if c.Success {
+				ok = "1"
+			}
+			rows = append(rows, [3]string{c.IPAddress, c.CheckType, ok})
+		}
+		if !reflect.DeepEqual(rows, tc.want) {
+			t.Errorf("%s: %v, want %v", tc.name, rows, tc.want)
+		}
+	}
+	if _, err := d.RegistryBreakdownList(ctx, withRun(eg, run1), "nope"); !errors.Is(err, ErrNotFound) {
+		t.Errorf("unknown key: %v", err)
+	}
+	for _, f := range []RegistryFilter{{}, {Level: LevelEgress}, {Family: "https"}, {Level: "sideways", Family: "https"}} {
+		if _, err := d.RegistryBreakdown(ctx, f); !errors.Is(err, ErrValidation) {
+			t.Errorf("%+v: %v", f, err)
+		}
+	}
+}
+
+func withRun(f RegistryFilter, run int64) RegistryFilter {
+	f.RunID = run
+	return f
+}

internal/db/queries_registry_levels_test.go

@@ -1,7 +1,9 @@
 package db
 
 import (
+	"errors"
 	"reflect"
+	"sort"
 	"testing"
 )
 
@@ -167,3 +169,102 @@ func TestRegistryLevelsLatestCycleAndManyAddresses(t *testing.T) {
 		t.Errorf("after delete: %+v", s)
 	}
 }
+
+// TestRegistryPageSlice covers the run / level / protocol filters of
+// ListRegistryPage together with the status and the subnet. Run 1 holds three
+// addresses (cycle 1), run 2 only a re-check of .1 (cycle 2) whose result
+// differs from run 1.
+func TestRegistryPageSlice(t *testing.T) {
+	d, ctx := newTestDB(t)
+	const a1, a2, a3 = "10.0.0.1", "10.0.0.2", "10.0.1.1"
+	submit(t, d, RunManual, a1, a2, a3)
+	for _, c := range []struct {
+		addr, source, typ string
+		ok                bool
+	}{
+		{a1, SourceEgress, "https", true}, {a1, SourceEgress, "icmp", true},
+		{a1, InboundSource(1), "tcp-22", true}, {a1, InboundSource(1), "tcp-443", false}, {a1, InboundSource(1), "tls-443", false},
+		{a2, SourceEgress, "https", false},
+		{a2, InboundSource(1), "tcp-443", true}, {a2, InboundSource(1), "tls-443", true}, {a2, InboundSource(1), "ssh", true},
+		{a3, InboundSource(1), "icmp", true},
+	} {
+		addCheck(t, d, c.addr, c.source, c.typ, c.addr, c.ok)
+	}
+	finish(t, d, a1, ResultPartial, -1)
+	finish(t, d, a2, ResultPartial, -1)
+	finish(t, d, a3, ResultPass, -1)
+	run1 := runs(t, d)[0].ID
+
+	submit(t, d, RunManual, a1)
+	addCheck(t, d, a1, SourceEgress, "https", a1, false)
+	addCheck(t, d, a1, InboundSource(1), "tcp-443", a1, true)
+	addCheck(t, d, a1, InboundSource(1), "tls-443", a1, true)
+	finish(t, d, a1, ResultPartial, -1)
+	run2 := runs(t, d)[0].ID
+
+	for _, tc := range []struct {
+		name string
+		f    RegistryFilter
+		want []string
+	}{
+		{"no filter", RegistryFilter{}, []string{a1, a2, a3}},
+		{"run 1", RegistryFilter{RunID: run1}, []string{a1, a2, a3}},
+		{"run 2", RegistryFilter{RunID: run2}, []string{a1}},
+		{"unknown run", RegistryFilter{RunID: 999}, nil},
+		{"egress, newest cycle", RegistryFilter{Level: LevelEgress}, []string{a1, a2}},
+		{"ingress, newest cycle", RegistryFilter{Level: LevelIngress}, []string{a1, a2, a3}},
+		{"verdict of the run", RegistryFilter{RunID: run1, LastResult: ResultPartial}, []string{a1, a2}},
+		{"cancelled in the run", RegistryFilter{RunID: run1, LastResult: ResultCancelled}, nil},
+		{"tcp-22 and tcp-443 are tcp", RegistryFilter{RunID: run1, Family: "tcp"}, []string{a1, a2}},
+		{"tcp partial", RegistryFilter{RunID: run1, Family: "tcp", LastResult: ResultPartial}, []string{a1}},
+		{"tcp pass", RegistryFilter{RunID: run1, Family: "tcp", LastResult: ResultPass}, []string{a2}},
+		{"tcp fail", RegistryFilter{RunID: run1, Family: "tcp", LastResult: ResultFail}, nil},
+		{"tls fail in run 1", RegistryFilter{RunID: run1, Family: "tls", LastResult: ResultFail}, []string{a1}},
+		{"tls pass in run 2", RegistryFilter{RunID: run2, Family: "tls", LastResult: ResultPass}, []string{a1}},
+		{"tls on egress is empty", RegistryFilter{RunID: run1, Level: LevelEgress, Family: "tls"}, nil},
+		{"egress status is of egress checks", RegistryFilter{RunID: run1, Level: LevelEgress, LastResult: ResultPass}, []string{a1}},
+		{"egress fail", RegistryFilter{RunID: run1, Level: LevelEgress, LastResult: ResultFail}, []string{a2}},
+		{"cancelled has no scope", RegistryFilter{Level: LevelEgress, LastResult: ResultCancelled}, nil},
+		{"icmp on ingress", RegistryFilter{RunID: run1, Level: LevelIngress, Family: "icmp"}, []string{a3}},
+		{"subnet, run and level", RegistryFilter{RunID: run1, Level: LevelIngress, Subnet: "10.0.0.0/24"}, []string{a1, a2}},
+		{"subnet with no checks of the level", RegistryFilter{RunID: run1, Level: LevelEgress, Subnet: "10.0.1.0/24"}, nil},
+		{"query and level", RegistryFilter{Query: ".2", Level: LevelEgress}, []string{a2}},
+	} {
+		page, total, err := d.ListRegistryPage(ctx, tc.f, 50, 0)
+		if err != nil {
+			t.Fatalf("%s: %v", tc.name, err)
+		}
+		var got []string
+		for _, s := range page {
+			got = append(got, s.IPAddress)
+		}
+		sort.Strings(got)
+		if total != len(tc.want) || !reflect.DeepEqual(got, tc.want) {
+			t.Errorf("%s: total=%d got=%v, want %v", tc.name, total, got, tc.want)
+		}
+	}
+
+	// The same address shows its own result in each run, narrowed to the scope.
+	for run, want := range map[int64]struct {
+		cycle  int
+		result LevelResult
+	}{
+		run1: {1, LevelResult{Total: 1, OK: 0, ByType: []TypeStat{{"tls", 1, 0}}}},
+		run2: {2, LevelResult{Total: 1, OK: 1, ByType: []TypeStat{{"tls", 1, 1}}}},
+	} {
+		page, _, err := d.ListRegistryPage(ctx, RegistryFilter{RunID: run, Query: a1, Family: "tls"}, 10, 0)
+		if err != nil || len(page) != 1 {
+			t.Fatalf("run %d: %v %+v", run, err, page)
+		}
+		s := page[0]
+		if s.LastCycleID != want.cycle || s.LastResult != ResultPartial || s.Egress.Total != 0 || !reflect.DeepEqual(s.Ingress, want.result) {
+			t.Errorf("run %d: cycle=%d result=%q egress=%+v ingress=%+v", run, s.LastCycleID, s.LastResult, s.Egress, s.Ingress)
+		}
+	}
+
+	for _, f := range []RegistryFilter{{Level: "sideways"}, {Family: "dns"}} {
+		if _, _, err := d.ListRegistryPage(ctx, f, 10, 0); !errors.Is(err, ErrValidation) {
+			t.Errorf("%+v: %v", f, err)
+		}
+	}
+}

internal/db/queries_runs.go

@@ -3,6 +3,7 @@ package db
 import (
 	"context"
 	"database/sql"
+	"encoding/json"
 	"fmt"
 	"net/netip"
 	"sort"
@@ -273,13 +274,25 @@ type RunCheck struct {
 
 // EachRunCheck calls fn for every check of the cycles that make up the run's
 // results (the latest cycle of each address in the run), in one pass over the
-// run_id index. fn must not call back into the DB (one connection).
-func (d *DB) EachRunCheck(ctx context.Context, runID int64, fn func(RunCheck)) error {
-	rows, err := d.QueryContext(ctx, `
+// run_id index. With registryIDs (not nil) only the checks of those addresses
+// are read. fn must not call back into the DB (one connection).
+func (d *DB) EachRunCheck(ctx context.Context, runID int64, registryIDs []int64, fn func(RunCheck)) error {
+	q := `
 		SELECT c.registry_id, c.source, c.check_type, c.target, c.success, c.validator_id, c.detail,
 			COALESCE(c.recorded_at, c.created_at), c.after_verdict
 		FROM checks c JOIN run_results r ON r.run_id=c.run_id AND r.registry_id=c.registry_id AND r.cycle_id=c.cycle_id
-		WHERE c.run_id=?`, runID)
+		WHERE c.run_id=?`
+	args := []any{runID}
+	if registryIDs != nil {
+		ids, err := json.Marshal(registryIDs)
+		if err != nil {
+			return err
+		}
+		// One JSON parameter, not an id per "?" (SQLite's bound-variable limit).
+		q += ` AND c.registry_id IN (SELECT value FROM json_each(?))`
+		args = append(args, string(ids))
+	}
+	rows, err := d.QueryContext(ctx, q, args...)
 	if err != nil {
 		return err
 	}

internal/db/queries_runs_test.go

@@ -120,9 +120,17 @@ func TestRecheckAfterFinalizeOpensNewRun(t *testing.T) {
 	}
 	// The checks of a run are the ones of its result cycle, nothing else.
 	var got []RunCheck
-	if err := d.EachRunCheck(ctx, first.ID, func(c RunCheck) { got = append(got, c) }); err != nil || len(got) != 1 || !got[0].Success {
+	if err := d.EachRunCheck(ctx, first.ID, nil, func(c RunCheck) { got = append(got, c) }); err != nil || len(got) != 1 || !got[0].Success {
 		t.Fatalf("run 1 checks: %+v %v", got, err)
 	}
+	// With a list of addresses only theirs are read; an empty list reads none.
+	got = nil
+	if err := d.EachRunCheck(ctx, first.ID, []int64{}, func(c RunCheck) { got = append(got, c) }); err != nil || len(got) != 0 {
+		t.Fatalf("empty address list must read no checks: %+v %v", got, err)
+	}
+	if err := d.EachRunCheck(ctx, first.ID, []int64{old[0].RegistryID}, func(c RunCheck) { got = append(got, c) }); err != nil || len(got) != 1 {
+		t.Fatalf("listed address: %+v %v", got, err)
+	}
 }
 
 // A re-check while the run is still open joins it and replaces the address's

internal/db/queries_scale_test.go

@@ -351,30 +351,36 @@ func TestMigration0009Indexes(t *testing.T) {
 	}
 }
 
-// TestRegistryLevelsQueryUsesIndex guards against a full scan of checks: both
-// the per-address MAX(cycle_id) and the join back must go through
-// idx_checks_registry_cycle.
+// TestRegistryLevelsQueryUsesIndex guards against a full scan of checks: the
+// per-address MAX(cycle_id) and the join back must go through
+// idx_checks_registry_cycle, the run's slice through idx_checks_run.
 func TestRegistryLevelsQueryUsesIndex(t *testing.T) {
 	d, ctx := newTestDB(t)
-	rows, err := d.QueryContext(ctx, "EXPLAIN QUERY PLAN "+registryLevelsQuery(3), 1, 2, 3)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer rows.Close()
-	var plan string
-	for rows.Next() {
-		var id, parent, unused int
-		var detail string
-		if err := rows.Scan(&id, &parent, &unused, &detail); err != nil {
+	for sl, index := range map[registrySlice]string{
+		{}: "idx_checks_registry_cycle",
+		{RunID: 1, Level: LevelIngress, Family: "tls"}: "idx_checks_run",
+	} {
+		q, args := registryLevelsQuery([]any{1, 2, 3}, sl)
+		rows, err := d.QueryContext(ctx, "EXPLAIN QUERY PLAN "+q, args...)
+		if err != nil {
 			t.Fatal(err)
 		}
-		plan += detail + "\n"
-		if strings.HasPrefix(detail, "SCAN") && strings.Contains(detail, "checks") {
-			t.Errorf("full scan of checks in plan:\n%s", plan)
+		var plan string
+		for rows.Next() {
+			var id, parent, unused int
+			var detail string
+			if err := rows.Scan(&id, &parent, &unused, &detail); err != nil {
+				t.Fatal(err)
+			}
+			plan += detail + "\n"
+			if strings.HasPrefix(detail, "SCAN") && strings.Contains(detail, "checks") {
+				t.Errorf("%+v: full scan of checks in plan:\n%s", sl, plan)
+			}
+		}
+		rows.Close()
+		if !strings.Contains(plan, index) {
+			t.Errorf("%+v: expected %s in plan:\n%s", sl, index, plan)
 		}
-	}
-	if !strings.Contains(plan, "idx_checks_registry_cycle") {
-		t.Errorf("expected idx_checks_registry_cycle in plan:\n%s", plan)
 	}
 }
 
@@ -417,6 +423,17 @@ func TestScaleSmoke6440(t *testing.T) {
 				t.Fatalf("upsert check: %v", err)
 			}
 		}
+		// The run's result of the address (as aggregation would record it).
+		if _, err := d.ExecContext(ctx, `
+			INSERT INTO run_results (run_id, registry_id, ip_address, cycle_id, verdict, aggregated_at)
+			SELECT run_id, registry_id, ip_address, cycle_id, 'partial', ? FROM ip_queue WHERE id=?
+		`, timeToDB(Now()), ip.ID); err != nil {
+			t.Fatalf("run result: %v", err)
+		}
+	}
+	var runID int64
+	if err := d.QueryRowContext(ctx, `SELECT run_id FROM run_results LIMIT 1`).Scan(&runID); err != nil {
+		t.Fatalf("run id: %v", err)
 	}
 
 	start = time.Now()
@@ -430,6 +447,25 @@ func TestScaleSmoke6440(t *testing.T) {
 	if _, total, err = d.ListRegistryPage(ctx, RegistryFilter{LastResult: ResultPass}, 100, 0); err != nil || total != 0 {
 		t.Fatalf("registry last_result filter: total=%d err=%v", total, err)
 	}
+	// All filters at once, the subnet covering the whole registry.
+	page, total, err = d.ListRegistryPage(ctx, RegistryFilter{
+		RunID: runID, Subnet: "10.0.0.0/8", Level: LevelIngress, Family: "tcp", LastResult: ResultPartial,
+	}, 50, 0)
+	if err != nil || total != 100 || len(page) != 50 || page[0].Egress.Total != 0 || page[0].Ingress.Total != 18 {
+		t.Fatalf("registry slice: total=%d len=%d err=%v", total, len(page), err)
+	}
+	// The chart and the list behind one of its rows: 100 addresses x 6 checks per site.
+	bf := RegistryFilter{RunID: runID, Subnet: "10.0.0.0/8", Level: LevelIngress, Family: "tcp", LastResult: ResultPartial}
+	bd, err := d.RegistryBreakdown(ctx, bf)
+	if err != nil || bd.Addresses != 100 || len(bd.Rows) != 3 || bd.Rows[0].Total != 600 || bd.Rows[0].OK != 500 {
+		t.Fatalf("breakdown: %+v err=%v", bd, err)
+	}
+	if l, err := d.RegistryBreakdownList(ctx, bf, bd.Rows[0].Key); err != nil || len(l) != 600 {
+		t.Fatalf("breakdown list: len=%d err=%v", len(l), err)
+	}
+	if bd, err = d.RegistryBreakdown(ctx, RegistryFilter{Level: LevelEgress, Family: "https"}); err != nil || bd.Addresses != 100 || len(bd.Rows) != 6 {
+		t.Fatalf("breakdown, newest cycle: %+v err=%v", bd, err)
+	}
 	registryDur := time.Since(start)
 
 	start = time.Now()