Add the Analytics section: check runs, analytics API and page
Runs (migration 0011): a run groups the cycles of one launch. It opens when an address enters an idle queue, takes everything submitted or re-checked while it is open and is finalized when all its addresses are done; a re-check after that opens a new run, so results of different runs never mix. check_runs, run_results (one result per address and run, with the verdict and the expected and stored check counts), subnets, run_id on ip_queue and checks. Existing data is split into runs at pauses of more than an hour; ingress checks get the validator that held the address (also at write time from now on). Analytics (internal/analytics): figures computed from the stored checks of the latest cycle of each address in the run, as facts next to the verdict: summary, reasons of partial, data quality, subnets, targets and the subnet x target matrix by check type, ingress by site, error classes, validators, and the address lists behind the indicators and error classes. API: analytics runs, report, lists (JSON or CSV), subnet list; run and subnet filters for the registry. Dashboard: /analytics matching the approved mockup (run selector, indicators with address lists and CSV, error-class dialogs, drill-down to the registry), subnet list on /settings. Sidebar: the control-api link state, theme toggle and logout moved to the top, the three dots next to the logo removed, sections grouped. Rebuilt bin/control-api and bin/admin-dashboard to match. Plan, summary and the updated README, API, USAGE, DASHBOARD and ADMIN_CLEANUP docs are in docs/. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
864208238f
commit
b7669c9e41
44 files changed
+4123
-62
No files matched your search
@@ -0,0 +1,98 @@
|
||||
-- Check runs (see docs/changes/2026-10-03_16-39_analytics-section-plan.md).
|
||||
--
|
||||
-- cycle_id counts per address, so it cannot tell one launch from another. A
|
||||
-- run groups the cycles of one launch: it opens when an address enters an idle
|
||||
-- queue, takes every address submitted or re-checked while it is open, and is
|
||||
-- finalized when all its queue rows are terminal. checks.run_id and
|
||||
-- ip_queue.run_id carry the membership; run_results keeps one row per address
|
||||
-- and run (its latest cycle) with the verdict, which ip_queue overwrites on a
|
||||
-- re-check.
|
||||
--
|
||||
-- No foreign keys on purpose: the manual cleanup in docs/ADMIN_CLEANUP.md
|
||||
-- deletes from these tables freely.
|
||||
|
||||
CREATE TABLE check_runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
kind TEXT NOT NULL DEFAULT 'manual', -- manual | auto
|
||||
state TEXT NOT NULL DEFAULT 'open', -- open | finalized
|
||||
started_at TIMESTAMP NOT NULL,
|
||||
finalized_at TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE run_results (
|
||||
run_id INTEGER NOT NULL,
|
||||
registry_id INTEGER NOT NULL,
|
||||
ip_address TEXT NOT NULL,
|
||||
cycle_id INTEGER NOT NULL,
|
||||
verdict TEXT NOT NULL, -- pass | partial | fail | cancelled
|
||||
verdict_derived INTEGER NOT NULL DEFAULT 0, -- 1: computed from checks, not stored by the orchestrator
|
||||
aggregated_at TIMESTAMP NOT NULL,
|
||||
expected_checks INTEGER, -- NULL when unknown
|
||||
recorded_checks INTEGER NOT NULL DEFAULT 0, -- checks stored when the verdict was made
|
||||
PRIMARY KEY (run_id, registry_id)
|
||||
);
|
||||
CREATE INDEX idx_run_results_registry ON run_results(registry_id);
|
||||
|
||||
CREATE TABLE subnets (
|
||||
cidr TEXT PRIMARY KEY,
|
||||
label TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
ALTER TABLE ip_queue ADD COLUMN run_id INTEGER;
|
||||
ALTER TABLE checks ADD COLUMN run_id INTEGER;
|
||||
CREATE INDEX idx_checks_run ON checks(run_id, registry_id, cycle_id);
|
||||
|
||||
-- ---- existing data -------------------------------------------------------
|
||||
-- Ingress checks never stored the validator. The one holding the address in a
|
||||
-- cycle is named by its fip_associated event.
|
||||
UPDATE checks SET validator_id = COALESCE((
|
||||
SELECT json_extract(e.payload, '$.validator_id') FROM events e
|
||||
WHERE e.event_type = 'fip_associated' AND e.registry_id = checks.registry_id AND e.cycle_id = checks.cycle_id
|
||||
ORDER BY e.id DESC LIMIT 1), '')
|
||||
WHERE source LIKE 'inbound-site-%' AND validator_id = '';
|
||||
|
||||
-- Runs of existing data: cycles whose last checks end less than an hour apart
|
||||
-- belong to one run.
|
||||
CREATE TEMP TABLE _bf_cyc AS
|
||||
WITH c AS (
|
||||
SELECT registry_id, cycle_id, MAX(julianday(checked_at)) AS t, MIN(checked_at) AS first_at, MAX(checked_at) AS last_at
|
||||
FROM checks GROUP BY registry_id, cycle_id
|
||||
), o AS (
|
||||
SELECT *, CASE WHEN t - LAG(t) OVER (ORDER BY t) > 60.0 / 1440.0 THEN 1 ELSE 0 END AS brk FROM c
|
||||
)
|
||||
SELECT *, 1 + SUM(brk) OVER (ORDER BY t ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS rid FROM o;
|
||||
|
||||
CREATE INDEX _bf_cyc_key ON _bf_cyc(registry_id, cycle_id);
|
||||
|
||||
INSERT INTO check_runs (id, kind, state, started_at, finalized_at)
|
||||
SELECT rid, 'manual', 'finalized', MIN(first_at), MAX(last_at) FROM _bf_cyc GROUP BY rid;
|
||||
|
||||
UPDATE checks SET run_id = (
|
||||
SELECT b.rid FROM _bf_cyc b WHERE b.registry_id = checks.registry_id AND b.cycle_id = checks.cycle_id);
|
||||
|
||||
UPDATE ip_queue SET run_id = (
|
||||
SELECT b.rid FROM _bf_cyc b WHERE b.registry_id = ip_queue.registry_id AND b.cycle_id = ip_queue.cycle_id);
|
||||
|
||||
-- One result per address and run: the latest cycle of the address in the run.
|
||||
-- The verdict is the orchestrator's when the queue row still holds that cycle,
|
||||
-- otherwise it is derived from the stored checks.
|
||||
INSERT INTO run_results (run_id, registry_id, ip_address, cycle_id, verdict, verdict_derived, aggregated_at, expected_checks, recorded_checks)
|
||||
SELECT l.rid, l.registry_id, r.ip_address, l.cycle_id,
|
||||
CASE WHEN q.id IS NOT NULL AND q.overall_result <> '' THEN q.overall_result
|
||||
WHEN s.ok = 0 THEN 'fail' WHEN s.ok = s.n THEN 'pass' ELSE 'partial' END,
|
||||
CASE WHEN q.id IS NOT NULL AND q.overall_result <> '' THEN 0 ELSE 1 END,
|
||||
COALESCE(CASE WHEN q.overall_result <> '' THEN q.aggregated_at END, s.last_at),
|
||||
(SELECT json_extract(e.payload, '$.checks') + json_extract(e.payload, '$.missing') FROM events e
|
||||
WHERE e.event_type = 'aggregated' AND e.registry_id = l.registry_id AND e.cycle_id = l.cycle_id
|
||||
ORDER BY e.id DESC LIMIT 1),
|
||||
COALESCE((SELECT json_extract(e.payload, '$.checks') FROM events e
|
||||
WHERE e.event_type = 'aggregated' AND e.registry_id = l.registry_id AND e.cycle_id = l.cycle_id
|
||||
ORDER BY e.id DESC LIMIT 1), s.n_before)
|
||||
FROM (SELECT rid, registry_id, MAX(cycle_id) AS cycle_id FROM _bf_cyc GROUP BY rid, registry_id) l
|
||||
JOIN ip_registry r ON r.id = l.registry_id
|
||||
JOIN (SELECT registry_id, cycle_id, COUNT(*) AS n, SUM(success) AS ok, MAX(checked_at) AS last_at,
|
||||
SUM(CASE WHEN after_verdict = 0 THEN 1 ELSE 0 END) AS n_before
|
||||
FROM checks GROUP BY registry_id, cycle_id) s ON s.registry_id = l.registry_id AND s.cycle_id = l.cycle_id
|
||||
LEFT JOIN ip_queue q ON q.registry_id = l.registry_id AND q.cycle_id = l.cycle_id;
|
||||
|
||||
DROP TABLE _bf_cyc;
|
||||
Reference in new issue
Block a user