Add the Analytics section: check runs, analytics API and page
Runs (migration 0011): a run groups the cycles of one launch. It opens when an address enters an idle queue, takes everything submitted or re-checked while it is open and is finalized when all its addresses are done; a re-check after that opens a new run, so results of different runs never mix. check_runs, run_results (one result per address and run, with the verdict and the expected and stored check counts), subnets, run_id on ip_queue and checks. Existing data is split into runs at pauses of more than an hour; ingress checks get the validator that held the address (also at write time from now on). Analytics (internal/analytics): figures computed from the stored checks of the latest cycle of each address in the run, as facts next to the verdict: summary, reasons of partial, data quality, subnets, targets and the subnet x target matrix by check type, ingress by site, error classes, validators, and the address lists behind the indicators and error classes. API: analytics runs, report, lists (JSON or CSV), subnet list; run and subnet filters for the registry. Dashboard: /analytics matching the approved mockup (run selector, indicators with address lists and CSV, error-class dialogs, drill-down to the registry), subnet list on /settings. Sidebar: the control-api link state, theme toggle and logout moved to the top, the three dots next to the logo removed, sections grouped. Rebuilt bin/control-api and bin/admin-dashboard to match. Plan, summary and the updated README, API, USAGE, DASHBOARD and ADMIN_CLEANUP docs are in docs/. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
864208238f
commit
b7669c9e41
44 files changed
+4123
-62
No files matched your search
@@ -21,6 +21,7 @@ func (d *DB) SeedQueue(ctx context.Context, addresses []string) error {
|
||||
defer tx.Rollback()
|
||||
|
||||
now := timeToDB(Now())
|
||||
runID := int64(0)
|
||||
for i, addr := range addresses {
|
||||
var exists bool
|
||||
if err := tx.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM ip_queue WHERE ip_address=?)`, addr).Scan(&exists); err != nil {
|
||||
@@ -37,11 +38,16 @@ func (d *DB) SeedQueue(ctx context.Context, addresses []string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if runID == 0 {
|
||||
if runID, err = openRunTx(ctx, tx, RunManual, now); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO ip_queue (ip_address, sequence, state, registry_id, cycle_id, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO ip_queue (ip_address, sequence, state, registry_id, cycle_id, run_id, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(ip_address) DO NOTHING
|
||||
`, addr, i, IPQueued, registryID, cycle, now, now); err != nil {
|
||||
`, addr, i, IPQueued, registryID, cycle, runID, now, now); err != nil {
|
||||
return fmt.Errorf("seed %s: %w", addr, err)
|
||||
}
|
||||
}
|
||||
@@ -194,16 +200,36 @@ func (d *DB) SetAggregating(ctx context.Context, ipID int64) error {
|
||||
|
||||
// FinishIP records the aggregated result and marks the IP done or failed.
|
||||
func (d *DB) FinishIP(ctx context.Context, ipID int64, result string) error {
|
||||
return d.FinishIPExpected(ctx, ipID, result, -1)
|
||||
}
|
||||
|
||||
// FinishIPExpected is FinishIP that also records, in the address's run, how
|
||||
// many checks were expected at the verdict (expected < 0: unknown) and
|
||||
// finalizes the run if this was its last open address.
|
||||
func (d *DB) FinishIPExpected(ctx context.Context, ipID int64, result string, expected int) error {
|
||||
state := IPDone
|
||||
if result == ResultFail {
|
||||
state = IPFailed
|
||||
}
|
||||
tx, err := d.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
now := timeToDB(Now())
|
||||
_, err := d.ExecContext(ctx, `
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE ip_queue SET state=?, overall_result=?, aggregated_at=?, updated_at=?
|
||||
WHERE id=?
|
||||
`, state, result, now, now, ipID)
|
||||
return err
|
||||
`, state, result, now, now, ipID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := upsertRunResultTx(ctx, tx, ipID, result, expected, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := finalizeRunsTx(ctx, tx, now); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// ReleaseFIP records that the floating IP has been disassociated and frees
|
||||
@@ -254,6 +280,9 @@ func (d *DB) MarkFIPOccupied(ctx context.Context, ipID int64, validatorID string
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := finalizeRunsTx(ctx, tx, now); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
@@ -304,6 +333,12 @@ func (d *DB) RequeueOrFail(ctx context.Context, ipID int64, validatorID string,
|
||||
state=?, retry_count=?, overall_result=?, aggregated_at=?, updated_at=?
|
||||
WHERE id=?
|
||||
`, nextState, retryCount, ResultFail, now, now, ipID)
|
||||
if err == nil {
|
||||
err = upsertRunResultTx(ctx, tx, ipID, ResultFail, -1, now)
|
||||
}
|
||||
if err == nil {
|
||||
err = finalizeRunsTx(ctx, tx, now)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -337,6 +372,12 @@ func (d *DB) RequeueOrFail(ctx context.Context, ipID int64, validatorID string,
|
||||
// sequence, so a batch's relative order is preserved and, critically,
|
||||
// resubmitting the same list later reproduces the same relative order.
|
||||
func (d *DB) SubmitIPs(ctx context.Context, addresses []string) (SubmitIPsResult, error) {
|
||||
return d.SubmitIPsAs(ctx, addresses, RunManual)
|
||||
}
|
||||
|
||||
// SubmitIPsAs is SubmitIPs that names the kind of run it opens when no run is
|
||||
// open (RunManual or RunAuto); an already open run is joined whatever the kind.
|
||||
func (d *DB) SubmitIPsAs(ctx context.Context, addresses []string, kind string) (SubmitIPsResult, error) {
|
||||
var result SubmitIPsResult
|
||||
if len(addresses) == 0 {
|
||||
return result, fmt.Errorf("addresses must not be empty: %w", ErrValidation)
|
||||
@@ -354,6 +395,17 @@ func (d *DB) SubmitIPs(ctx context.Context, addresses []string) (SubmitIPsResult
|
||||
}
|
||||
|
||||
now := timeToDB(Now())
|
||||
runID := int64(0)
|
||||
ensureRun := func() (int64, error) {
|
||||
if runID == 0 {
|
||||
id, err := openRunTx(ctx, tx, kind, now)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
runID = id
|
||||
}
|
||||
return runID, nil
|
||||
}
|
||||
for i, addr := range addresses {
|
||||
seq := base + i
|
||||
|
||||
@@ -369,10 +421,14 @@ func (d *DB) SubmitIPs(ctx context.Context, addresses []string) (SubmitIPsResult
|
||||
if cErr != nil {
|
||||
return result, cErr
|
||||
}
|
||||
rid, rErr := ensureRun()
|
||||
if rErr != nil {
|
||||
return result, rErr
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO ip_queue (ip_address, sequence, state, registry_id, cycle_id, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`, addr, seq, IPQueued, registryID, cycle, now, now); err != nil {
|
||||
INSERT INTO ip_queue (ip_address, sequence, state, registry_id, cycle_id, run_id, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`, addr, seq, IPQueued, registryID, cycle, rid, now, now); err != nil {
|
||||
return result, fmt.Errorf("insert %s: %w", addr, err)
|
||||
}
|
||||
result.Added = append(result.Added, addr)
|
||||
@@ -389,14 +445,18 @@ func (d *DB) SubmitIPs(ctx context.Context, addresses []string) (SubmitIPsResult
|
||||
if cErr != nil {
|
||||
return result, cErr
|
||||
}
|
||||
rid, rErr := ensureRun()
|
||||
if rErr != nil {
|
||||
return result, rErr
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE ip_queue SET
|
||||
state=?, sequence=?, owner_validator_id=NULL, fip_id='', retry_count=0,
|
||||
attempt_number=attempt_number+1, cycle_id=?, lease_expires_at=NULL, egress_complete=0,
|
||||
overall_result='',
|
||||
overall_result='', run_id=?,
|
||||
assigned_at=NULL, checking_started_at=NULL, fip_associated_at=NULL, aggregated_at=NULL, fip_released_at=NULL, updated_at=?
|
||||
WHERE ip_address=?
|
||||
`, IPQueued, seq, cycle, now, addr); err != nil {
|
||||
`, IPQueued, seq, cycle, rid, now, addr); err != nil {
|
||||
return result, fmt.Errorf("requeue %s: %w", addr, err)
|
||||
}
|
||||
result.Requeued = append(result.Requeued, addr)
|
||||
@@ -431,7 +491,12 @@ func (d *DB) SubmitIPs(ctx context.Context, addresses []string) (SubmitIPsResult
|
||||
// closed by the single-connection transactional UPDATE below).
|
||||
func (d *DB) CancelIP(ctx context.Context, ipID int64) error {
|
||||
now := timeToDB(Now())
|
||||
res, err := d.ExecContext(ctx, `
|
||||
tx, err := d.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
res, err := tx.ExecContext(ctx, `
|
||||
UPDATE ip_queue SET
|
||||
state=?, overall_result=?, aggregated_at=?, owner_validator_id=NULL, fip_id='',
|
||||
lease_expires_at=NULL, updated_at=?
|
||||
@@ -443,7 +508,13 @@ func (d *DB) CancelIP(ctx context.Context, ipID int64) error {
|
||||
if n, _ := res.RowsAffected(); n == 0 {
|
||||
return fmt.Errorf("ip_id %d already finished: %w", ipID, ErrInvalidState)
|
||||
}
|
||||
return nil
|
||||
if err := upsertRunResultTx(ctx, tx, ipID, ResultCancelled, -1, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := finalizeRunsTx(ctx, tx, now); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// DeleteIP permanently removes an ip_queue row, along with its full check
|
||||
@@ -463,6 +534,9 @@ func (d *DB) DeleteIP(ctx context.Context, ipID int64) error {
|
||||
if err := deleteIPTx(ctx, tx, ipID); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := finalizeRunsTx(ctx, tx, timeToDB(Now())); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
@@ -497,6 +571,9 @@ func (d *DB) DeleteIPs(ctx context.Context, addresses []string) (DeleteIPsResult
|
||||
result.Deleted = append(result.Deleted, addr)
|
||||
}
|
||||
|
||||
if err := finalizeRunsTx(ctx, tx, timeToDB(Now())); err != nil {
|
||||
return result, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
@@ -592,6 +669,9 @@ func (d *DB) ClearAllIPs(ctx context.Context) ([]string, error) {
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM ip_queue`); err != nil {
|
||||
return nil, fmt.Errorf("delete ip_queue rows: %w", err)
|
||||
}
|
||||
if err := finalizeRunsTx(ctx, tx, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user