cloud-ip-validator/internal/analytics/analytics.go

// Package analytics turns the stored checks of one finished run into the
// numbers behind the dashboard's analytics page. It works on facts: every
// check stored for the latest cycle of each address in the run, whenever it
// arrived. The verdict is shown next to those facts, never mixed into them.
package analytics

import (
	"net/netip"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"

	"cloudipvalidator/internal/db"
)

// Input is everything Compute needs, already read from the database.
type Input struct {
	Run       db.CheckRun
	Results   []db.RunResult
	Subnets   []db.Subnet
	SiteNames map[int]string // site index -> site id
	Rechecked int            // addresses with more than one cycle in the run
	// Each feeds every check of the run's result cycles to fn.
	Each func(fn func(db.RunCheck)) error
}

// Report is the data of the analytics page for one run.
type Report struct {
	Run        RunInfo                `json:"run"`
	Summary    Summary                `json:"summary"`
	Reasons    []Reason               `json:"reasons"`
	Quality    Quality                `json:"quality"`
	Subnets    []SubnetRow            `json:"subnets"`
	Targets    TargetsBlock           `json:"targets"`
	Matrix     map[string][]MatrixRow `json:"matrix"`
	Sites      SitesBlock             `json:"sites"`
	Errors     []ErrorClass           `json:"errors"`
	Validators []ValidatorRow         `json:"validators"`
}

type RunInfo struct {
	ID          int64      `json:"id"`
	Kind        string     `json:"kind"`
	State       string     `json:"state"`
	StartedAt   time.Time  `json:"started_at"`
	FinalizedAt *time.Time `json:"finalized_at"`
	DurationSec int        `json:"duration_seconds"`
	Rechecked   int        `json:"rechecked"`
}

type Summary struct {
	Addresses      int `json:"addresses"`
	Pass           int `json:"pass"`
	Partial        int `json:"partial"`
	Fail           int `json:"fail"`
	Cancelled      int `json:"cancelled"`
	EgressOK       int `json:"egress_ok"`
	IngressOK      int `json:"ingress_ok"`
	EgressHTTPSAny int `json:"egress_https_any_failed"`
	EgressHTTPSAll int `json:"egress_https_all_failed"`
	// EgressHTTPSAllTargets counts the addresses that failed https to every
	// target of the full set (as many checks as the best-covered address).
	EgressHTTPSAllTargets int     `json:"egress_https_all_targets_failed"`
	IngressSSHAny         int     `json:"ingress_ssh_any_failed"`
	IngressSSHAll         int     `json:"ingress_ssh_all_failed"`
	PerMinute             float64 `json:"addresses_per_minute"`
}

type Reason struct {
	Name  string `json:"name"`
	Count int    `json:"count"`
}

// Quality is the data-quality block: how the verdict relates to the checks.
type Quality struct {
	LateFailedAtPass          int `json:"late_failed_checks_at_pass"`
	LateFailedAtPassAddresses int `json:"late_failed_addresses_at_pass"`
	IngressFailed             int `json:"ingress_failed_checks"`
	IngressFailedLate         int `json:"ingress_failed_late"`
	Incomplete                int `json:"incomplete_addresses"`
	PassWithFailed            int `json:"pass_with_failed_addresses"`
	PassByFacts               int `json:"pass_by_facts"`
}

type SubnetRow struct {
	CIDR      string `json:"cidr"`
	Label     string `json:"label,omitempty"`
	Addresses int    `json:"addresses"`
	Pass      int    `json:"pass"`
	EgressOK  int    `json:"egress_ok"`
	IngressOK int    `json:"ingress_ok"`
}

type TargetsBlock struct {
	Types   []string         `json:"types"`
	Targets []string         `json:"targets"`
	Failed  map[string][]int `json:"failed"` // type -> failed addresses per target, in Targets order
}

type MatrixRow struct {
	CIDR    string `json:"cidr"`
	Partial int    `json:"partial"`
	Percent []int  `json:"percent"` // per target, in Targets order
}

type SitesBlock struct {
	Types []string  `json:"types"`
	Rows  []SiteRow `json:"rows"`
}

type SiteRow struct {
	Site  string     `json:"site"`
	Stats []SiteStat `json:"stats"` // per type, in Types order
}

type SiteStat struct {
	Total int `json:"total"`
	OK    int `json:"ok"`
}

type ErrorClass struct {
	Name  string `json:"name"`
	Count int    `json:"count"`
}

type ValidatorRow struct {
	Validator string `json:"validator"`
	Total     int    `json:"total"`
	OK        int    `json:"ok"`
}

type typeStat struct{ n, ok int }

type failedIngress struct {
	class, site, validator string
	late                   bool
}

// addr is everything known about one address of the run.
type addr struct {
	res     db.RunResult
	subnet  string
	egress  typeStat
	ingress typeStat
	stored  int
	https   struct {
		typeStat
		validator     string
		failedTargets []string
	}
	ssh struct {
		typeStat
		sites []string
		errs  map[string]bool
	}
	failedTargets map[string]bool // family\x00target -> failed
	failedIngress []failedIngress
	lateFailed    int
}

// Analysis is a computed report plus the per-address data the lists are cut from.
type Analysis struct {
	Report    Report
	addrs     []*addr
	siteNames map[int]string
}

// Compute reads the checks of the run once and builds the report.
func Compute(in Input) (*Analysis, error) {
	byReg := make(map[int64]*addr, len(in.Results))
	var addrs []*addr
	subnetOf := newSubnetMatcher(in.Subnets)
	for _, r := range in.Results {
		a := &addr{res: r, subnet: subnetOf(r.IPAddress), failedTargets: map[string]bool{}}
		a.ssh.errs = map[string]bool{}
		byReg[r.RegistryID] = a
		addrs = append(addrs, a)
	}
	siteName := func(source string) string {
		idx, _ := strconv.Atoi(strings.TrimPrefix(source, "inbound-site-"))
		if n := in.SiteNames[idx]; n != "" {
			return n
		}
		return "site-" + strconv.Itoa(idx)
	}

	type key struct{ site, typ string }
	siteStats := map[key]*typeStat{}
	siteTypes := map[string]bool{}
	egressTypes := map[string]bool{}
	valHTTPS := map[string]*typeStat{}
	targetSet := map[string]bool{}
	errCount := map[string]int{}

	err := in.Each(func(c db.RunCheck) {
		a := byReg[c.RegistryID]
		if a == nil || a.res.Verdict == db.ResultCancelled {
			return // a cancelled address was stopped, its checks say nothing
		}
		a.stored++
		late := c.AfterVerdict || c.RecordedAt.After(a.res.AggregatedAt)
		family := db.CheckFamily(c.CheckType)
		switch db.CheckLevel(c.Source) {
		case db.LevelEgress:
			a.egress.n++
			if c.Success {
				a.egress.ok++
			}
			egressTypes[family] = true
			target := normalizeTarget(c.Target)
			targetSet[target] = true
			if !c.Success {
				a.failedTargets[family+"\x00"+target] = true
			}
			if family == "https" {
				a.https.n++
				a.https.validator = c.ValidatorID
				if c.Success {
					a.https.ok++
				} else {
					a.https.failedTargets = append(a.https.failedTargets, target)
				}
				v := valHTTPS[c.ValidatorID]
				if v == nil {
					v = &typeStat{}
					valHTTPS[c.ValidatorID] = v
				}
				v.n++
				if c.Success {
					v.ok++
				}
			}
		case db.LevelIngress:
			a.ingress.n++
			if c.Success {
				a.ingress.ok++
			}
			site := siteName(c.Source)
			siteTypes[family] = true
			ss := siteStats[key{site, family}]
			if ss == nil {
				ss = &typeStat{}
				siteStats[key{site, family}] = ss
			}
			ss.n++
			if c.Success {
				ss.ok++
			}
			if family == "ssh" {
				a.ssh.n++
				if c.Success {
					a.ssh.ok++
				}
			}
			if !c.Success {
				class := ErrorClassOf(c.CheckType, c.Detail)
				errCount[class]++
				a.failedIngress = append(a.failedIngress, failedIngress{class: class, site: site, validator: c.ValidatorID, late: late})
				if family == "ssh" {
					a.ssh.sites = append(a.ssh.sites, site)
					a.ssh.errs[errorReason(c.CheckType, c.Detail)] = true
				}
			}
		}
		if late && !c.Success {
			a.lateFailed++
		}
	})
	if err != nil {
		return nil, err
	}

	rep := Report{Matrix: map[string][]MatrixRow{}}
	rep.Run = RunInfo{ID: in.Run.ID, Kind: in.Run.Kind, State: in.Run.State, StartedAt: in.Run.StartedAt,
		FinalizedAt: in.Run.FinalizedAt, Rechecked: in.Rechecked}
	if in.Run.FinalizedAt != nil {
		rep.Run.DurationSec = int(in.Run.FinalizedAt.Sub(in.Run.StartedAt).Seconds())
	}

	maxHTTPS := 0
	for _, a := range addrs {
		if a.https.n > maxHTTPS {
			maxHTTPS = a.https.n
		}
	}

	reasonCount := map[string]int{}
	type subAgg struct {
		n, pass, eg, ing, partial int
		failed                    map[string]int
	}
	subs := map[string]*subAgg{}
	sum := &rep.Summary
	for _, a := range addrs {
		v := a.res.Verdict
		if v == db.ResultCancelled {
			sum.Cancelled++
			continue
		}
		sum.Addresses++
		switch v {
		case db.ResultPass:
			sum.Pass++
		case db.ResultPartial:
			sum.Partial++
		case db.ResultFail:
			sum.Fail++
		}
		egOK := a.egress.n > 0 && a.egress.ok == a.egress.n
		inOK := a.ingress.n > 0 && a.ingress.ok == a.ingress.n
		if egOK {
			sum.EgressOK++
		}
		if inOK {
			sum.IngressOK++
		}
		if a.https.n > 0 && a.https.ok < a.https.n {
			sum.EgressHTTPSAny++
			if a.https.ok == 0 {
				sum.EgressHTTPSAll++
				if a.https.n == maxHTTPS {
					sum.EgressHTTPSAllTargets++
				}
			}
		}
		if a.ssh.n > 0 && a.ssh.ok < a.ssh.n {
			sum.IngressSSHAny++
			if a.ssh.ok == 0 {
				sum.IngressSSHAll++
			}
		}

		egFail := a.egress.ok < a.egress.n
		inFail := a.ingress.ok < a.ingress.n
		incomplete := a.res.ExpectedChecks >= 0 && a.stored < a.res.ExpectedChecks
		if incomplete {
			rep.Quality.Incomplete++
		}
		if v == db.ResultPartial {
			reasonCount[reasonName(egFail, inFail, incomplete)]++
		}
		if v == db.ResultPass {
			if a.egress.ok < a.egress.n || a.ingress.ok < a.ingress.n {
				rep.Quality.PassWithFailed++
				rep.Quality.LateFailedAtPass += a.lateFailed
				rep.Quality.LateFailedAtPassAddresses++
			}
		}

		sa := subs[a.subnet]
		if sa == nil {
			sa = &subAgg{failed: map[string]int{}}
			subs[a.subnet] = sa
		}
		sa.n++
		if v == db.ResultPass {
			sa.pass++
		}
		if egOK {
			sa.eg++
		}
		if inOK {
			sa.ing++
		}
		if v == db.ResultPartial {
			sa.partial++
			for k := range a.failedTargets {
				sa.failed[k]++
			}
		}
	}
	rep.Quality.PassByFacts = sum.Pass - rep.Quality.PassWithFailed
	if sum.Addresses > 0 && rep.Run.DurationSec > 0 {
		sum.PerMinute = float64(sum.Addresses) / (float64(rep.Run.DurationSec) / 60)
	}
	for _, a := range addrs {
		for _, f := range a.failedIngress {
			rep.Quality.IngressFailed++
			if f.late {
				rep.Quality.IngressFailedLate++
			}
		}
	}

	for _, name := range reasonOrder {
		if n := reasonCount[name]; n > 0 {
			rep.Reasons = append(rep.Reasons, Reason{Name: name, Count: n})
		}
	}

	// Subnets: worst first is the page's job; the report lists them by size.
	labels := map[string]string{}
	for _, s := range in.Subnets {
		labels[s.CIDR] = s.Label
	}
	for cidr, sa := range subs {
		rep.Subnets = append(rep.Subnets, SubnetRow{CIDR: cidr, Label: labels[cidr], Addresses: sa.n, Pass: sa.pass, EgressOK: sa.eg, IngressOK: sa.ing})
	}
	sort.Slice(rep.Subnets, func(i, j int) bool {
		if rep.Subnets[i].Addresses != rep.Subnets[j].Addresses {
			return rep.Subnets[i].Addresses > rep.Subnets[j].Addresses
		}
		return rep.Subnets[i].CIDR < rep.Subnets[j].CIDR
	})

	// Targets and the subnet x target matrix, per egress check family.
	types := sortedKeys(egressTypes)
	rep.Targets.Types = types
	failedAddrs := map[string]int{} // family\x00target -> addresses
	for _, a := range addrs {
		if a.res.Verdict == db.ResultCancelled {
			continue
		}
		for k := range a.failedTargets {
			failedAddrs[k]++
		}
	}
	targets := sortedKeys(targetSet)
	lead := ""
	if len(types) > 0 {
		lead = types[0]
		for _, t := range types {
			if t == "https" {
				lead = t
			}
		}
	}
	sort.SliceStable(targets, func(i, j int) bool {
		fi, fj := failedAddrs[lead+"\x00"+targets[i]], failedAddrs[lead+"\x00"+targets[j]]
		if fi != fj {
			return fi > fj
		}
		return targets[i] < targets[j]
	})
	rep.Targets.Targets = targets
	rep.Targets.Failed = map[string][]int{}
	for _, t := range types {
		row := make([]int, len(targets))
		for i, tg := range targets {
			row[i] = failedAddrs[t+"\x00"+tg]
		}
		rep.Targets.Failed[t] = row
	}
	for _, t := range types {
		var rows []MatrixRow
		for cidr, sa := range subs {
			if sa.partial == 0 {
				continue
			}
			pc := make([]int, len(targets))
			for i, tg := range targets {
				pc[i] = int(float64(sa.failed[t+"\x00"+tg])/float64(sa.partial)*100 + 0.5)
			}
			rows = append(rows, MatrixRow{CIDR: cidr, Partial: sa.partial, Percent: pc})
		}
		sort.Slice(rows, func(i, j int) bool {
			if rows[i].Partial != rows[j].Partial {
				return rows[i].Partial > rows[j].Partial
			}
			return rows[i].CIDR < rows[j].CIDR
		})
		rep.Matrix[t] = rows
	}

	// Sites.
	rep.Sites.Types = sortedKeys(siteTypes)
	names := map[string]bool{}
	for k := range siteStats {
		names[k.site] = true
	}
	siteList := sortedKeys(names)
	sort.Slice(siteList, func(i, j int) bool {
		return siteIndexOf(in.SiteNames, siteList[i]) < siteIndexOf(in.SiteNames, siteList[j])
	})
	for _, s := range siteList {
		row := SiteRow{Site: s}
		for _, t := range rep.Sites.Types {
			st := siteStats[key{s, t}]
			if st == nil {
				st = &typeStat{}
			}
			row.Stats = append(row.Stats, SiteStat{Total: st.n, OK: st.ok})
		}
		rep.Sites.Rows = append(rep.Sites.Rows, row)
	}

	for name, n := range errCount {
		rep.Errors = append(rep.Errors, ErrorClass{Name: name, Count: n})
	}
	sort.Slice(rep.Errors, func(i, j int) bool {
		if rep.Errors[i].Count != rep.Errors[j].Count {
			return rep.Errors[i].Count > rep.Errors[j].Count
		}
		return rep.Errors[i].Name < rep.Errors[j].Name
	})

	for id, st := range valHTTPS {
		rep.Validators = append(rep.Validators, ValidatorRow{Validator: id, Total: st.n, OK: st.ok})
	}
	sort.Slice(rep.Validators, func(i, j int) bool {
		a, b := validatorNumber(rep.Validators[i].Validator), validatorNumber(rep.Validators[j].Validator)
		if a != b {
			return a < b
		}
		return rep.Validators[i].Validator < rep.Validators[j].Validator
	})

	return &Analysis{Report: rep, addrs: addrs, siteNames: in.SiteNames}, nil
}

var reasonOrder = []string{
	"Только egress",
	"Ingress и egress",
	"Egress и неполный набор",
	"Ingress, egress и неполный набор",
	"Только неполный набор",
	"Только ingress",
	"Ingress и неполный набор",
	"Прочее",
}

func reasonName(egress, ingress, incomplete bool) string {
	switch {
	case egress && ingress && incomplete:
		return "Ingress, egress и неполный набор"
	case egress && ingress:
		return "Ingress и egress"
	case egress && incomplete:
		return "Egress и неполный набор"
	case egress:
		return "Только egress"
	case ingress && incomplete:
		return "Ingress и неполный набор"
	case ingress:
		return "Только ingress"
	case incomplete:
		return "Только неполный набор"
	}
	return "Прочее"
}

func sortedKeys(m map[string]bool) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}

func siteIndexOf(names map[int]string, site string) int {
	for i, n := range names {
		if n == site {
			return i
		}
	}
	if n, err := strconv.Atoi(strings.TrimPrefix(site, "site-")); err == nil {
		return n
	}
	return 1 << 20
}

// validatorNumber is the trailing number of a validator id ("vkiplab-v12" ->
// 12), or 1<<20 when there is none, so numbered validators sort naturally.
func validatorNumber(id string) int {
	i := len(id)
	for i > 0 && id[i-1] >= '0' && id[i-1] <= '9' {
		i--
	}
	if i == len(id) {
		return 1 << 20
	}
	n, _ := strconv.Atoi(id[i:])
	return n
}

// ShortValidator is the validator id as the page shows it: "vkiplab-v12" ->
// "v12"; ids without a number stay whole.
func ShortValidator(id string) string {
	n := validatorNumber(id)
	if n == 1<<20 {
		return id
	}
	return "v" + strconv.Itoa(n)
}

// normalizeTarget is the host of an egress target: https://host/path -> host.
func normalizeTarget(t string) string {
	if u, err := url.Parse(t); err == nil && u.Host != "" {
		return u.Hostname()
	}
	return strings.TrimSuffix(t, "/")
}

// newSubnetMatcher returns a function that maps an address to the most
// specific configured subnet. With no subnets configured addresses group by
// /24 (/64 for IPv6). An address outside the list goes to "прочие".
func newSubnetMatcher(subnets []db.Subnet) func(string) string {
	type entry struct {
		p    netip.Prefix
		name string
	}
	var list []entry
	for _, s := range subnets {
		if p, err := netip.ParsePrefix(s.CIDR); err == nil {
			list = append(list, entry{p.Masked(), p.Masked().String()})
		}
	}
	sort.Slice(list, func(i, j int) bool { return list[i].p.Bits() > list[j].p.Bits() })
	return func(ip string) string {
		a, err := netip.ParseAddr(ip)
		if err != nil {
			return "прочие"
		}
		if len(list) == 0 {
			bits := 24
			if a.Is6() {
				bits = 64
			}
			p, _ := a.Prefix(bits)
			return p.String()
		}
		for _, e := range list {
			if e.p.Contains(a) {
				return e.name
			}
		}
		return "прочие"
	}
}

// ErrorClassOf names the class of a failed ingress check: the check type and
// the reason, e.g. "SSH: таймаут", "ICMP: нет ответа".
func ErrorClassOf(checkType, detail string) string {
	return strings.ToUpper(checkType) + ": " + errorReason(checkType, detail)
}

func errorReason(checkType, detail string) string {
	d := strings.ToLower(detail)
	switch {
	case strings.Contains(d, "unexpected banner prefix"):
		if strings.Contains(d, "not allo") {
			return "баннер «Not allowed»"
		}
		return "неожиданный баннер"
	case strings.Contains(d, "no route to host"):
		return "нет маршрута"
	case strings.Contains(d, "time exceeded"):
		return "time exceeded"
	case strings.Contains(d, "connection refused"):
		return "отказ в соединении"
	case strings.Contains(d, "timeout") || strings.Contains(d, "deadline exceeded"):
		if strings.EqualFold(checkType, "icmp") {
			return "нет ответа"
		}
		return "таймаут"
	}
	if strings.EqualFold(checkType, "icmp") {
		return "нет ответа"
	}
	return "прочее"
}