// Package analytics turns the stored checks of one finished run into the // numbers behind the dashboard's analytics page. It works on facts: every // check stored for the latest cycle of each address in the run, whenever it // arrived. The verdict is shown next to those facts, never mixed into them. package analytics import ( "net/netip" "net/url" "sort" "strconv" "strings" "time" "cloudipvalidator/internal/db" ) // Input is everything Compute needs, already read from the database. type Input struct { Run db.CheckRun Results []db.RunResult Subnets []db.Subnet SiteNames map[int]string // site index -> site id Rechecked int // addresses with more than one cycle in the run // Each feeds every check of the run's result cycles to fn. Each func(fn func(db.RunCheck)) error } // Report is the data of the analytics page for one run. type Report struct { Run RunInfo `json:"run"` Summary Summary `json:"summary"` Reasons []Reason `json:"reasons"` Quality Quality `json:"quality"` Subnets []SubnetRow `json:"subnets"` Targets TargetsBlock `json:"targets"` Matrix map[string][]MatrixRow `json:"matrix"` Sites SitesBlock `json:"sites"` Errors []ErrorClass `json:"errors"` Validators []ValidatorRow `json:"validators"` } type RunInfo struct { ID int64 `json:"id"` Kind string `json:"kind"` State string `json:"state"` StartedAt time.Time `json:"started_at"` FinalizedAt *time.Time `json:"finalized_at"` DurationSec int `json:"duration_seconds"` Rechecked int `json:"rechecked"` } type Summary struct { Addresses int `json:"addresses"` Pass int `json:"pass"` Partial int `json:"partial"` Fail int `json:"fail"` Cancelled int `json:"cancelled"` EgressOK int `json:"egress_ok"` IngressOK int `json:"ingress_ok"` EgressHTTPSAny int `json:"egress_https_any_failed"` EgressHTTPSAll int `json:"egress_https_all_failed"` // EgressHTTPSAllTargets counts the addresses that failed https to every // target of the full set (as many checks as the best-covered address). EgressHTTPSAllTargets int `json:"egress_https_all_targets_failed"` IngressSSHAny int `json:"ingress_ssh_any_failed"` IngressSSHAll int `json:"ingress_ssh_all_failed"` PerMinute float64 `json:"addresses_per_minute"` } type Reason struct { Name string `json:"name"` Count int `json:"count"` } // Quality is the data-quality block: how the verdict relates to the checks. type Quality struct { LateFailedAtPass int `json:"late_failed_checks_at_pass"` LateFailedAtPassAddresses int `json:"late_failed_addresses_at_pass"` IngressFailed int `json:"ingress_failed_checks"` IngressFailedLate int `json:"ingress_failed_late"` Incomplete int `json:"incomplete_addresses"` PassWithFailed int `json:"pass_with_failed_addresses"` PassByFacts int `json:"pass_by_facts"` } type SubnetRow struct { CIDR string `json:"cidr"` Label string `json:"label,omitempty"` Addresses int `json:"addresses"` Pass int `json:"pass"` EgressOK int `json:"egress_ok"` IngressOK int `json:"ingress_ok"` } type TargetsBlock struct { Types []string `json:"types"` Targets []string `json:"targets"` Failed map[string][]int `json:"failed"` // type -> failed addresses per target, in Targets order } type MatrixRow struct { CIDR string `json:"cidr"` Partial int `json:"partial"` Percent []int `json:"percent"` // per target, in Targets order } type SitesBlock struct { Types []string `json:"types"` Rows []SiteRow `json:"rows"` } type SiteRow struct { Site string `json:"site"` Stats []SiteStat `json:"stats"` // per type, in Types order } type SiteStat struct { Total int `json:"total"` OK int `json:"ok"` } type ErrorClass struct { Name string `json:"name"` Count int `json:"count"` } type ValidatorRow struct { Validator string `json:"validator"` Total int `json:"total"` OK int `json:"ok"` } type typeStat struct{ n, ok int } type failedIngress struct { class, site, validator string late bool } // addr is everything known about one address of the run. type addr struct { res db.RunResult subnet string egress typeStat ingress typeStat stored int https struct { typeStat validator string failedTargets []string } ssh struct { typeStat sites []string errs map[string]bool } failedTargets map[string]bool // family\x00target -> failed failedIngress []failedIngress lateFailed int } // Analysis is a computed report plus the per-address data the lists are cut from. type Analysis struct { Report Report addrs []*addr siteNames map[int]string } // Compute reads the checks of the run once and builds the report. func Compute(in Input) (*Analysis, error) { byReg := make(map[int64]*addr, len(in.Results)) var addrs []*addr subnetOf := newSubnetMatcher(in.Subnets) for _, r := range in.Results { a := &addr{res: r, subnet: subnetOf(r.IPAddress), failedTargets: map[string]bool{}} a.ssh.errs = map[string]bool{} byReg[r.RegistryID] = a addrs = append(addrs, a) } siteName := func(source string) string { idx, _ := strconv.Atoi(strings.TrimPrefix(source, "inbound-site-")) if n := in.SiteNames[idx]; n != "" { return n } return "site-" + strconv.Itoa(idx) } type key struct{ site, typ string } siteStats := map[key]*typeStat{} siteTypes := map[string]bool{} egressTypes := map[string]bool{} valHTTPS := map[string]*typeStat{} targetSet := map[string]bool{} errCount := map[string]int{} err := in.Each(func(c db.RunCheck) { a := byReg[c.RegistryID] if a == nil || a.res.Verdict == db.ResultCancelled { return // a cancelled address was stopped, its checks say nothing } a.stored++ late := c.AfterVerdict || c.RecordedAt.After(a.res.AggregatedAt) family := db.CheckFamily(c.CheckType) switch db.CheckLevel(c.Source) { case db.LevelEgress: a.egress.n++ if c.Success { a.egress.ok++ } egressTypes[family] = true target := normalizeTarget(c.Target) targetSet[target] = true if !c.Success { a.failedTargets[family+"\x00"+target] = true } if family == "https" { a.https.n++ a.https.validator = c.ValidatorID if c.Success { a.https.ok++ } else { a.https.failedTargets = append(a.https.failedTargets, target) } v := valHTTPS[c.ValidatorID] if v == nil { v = &typeStat{} valHTTPS[c.ValidatorID] = v } v.n++ if c.Success { v.ok++ } } case db.LevelIngress: a.ingress.n++ if c.Success { a.ingress.ok++ } site := siteName(c.Source) siteTypes[family] = true ss := siteStats[key{site, family}] if ss == nil { ss = &typeStat{} siteStats[key{site, family}] = ss } ss.n++ if c.Success { ss.ok++ } if family == "ssh" { a.ssh.n++ if c.Success { a.ssh.ok++ } } if !c.Success { class := ErrorClassOf(c.CheckType, c.Detail) errCount[class]++ a.failedIngress = append(a.failedIngress, failedIngress{class: class, site: site, validator: c.ValidatorID, late: late}) if family == "ssh" { a.ssh.sites = append(a.ssh.sites, site) a.ssh.errs[errorReason(c.CheckType, c.Detail)] = true } } } if late && !c.Success { a.lateFailed++ } }) if err != nil { return nil, err } rep := Report{Matrix: map[string][]MatrixRow{}} rep.Run = RunInfo{ID: in.Run.ID, Kind: in.Run.Kind, State: in.Run.State, StartedAt: in.Run.StartedAt, FinalizedAt: in.Run.FinalizedAt, Rechecked: in.Rechecked} if in.Run.FinalizedAt != nil { rep.Run.DurationSec = int(in.Run.FinalizedAt.Sub(in.Run.StartedAt).Seconds()) } maxHTTPS := 0 for _, a := range addrs { if a.https.n > maxHTTPS { maxHTTPS = a.https.n } } reasonCount := map[string]int{} type subAgg struct { n, pass, eg, ing, partial int failed map[string]int } subs := map[string]*subAgg{} sum := &rep.Summary for _, a := range addrs { v := a.res.Verdict if v == db.ResultCancelled { sum.Cancelled++ continue } sum.Addresses++ switch v { case db.ResultPass: sum.Pass++ case db.ResultPartial: sum.Partial++ case db.ResultFail: sum.Fail++ } egOK := a.egress.n > 0 && a.egress.ok == a.egress.n inOK := a.ingress.n > 0 && a.ingress.ok == a.ingress.n if egOK { sum.EgressOK++ } if inOK { sum.IngressOK++ } if a.https.n > 0 && a.https.ok < a.https.n { sum.EgressHTTPSAny++ if a.https.ok == 0 { sum.EgressHTTPSAll++ if a.https.n == maxHTTPS { sum.EgressHTTPSAllTargets++ } } } if a.ssh.n > 0 && a.ssh.ok < a.ssh.n { sum.IngressSSHAny++ if a.ssh.ok == 0 { sum.IngressSSHAll++ } } egFail := a.egress.ok < a.egress.n inFail := a.ingress.ok < a.ingress.n incomplete := a.res.ExpectedChecks >= 0 && a.stored < a.res.ExpectedChecks if incomplete { rep.Quality.Incomplete++ } if v == db.ResultPartial { reasonCount[reasonName(egFail, inFail, incomplete)]++ } if v == db.ResultPass { if a.egress.ok < a.egress.n || a.ingress.ok < a.ingress.n { rep.Quality.PassWithFailed++ rep.Quality.LateFailedAtPass += a.lateFailed rep.Quality.LateFailedAtPassAddresses++ } } sa := subs[a.subnet] if sa == nil { sa = &subAgg{failed: map[string]int{}} subs[a.subnet] = sa } sa.n++ if v == db.ResultPass { sa.pass++ } if egOK { sa.eg++ } if inOK { sa.ing++ } if v == db.ResultPartial { sa.partial++ for k := range a.failedTargets { sa.failed[k]++ } } } rep.Quality.PassByFacts = sum.Pass - rep.Quality.PassWithFailed if sum.Addresses > 0 && rep.Run.DurationSec > 0 { sum.PerMinute = float64(sum.Addresses) / (float64(rep.Run.DurationSec) / 60) } for _, a := range addrs { for _, f := range a.failedIngress { rep.Quality.IngressFailed++ if f.late { rep.Quality.IngressFailedLate++ } } } for _, name := range reasonOrder { if n := reasonCount[name]; n > 0 { rep.Reasons = append(rep.Reasons, Reason{Name: name, Count: n}) } } // Subnets: worst first is the page's job; the report lists them by size. labels := map[string]string{} for _, s := range in.Subnets { labels[s.CIDR] = s.Label } for cidr, sa := range subs { rep.Subnets = append(rep.Subnets, SubnetRow{CIDR: cidr, Label: labels[cidr], Addresses: sa.n, Pass: sa.pass, EgressOK: sa.eg, IngressOK: sa.ing}) } sort.Slice(rep.Subnets, func(i, j int) bool { if rep.Subnets[i].Addresses != rep.Subnets[j].Addresses { return rep.Subnets[i].Addresses > rep.Subnets[j].Addresses } return rep.Subnets[i].CIDR < rep.Subnets[j].CIDR }) // Targets and the subnet x target matrix, per egress check family. types := sortedKeys(egressTypes) rep.Targets.Types = types failedAddrs := map[string]int{} // family\x00target -> addresses for _, a := range addrs { if a.res.Verdict == db.ResultCancelled { continue } for k := range a.failedTargets { failedAddrs[k]++ } } targets := sortedKeys(targetSet) lead := "" if len(types) > 0 { lead = types[0] for _, t := range types { if t == "https" { lead = t } } } sort.SliceStable(targets, func(i, j int) bool { fi, fj := failedAddrs[lead+"\x00"+targets[i]], failedAddrs[lead+"\x00"+targets[j]] if fi != fj { return fi > fj } return targets[i] < targets[j] }) rep.Targets.Targets = targets rep.Targets.Failed = map[string][]int{} for _, t := range types { row := make([]int, len(targets)) for i, tg := range targets { row[i] = failedAddrs[t+"\x00"+tg] } rep.Targets.Failed[t] = row } for _, t := range types { var rows []MatrixRow for cidr, sa := range subs { if sa.partial == 0 { continue } pc := make([]int, len(targets)) for i, tg := range targets { pc[i] = int(float64(sa.failed[t+"\x00"+tg])/float64(sa.partial)*100 + 0.5) } rows = append(rows, MatrixRow{CIDR: cidr, Partial: sa.partial, Percent: pc}) } sort.Slice(rows, func(i, j int) bool { if rows[i].Partial != rows[j].Partial { return rows[i].Partial > rows[j].Partial } return rows[i].CIDR < rows[j].CIDR }) rep.Matrix[t] = rows } // Sites. rep.Sites.Types = sortedKeys(siteTypes) names := map[string]bool{} for k := range siteStats { names[k.site] = true } siteList := sortedKeys(names) sort.Slice(siteList, func(i, j int) bool { return siteIndexOf(in.SiteNames, siteList[i]) < siteIndexOf(in.SiteNames, siteList[j]) }) for _, s := range siteList { row := SiteRow{Site: s} for _, t := range rep.Sites.Types { st := siteStats[key{s, t}] if st == nil { st = &typeStat{} } row.Stats = append(row.Stats, SiteStat{Total: st.n, OK: st.ok}) } rep.Sites.Rows = append(rep.Sites.Rows, row) } for name, n := range errCount { rep.Errors = append(rep.Errors, ErrorClass{Name: name, Count: n}) } sort.Slice(rep.Errors, func(i, j int) bool { if rep.Errors[i].Count != rep.Errors[j].Count { return rep.Errors[i].Count > rep.Errors[j].Count } return rep.Errors[i].Name < rep.Errors[j].Name }) for id, st := range valHTTPS { rep.Validators = append(rep.Validators, ValidatorRow{Validator: id, Total: st.n, OK: st.ok}) } sort.Slice(rep.Validators, func(i, j int) bool { a, b := validatorNumber(rep.Validators[i].Validator), validatorNumber(rep.Validators[j].Validator) if a != b { return a < b } return rep.Validators[i].Validator < rep.Validators[j].Validator }) return &Analysis{Report: rep, addrs: addrs, siteNames: in.SiteNames}, nil } var reasonOrder = []string{ "Только egress", "Ingress и egress", "Egress и неполный набор", "Ingress, egress и неполный набор", "Только неполный набор", "Только ingress", "Ingress и неполный набор", "Прочее", } func reasonName(egress, ingress, incomplete bool) string { switch { case egress && ingress && incomplete: return "Ingress, egress и неполный набор" case egress && ingress: return "Ingress и egress" case egress && incomplete: return "Egress и неполный набор" case egress: return "Только egress" case ingress && incomplete: return "Ingress и неполный набор" case ingress: return "Только ingress" case incomplete: return "Только неполный набор" } return "Прочее" } func sortedKeys(m map[string]bool) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k) } sort.Strings(out) return out } func siteIndexOf(names map[int]string, site string) int { for i, n := range names { if n == site { return i } } if n, err := strconv.Atoi(strings.TrimPrefix(site, "site-")); err == nil { return n } return 1 << 20 } // validatorNumber is the trailing number of a validator id ("vkiplab-v12" -> // 12), or 1<<20 when there is none, so numbered validators sort naturally. func validatorNumber(id string) int { i := len(id) for i > 0 && id[i-1] >= '0' && id[i-1] <= '9' { i-- } if i == len(id) { return 1 << 20 } n, _ := strconv.Atoi(id[i:]) return n } // ShortValidator is the validator id as the page shows it: "vkiplab-v12" -> // "v12"; ids without a number stay whole. func ShortValidator(id string) string { n := validatorNumber(id) if n == 1<<20 { return id } return "v" + strconv.Itoa(n) } // normalizeTarget is the host of an egress target: https://host/path -> host. func normalizeTarget(t string) string { if u, err := url.Parse(t); err == nil && u.Host != "" { return u.Hostname() } return strings.TrimSuffix(t, "/") } // newSubnetMatcher returns a function that maps an address to the most // specific configured subnet. With no subnets configured addresses group by // /24 (/64 for IPv6). An address outside the list goes to "прочие". func newSubnetMatcher(subnets []db.Subnet) func(string) string { type entry struct { p netip.Prefix name string } var list []entry for _, s := range subnets { if p, err := netip.ParsePrefix(s.CIDR); err == nil { list = append(list, entry{p.Masked(), p.Masked().String()}) } } sort.Slice(list, func(i, j int) bool { return list[i].p.Bits() > list[j].p.Bits() }) return func(ip string) string { a, err := netip.ParseAddr(ip) if err != nil { return "прочие" } if len(list) == 0 { bits := 24 if a.Is6() { bits = 64 } p, _ := a.Prefix(bits) return p.String() } for _, e := range list { if e.p.Contains(a) { return e.name } } return "прочие" } } // ErrorClassOf names the class of a failed ingress check: the check type and // the reason, e.g. "SSH: таймаут", "ICMP: нет ответа". func ErrorClassOf(checkType, detail string) string { return strings.ToUpper(checkType) + ": " + errorReason(checkType, detail) } func errorReason(checkType, detail string) string { d := strings.ToLower(detail) switch { case strings.Contains(d, "unexpected banner prefix"): if strings.Contains(d, "not allo") { return "баннер «Not allowed»" } return "неожиданный баннер" case strings.Contains(d, "no route to host"): return "нет маршрута" case strings.Contains(d, "time exceeded"): return "time exceeded" case strings.Contains(d, "connection refused"): return "отказ в соединении" case strings.Contains(d, "timeout") || strings.Contains(d, "deadline exceeded"): if strings.EqualFold(checkType, "icmp") { return "нет ответа" } return "таймаут" } if strings.EqualFold(checkType, "icmp") { return "нет ответа" } return "прочее" }