cloud-ip-validator/internal/httpapi/handlers_analytics.go

package httpapi

import (
	"bytes"
	"encoding/csv"
	"fmt"
	"net/http"
	"net/netip"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"cloudipvalidator/internal/analytics"
	"cloudipvalidator/internal/db"
)

// analyticsRunDTO is one entry of the run selector.
type analyticsRunDTO struct {
	ID          int64      `json:"id"`
	Kind        string     `json:"kind"`
	State       string     `json:"state"`
	StartedAt   time.Time  `json:"started_at"`
	FinalizedAt *time.Time `json:"finalized_at"`
	Addresses   int        `json:"addresses"`
	Pass        int        `json:"pass"`
	Partial     int        `json:"partial"`
	Fail        int        `json:"fail"`
	Cancelled   int        `json:"cancelled"`
	// Total is the number of queue rows of the run, Pending those still being
	// processed (only an open run has any).
	Total   int `json:"total"`
	Pending int `json:"pending"`
}

type subnetDTO struct {
	CIDR  string `json:"cidr"`
	Label string `json:"label,omitempty"`
}

type subnetsDTO struct {
	Subnets []subnetDTO `json:"subnets"`
}

// analyticsCache keeps the computed analysis of finalized runs, per run and
// subnet (the subnet narrows the report; "" is the whole run). An entry is
// valid while the run's data version (checks written, results) is unchanged;
// the subnet list is part of the version because it changes the grouping. At
// most analyticsCacheMax entries are kept, the least recently used one goes
// first, so trying many subnets does not grow the memory without limit.
type analyticsCache struct {
	mu      sync.Mutex
	entries map[analyticsKey]analyticsEntry
	clock   uint64
}

const analyticsCacheMax = 16

type analyticsKey struct {
	run    int64
	subnet string
}

type analyticsEntry struct {
	version string
	an      *analytics.Analysis
	used    uint64
}

func (s *Server) handleAnalyticsRuns(w http.ResponseWriter, r *http.Request) {
	runs, err := s.DB.ListRuns(r.Context())
	if err != nil {
		writeDBError(w, err)
		return
	}
	out := make([]analyticsRunDTO, 0, len(runs))
	for _, x := range runs {
		out = append(out, analyticsRunDTO{
			ID: x.ID, Kind: x.Kind, State: x.State, StartedAt: x.StartedAt, FinalizedAt: x.FinalizedAt,
			Addresses: x.Addresses, Pass: x.Pass, Partial: x.Partial, Fail: x.Fail, Cancelled: x.Cancelled,
			Total: x.Total, Pending: x.Pending,
		})
	}
	writeJSON(w, http.StatusOK, out)
}

// analysisFor returns the analysis of the run named by the {id} of the path,
// narrowed to the ?subnet= (a CIDR) when given; see analysisByID.
func (s *Server) analysisFor(w http.ResponseWriter, r *http.Request) *analytics.Analysis {
	id, err := strconv.ParseInt(r.PathValue("id"), 10, 64)
	if err != nil || id <= 0 {
		writeError(w, http.StatusBadRequest, "invalid run id")
		return nil
	}
	var subnet netip.Prefix
	if v := strings.TrimSpace(r.URL.Query().Get("subnet")); v != "" {
		if subnet, err = netip.ParsePrefix(v); err != nil {
			writeError(w, http.StatusBadRequest, "invalid subnet "+strconv.Quote(v)+" (a CIDR such as 203.0.113.0/24 is expected)")
			return nil
		}
		subnet = subnet.Masked()
	}
	return s.analysisByID(w, r, id, subnet)
}

// analysisByID returns the analysis of a finalized run (of the addresses
// inside subnet, unless it is the zero prefix), from the cache when the run's
// data has not changed since it was computed. It writes the error response
// itself and returns nil when it cannot.
func (s *Server) analysisByID(w http.ResponseWriter, r *http.Request, id int64, subnet netip.Prefix) *analytics.Analysis {
	ctx := r.Context()
	run, err := s.DB.GetRun(ctx, id)
	if err != nil {
		writeDBError(w, err)
		return nil
	}
	if run.State != db.RunFinalized {
		writeError(w, http.StatusConflict, "run is still open: analytics are available for finished runs")
		return nil
	}
	data, err := s.DB.RunDataVersion(ctx, id)
	if err != nil {
		writeDBError(w, err)
		return nil
	}
	subnets, err := s.DB.ListSubnets(ctx)
	if err != nil {
		writeDBError(w, err)
		return nil
	}
	var sb strings.Builder
	for _, x := range subnets {
		sb.WriteString(x.CIDR + "|" + x.Label + ";")
	}
	version := data + "#" + sb.String()

	key := analyticsKey{run: id}
	if subnet.IsValid() {
		key.subnet = subnet.String()
	}
	s.analytics.mu.Lock()
	defer s.analytics.mu.Unlock()
	s.analytics.clock++
	if e, ok := s.analytics.entries[key]; ok && e.version == version {
		e.used = s.analytics.clock
		s.analytics.entries[key] = e
		return e.an
	}
	an, err := analytics.Load(ctx, s.DB, id, subnet)
	if err != nil {
		writeDBError(w, err)
		return nil
	}
	if s.analytics.entries == nil {
		s.analytics.entries = map[analyticsKey]analyticsEntry{}
	}
	if _, ok := s.analytics.entries[key]; !ok && len(s.analytics.entries) >= analyticsCacheMax {
		var oldest analyticsKey
		least := ^uint64(0)
		for k, e := range s.analytics.entries {
			if e.used < least {
				oldest, least = k, e.used
			}
		}
		delete(s.analytics.entries, oldest)
	}
	s.analytics.entries[key] = analyticsEntry{version: version, an: an, used: s.analytics.clock}
	return an
}

// handleAnalyticsRun serves the report of a finished run, narrowed to
// ?subnet= (a CIDR) when given.
func (s *Server) handleAnalyticsRun(w http.ResponseWriter, r *http.Request) {
	if an := s.analysisFor(w, r); an != nil {
		writeJSON(w, http.StatusOK, an.Report)
	}
}

var nonSlug = regexp.MustCompile(`[^a-z0-9]+`)

// handleAnalyticsList serves the address table (of the ?subnet= when given) behind one indicator
// (kind = egress_https_any|egress_https_all|ingress_ssh_any|ingress_ssh_all),
// the addresses of one verdict (kind = verdict_pass|verdict_partial|verdict_fail)
// or one ingress error class (kind = error, ?class=...), as JSON or, with
// ?format=csv, as a downloadable CSV file.
func (s *Server) handleAnalyticsList(w http.ResponseWriter, r *http.Request) {
	an := s.analysisFor(w, r)
	if an == nil {
		return
	}
	kind, class := r.PathValue("kind"), r.URL.Query().Get("class")
	list, err := an.List(kind, class)
	if err != nil {
		writeError(w, http.StatusNotFound, err.Error())
		return
	}
	if r.URL.Query().Get("format") != "csv" {
		writeJSON(w, http.StatusOK, list)
		return
	}
	name := kind
	if kind == analytics.ListError {
		if slug := strings.Trim(nonSlug.ReplaceAllString(strings.ToLower(class), "-"), "-"); slug != "" {
			name += "-" + slug
		} else {
			name += "-class"
		}
	}
	writeCSV(w, list.Columns, list.Rows, fmt.Sprintf("%s_run%s.csv", name, r.PathValue("id")))
}

// writeCSV sends a table as a downloadable CSV file.
func writeCSV(w http.ResponseWriter, columns []string, rows [][]string, filename string) {
	var buf bytes.Buffer
	buf.WriteString("\xef\xbb\xbf") // UTF-8 BOM, so Excel opens the file as UTF-8
	cw := csv.NewWriter(&buf)
	cw.UseCRLF = true
	_ = cw.Write(columns)
	_ = cw.WriteAll(rows)
	w.Header().Set("Content-Type", "text/csv; charset=utf-8")
	w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename))
	w.WriteHeader(http.StatusOK)
	_, _ = w.Write(buf.Bytes())
}

// compareFor loads the two runs named by ?base=A&target=B (the older and the
// newer one) and compares them. It writes the error response itself and
// returns nil when it cannot: 400 for a missing or malformed id or the same
// run twice, 404 for an unknown run, 409 for one that is still open.
func (s *Server) compareFor(w http.ResponseWriter, r *http.Request) (c *analytics.Comparison, base, target int64) {
	ids := [2]int64{}
	for i, name := range []string{"base", "target"} {
		id, err := strconv.ParseInt(r.URL.Query().Get(name), 10, 64)
		if err != nil || id <= 0 {
			writeError(w, http.StatusBadRequest, "invalid or missing "+name+" run id")
			return nil, 0, 0
		}
		ids[i] = id
	}
	if ids[0] == ids[1] {
		writeError(w, http.StatusBadRequest, "base and target must be different runs")
		return nil, 0, 0
	}
	a := s.analysisByID(w, r, ids[0], netip.Prefix{})
	if a == nil {
		return nil, 0, 0
	}
	b := s.analysisByID(w, r, ids[1], netip.Prefix{})
	if b == nil {
		return nil, 0, 0
	}
	return analytics.Compare(a, b), ids[0], ids[1]
}

// handleAnalyticsCompare serves the comparison of two finished runs:
// ?base=A (older) &target=B (newer).
func (s *Server) handleAnalyticsCompare(w http.ResponseWriter, r *http.Request) {
	if c, _, _ := s.compareFor(w, r); c != nil {
		writeJSON(w, http.StatusOK, c)
	}
}

// handleAnalyticsCompareList serves the address table of one group of the
// comparison (new|left|common|changed|same|entered|exited), narrowed by
// ?indicator=... and ?from=...&to=... (verdicts), as JSON or, with
// ?format=csv, as a downloadable CSV file.
func (s *Server) handleAnalyticsCompareList(w http.ResponseWriter, r *http.Request) {
	c, base, target := s.compareFor(w, r)
	if c == nil {
		return
	}
	q := r.URL.Query()
	group := r.PathValue("group")
	f := analytics.CompareFilter{Indicator: q.Get("indicator"), From: q.Get("from"), To: q.Get("to")}
	list, err := c.List(group, f)
	if err != nil {
		writeError(w, http.StatusNotFound, err.Error())
		return
	}
	if q.Get("format") != "csv" {
		writeJSON(w, http.StatusOK, list)
		return
	}
	name := "compare_" + group
	if f.Indicator != "" {
		name += "_" + f.Indicator
	}
	if f.From != "" {
		name += "_" + f.From + "-" + f.To
	}
	writeCSV(w, list.Columns, list.Rows, fmt.Sprintf("%s_run%d-%d.csv", name, base, target))
}

func (s *Server) handleConfigGetSubnets(w http.ResponseWriter, r *http.Request) {
	list, err := s.DB.ListSubnets(r.Context())
	if err != nil {
		writeDBError(w, err)
		return
	}
	out := subnetsDTO{Subnets: make([]subnetDTO, 0, len(list))}
	for _, x := range list {
		out.Subnets = append(out.Subnets, subnetDTO{CIDR: x.CIDR, Label: x.Label})
	}
	writeJSON(w, http.StatusOK, out)
}

// handleConfigPutSubnets replaces the whole subnet list. The list groups the
// addresses on the analytics page; with none configured they group by /24.
func (s *Server) handleConfigPutSubnets(w http.ResponseWriter, r *http.Request) {
	var req subnetsDTO
	if err := readJSON(r, &req); err != nil {
		writeError(w, http.StatusBadRequest, "invalid body: "+err.Error())
		return
	}
	in := make([]db.Subnet, 0, len(req.Subnets))
	for _, x := range req.Subnets {
		in = append(in, db.Subnet{CIDR: strings.TrimSpace(x.CIDR), Label: x.Label})
	}
	if err := s.DB.ReplaceSubnets(r.Context(), in); err != nil {
		writeDBError(w, err)
		return
	}
	s.handleConfigGetSubnets(w, r)
}