cloud-ip-validator/internal/orchestrator/verdict_integrity_test.go

package orchestrator

import (
	"context"
	"testing"
	"time"

	"cloudipvalidator/internal/db"
)

func chk(src, ct string, ok bool) db.Check {
	return db.Check{Source: src, CheckType: ct, Success: ok}
}

func TestComputeVerdict(t *testing.T) {
	in1 := db.InboundSource(1)
	cases := []struct {
		name     string
		checks   []db.Check
		expected int
		want     string
		missing  int
	}{
		{"all pass, complete", []db.Check{chk(db.SourceEgress, "https", true), chk(in1, "icmp", true)}, 2, db.ResultPass, 0},
		{"egress failure", []db.Check{chk(db.SourceEgress, "https", false), chk(in1, "icmp", true)}, 2, db.ResultPartial, 0},
		{"ingress failure", []db.Check{chk(db.SourceEgress, "https", true), chk(in1, "ssh", false)}, 2, db.ResultPartial, 0},
		{"all recorded pass but one missing", []db.Check{chk(db.SourceEgress, "https", true)}, 2, db.ResultPartial, 1},
		{"all fail", []db.Check{chk(db.SourceEgress, "https", false), chk(in1, "icmp", false)}, 2, db.ResultFail, 0},
		{"nothing recorded", nil, 2, db.ResultFail, 2},
		{"more recorded than expected", []db.Check{chk(db.SourceEgress, "https", true), chk(in1, "icmp", true)}, 1, db.ResultPass, 0},
	}
	for _, c := range cases {
		got, _, missing := computeVerdict(c.checks, c.expected, true)
		if got != c.want || missing != c.missing {
			t.Errorf("%s: got %s missing=%d, want %s missing=%d", c.name, got, missing, c.want, c.missing)
		}
	}
}

// The window starts when checking starts: an address handed out long ago but
// that only just began checking must not be cut off.
func TestAggregationWindowCountsFromCheckingStart(t *testing.T) {
	ctx := context.Background()
	o, _, _ := newTestOrchestrator(t, 180)
	deadline := time.Now().Add(-120 * time.Second)
	long, recent := time.Now().Add(-10*time.Minute), time.Now()

	item := db.IPQueueItem{AssignedAt: &long, CheckingStartedAt: &recent}
	ready, err := o.isReadyToAggregate(ctx, item, deadline, nil)
	if err != nil || ready {
		t.Fatalf("recent checking start must not be cut off by an old assignment: ready=%v err=%v", ready, err)
	}

	started := time.Now().Add(-3 * time.Minute)
	item.CheckingStartedAt = &started
	if ready, err := o.isReadyToAggregate(ctx, item, deadline, nil); err != nil || !ready {
		t.Fatalf("window elapsed since checking start: ready=%v err=%v", ready, err)
	}

	// Rows from before checking_started_at existed fall back to assigned_at.
	item.CheckingStartedAt = nil
	if ready, err := o.isReadyToAggregate(ctx, item, deadline, nil); err != nil || !ready {
		t.Fatalf("fallback to assigned_at: ready=%v err=%v", ready, err)
	}
}

// After the verdict nothing can change the stored checks, and the verdict is
// exactly what the stored checks give when computed again.
func TestVerdictMatchesStoredChecksAndLateResultIsDropped(t *testing.T) {
	ctx := context.Background()
	o, d, mock := newTestOrchestrator(t, 180)
	mock.Seed("fip-1", "1.2.3.4", "svc-project")
	_ = d.RegisterValidator(ctx, "validator-1", "host-1", "port-1", "v0.1")
	_ = d.SeedQueue(ctx, []string{"1.2.3.4"})

	o.Tick(ctx)
	ip, _ := d.GetIPByAddress(ctx, "1.2.3.4")
	_ = o.SelfCheckResult(ctx, "validator-1", ip.ID, true, "ok")
	ip, _ = d.GetIP(ctx, ip.ID)

	rec := func(src, ct string, ok bool) bool {
		t.Helper()
		written, err := o.RecordCheckIfOpen(ctx, db.Check{
			IPID: ip.ID, IPAddress: ip.IPAddress, AttemptNumber: ip.AttemptNumber, ValidatorID: "validator-1",
			Source: src, CheckType: ct, Target: "t", Success: ok, CheckedAt: db.Now(),
		})
		if err != nil {
			t.Fatal(err)
		}
		return written
	}
	if !rec(db.SourceEgress, "https", true) || !rec(db.InboundSource(1), "tcp-22", false) {
		t.Fatal("results while checking must be stored")
	}
	_ = o.MarkEgressComplete(ctx, ip.ID)
	o.Cfg.CheckingWindowSeconds = 0
	time.Sleep(5 * time.Millisecond)
	o.Tick(ctx)

	ip, _ = d.GetIP(ctx, ip.ID)
	if ip.State != db.IPDone {
		t.Fatalf("expected done, got %s", ip.State)
	}

	// A late result: a new check, and an attempt to flip a stored one.
	if rec(db.InboundSource(2), "icmp", true) {
		t.Fatal("new check after the verdict was stored")
	}
	if rec(db.InboundSource(1), "tcp-22", true) {
		t.Fatal("overwrite after the verdict was accepted")
	}

	stored, err := d.ListChecksForAttempt(ctx, ip.ID, ip.AttemptNumber)
	if err != nil {
		t.Fatal(err)
	}
	if len(stored) != 2 {
		t.Fatalf("stored checks changed after the verdict: %d rows", len(stored))
	}
	expected, err := o.expectedCheckCount(ctx)
	if err != nil {
		t.Fatal(err)
	}
	if got, _, _ := computeVerdict(stored, expected, o.Agg.MissingCountsAsFail); got != ip.OverallResult {
		t.Fatalf("recomputed verdict %s differs from stored %s", got, ip.OverallResult)
	}
}