ayurishchevandClaude Sonnet 5
cb935fef0d
Collectors record the result of every source (schema v3, table source_status); /health reports failing sources (failures in a row >= source_failure_threshold) and turns degraded; GET /sources shows the full state; runs end with a summary log line instead of "data saved". Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
192 lines
10 KiB
Python
192 lines
10 KiB
Python
import datetime
|
|
import os
|
|
import sys
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
import api_server
|
|
import cidr_collector as cc
|
|
import db
|
|
from storage import load_json, save_json_atomic
|
|
|
|
KEY = {"X-API-Key": "secret"}
|
|
|
|
|
|
@pytest.fixture
|
|
def env(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(cc, "CONFIG_FILE", str(tmp_path / "config.json"))
|
|
monkeypatch.setattr(cc, "DATA_FILE", str(tmp_path / "data.json"))
|
|
monkeypatch.setattr(cc, "FQDN_DATA_FILE", str(tmp_path / "fqdn_data.json"))
|
|
monkeypatch.setattr(cc, "DB_FILE", str(tmp_path / "ripe.db"))
|
|
monkeypatch.setattr(cc, "RECREATED_FILE", str(tmp_path / "db_recreated.json"))
|
|
monkeypatch.setattr(cc, "RESTORE_FILE", str(tmp_path / "last_restore.json"))
|
|
monkeypatch.setattr(cc, "BACKUP_DIR", str(tmp_path / "backups"))
|
|
monkeypatch.setenv("RIPE_API_TOKEN", "secret")
|
|
save_json_atomic(cc.CONFIG_FILE, {"asns": [], "fqdns": [], "ttl_days": 90})
|
|
return TestClient(api_server.app)
|
|
|
|
|
|
def test_manage_asns_and_fqdns(env):
|
|
# Без ключа - 401; ASN добавляется один раз (201, затем 200)
|
|
assert env.post("/asns", json={"asn": 62041}).status_code == 401
|
|
assert env.post("/asns", json={"asn": 62041}, headers=KEY).status_code == 201
|
|
assert env.post("/asns", json={"asn": 62041}, headers=KEY).status_code == 200
|
|
assert env.get("/asns").json() == {"asns": [62041]}
|
|
assert env.post("/asns", json={"asn": 0}, headers=KEY).status_code == 422
|
|
|
|
# FQDN нормализуется; IP-литералы и неверные метки отклоняются
|
|
resp = env.post("/fqdns", json={"fqdn": " Example.COM. "}, headers=KEY)
|
|
assert resp.status_code == 201 and resp.json()["fqdn"] == "example.com"
|
|
for bad in ("1.2.3.4", "bad_name.com", "-a.com", "a..com"):
|
|
assert env.post("/fqdns", json={"fqdn": bad}, headers=KEY).status_code == 422
|
|
assert env.get("/fqdns").json() == {"fqdns": ["example.com"]}
|
|
|
|
# Удаление: 404 для неизвестного, 200 для существующего, конфиг обновлён
|
|
assert env.delete("/asns/1", headers=KEY).status_code == 404
|
|
assert env.delete("/fqdns/example.com", headers=KEY).status_code == 200
|
|
assert env.delete("/asns/62041", headers=KEY).status_code == 200
|
|
assert load_json(cc.CONFIG_FILE, {})["asns"] == [] and load_json(cc.CONFIG_FILE, {})["fqdns"] == []
|
|
|
|
|
|
def sources():
|
|
with db.session() as conn:
|
|
return {s for (s,) in conn.execute("SELECT DISTINCT source FROM addresses")}
|
|
|
|
|
|
def test_removed_source_data_purge_and_ttl(env, monkeypatch):
|
|
now = datetime.datetime.now()
|
|
with db.session() as conn:
|
|
for source, days_ago in (("1", 1), ("2", 100), ("3", 1)):
|
|
stamp = (now - datetime.timedelta(days=days_ago)).isoformat(timespec="seconds")
|
|
conn.execute("INSERT INTO addresses VALUES ('asn', ?, '10.0.0.0/24', ?, ?)", (source, stamp, stamp))
|
|
save_json_atomic(cc.CONFIG_FILE, {"asns": [1, 2, 3], "ttl_days": 90})
|
|
|
|
# Без purge данные остаются, purge=true удаляет сразу
|
|
assert env.delete("/asns/2", headers=KEY).status_code == 200
|
|
assert env.delete("/asns/3?purge=true", headers=KEY).json() == {"asn": 3, "removed": True, "purged": True}
|
|
assert sources() == {"1", "2"}
|
|
|
|
# Сбор: у неопрашиваемого источника 2 адрес просрочен (100 > 90 дней) - исчезает
|
|
collector = cc.CIDRCollector()
|
|
monkeypatch.setattr(collector, "fetch_prefixes", lambda asn: ["10.0.0.0/24"])
|
|
collector.run_collection()
|
|
assert sources() == {"1"}
|
|
|
|
|
|
def test_addresses_diff(env):
|
|
now = datetime.datetime.now()
|
|
with db.session() as conn, db.transaction(conn):
|
|
db.merge_source(conn, "asn", "1", {"1.0.0.0/24", "2001:db8::/32"}, now, 90)
|
|
db.merge_source(conn, "fqdn", "example.com", {"9.9.9.9"}, now, 90)
|
|
horizon = dict(conn.execute("SELECT key, value FROM meta"))["horizon_ts"]
|
|
|
|
body = env.get("/addresses/diff", params={"since": "0"}).json()
|
|
assert body["added"] == ["1.0.0.0/24", "2001:db8::/32", "9.9.9.9"] and body["removed"] == []
|
|
assert env.get("/addresses/diff", params={"since": "0", "type": "cidr", "ip_version": "6"}).json()["added"] == [
|
|
"2001:db8::/32"]
|
|
assert env.get("/addresses").headers["X-Changes-Cursor"] == str(body["cursor"])
|
|
assert env.get("/addresses", params={"format": "nftables"}).headers["X-Changes-Cursor"] == str(body["cursor"])
|
|
# Время вместо курсора; с полученным курсором изменений нет
|
|
assert env.get("/addresses/diff", params={"since": horizon}).json()["added"] == body["added"]
|
|
assert env.get("/addresses/diff", params={"since": body["cursor"]}).json()["added"] == []
|
|
|
|
# Неверное значение - 400; вне журнала (курсор из будущего, время до создания журнала) - 410
|
|
assert env.get("/addresses/diff", params={"since": "yesterday"}).status_code == 400
|
|
assert env.get("/addresses/diff", params={"since": body["cursor"] + 1}).status_code == 410
|
|
assert env.get("/addresses/diff", params={"since": "2000-01-01T00:00:00Z"}).status_code == 410
|
|
assert env.get("/addresses/diff").status_code == 422
|
|
|
|
|
|
def test_input_hardening(env):
|
|
# Нелатинский ключ - 401 (раньше compare_digest падал: 500)
|
|
assert env.post("/collect", headers={"X-API-Key": "é".encode("latin-1")}).status_code == 401
|
|
|
|
# Некорректный since - 400: юникод-цифра, слишком длинное число, крайние даты с поясом (раньше 500)
|
|
for bad in ("²", "1" * 5000, "1" * 31, "0001-01-01T00:00:00+05:00", "9999-12-31T23:59:59-05:00"):
|
|
assert env.get("/addresses/diff", params={"since": bad}).status_code == 400, bad
|
|
# Корректный, но выходящий за журнал курсор - 410, а не 400
|
|
assert env.get("/addresses/diff", params={"since": "9" * 30}).status_code == 410
|
|
|
|
|
|
def test_recreated_database_is_withheld(env):
|
|
cc.add_to_config_list("asns", 62041)
|
|
with open(cc.DB_FILE, "wb") as f: # порча базы, копий нет
|
|
f.write(b"garbage" * 1000)
|
|
|
|
# Пока данные не собраны заново, пустой список не отдаётся (503 + Retry-After)
|
|
for url in ("/addresses", "/addresses/diff?since=0"):
|
|
response = env.get(url)
|
|
assert response.status_code == 503 and response.headers["Retry-After"], url
|
|
assert env.get("/addresses", params={"type": "fqdn"}).json() == [] # тип без источников не блокируется
|
|
health = env.get("/health").json()
|
|
assert health["status"] == "degraded" and health["db_recreated"]["pending"] is True
|
|
assert "quarantine" not in health["db_recreated"] # пути наружу не отдаются
|
|
|
|
# Сборщик собрал данные: выдача возобновляется, метка снята
|
|
with db.session() as conn:
|
|
with db.transaction(conn):
|
|
db.merge_source(conn, "asn", "62041", {"1.0.0.0/24"}, datetime.datetime.now(), 90)
|
|
db.settle_recreated(conn)
|
|
assert env.get("/addresses").json() == ["1.0.0.0/24"]
|
|
assert env.get("/health").json()["db_recreated"] is None
|
|
|
|
|
|
def test_health_hides_internal_paths(env):
|
|
save_json_atomic(cc.RESTORE_FILE, {"at": "2026-09-21T10:00:00", "backup": "/data/backups/ripe-20260921T100000Z.db",
|
|
"quarantine": "/data/ripe.db.corrupt-1", "error": "file is not a database"})
|
|
# Наружу - время и имя файла копии, без каталогов, карантина и текста ошибки
|
|
assert env.get("/health").json()["last_restore"] == {"at": "2026-09-21T10:00:00",
|
|
"backup": "ripe-20260921T100000Z.db"}
|
|
assert json_dump_has_no_paths(env.get("/health").text)
|
|
|
|
|
|
def json_dump_has_no_paths(text):
|
|
return "/data" not in text and "corrupt" not in text and "not a database" not in text
|
|
|
|
|
|
def test_health_reports_failing_sources(env):
|
|
cc.add_to_config_list("asns", 62041)
|
|
cc.add_to_config_list("fqdns", "example.com")
|
|
save_json_atomic(cc.STATUS_FILE, {"updated_at": datetime.datetime.now().isoformat(), "jobs": {}}) # демон "жив"
|
|
now = datetime.datetime.now()
|
|
|
|
def fail(times):
|
|
for _ in range(times):
|
|
with db.session() as conn, db.transaction(conn):
|
|
db.record_source_result(conn, "asn", "62041", now, "http_5xx")
|
|
|
|
# До порога (3 сбоя подряд) источник не считается неисправным
|
|
fail(2)
|
|
health = env.get("/health").json()
|
|
assert health["status"] == "ok" and health["sources"] == {"asn": {"total": 1, "failing": []},
|
|
"fqdn": {"total": 1, "failing": []}}
|
|
|
|
# На пороге: degraded, источник в списке, наружу - только категория ошибки
|
|
fail(1)
|
|
health = env.get("/health").json()
|
|
assert health["status"] == "degraded"
|
|
assert health["sources"]["asn"]["failing"] == [{"source": "62041", "failures": 3, "last_success": None,
|
|
"error_kind": "http_5xx"}]
|
|
|
|
# Полная картина: адреса источника и состояние; неопрошенный источник - пустое состояние
|
|
with db.session() as conn, db.transaction(conn):
|
|
db.merge_source(conn, "asn", "62041", {"1.0.0.0/24", "2.0.0.0/24"}, now, 90)
|
|
report = env.get("/sources").json()
|
|
assert report["failure_threshold"] == 3
|
|
asn, fqdn = report["sources"]
|
|
assert (asn["kind"], asn["source"], asn["addresses"], asn["failures"]) == ("asn", "62041", 2, 3)
|
|
assert (fqdn["source"], fqdn["addresses"], fqdn["last_attempt"], fqdn["failures"]) == ("example.com", 0, None, 0)
|
|
|
|
# Успех сбрасывает счётчик, /health снова ok
|
|
with db.session() as conn, db.transaction(conn):
|
|
db.record_source_result(conn, "asn", "62041", now)
|
|
assert env.get("/health").json()["status"] == "ok"
|
|
|
|
# Порог настраивается ключом source_failure_threshold
|
|
cc.update_config(lambda config: config.update(source_failure_threshold=1))
|
|
fail(1)
|
|
assert env.get("/health").json()["status"] == "degraded"
|