ayurishchevandClaude Sonnet 5
cb935fef0d
Collectors record the result of every source (schema v3, table source_status); /health reports failing sources (failures in a row >= source_failure_threshold) and turns degraded; GET /sources shows the full state; runs end with a summary log line instead of "data saved". Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
211 lines
10 KiB
Python
211 lines
10 KiB
Python
import datetime
|
|
import os
|
|
import sys
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
import api_server
|
|
import cidr_collector as cc
|
|
import db
|
|
from storage import StorageError, load_json
|
|
|
|
NOW = datetime.datetime.now()
|
|
|
|
|
|
@pytest.fixture
|
|
def files(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(cc, "CONFIG_FILE", str(tmp_path / "config.json"))
|
|
monkeypatch.setattr(cc, "DATA_FILE", str(tmp_path / "data.json"))
|
|
monkeypatch.setattr(cc, "FQDN_DATA_FILE", str(tmp_path / "fqdn_data.json"))
|
|
monkeypatch.setattr(cc, "DB_FILE", str(tmp_path / "ripe.db"))
|
|
return tmp_path
|
|
|
|
|
|
def stored(kind="asn"):
|
|
with db.session() as conn:
|
|
return sorted(db.get_values(conn, kind))
|
|
|
|
|
|
def test_ttl_merge_and_failure_safety(files, monkeypatch):
|
|
(files / "config.json").write_text('{"asns": [1], "ttl_days": 90}')
|
|
collector = cc.CIDRCollector()
|
|
old = (NOW - datetime.timedelta(days=100)).isoformat(timespec="seconds")
|
|
with db.session() as conn:
|
|
conn.execute("INSERT INTO addresses VALUES ('asn', '1', '9.9.9.0/24', ?, ?)", (old, old))
|
|
|
|
# Ошибка источника: ничего не удаляется, даже просроченное
|
|
monkeypatch.setattr(collector, "fetch_prefixes", lambda asn: None)
|
|
collector.run_collection()
|
|
assert stored() == ["9.9.9.0/24"]
|
|
|
|
# Успешный ответ: просроченный удаляется, новый добавляется
|
|
monkeypatch.setattr(collector, "fetch_prefixes", lambda asn: ["1.1.1.0/24"])
|
|
collector.run_collection()
|
|
assert stored() == ["1.1.1.0/24"]
|
|
|
|
|
|
def test_corrupted_storage_is_preserved(files):
|
|
# Битый JSON (конфигурация)
|
|
path = files / "data.json"
|
|
path.write_text("{broken")
|
|
with pytest.raises(StorageError):
|
|
load_json(str(path), {})
|
|
assert not path.exists()
|
|
assert len(list(files.glob("data.json.corrupt-*"))) == 1
|
|
|
|
# Битая база без копий: оригинал убран в сторону, создана новая пустая база и поставлена метка пересоздания
|
|
(files / "ripe.db").write_bytes(b"this is not a sqlite database" * 100)
|
|
with db.session() as conn:
|
|
assert db.get_values(conn) == []
|
|
assert len(list(files.glob("ripe.db.corrupt-*"))) == 1
|
|
assert (files / "db_recreated.json").exists()
|
|
|
|
|
|
def test_post_schedule_auth(files, monkeypatch):
|
|
(files / "config.json").write_text('{"asns": [], "fqdns": []}')
|
|
client = TestClient(api_server.app)
|
|
body = {"type": "asn", "cron": "*/5 * * * *"}
|
|
|
|
monkeypatch.delenv("RIPE_API_TOKEN", raising=False)
|
|
assert client.post("/schedule", json=body).status_code == 503
|
|
|
|
monkeypatch.setenv("RIPE_API_TOKEN", "secret")
|
|
assert client.post("/schedule", json=body).status_code == 401
|
|
assert client.post("/schedule", json=body, headers={"X-API-Key": "wrong"}).status_code == 401
|
|
assert client.post("/schedule", json=body, headers={"X-API-Key": "secret"}).status_code == 200
|
|
assert load_json(cc.CONFIG_FILE, {})["schedule"]["asn"] == "*/5 * * * *"
|
|
assert client.get("/addresses").status_code == 200
|
|
|
|
|
|
def test_fqdn_keeps_only_global_addresses(files, monkeypatch):
|
|
answers = [(2, 1, 6, "", (address, 0)) for address in
|
|
("93.184.216.34", "127.0.0.1", "10.0.0.5", "0.0.0.0", "fe80::1%eth0", "2606:2800:220:1::1")]
|
|
monkeypatch.setattr(cc.socket, "getaddrinfo", lambda *args, **kwargs: answers)
|
|
|
|
(files / "config.json").write_text('{"fqdns": ["x.example"]}')
|
|
assert sorted(cc.FQDNCollector().resolve_fqdn("x.example")) == ["2606:2800:220:1::1", "93.184.216.34"]
|
|
|
|
# Внутренние имена: фильтр отключается, зона IPv6 отбрасывается
|
|
(files / "config.json").write_text('{"fqdns": ["x.example"], "allow_non_global_ips": true}')
|
|
everything = cc.FQDNCollector().resolve_fqdn("x.example")
|
|
assert {"127.0.0.1", "10.0.0.5", "fe80::1"} <= set(everything) and len(everything) == 6
|
|
|
|
# Глобальных адресов нет - результат пуст, как при ошибке DNS (сбор ничего не удаляет)
|
|
monkeypatch.setattr(cc.socket, "getaddrinfo", lambda *args, **kwargs: [answers[1]])
|
|
(files / "config.json").write_text('{"fqdns": ["x.example"]}')
|
|
assert cc.FQDNCollector().resolve_fqdn("x.example") == []
|
|
|
|
|
|
def test_ripestat_session_retries_and_sourceapp(files, monkeypatch):
|
|
calls = []
|
|
real_make_session = cc.make_session
|
|
|
|
class Response:
|
|
def raise_for_status(self):
|
|
pass
|
|
|
|
def json(self):
|
|
return {"data": {"prefixes": [{"prefix": "1.0.0.0/24"}]}}
|
|
|
|
class Session:
|
|
def get(self, url, params, timeout):
|
|
calls.append((params, timeout))
|
|
return Response()
|
|
|
|
def close(self):
|
|
pass
|
|
|
|
monkeypatch.setattr(cc, "make_session", lambda: Session())
|
|
assert cc.CIDRCollector().fetch_prefixes(1) == ["1.0.0.0/24"]
|
|
assert calls[0][0] == {"resource": "AS1", "sourceapp": "ripe-cidr-collector"} and calls[0][1] == cc.RIPESTAT_TIMEOUT
|
|
|
|
# Своё имя приложения из config.json (например, с контактом)
|
|
(files / "config.json").write_text('{"asns": [], "ripestat_sourceapp": "my-app admin@example.org"}')
|
|
assert cc.CIDRCollector().fetch_prefixes(2) == ["1.0.0.0/24"]
|
|
assert calls[1][0]["sourceapp"] == "my-app admin@example.org"
|
|
|
|
# После исчерпания повторов источник пропускается (None), сбор ничего не удаляет
|
|
collector = cc.CIDRCollector()
|
|
monkeypatch.setattr(collector, "sourceapp", "x")
|
|
Session.get = lambda self, url, params, timeout: (_ for _ in ()).throw(cc.requests.exceptions.RetryError("x"))
|
|
assert collector.fetch_prefixes(3) is None
|
|
|
|
# Настройка сессии: повторы на сбои и коды 429/5xx, пауза Retry-After ограничена потолком
|
|
retry = real_make_session().get_adapter("https://stat.ripe.net").max_retries
|
|
assert retry.total == cc.RIPESTAT_RETRIES and 503 in retry.status_forcelist and 429 in retry.status_forcelist
|
|
|
|
class Response: # сервер просит подождать час: пауза ограничена потолком
|
|
headers = {"Retry-After": "3600"}
|
|
assert retry.get_retry_after(Response()) == cc.MAX_RETRY_AFTER
|
|
|
|
|
|
def test_source_status_is_recorded(files, monkeypatch):
|
|
(files / "config.json").write_text('{"asns": [1, 2], "fqdns": ["ok.example", "dead.example", "private.example"]}')
|
|
|
|
def statuses():
|
|
with db.session() as conn:
|
|
return db.source_statuses(conn)
|
|
|
|
# ASN 1 отвечает, ASN 2 сбоит трижды подряд: счётчик растёт, last_success не появляется
|
|
def fetch_all(collector, broken):
|
|
def fetch(asn):
|
|
if asn in broken:
|
|
collector.errors[str(asn)] = "http_5xx"
|
|
return None
|
|
return ["1.0.0.0/24"]
|
|
return fetch
|
|
|
|
for _ in range(3):
|
|
collector = cc.CIDRCollector()
|
|
monkeypatch.setattr(collector, "fetch_prefixes", fetch_all(collector, {2}))
|
|
collector.run_collection()
|
|
state = statuses()
|
|
assert state["asn", "1"]["failures"] == 0 and state["asn", "1"]["last_success"]
|
|
assert state["asn", "2"]["failures"] == 3 and state["asn", "2"]["error_kind"] == "http_5xx"
|
|
assert state["asn", "2"]["last_success"] is None
|
|
|
|
# Успех сбрасывает счётчик; источник, удалённый из конфигурации, теряет состояние
|
|
collector = cc.CIDRCollector()
|
|
monkeypatch.setattr(collector, "fetch_prefixes", fetch_all(collector, set()))
|
|
collector.run_collection()
|
|
assert statuses()["asn", "2"]["failures"] == 0 and statuses()["asn", "2"]["error_kind"] is None
|
|
(files / "config.json").write_text('{"asns": [1], "fqdns": ["ok.example", "dead.example", "private.example"]}')
|
|
collector = cc.CIDRCollector()
|
|
monkeypatch.setattr(collector, "fetch_prefixes", lambda asn: ["1.0.0.0/24"])
|
|
collector.run_collection()
|
|
assert ("asn", "2") not in statuses()
|
|
|
|
# FQDN: DNS-сбой и «только неглобальные адреса» - разные категории
|
|
def getaddrinfo(name, *args, **kwargs):
|
|
if name == "dead.example":
|
|
raise cc.socket.gaierror("no such host")
|
|
address = "93.184.216.34" if name == "ok.example" else "10.0.0.5"
|
|
return [(2, 1, 6, "", (address, 0))]
|
|
|
|
monkeypatch.setattr(cc.socket, "getaddrinfo", getaddrinfo)
|
|
cc.FQDNCollector().run_collection()
|
|
state = statuses()
|
|
assert state["fqdn", "ok.example"]["failures"] == 0
|
|
assert state["fqdn", "dead.example"]["error_kind"] == "dns"
|
|
assert state["fqdn", "private.example"]["error_kind"] == "no_global_addresses"
|
|
|
|
# Категории ошибок (текст наружу не отдаётся)
|
|
requests = cc.requests.exceptions
|
|
assert cc.classify_error(requests.ConnectTimeout("x")) == "timeout"
|
|
assert cc.classify_error(requests.ConnectionError("Read timed out")) == "timeout"
|
|
assert cc.classify_error(requests.ConnectionError("refused")) == "connection"
|
|
assert cc.classify_error(requests.RetryError("too many 429 error responses")) == "http_429"
|
|
assert cc.classify_error(requests.HTTPError(response=type("R", (), {"status_code": 503})())) == "http_5xx"
|
|
assert cc.classify_error(requests.HTTPError(response=type("R", (), {"status_code": 404})())) == "http_4xx"
|
|
assert cc.classify_error(ValueError("bad json")) == "invalid_response" and cc.classify_error(KeyError()) == "unknown"
|
|
|
|
# База версии 2 (без таблицы состояния) обновляется при открытии, данные остаются
|
|
with db.session() as conn:
|
|
conn.execute("DROP TABLE source_status")
|
|
conn.execute("PRAGMA user_version = 2")
|
|
with db.session() as conn:
|
|
assert db.source_statuses(conn) == {} and db.get_values(conn, "asn") == ["1.0.0.0/24"]
|