commit 6563d755d82705048e30f7afcb1fc512452e79a7 Author: ayurishchev Date: Tue Jul 21 15:05:52 2026 +0300 build v_1.0.1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b2a7d35 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +artifacts +.claude +sources diff --git a/README.md b/README.md new file mode 100644 index 0000000..73b57aa --- /dev/null +++ b/README.md @@ -0,0 +1,231 @@ +# VPNaaS-миграция: Neutron → Sprut (`ipsec_migrator`) + +Инструмент для миграции VPNaaS (IPsec site-to-site) с Neutron-роутера OpenStack на «продвинутый» роутер (DC Router) во внешней SDN **Sprut** (`infra.mail.ru:9696`). + +Инструмент роутероцентричен: каждый Neutron-роутер из входного CSV сопоставлен с advanced-роутером в Sprut. Переносятся все IPsec-туннели, IKE/IPsec policy и endpoint groups, связанные с этим Neutron-роутером; объекты других роутеров того же OpenStack-проекта не затрагиваются. + +В репозитории две реализации с идентичным CLI, логикой и форматом `audit.json`: + +- **`ipsec_migrator_v2.sh`** — исходный bash-скрипт. Требует бинари `openstack`/`jq`/`curl`. +- **`ipsec_migrator/`** — Python-порт, актуальная реализация. Не использует бинарь `openstack` (ходит в Keystone/Neutron напрямую по REST) и не требует `jq`/`curl`. См. раздел [Python-порт](#python-порт-ipsec_migrator) ниже. + +## Статус проекта + +Python-порт — рекомендуемая реализация для новых миграций. История: + +1. Faithful-порт bash-скрипта на Python через subprocess-обёртку над `openstack` CLI (см. `port_summary.md`). +2. Переход с `openstack` CLI на прямые REST-вызовы к Keystone/Neutron (см. `session_2026-07-21_summary.md`). +3. Добавлено автосоздание DC Router в Sprut, когда `advanced_router_id` в CSV пуст (см. [Авто-создание целевого роутера](#авто-создание-целевого-advanced-роутера) ниже). + +Пункт 3 прошёл боевую проверку 2026-07-21: полный цикл (`--audit-only` → ревью → `--from-audit`) выполнен против настоящего OpenStack-тенанта и Sprut — DC Router + публичный интерфейс, IKE/IPsec policy, endpoint groups, VPN service и IPsec site connection созданы успешно для одного роутера. В ходе этого прогона найдены и исправлены три бага, не проявлявшиеся в офлайн-тестах (подробности и точные фиксы — в `session_2026-07-21_summary.md` и в разделе про авто-создание ниже): + +- Задвоенный `/v2.0` в пути запроса списка сетей Sprut (`.../v2.0/v2.0/networks`) — 404 при любом не-`--audit-only` прогоне. +- Публичная сеть в реальном тенанте называется `internet` (нижний регистр), а сравнение было регистрозависимым и искало `Internet`. +- Sprut отклоняет `subnet_id`/`ip_address` при подключении интерфейса к внешней сети (`400 Bad dc_interface request`) — `subnet_id` для такого интерфейса отправлять не нужно, Sprut назначает его сам. + +## Требования (bash-скрипт `ipsec_migrator_v2.sh`) + +- `bash` ≥ 4.3 (используются ассоциативные массивы и namerefs) +- `openstack` CLI, аутентифицированный в нужном проекте (переменные окружения `OS_*` / `clouds.yaml`) +- `jq` +- `curl` +- Сетевой доступ к `https://infra.mail.ru:9696` + +Скрипт проверяет версию bash и наличие всех трёх бинарей при старте и завершается с понятной ошибкой, если что-то не так. + +Требования для Python-порта — в его собственном разделе ниже; он ничего из списка выше не требует. + +## Формат входного CSV + +``` +neutron_router_id,advanced_router_id +neutron_router_id2,advanced_router_id2 +... +``` + +Пустые строки пропускаются; строки без второй колонки — с предупреждением, но не останавливая скрипт. + +Python-порт дополнительно поддерживает пустой `advanced_router_id` (авто-создание DC Router) и необязательные 3-ю/4-ю колонки — см. [Авто-создание целевого роутера](#авто-создание-целевого-advanced-роутера). + +## Режимы запуска + +### 1. Полный прогон (аудит + настройка Sprut) + +```bash +./ipsec_migrator_v2.sh [output.json] [--dry-run] +python3 -m ipsec_migrator [output.json] [--dry-run] +``` + +Выполняет всё последовательно: аудит конфигурации Neutron (STAGE 1) → сбор существующих объектов в Sprut (STAGE 2) → создание недостающих объектов и site connection'ов (STAGE 3–4). `output.json` необязателен, по умолчанию — `vpnaas_audit_.json` в текущей директории. + +### 2. Только аудит (без обращений к Sprut на запись) + +```bash +./ipsec_migrator_v2.sh --audit-only [output.json] +python3 -m ipsec_migrator --audit-only [output.json] +``` + +Выполняет только STAGE 1: проверяет, что Neutron- и advanced-роутеры существуют, собирает полную конфигурацию VPNaaS из Neutron и пишет её в JSON-файл. STAGE 2–4 (создание объектов в Sprut) не запускаются — скрипт завершается сразу после записи аудита. + +### 3. Dry-run по ранее собранному аудиту + +```bash +./ipsec_migrator_v2.sh --from-audit --dry-run +python3 -m ipsec_migrator --from-audit --dry-run +``` + +Пропускает повторный опрос OpenStack: все данные (IKE/IPsec policy, endpoint groups, VPN service, site connections) берутся из ранее сохранённого `audit.json`. Заново получает Keystone-токен и перепроверяет, что advanced-роутеры из аудита всё ещё существуют в Sprut (аудит мог быть сделан заранее). Дальше выполняются STAGE 2–4 в режиме симуляции: `GET`-запросы к Sprut выполняются по-настоящему (для сверки, что уже существует), но ни один объект не создаётся — вместо `POST` в лог пишется, что было бы отправлено (с маскированным PSK). + +### 4. Реальная настройка продвинутого роутера по аудиту + +```bash +./ipsec_migrator_v2.sh --from-audit +python3 -m ipsec_migrator --from-audit +``` + +То же самое, что режим 3, но без `--dry-run` — объекты реально создаются в Sprut (STAGE 2–4 выполняются по-боевому). + +`--audit-only` и `--from-audit` взаимоисключающие — совместное указание завершает скрипт с ошибкой. + +**Рекомендуемый порядок для боевого прогона:** `--audit-only` → ревью получившегося `audit.json` → `--from-audit` (без `--dry-run`). Так реальное создание объектов в Sprut — отдельный, осознанный шаг после проверки того, что именно будет создано. + +## Что делает каждый STAGE + +| Stage | Действие | +|---|---| +| **STAGE 1** (STEP 1–10) | Читает CSV, проверяет существование роутеров в OpenStack и Sprut (STEP 3; для Python-порта здесь же — авто-создание DC Router, если `advanced_router_id` пуст), собирает все IKE/IPsec policy, endpoint groups, VPN services и IPsec site connections по каждому роутеру из CSV, пишет самодостаточный JSON-аудит (атомарная запись, права `600`). | +| **STAGE 2** | Забирает текущий список объектов из Sprut API (`/vpn/ikepolicies`, `/vpn/ipsecpolicies`, `/vpn/endpoint-groups`, `/vpn/vpnservices`, `/vpn/ipsec-site-connections`) — основа для идемпотентной сверки. | +| **STAGE 3** | Сравнивает Neutron-объекты с уже существующими в Sprut (по имени / router_id / набору endpoints) и создаёт недостающие: IKE policy → IPsec policy → Endpoint Groups → VPN service. | +| **STAGE 4** | Создаёт IPsec site connections в Sprut, транслируя все связанные ID через карты соответствия из STAGE 3. Идемпотентно по полю `name`. | + +## Формат audit.json + +```json +{ + "audit_metadata": { "generated_at": "...", "input_file": "routers.csv", "stage": "STAGE1_AUDIT" }, + "subnets": { "": "" }, + "routers": [ + { + "neutron_router_id": "...", + "advanced_router_id": "...", + "pending_dc_router": { "name": "...", "description": "...", "availability_zone": "...", "flavor": "..." }, + "vpn_service": { "...": "полный объект vpnservice из Neutron API, либо null" }, + "ipsec_site_connections": [ + { + "id": "...", + "raw": { "...": "полный объект ipsec_site_connection из Neutron API" }, + "ike_policy": { "...": "полный объект ikepolicy" }, + "ipsec_policy": { "...": "полный объект ipsecpolicy" }, + "local_endpoint_group": { "id": "...", "raw": {}, "resolved_endpoints": ["10.0.0.0/24"], "already_migrated": false }, + "peer_endpoint_group": { "...": "та же структура" } + } + ] + } + ] +} +``` + +`advanced_router_id` равен `null`, а `pending_dc_router` заполнен, только когда CSV-строка не задавала advanced-роутер и он ещё не создан (Python-порт, `--audit-only`) — см. [Авто-создание целевого роутера](#авто-создание-целевого-advanced-роутера). + +Это тот же файл, что пишет режим `--audit-only` / полный прогон, и его же читает `--from-audit`. `--from-audit` проверяет `audit_metadata.stage == "STAGE1_AUDIT"` перед использованием. + +## Безопасность + +- **Файл аудита содержит PSK в открытом виде** (`ipsec_site_connections[].raw.psk` в Python-порте / `..."Pre-shared Key"` в bash-выводе). Пишется атомарно с правами `600` (только владелец). Храните и передавайте его как секрет. +- В stdout-логах PSK всегда маскируется (`redact_psk`), в т.ч. в теле запросов к Sprut и в ответах, содержащих `psk`. +- В `--dry-run` не-`GET` запросы к Sprut не отправляются — в лог пишется только предполагаемый метод/URL/тело (с маскированным PSK). `GET`-запросы (в т.ч. проверка существования advanced-роутера и поиск публичной сети для авто-создания DC Router) выполняются по-настоящему даже в `--dry-run`. +- HTTP-код ответа Sprut/Neutron всегда проверяется; не-2xx останавливает скрипт с ошибкой вместо тихого продолжения. + +## Известные ограничения + +- Токен Keystone не обновляется в течение STAGE 2–4 — для очень долгих миграций возможен `401` в середине прогона. +- Сравнение endpoint group'ов чувствительно к порядку адресов в массиве. +- Совпадающие имена IKE/IPsec policy у разных объектов в Neutron могут сломать сопоставление в Sprut. То же верно и для DC Router: авто-создание идемпотентно по имени Neutron-роутера — два разных роутера с одинаковым именем будут ошибочно сведены к одному DC Router. +- Между проверкой существования объекта (`GET`) и его созданием (`POST`) есть окно гонки — не запускайте миграцию параллельно против одного проекта. + +## Python-порт (`ipsec_migrator/`) + +Порт `ipsec_migrator_v2.sh` на Python, запуск `python3 -m ipsec_migrator `, тот же CLI и формат `audit.json`, что описаны выше. Ключевое отличие от bash-версии — способ обращения к OpenStack и Sprut: порт **не использует бинарь `openstack`** (и не требует `jq`/`curl`), а ходит в Keystone/Neutron/Sprut напрямую по REST через `requests`. + +Требования: + +- `python3`, `requests` — больше ничего не нужно. +- Аутентификация — только переменные окружения `OS_*` (password auth, + `clouds.yaml` не поддерживается): `OS_AUTH_URL`, `OS_USERNAME`, + `OS_PASSWORD`, `OS_USER_DOMAIN_NAME` (или `OS_USER_DOMAIN_ID`), + `OS_PROJECT_ID` (либо `OS_PROJECT_NAME` + `OS_PROJECT_DOMAIN_NAME`/`_ID`); + опционально `OS_REGION_NAME`, `OS_INTERFACE` (по умолчанию `public`) — + см. `test_openrc.sh` для примера. Эндпоинт Neutron резолвится из каталога + сервисов Keystone, а не хардкодится (с нормализацией хвостового `/v2.0`/`/v2` — + разные облака отдают его в каталоге по-разному). +- Списки Neutron-объектов (routers, subnets, ike/ipsec policies, endpoint groups, + vpn services, ipsec site connections) собираются с пагинацией — порт идёт по + `_links` (`rel: next`), пока не соберёт все страницы, а не только + первую. + +**Несовместимость `--from-audit`:** внутри `audit.json` вложенные объекты +(`raw`, `ike_policy`, `ipsec_policy`, endpoint group'ы) хранят имена полей +как их отдаёт Neutron API (`id`, `name`, `admin_state_up`, ...), а не +человекочитаемые заголовки колонок `openstack ... -f json` (`ID`, `Name`, +`State`, ...). Audit-файл, сгенерированный более ранней версией порта (той, +что ходила в OpenStack через `openstack` CLI), нельзя скормить в +`--from-audit` этой версии — нужно перегенерировать аудит заново. + +### Авто-создание целевого (advanced) роутера + +В `input.csv` для порта вторая колонка (`advanced_router_id`) может быть +пустой: + +``` +neutron_router_id1,advanced_router_id1 +neutron_router_id2,,MS1,standard +neutron_router_id3 +``` + +Если `advanced_router_id` пуст, порт сам создаёт DC Router в Sprut +(`POST /direct_connect/dc_routers`, `enable_snat: true`, `name`/`description` +берутся из самого Neutron-роутера) и подключает к нему публичный интерфейс +на внешнюю сеть Sprut (`POST /direct_connect/dc_interfaces`; сеть ищется по +имени `internet`/`Internet` без учёта регистра — реальное имя в проверенном +тенанте оказалось строчным `internet`). `subnet_id` для этого интерфейса не +указывается: Sprut отклоняет `subnet_id`/`ip_address` для интерфейса на +внешнюю сеть (`400 Bad dc_interface request: Specifying subnet_id or +ip_address for external network is restricted`) и назначает адрес сам. +Обе операции (DC Router, DC Interface) идемпотентны — по имени и по паре +`(dc_router_id, network_id)` соответственно — повторный прогон не создаёт +дублей. Точные пути и схемы запросов сверены с `neutron-sprut-api.json` +в корне репозитория. + +- `availability_zone` и `flavor` (обязательные поля Sprut, `flavor` — одно из + `basic`/`standard`/`advanced`) берутся из 3-й/4-й колонки CSV; если их там + нет — порт спросит интерактивно (`input()`). Для неинтерактивных запусков + (cron/CI) указывайте их в CSV заранее — иначе скрипт завершится ошибкой + вместо зависания на `stdin`. +- В режиме `--audit-only` DC Router **не создаётся** (режим по-прежнему + ничего не пишет в Sprut) — вместо этого в `audit.json` записывается + `"advanced_router_id": null` и блок `"pending_dc_router"` с именем, + описанием, AZ и flavor. Реальное создание происходит при последующем + `--from-audit` (без `--dry-run`) или в полном прогоне. +- Если в Sprut найдено несколько сетей с именем `internet`/`Internet` без + учёта регистра, используется первая с предупреждением в stderr — сознательный + упрощённый выбор (без дополнительной проверки `router:external`), см. + «Известные ограничения» выше про аналогичный риск коллизии имён. +- Реальное (не `--dry-run`) создание роутера и интерфейса — необратимое + изменение инфраструктуры Sprut, выполняется как часть обычного потока + STEP 3 (внутри STAGE 1). + +Статус: боевой цикл (`--audit-only` → `--from-audit`) успешно прогнан против +реального Sprut 2026-07-21 — детали в `session_2026-07-21_summary.md`. + +## Файлы в репозитории + +| Файл/директория | Назначение | +|---|---| +| `ipsec_migrator_v2.sh` | Исходный bash-скрипт миграции | +| `ipsec_migrator/` | Python-порт (пакет), запуск `python3 -m ipsec_migrator` | +| `input.csv` | Пример/рабочий входной CSV для текущей миграции | +| `test_openrc.sh` | Пример переменных окружения `OS_*` для аутентификации в OpenStack | +| `neutron-sprut-api.json` | OpenAPI-спека Sprut API — источник истины для путей/схем запросов (онлайн-документация `cloud.vk.com` недоступна из окружения разработки) | +| `port_summary.md` | Как и с какими оговорками бэш-скрипт был портирован на Python (2026-07-20) | +| `session_2026-07-21_summary.md` | Переход Python-порта на прямые Keystone/Neutron REST-вызовы и добавление авто-создания DC Router, включая боевую проверку и найденные баги (2026-07-21) | +| `vpnaas_audit_*.json` | Результаты прогонов (`--audit-only`/полный прогон) — содержат PSK, права `600` | diff --git a/input.csv b/input.csv new file mode 100644 index 0000000..192a942 --- /dev/null +++ b/input.csv @@ -0,0 +1 @@ +625d6d56-b2dc-4c83-b96f-e4e28af6b928,,MS1,standard diff --git a/ipsec_migrator/__init__.py b/ipsec_migrator/__init__.py new file mode 100644 index 0000000..8c0d5d5 --- /dev/null +++ b/ipsec_migrator/__init__.py @@ -0,0 +1 @@ +__version__ = "2.0.0" diff --git a/ipsec_migrator/__main__.py b/ipsec_migrator/__main__.py new file mode 100644 index 0000000..dbdd066 --- /dev/null +++ b/ipsec_migrator/__main__.py @@ -0,0 +1,6 @@ +import sys + +from .cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ipsec_migrator/__pycache__/__init__.cpython-311.pyc b/ipsec_migrator/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..a8cb484 Binary files /dev/null and b/ipsec_migrator/__pycache__/__init__.cpython-311.pyc differ diff --git a/ipsec_migrator/__pycache__/__main__.cpython-311.pyc b/ipsec_migrator/__pycache__/__main__.cpython-311.pyc new file mode 100644 index 0000000..8b6cc26 Binary files /dev/null and b/ipsec_migrator/__pycache__/__main__.cpython-311.pyc differ diff --git a/ipsec_migrator/__pycache__/audit.cpython-311.pyc b/ipsec_migrator/__pycache__/audit.cpython-311.pyc new file mode 100644 index 0000000..2b8ec45 Binary files /dev/null and b/ipsec_migrator/__pycache__/audit.cpython-311.pyc differ diff --git a/ipsec_migrator/__pycache__/cli.cpython-311.pyc b/ipsec_migrator/__pycache__/cli.cpython-311.pyc new file mode 100644 index 0000000..14f308b Binary files /dev/null and b/ipsec_migrator/__pycache__/cli.cpython-311.pyc differ diff --git a/ipsec_migrator/__pycache__/constants.cpython-311.pyc b/ipsec_migrator/__pycache__/constants.cpython-311.pyc new file mode 100644 index 0000000..d060d21 Binary files /dev/null and b/ipsec_migrator/__pycache__/constants.cpython-311.pyc differ diff --git a/ipsec_migrator/__pycache__/keystone_auth.cpython-311.pyc b/ipsec_migrator/__pycache__/keystone_auth.cpython-311.pyc new file mode 100644 index 0000000..4b62e4d Binary files /dev/null and b/ipsec_migrator/__pycache__/keystone_auth.cpython-311.pyc differ diff --git a/ipsec_migrator/__pycache__/models.cpython-311.pyc b/ipsec_migrator/__pycache__/models.cpython-311.pyc new file mode 100644 index 0000000..3eb2d74 Binary files /dev/null and b/ipsec_migrator/__pycache__/models.cpython-311.pyc differ diff --git a/ipsec_migrator/__pycache__/neutron_client.cpython-311.pyc b/ipsec_migrator/__pycache__/neutron_client.cpython-311.pyc new file mode 100644 index 0000000..08c1dea Binary files /dev/null and b/ipsec_migrator/__pycache__/neutron_client.cpython-311.pyc differ diff --git a/ipsec_migrator/__pycache__/redact.cpython-311.pyc b/ipsec_migrator/__pycache__/redact.cpython-311.pyc new file mode 100644 index 0000000..2b2fa29 Binary files /dev/null and b/ipsec_migrator/__pycache__/redact.cpython-311.pyc differ diff --git a/ipsec_migrator/__pycache__/sprut_client.cpython-311.pyc b/ipsec_migrator/__pycache__/sprut_client.cpython-311.pyc new file mode 100644 index 0000000..40f13ec Binary files /dev/null and b/ipsec_migrator/__pycache__/sprut_client.cpython-311.pyc differ diff --git a/ipsec_migrator/__pycache__/sync.cpython-311.pyc b/ipsec_migrator/__pycache__/sync.cpython-311.pyc new file mode 100644 index 0000000..ad46bde Binary files /dev/null and b/ipsec_migrator/__pycache__/sync.cpython-311.pyc differ diff --git a/ipsec_migrator/__pycache__/tables.cpython-311.pyc b/ipsec_migrator/__pycache__/tables.cpython-311.pyc new file mode 100644 index 0000000..8d2a061 Binary files /dev/null and b/ipsec_migrator/__pycache__/tables.cpython-311.pyc differ diff --git a/ipsec_migrator/audit.py b/ipsec_migrator/audit.py new file mode 100644 index 0000000..edf82e4 --- /dev/null +++ b/ipsec_migrator/audit.py @@ -0,0 +1,763 @@ +"""STAGE 1: collecting Neutron VPNaaS configuration into an audit JSON file, +plus the --from-audit path that reconstructs the same in-memory state from a +previously written audit file instead of re-querying OpenStack. + +This module owns the audit JSON schema definition (to_audit_dict / +load_from_audit_json), so it's defined exactly once and shared by both the +write path (STEP 10) and the read path (--from-audit). +""" +from __future__ import annotations + +import json +import os +import re +import sys +from datetime import datetime, timezone + +from . import keystone_auth, neutron_client +from .models import MigrationState +from .neutron_client import NeutronClient +from .redact import redact_psk +from .sprut_client import SprutClient +from .tables import print_kv_table + +_UUID_LIKE = re.compile(r"^[0-9a-fA-F-]{36}$") + + +def _dump(obj) -> str: + return json.dumps(obj, indent=2) + + +def _rule() -> None: + print("*" * 86) + + +# -------------------------------------------------------------------------- +# STEP 1: read the input CSV and build the neutron -> advanced router map. +# A row's advanced_router_id may be blank -- that router will be created in +# Sprut later (STEP 3). Optional 3rd/4th columns (availability_zone, +# flavor) only matter for such rows; they're a hint for that creation, +# used instead of the interactive prompt when present. +# -------------------------------------------------------------------------- +def read_input_csv(input_csv: str) -> tuple[dict[str, str], dict[str, dict]]: + print(f"Executing STEP 1: Reading config file {input_csv}") + + mapping: dict[str, str] = {} + creation_hints: dict[str, dict] = {} + try: + handle = open(input_csv, encoding="utf-8") + except OSError as exc: + print(f"Error: could not open input file '{input_csv}': {exc}", file=sys.stderr) + sys.exit(1) + + with handle: + for raw_line in handle: + line = raw_line.rstrip("\n").rstrip("\r") + fields = [f.strip() for f in line.split(",")] + neutron_router = fields[0] if fields else "" + advanced_router = fields[1] if len(fields) > 1 else "" + availability_zone = fields[2] if len(fields) > 2 else "" + flavor = fields[3] if len(fields) > 3 else "" + + if not neutron_router: + continue + mapping[neutron_router] = advanced_router + if not advanced_router: + hint = {} + if availability_zone: + hint["availability_zone"] = availability_zone + if flavor: + hint["flavor"] = flavor + if hint: + creation_hints[neutron_router] = hint + print( + f"Neutron router '{neutron_router}' has no advanced router — " + "a DC Router will be created for it in Sprut.", + file=sys.stderr, + ) + + print("Reading values from config:") + print() + print_kv_table( + {k: (v or "") for k, v in mapping.items()}, "Neutron router", "Advanced Router" + ) + print() + + print("STEP 1 complete (config read)") + _rule() + return mapping, creation_hints + + +# -------------------------------------------------------------------------- +# STEP 2: verify every Neutron router from the CSV exists in OpenStack. +# -------------------------------------------------------------------------- +def check_neutron_routers_exist(state: MigrationState, nc: NeutronClient) -> None: + print("Executing STEP 2: Checking Neutron routers") + + listing = neutron_client.router_list(nc) + state.neutron_router_by_id = {entry["id"]: entry for entry in listing} + existing_ids = set(state.neutron_router_by_id) + + for neutron_router_id in state.neutron_to_adv_router: + if neutron_router_id not in existing_ids: + # No stderr redirect here -- matches the bash original's + # (inconsistent, but preserved) use of stdout for this message. + print(f"Error: Neutron router ID {neutron_router_id} not found in OpenStack tenant.") + sys.exit(1) + print(f"Neutron router ID {neutron_router_id} exists in OpenStack tenant.") + + print("STEP 2 complete (Neutron routers checked)") + _rule() + + +_DC_FLAVORS = ("basic", "standard", "advanced") + + +def _require_interactive(neutron_router_id: str, field_name: str) -> None: + if not sys.stdin.isatty(): + print( + f"Error: router '{neutron_router_id}' needs a DC Router created but '{field_name}' " + "isn't set in the CSV and stdin is not interactive. Add it as a CSV column " + "(3rd=availability_zone, 4th=flavor) or run interactively.", + file=sys.stderr, + ) + sys.exit(1) + + +def _resolve_az_flavor(neutron_router_id: str, hints: dict[str, dict]) -> tuple[str, str]: + hint = hints.get(neutron_router_id, {}) + + availability_zone = hint.get("availability_zone") or "" + if not availability_zone: + _require_interactive(neutron_router_id, "availability_zone") + while not availability_zone: + availability_zone = input(f"Router '{neutron_router_id}': enter availability_zone: ").strip() + + flavor = hint.get("flavor") or "" + if flavor not in _DC_FLAVORS: + _require_interactive(neutron_router_id, "flavor") + while flavor not in _DC_FLAVORS: + flavor = input(f"Router '{neutron_router_id}': enter flavor ({'/'.join(_DC_FLAVORS)}): ").strip() + + return availability_zone, flavor + + +def _find_internet_network(sprut_client: SprutClient) -> str: + """Locates Sprut's public network (name "internet"/"Internet"), for + attaching a DC Router's public interface. Looked up once per run, + lazily, only when at least one router actually needs creation. + + Deliberately does not resolve or send a subnet_id: Sprut rejects + dc_interface creation with subnet_id set for an external network + ("Specifying subnet_id or ip_address for external network is + restricted", confirmed against the real API) -- it auto-assigns one. + """ + networks = sprut_client.get("/networks?limit=10000").get("networks", []) + matches = [n for n in networks if (n.get("name") or "").lower() == "internet"] + if not matches: + print("Error: no Sprut network named 'Internet' found (needed for the public interface).", file=sys.stderr) + sys.exit(1) + if len(matches) > 1: + print( + f"Warning: {len(matches)} Sprut networks named 'Internet' found -- " + f"using the first one ({matches[0].get('id')}).", + file=sys.stderr, + ) + return matches[0]["id"] + + +def _find_or_create_dc_router( + neutron_router_id: str, + name: str, + description: str, + availability_zone: str, + flavor: str, + sprut_client: SprutClient, + existing_dc_routers_by_name: dict[str, str], +) -> str: + sprut_id = existing_dc_routers_by_name.get(name) + if sprut_id: + print(f"DC Router '{name}' already exists in Sprut (id {sprut_id}) -- reusing") + return sprut_id + + print(f"Creating DC Router '{name}' in Sprut (az={availability_zone}, flavor={flavor}, enable_snat=true)") + request_body = { + "dc_router": { + "availability_zone": availability_zone, + "flavor": flavor, + "enable_snat": True, + "name": name, + "description": description, + } + } + response = sprut_client.post("/direct_connect/dc_routers", request_body) + sprut_id = (response.get("dc_router") or {}).get("id") + if sprut_client.dry_run and not sprut_id: + sprut_id = f"DRY-RUN-{neutron_router_id}" + return sprut_id + + +def _find_or_create_dc_interface( + dc_router_id: str, network_id: str, name: str, sprut_client: SprutClient +) -> str: + existing = sprut_client.get("/direct_connect/dc_interfaces").get("dc_interfaces", []) + match = next( + (i for i in existing if i.get("dc_router_id") == dc_router_id and i.get("network_id") == network_id), + None, + ) + if match: + print(f"Public interface already attached to DC Router {dc_router_id} (id {match.get('id')}) -- reusing") + return match["id"] + + print(f"Attaching public interface (network 'Internet') to DC Router {dc_router_id}") + # subnet_id deliberately omitted -- Sprut rejects it for external + # networks (see _find_internet_network's docstring). + request_body = { + "dc_interface": { + "dc_router_id": dc_router_id, + "network_id": network_id, + "name": name, + } + } + response = sprut_client.post("/direct_connect/dc_interfaces", request_body) + sprut_id = (response.get("dc_interface") or {}).get("id") + if sprut_client.dry_run and not sprut_id: + sprut_id = f"DRY-RUN-iface-{dc_router_id}" + return sprut_id + + +def _ensure_dc_router( + neutron_router_id: str, + name: str, + description: str, + availability_zone: str, + flavor: str, + sprut_client: SprutClient, + existing_dc_routers_by_name: dict[str, str], + internet_network_id: str, +) -> str: + """Creates (or reuses, by name) a DC Router and attaches its public + interface. Shared by the fresh-collect path (STEP 3) and the + --from-audit path, which both need identical create-or-reuse logic.""" + sprut_id = _find_or_create_dc_router( + neutron_router_id, name, description, availability_zone, flavor, sprut_client, existing_dc_routers_by_name + ) + _find_or_create_dc_interface(sprut_id, internet_network_id, f"{name}-public", sprut_client) + existing_dc_routers_by_name[name] = sprut_id + return sprut_id + + +# -------------------------------------------------------------------------- +# STEP 3: verify every advanced router exists in Sprut, using the Keystone +# token issued once up front by collect(). For CSV rows with a blank +# advanced_router_id, either creates the DC Router now (full run) or just +# records what it would need (--audit-only, which must not write to +# Sprut). Returns the SprutClient constructed with that token, reused by +# STAGE 2-4 (the token is never refreshed after this point). +# -------------------------------------------------------------------------- +def check_advanced_routers_exist( + state: MigrationState, token: str, sprut_base_url: str, dry_run: bool, audit_only: bool +) -> SprutClient: + print("Executing STEP 3: Checking Advanced routers") + + sprut_client = SprutClient(sprut_base_url, token, dry_run) + + dc_routers = sprut_client.get("/direct_connect/dc_routers").get("dc_routers", []) + existing_ids = {entry["id"] for entry in dc_routers} + existing_by_name = {entry["name"]: entry["id"] for entry in dc_routers if entry.get("name")} + + internet_network_id: str | None = None + + for neutron_router_id, advanced_router_id in state.neutron_to_adv_router.items(): + if advanced_router_id: + if advanced_router_id not in existing_ids: + print(f"Error: Advanced router ID {advanced_router_id} not found in SDN.") + sys.exit(1) + print(f"Advanced router ID {advanced_router_id} exists in SDN.") + continue + + neutron_router = state.neutron_router_by_id.get(neutron_router_id, {}) + name = neutron_router.get("name") or neutron_router_id + description = neutron_router.get("description") or "" + availability_zone, flavor = _resolve_az_flavor(neutron_router_id, state.router_creation_hints) + + if audit_only: + state.pending_dc_router_by_neutron_id[neutron_router_id] = { + "name": name, + "description": description, + "availability_zone": availability_zone, + "flavor": flavor, + } + print( + f"Neutron router '{neutron_router_id}': no advanced router -- recorded as pending " + "DC Router creation in the audit (no Sprut writes in --audit-only)." + ) + continue + + if internet_network_id is None: + internet_network_id = _find_internet_network(sprut_client) + + state.neutron_to_adv_router[neutron_router_id] = _ensure_dc_router( + neutron_router_id, + name, + description, + availability_zone, + flavor, + sprut_client, + existing_by_name, + internet_network_id, + ) + + print("STEP 3 complete (Advanced routers checked)") + _rule() + return sprut_client + + +def recheck_advanced_routers_from_audit(state: MigrationState, sprut_client: SprutClient) -> None: + """Re-verifies advanced routers recorded in a --from-audit file still + exist in Sprut, since the audit may have been generated earlier. Also + creates any DC Router left pending by a prior --audit-only run, using + the name/description/az/flavor cached in that audit's + "pending_dc_router" block -- Neutron is never re-queried here.""" + print("Issuing OpenStack token and re-checking advanced routers in SDN...") + + dc_routers = sprut_client.get("/direct_connect/dc_routers").get("dc_routers", []) + existing_ids = {entry["id"] for entry in dc_routers} + existing_by_name = {entry["name"]: entry["id"] for entry in dc_routers if entry.get("name")} + + internet_network_id: str | None = None + + for neutron_router_id, advanced_router_id in state.neutron_to_adv_router.items(): + if advanced_router_id: + if advanced_router_id not in existing_ids: + print( + f"Error: Advanced router ID {advanced_router_id} (from audit file) not found in SDN.", + file=sys.stderr, + ) + sys.exit(1) + print(f"Advanced router ID {advanced_router_id} exists in SDN.") + continue + + pending = state.pending_dc_router_by_neutron_id.get(neutron_router_id) + if not pending: + print( + f"Error: router '{neutron_router_id}' has no advanced_router_id and no " + "pending_dc_router info in the audit file -- file is incompatible or corrupted.", + file=sys.stderr, + ) + sys.exit(1) + + if internet_network_id is None: + internet_network_id = _find_internet_network(sprut_client) + + state.neutron_to_adv_router[neutron_router_id] = _ensure_dc_router( + neutron_router_id, + pending["name"], + pending.get("description", ""), + pending["availability_zone"], + pending["flavor"], + sprut_client, + existing_by_name, + internet_network_id, + ) + + +# -------------------------------------------------------------------------- +# STEP 4-6: IPsec policies, IKE policies, Endpoint groups (project-wide). +# One list call each -- Neutron's list endpoints return full objects, so no +# per-item show call is needed (unlike the old CLI-based version). +# -------------------------------------------------------------------------- +def collect_ipsec_policies(state: MigrationState, nc: NeutronClient) -> None: + print("Executing STEP 4: Collecting info about IPsec policies") + + for details in neutron_client.ipsec_policy_list(nc): + state.neutron_ipsec_policy_by_id[details["id"]] = details + + print("Stored IPsec policies:") + for policy_id, details in state.neutron_ipsec_policy_by_id.items(): + print(f"Policy ID: {policy_id}") + print(f"Details: {_dump(details)}") + print() + + print("STEP 4 complete: Info about IPsec policies stored") + _rule() + + +def collect_ike_policies(state: MigrationState, nc: NeutronClient) -> None: + print("Executing STEP 5: Collecting info about IKE policies") + + for details in neutron_client.ike_policy_list(nc): + state.neutron_ike_policy_by_id[details["id"]] = details + + print("Stored IKE policies:") + for policy_id, details in state.neutron_ike_policy_by_id.items(): + print(f"Policy ID: {policy_id}") + print(f"Details: {_dump(details)}") + print() + + print("STEP 5 complete: Info about IKE policies stored") + _rule() + + +def collect_endpoint_groups(state: MigrationState, nc: NeutronClient) -> None: + print("Executing STEP 6: Collecting info about Endpoint Groups") + + for details in neutron_client.endpoint_group_list(nc): + eg_id = details["id"] + state.neutron_endpoint_group_by_id[eg_id] = details + + if details.get("sdn") == "sprut": + print(f"Endpoint Group {eg_id} has sdn 'sprut' with the following network addresses:") + for address in details.get("endpoints") or []: + print(address) + + print("Stored Endpoint Groups:") + for eg_id, details in state.neutron_endpoint_group_by_id.items(): + print(f"Endpoint Group ID: {eg_id}") + print(f"Details: {_dump(details)}") + print() + + print("STEP 6 complete: Info about Endpoint Groups stored") + _rule() + + +# -------------------------------------------------------------------------- +# STEP 7: subnet id -> CIDR map, excluding ext-subnet* names. +# -------------------------------------------------------------------------- +def collect_subnets(state: MigrationState, nc: NeutronClient) -> None: + print("Executing STEP 7: Creating a map of Subnet ID to Subnet Address") + + for entry in neutron_client.subnet_list(nc): + name = entry.get("name") or "" + if name.startswith("ext-subnet"): + continue + state.subnet_id_to_subnet_address[entry["id"]] = entry.get("cidr") + + print("Stored Subnet ID to Subnet Address mapping:") + print_kv_table(state.subnet_id_to_subnet_address, "Subnet ID", "Subnet Address", width1=36, width2=18) + print() + + print("STEP 7 complete: Subnet ID to Subnet Address map created") + _rule() + + +# -------------------------------------------------------------------------- +# STEP 8: router <-> vpn service maps. router_id_to_vpn_service_id is +# filtered to in-scope (CSV) routers; the reverse map is deliberately left +# project-wide since STEP 9 needs it to resolve every connection's owning +# router before in-scope narrowing happens. +# -------------------------------------------------------------------------- +def collect_vpn_services(state: MigrationState, nc: NeutronClient) -> None: + print("Executing STEP 8: Collecting info about vpn services") + listing = neutron_client.vpn_service_list(nc) + + for entry in listing: + router_id = entry["router_id"] + vpn_id = entry["id"] + state.router_id_to_vpn_service_id.setdefault(router_id, vpn_id) + state.neutron_vpn_service_by_id.setdefault(vpn_id, entry) + + print("Remove router_ids that are not in config") + for router_id in list(state.router_id_to_vpn_service_id): + if router_id not in state.neutron_to_adv_router: + del state.router_id_to_vpn_service_id[router_id] + + print_kv_table(state.router_id_to_vpn_service_id, "Router ID", "VPN service ID") + + print("build backwards dictionary") + for entry in listing: + state.vpn_service_id_to_router_id[entry["id"]] = entry["router_id"] + + print_kv_table(state.vpn_service_id_to_router_id, "VPN service ID", "Router ID") + + print("STEP 8 complete (Router ID to VPN Service ID map built)") + _rule() + + +# -------------------------------------------------------------------------- +# STEP 9: ipsec site connections + router -> connection-ids map. +# -------------------------------------------------------------------------- +def collect_ipsec_site_connections(state: MigrationState, nc: NeutronClient) -> None: + print("Executing STEP 9: Collecting info about ipsec") + + for details in neutron_client.ipsec_site_connection_list(nc): + conn_id = details["id"] + + vpn_service_id = details.get("vpnservice_id") + router_id = state.vpn_service_id_to_router_id.get(vpn_service_id, "") + state.neutron_router_to_ipsec_ids.setdefault(router_id, []).append(conn_id) + state.neutron_connection_by_id[conn_id] = details + + print("Stored IPsec connections:") + for conn_id, details in state.neutron_connection_by_id.items(): + print(f"ID: {conn_id}") + print(f"Details: {_dump(redact_psk(details))}") + print() + + print("Neutron Router to IPsec IDs mapping:") + for router_id, conn_ids in state.neutron_router_to_ipsec_ids.items(): + print(f"Router ID: {router_id}") + print(f"IPsec IDs: {', '.join(conn_ids)}") + + print("STEP 9 complete: Info about ipsec stored") + _rule() + + +# -------------------------------------------------------------------------- +# In-scope computation: only routers/objects tied to the input CSV are +# actually created in Sprut by STAGE 3/4, even though STEP 4-9 collect +# project-wide (needed for matching against what Sprut already has). +# -------------------------------------------------------------------------- +def compute_in_scope(state: MigrationState) -> None: + for neutron_router_id in state.neutron_to_adv_router: + for conn_id in state.neutron_router_to_ipsec_ids.get(neutron_router_id, []): + if not conn_id: + continue + state.in_scope_connection_ids.add(conn_id) + + conn_raw = state.neutron_connection_by_id[conn_id] + ike_id = conn_raw.get("ikepolicy_id") + ipsec_id = conn_raw.get("ipsecpolicy_id") + local_eg_id = conn_raw.get("local_ep_group_id") + peer_eg_id = conn_raw.get("peer_ep_group_id") + + if ike_id: + state.in_scope_ike_policy_ids.add(ike_id) + if ipsec_id: + state.in_scope_ipsec_policy_ids.add(ipsec_id) + if local_eg_id: + state.in_scope_endpoint_group_ids.add(local_eg_id) + if peer_eg_id: + state.in_scope_endpoint_group_ids.add(peer_eg_id) + + print(f"In-scope IPsec site connections for this migration: {len(state.in_scope_connection_ids)}") + + +# -------------------------------------------------------------------------- +# Endpoint-group helpers, shared by STEP 10 (audit assembly) and STAGE 3 +# (endpoint-group compare&create). +# -------------------------------------------------------------------------- +def resolve_endpoint_addresses(details: dict, subnet_map: dict[str, str]) -> list[str]: + name = details.get("name") + converted: list[str] = [] + for endpoint in details.get("endpoints") or []: + if _UUID_LIKE.match(endpoint): + print(f"Converting subnet UUID {endpoint} in {name} endpoints", file=sys.stderr) + address = subnet_map.get(endpoint) + if address: + print(f"Converted to {address}", file=sys.stderr) + converted.append(address) + else: + print(f"Warning: Subnet ID {endpoint} not found in subnet_id_to_subnet_address map.", file=sys.stderr) + else: + converted.append(endpoint) + return converted + + +def build_endpoint_group_block(raw: dict | None, subnet_map: dict[str, str]) -> dict | None: + if not raw: + return None + resolved = resolve_endpoint_addresses(raw, subnet_map) + return { + "id": raw.get("id"), + "raw": raw, + "resolved_endpoints": resolved, + "already_migrated": raw.get("sdn") == "sprut", + } + + +# -------------------------------------------------------------------------- +# STEP 10: assemble and write the audit JSON. +# -------------------------------------------------------------------------- +def to_audit_dict(state: MigrationState, input_file: str) -> dict: + router_blocks = [] + for neutron_router_id, advanced_router_id in state.neutron_to_adv_router.items(): + vpn_id = state.router_id_to_vpn_service_id.get(neutron_router_id) + vpn_service_json = state.neutron_vpn_service_by_id.get(vpn_id) if vpn_id else None + + connection_blocks = [] + for conn_id in state.neutron_router_to_ipsec_ids.get(neutron_router_id, []): + if not conn_id: + continue + conn_raw = state.neutron_connection_by_id[conn_id] + + ike_id = conn_raw.get("ikepolicy_id") + ipsec_id = conn_raw.get("ipsecpolicy_id") + local_eg_id = conn_raw.get("local_ep_group_id") + peer_eg_id = conn_raw.get("peer_ep_group_id") + + ike_json = state.neutron_ike_policy_by_id.get(ike_id) + ipsec_json = state.neutron_ipsec_policy_by_id.get(ipsec_id) + + local_eg_raw = state.neutron_endpoint_group_by_id.get(local_eg_id) + peer_eg_raw = state.neutron_endpoint_group_by_id.get(peer_eg_id) + + connection_blocks.append( + { + "id": conn_raw.get("id"), + "raw": conn_raw, + "ike_policy": ike_json, + "ipsec_policy": ipsec_json, + "local_endpoint_group": build_endpoint_group_block(local_eg_raw, state.subnet_id_to_subnet_address), + "peer_endpoint_group": build_endpoint_group_block(peer_eg_raw, state.subnet_id_to_subnet_address), + } + ) + + router_blocks.append( + { + "neutron_router_id": neutron_router_id, + "advanced_router_id": advanced_router_id or None, + "pending_dc_router": state.pending_dc_router_by_neutron_id.get(neutron_router_id), + "vpn_service": vpn_service_json, + "ipsec_site_connections": connection_blocks, + } + ) + + return { + "audit_metadata": { + "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "input_file": input_file, + "stage": "STAGE1_AUDIT", + }, + "subnets": dict(state.subnet_id_to_subnet_address), + "routers": router_blocks, + } + + +def write_audit_json(data: dict, output_file: str) -> None: + tmp_path = f"{output_file}.tmp" + with open(tmp_path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2) + + try: + with open(tmp_path, encoding="utf-8") as f: + json.load(f) + except (OSError, json.JSONDecodeError): + print("Error: failed to assemble valid JSON audit output", file=sys.stderr) + os.remove(tmp_path) + sys.exit(1) + + os.replace(tmp_path, output_file) + os.chmod(output_file, 0o600) + print(f"Audit JSON written to {output_file} (permissions restricted to owner: contains PSK values)") + + +# -------------------------------------------------------------------------- +# --from-audit: reconstruct MigrationState from a previously written audit +# file, without any OpenStack query. Pure (no network I/O) -- token +# reissuance and the Sprut recheck are the orchestration's job (cli.py), +# via recheck_advanced_routers_from_audit() above. +# -------------------------------------------------------------------------- +def load_from_audit_json(audit_file: str) -> MigrationState: + print(f"Loading audit data from {audit_file} (skipping OpenStack collection)") + + try: + with open(audit_file, encoding="utf-8") as f: + data = json.load(f) + except (OSError, json.JSONDecodeError): + print(f"Error: '{audit_file}' is not valid JSON.", file=sys.stderr) + sys.exit(1) + + schema_stage = (data.get("audit_metadata") or {}).get("stage") + if schema_stage != "STAGE1_AUDIT": + print( + f"Error: '{audit_file}' does not look like a STAGE1 audit file " + "(audit_metadata.stage != STAGE1_AUDIT).", + file=sys.stderr, + ) + sys.exit(1) + + state = MigrationState() + + for subnet_id, subnet_cidr in (data.get("subnets") or {}).items(): + if not subnet_id: + continue + state.subnet_id_to_subnet_address[subnet_id] = subnet_cidr + + for router_json in data.get("routers") or []: + neutron_router_id = router_json.get("neutron_router_id") + advanced_router_id = router_json.get("advanced_router_id") or "" + if not neutron_router_id: + continue + state.neutron_to_adv_router[neutron_router_id] = advanced_router_id + + if not advanced_router_id: + pending = router_json.get("pending_dc_router") + if pending: + state.pending_dc_router_by_neutron_id[neutron_router_id] = pending + + vpn_service_json = router_json.get("vpn_service") + if vpn_service_json: + vpn_service_id = vpn_service_json.get("id") + state.router_id_to_vpn_service_id[neutron_router_id] = vpn_service_id + + for conn_json in router_json.get("ipsec_site_connections") or []: + conn_id = conn_json.get("id") + if not conn_id: + continue + + state.neutron_connection_by_id[conn_id] = conn_json.get("raw") + state.in_scope_connection_ids.add(conn_id) + + ike_json = conn_json.get("ike_policy") + if ike_json: + ike_id = ike_json.get("id") + state.neutron_ike_policy_by_id[ike_id] = ike_json + state.in_scope_ike_policy_ids.add(ike_id) + + ipsec_json = conn_json.get("ipsec_policy") + if ipsec_json: + ipsec_id = ipsec_json.get("id") + state.neutron_ipsec_policy_by_id[ipsec_id] = ipsec_json + state.in_scope_ipsec_policy_ids.add(ipsec_id) + + local_eg_json = conn_json.get("local_endpoint_group") + if local_eg_json: + local_eg_id = local_eg_json.get("id") + state.neutron_endpoint_group_by_id[local_eg_id] = local_eg_json.get("raw") + state.in_scope_endpoint_group_ids.add(local_eg_id) + + peer_eg_json = conn_json.get("peer_endpoint_group") + if peer_eg_json: + peer_eg_id = peer_eg_json.get("id") + state.neutron_endpoint_group_by_id[peer_eg_id] = peer_eg_json.get("raw") + state.in_scope_endpoint_group_ids.add(peer_eg_id) + + print( + f"Loaded from audit: {len(state.neutron_to_adv_router)} router(s), " + f"{len(state.in_scope_connection_ids)} in-scope IPsec site connection(s)." + ) + + return state + + +# -------------------------------------------------------------------------- +# Full STAGE 1 orchestration (STEP 1-9 + in-scope computation). STEP 10 +# (write) is invoked separately by the caller so --audit-only and the full +# run can share this without duplicating the write step. +# -------------------------------------------------------------------------- +def collect( + input_csv: str, sprut_base_url: str, dry_run: bool, audit_only: bool +) -> tuple[MigrationState, SprutClient]: + state = MigrationState() + state.neutron_to_adv_router, state.router_creation_hints = read_input_csv(input_csv) + + # Authenticate once, up front: STEP 2 now needs a Neutron token just as + # much as STEP 3 needs one for Sprut, since neither goes through the + # `openstack` CLI's own per-call auth resolution anymore. + token, catalog = keystone_auth.authenticate() + neutron_base_url = keystone_auth.resolve_endpoint(catalog, service_type="network") + nc = NeutronClient(neutron_base_url, token) + + check_neutron_routers_exist(state, nc) + sprut_client = check_advanced_routers_exist(state, token, sprut_base_url, dry_run, audit_only) + + collect_ipsec_policies(state, nc) + collect_ike_policies(state, nc) + collect_endpoint_groups(state, nc) + collect_subnets(state, nc) + collect_vpn_services(state, nc) + collect_ipsec_site_connections(state, nc) + compute_in_scope(state) + + return state, sprut_client diff --git a/ipsec_migrator/cli.py b/ipsec_migrator/cli.py new file mode 100644 index 0000000..8d73452 --- /dev/null +++ b/ipsec_migrator/cli.py @@ -0,0 +1,153 @@ +"""Argument parsing and top-level orchestration of STAGE 1-4. + +Mirrors the bash original's literal top-to-bottom execution order: banner +and the "STAGE 1: Collecting information" header print before anything +else, then argument parsing, then mode dispatch. The `openstack` CLI +binary check that used to run here is gone -- OpenStack access is now +direct Neutron/Keystone REST calls (`keystone_auth.py`, `neutron_client.py`), +so no external binary is required at all. +""" +from __future__ import annotations + +import argparse +import os +import sys + +from . import audit, keystone_auth, sync +from .constants import BANNER, SPRUT_API_BASE, USAGE +from .sprut_client import SprutClient + + +def print_banner() -> None: + print(BANNER) + + +def print_usage() -> None: + print(USAGE) + + +def _rule() -> None: + print("*" * 86) + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--dry-run", action="store_true", dest="dry_run") + parser.add_argument("--audit-only", action="store_true", dest="audit_only") + parser.add_argument("--from-audit", dest="from_audit", default=None) + parser.add_argument("-h", "--help", action="store_true", dest="help") + parser.add_argument("positional", nargs="*") + + args = parser.parse_args(argv) + + if args.help: + print_usage() + sys.exit(0) + + audit_only_seen = args.audit_only + from_audit_seen = args.from_audit is not None + + if audit_only_seen and from_audit_seen: + print("Error: --audit-only and --from-audit are mutually exclusive.", file=sys.stderr) + sys.exit(1) + + if from_audit_seen: + mode = "from-audit" + elif audit_only_seen: + mode = "audit-only" + else: + mode = "full" + + input_csv = None + output_file = None + + if mode == "from-audit": + if not args.from_audit: + print("Error: --from-audit requires a path to an audit JSON file.", file=sys.stderr) + print_usage() + sys.exit(1) + if not os.path.isfile(args.from_audit): + print(f"Error: audit file '{args.from_audit}' not found.", file=sys.stderr) + sys.exit(1) + else: + if not args.positional: + print("Error: No input file provided.") + print_usage() + sys.exit(1) + input_csv = args.positional[0] + if len(args.positional) > 1: + output_file = args.positional[1] + + return argparse.Namespace( + mode=mode, + dry_run=args.dry_run, + from_audit=args.from_audit, + input_csv=input_csv, + output_file=output_file, + ) + + +def _default_output_file() -> str: + from datetime import datetime + + return f"vpnaas_audit_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + + +def main(argv: list[str] | None = None) -> int: + # Python buffers stdout in blocks (not line-by-line) whenever it isn't a + # TTY, while stderr stays unbuffered -- under redirection (e.g. `2>&1 | + # tee log`) that visibly reorders stdout/stderr lines relative to each + # other. Bash's `echo` has no such issue (each call is its own unbuffered + # write), so line-buffer stdout here to keep interleaving faithful. + sys.stdout.reconfigure(line_buffering=True) + + print_banner() + print("{====================STAGE 1: Collecting information====================}") + print() + print() + + args = parse_args(argv if argv is not None else sys.argv[1:]) + + if args.dry_run: + print("Running in --dry-run mode: no objects will be created in Sprut.") + + if args.mode == "from-audit": + state = audit.load_from_audit_json(args.from_audit) + token, _catalog = keystone_auth.authenticate() + sprut_client = SprutClient(SPRUT_API_BASE, token, args.dry_run) + audit.recheck_advanced_routers_from_audit(state, sprut_client) + else: + state, sprut_client = audit.collect( + args.input_csv, SPRUT_API_BASE, args.dry_run, audit_only=(args.mode == "audit-only") + ) + + output_file = args.output_file or _default_output_file() + print(f"Executing STEP 10: Writing audit JSON to {output_file}") + audit_data = audit.to_audit_dict(state, args.input_csv) + audit.write_audit_json(audit_data, output_file) + print("STEP 10 complete: audit JSON exported") + _rule() + + print("{====================STAGE 1: COMPLETE====================}") + print() + + if args.mode == "audit-only": + print("Mode --audit-only: stopping after STAGE 1 (Sprut configuration stages skipped).") + return 0 + + print("{====================STAGE 2: Collecting Existing Sprut Objects====================}") + sprut_state = sync.collect_sprut_state(sprut_client) + print("STAGE 2 complete: Sprut objects collected") + _rule() + + print("{====================STAGE 3: Comparing and Creating Missing Objects in Sprut====================}") + correspondence = sync.compare_and_create_all(state, sprut_client, sprut_state) + print("STAGE 3 complete: Missing Sprut objects created") + _rule() + + print("{====================STAGE 4: Creating IPsec Site Connections in Sprut====================}") + sync.create_ipsec_site_connections(state, sprut_client, sprut_state, correspondence) + print("STAGE 4 complete: IPsec site connections created in Sprut") + _rule() + + return 0 diff --git a/ipsec_migrator/constants.py b/ipsec_migrator/constants.py new file mode 100644 index 0000000..962b58f --- /dev/null +++ b/ipsec_migrator/constants.py @@ -0,0 +1,51 @@ +"""Fixed text constants: banner, usage text, Sprut API base URL. Reproduced +verbatim from the bash original (ipsec_migrator_v2.sh) except for the +invocation name, which is "python -m ipsec_migrator" here instead of "$0" +since this port has no console-script entry point. +""" + +SPRUT_API_BASE = "https://infra.mail.ru:9696/v2.0" + +PROG = "python -m ipsec_migrator" + +BANNER = r""" +██╗██████╗ ███████╗███████╗ ██████╗ ██╗ ██╗██████╗ ███╗ ██╗ +██║██╔══██╗██╔════╝██╔════╝██╔════╝ ██║ ██║██╔══██╗████╗ ██║ +██║██████╔╝███████╗█████╗ ██║ ██║ ██║██████╔╝██╔██╗ ██║ +██║██╔═══╝ ╚════██║██╔══╝ ██║ ╚██╗ ██╔╝██╔═══╝ ██║╚██╗██║ +██║██║ ███████║███████╗╚██████╗ ╚████╔╝ ██║ ██║ ╚████║ +╚═╝╚═╝ ╚══════╝╚══════╝ ╚═════╝ ╚═══╝ ╚═╝ ╚═╝ ╚═══╝ + +███╗ ███╗██╗ ██████╗ ██████╗ █████╗ ████████╗██╗ ██████╗ ███╗ ██╗ ███████╗ ██████╗██████╗ ██╗██████╗ ████████╗ +████╗ ████║██║██╔════╝ ██╔══██╗██╔══██╗╚══██╔══╝██║██╔═══██╗████╗ ██║ ██╔════╝██╔════╝██╔══██╗██║██╔══██╗╚══██╔══╝ +██╔████╔██║██║██║ ███╗██████╔╝███████║ ██║ ██║██║ ██║██╔██╗ ██║ ███████╗██║ ██████╔╝██║██████╔╝ ██║ +██║╚██╔╝██║██║██║ ██║██╔══██╗██╔══██║ ██║ ██║██║ ██║██║╚██╗██║ ╚════██║██║ ██╔══██╗██║██╔═══╝ ██║ +██║ ╚═╝ ██║██║╚██████╔╝██║ ██║██║ ██║ ██║ ██║╚██████╔╝██║ ╚████║ ███████║╚██████╗██║ ██║██║██║ ██║ +╚═╝ ╚═╝╚═╝ ╚═════╝ ╚═╝ ╚═╝╚═╝ ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═══╝ ╚══════╝ ╚═════╝╚═╝ ╚═╝╚═╝╚═╝ ╚═╝ + + +Input file format: + +,,, +XXXX6d56-b2dc-4c83-b96f-e4e28af6XXXX,,MS1,standard + +where DC_Router_UUID - optional (script will create a new one (DC_Router) if UUID won't be provided) + +""" + +USAGE = f""" + +Usage: + + Prepare input CSV file then use option: + + 1. Full run (audit Neutron + configure Sprut): + {PROG} [output.json] [--dry-run] + + 2. Audit only (collect Neutron VPNaaS config, write JSON, skip Sprut entirely): + {PROG} --audit-only [output.json] + + 3. Configure Sprut from a previously generated audit JSON (no OpenStack re-query): + {PROG} --from-audit [--dry-run] + +""" diff --git a/ipsec_migrator/keystone_auth.py b/ipsec_migrator/keystone_auth.py new file mode 100644 index 0000000..d3a318a --- /dev/null +++ b/ipsec_migrator/keystone_auth.py @@ -0,0 +1,129 @@ +"""Keystone v3 password authentication via `requests`, replacing the +`openstack token issue` subprocess call. + +Mirrors the auth flow the `openstack` CLI itself performs when reading +`OS_*` environment variables: POST a project-scoped password auth request +to `{OS_AUTH_URL}/auth/tokens`, read the token back from the +`X-Subject-Token` response header, and keep the returned service catalog +around so the Neutron endpoint can be resolved from it (same as the CLI +does) instead of being hardcoded. + +No `clouds.yaml` support -- the only auth pattern actually in use here +(`test_openrc.sh`) is password auth via `OS_*` env vars. +""" +from __future__ import annotations + +import os +import sys + +import requests + +_REQUIRED = ("OS_AUTH_URL", "OS_USERNAME", "OS_PASSWORD") + + +def _env(name: str) -> str | None: + value = os.environ.get(name) + return value if value else None + + +def _user_domain() -> dict: + if domain_id := _env("OS_USER_DOMAIN_ID"): + return {"id": domain_id} + return {"name": _env("OS_USER_DOMAIN_NAME") or "Default"} + + +def _project_scope() -> dict: + if project_id := _env("OS_PROJECT_ID"): + return {"id": project_id} + + project_name = _env("OS_PROJECT_NAME") + if not project_name: + print( + "Error: neither OS_PROJECT_ID nor OS_PROJECT_NAME is set.", + file=sys.stderr, + ) + sys.exit(1) + + if project_domain_id := _env("OS_PROJECT_DOMAIN_ID"): + domain = {"id": project_domain_id} + else: + domain = {"name": _env("OS_PROJECT_DOMAIN_NAME") or "Default"} + + return {"name": project_name, "domain": domain} + + +def authenticate() -> tuple[str, list[dict]]: + """Returns (token, service_catalog).""" + missing = [name for name in _REQUIRED if not _env(name)] + if missing: + print(f"Error: required environment variable(s) not set: {', '.join(missing)}", file=sys.stderr) + sys.exit(1) + + auth_url = _env("OS_AUTH_URL").rstrip("/") + body = { + "auth": { + "identity": { + "methods": ["password"], + "password": { + "user": { + "name": _env("OS_USERNAME"), + "domain": _user_domain(), + "password": _env("OS_PASSWORD"), + } + }, + }, + "scope": {"project": _project_scope()}, + } + } + + try: + response = requests.post(f"{auth_url}/auth/tokens", json=body, timeout=30.0) + except requests.exceptions.RequestException as exc: + print(f"Error: Keystone authentication request failed: {exc}", file=sys.stderr) + sys.exit(1) + + if not 200 <= response.status_code < 300: + print(f"Error: Keystone authentication failed (HTTP {response.status_code})", file=sys.stderr) + print(f"Response: {response.text}", file=sys.stderr) + sys.exit(1) + + token = response.headers.get("X-Subject-Token") + if not token: + print("Error: Keystone response did not include X-Subject-Token.", file=sys.stderr) + sys.exit(1) + + catalog = (response.json().get("token") or {}).get("catalog") or [] + return token, catalog + + +def resolve_endpoint(catalog: list[dict], service_type: str = "network") -> str: + interface = _env("OS_INTERFACE") or "public" + region = _env("OS_REGION_NAME") + + service = next((entry for entry in catalog if entry.get("type") == service_type), None) + if service is None: + print(f"Error: service type '{service_type}' not found in Keystone catalog.", file=sys.stderr) + sys.exit(1) + + candidates = [ep for ep in service.get("endpoints", []) if ep.get("interface") == interface] + if region: + region_matches = [ep for ep in candidates if ep.get("region") == region] + if region_matches: + candidates = region_matches + + if not candidates: + print( + f"Error: no '{interface}' endpoint for service type '{service_type}' in Keystone catalog" + + (f" (region {region})" if region else ""), + file=sys.stderr, + ) + sys.exit(1) + + url = candidates[0]["url"].rstrip("/") + # Neutron's catalog entry is registered with the /v2.0 suffix already on + # some clouds and without it on others; neutron_client.py always adds + # /v2.0/... itself, so strip a pre-existing suffix here to avoid + # double-versioned URLs (.../v2.0/v2.0/routers). + if service_type == "network" and (url.endswith("/v2.0") or url.endswith("/v2")): + url = url.rsplit("/", 1)[0] + return url diff --git a/ipsec_migrator/models.py b/ipsec_migrator/models.py new file mode 100644 index 0000000..b32d688 --- /dev/null +++ b/ipsec_migrator/models.py @@ -0,0 +1,53 @@ +"""In-memory state shared between STAGE 1 collection/loading and STAGE 3/4. + +Replaces the bash original's `declare -gA` global associative-array soup +with one explicit, typed object. Populated either by audit.collect() (real +OpenStack/Sprut queries) or audit.load_from_audit_json() (parsing a +previously written audit file) -- both paths produce the same shape so +STAGE 3/4 don't need to know which one ran. +""" +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class MigrationState: + neutron_to_adv_router: dict[str, str] = field(default_factory=dict) + subnet_id_to_subnet_address: dict[str, str] = field(default_factory=dict) + + # STEP 2: full Neutron router objects (id -> raw dict), so a router + # whose advanced_router_id is blank in the CSV can be auto-created in + # Sprut using this router's own name/description, without a second + # per-item show call. + neutron_router_by_id: dict[str, dict] = field(default_factory=dict) + # STEP 1: optional availability_zone/flavor CSV columns (3rd/4th), + # keyed by neutron_router_id, only present for rows whose + # advanced_router_id was blank. + router_creation_hints: dict[str, dict] = field(default_factory=dict) + # STEP 3 (collect()) or --from-audit load: resolved name/description/ + # availability_zone/flavor for routers still awaiting DC Router + # creation. Populated either freshly (collect()) or from a previously + # written audit.json's "pending_dc_router" block (--from-audit). + pending_dc_router_by_neutron_id: dict[str, dict] = field(default_factory=dict) + + neutron_ike_policy_by_id: dict[str, dict] = field(default_factory=dict) + neutron_ipsec_policy_by_id: dict[str, dict] = field(default_factory=dict) + neutron_endpoint_group_by_id: dict[str, dict] = field(default_factory=dict) + + # STEP 8: filtered to in-scope (CSV) routers only. + router_id_to_vpn_service_id: dict[str, str] = field(default_factory=dict) + # STEP 8: reverse map, deliberately NOT filtered -- stays project-wide + # because STEP 9 needs it to resolve every connection's owning router + # before in-scope narrowing happens. Do not filter this one. + vpn_service_id_to_router_id: dict[str, str] = field(default_factory=dict) + neutron_vpn_service_by_id: dict[str, dict] = field(default_factory=dict) + + # STEP 9: router id -> list of ipsec site connection ids it owns. + neutron_router_to_ipsec_ids: dict[str, list[str]] = field(default_factory=dict) + neutron_connection_by_id: dict[str, dict] = field(default_factory=dict) + + in_scope_connection_ids: set[str] = field(default_factory=set) + in_scope_ike_policy_ids: set[str] = field(default_factory=set) + in_scope_ipsec_policy_ids: set[str] = field(default_factory=set) + in_scope_endpoint_group_ids: set[str] = field(default_factory=set) diff --git a/ipsec_migrator/neutron_client.py b/ipsec_migrator/neutron_client.py new file mode 100644 index 0000000..e55f0f5 --- /dev/null +++ b/ipsec_migrator/neutron_client.py @@ -0,0 +1,109 @@ +"""REST client for the Neutron API, replacing the `openstack` CLI subprocess +wrapper (`openstack_client.py`, removed). + +Mirrors `SprutClient`'s shape: a single `requests.Session`, token/base URL +fixed at construction, `.get()`/`.post()` raising a clear error and exiting +on a non-2xx response. + +Every list function below does exactly one GET and returns the collection +unwrapped -- no per-item `show` call. Unlike the `openstack` CLI's table +output (which truncates to a handful of display columns unless `--long` is +given), Neutron's list endpoints return the full attribute set for every +item by default, so the per-item `show` calls the CLI-based version needed +are redundant here. +""" +from __future__ import annotations + +import sys +from typing import Any + +import requests + + +class NeutronClient: + def __init__(self, base_url: str, token: str, timeout: float = 30.0): + self.base_url = base_url.rstrip("/") + self.timeout = timeout + self._session = requests.Session() + self._session.headers.update({"X-Auth-Token": token}) + + def call(self, method: str, path: str, data: dict | None = None) -> Any: + url = f"{self.base_url}{path}" + try: + response = self._session.request( + method, url, json=data if data is not None else None, timeout=self.timeout + ) + except requests.exceptions.RequestException as exc: + print(f"Error: Neutron API {method} {url} failed: {exc}", file=sys.stderr) + sys.exit(1) + + if not 200 <= response.status_code < 300: + print(f"Error: Neutron API {method} {url} returned HTTP {response.status_code}", file=sys.stderr) + print(f"Response: {response.text}", file=sys.stderr) + sys.exit(1) + + if not response.text: + return {} + return response.json() + + def get(self, path: str) -> Any: + return self.call("GET", path) + + def get_absolute(self, url: str) -> Any: + """Like get(), but for an already-absolute URL (pagination "next" + links come back as full URLs, not paths relative to base_url).""" + try: + response = self._session.get(url, timeout=self.timeout) + except requests.exceptions.RequestException as exc: + print(f"Error: Neutron API GET {url} failed: {exc}", file=sys.stderr) + sys.exit(1) + + if not 200 <= response.status_code < 300: + print(f"Error: Neutron API GET {url} returned HTTP {response.status_code}", file=sys.stderr) + print(f"Response: {response.text}", file=sys.stderr) + sys.exit(1) + + return response.json() if response.text else {} + + def list_all(self, path: str, collection_key: str) -> list[dict]: + """GETs a collection and follows Neutron's pagination "next" links + (`_links` with `rel: next`) until exhausted. Plain + deployments without pagination enabled just return one page with no + `_links` key, so this is a no-op there.""" + items: list[dict] = [] + body = self.get(path) + while True: + items.extend(body.get(collection_key, [])) + links = body.get(f"{collection_key}_links", []) + next_url = next((link["href"] for link in links if link.get("rel") == "next"), None) + if not next_url: + return items + body = self.get_absolute(next_url) + + +def router_list(client: NeutronClient) -> list[dict]: + return client.list_all("/v2.0/routers", "routers") + + +def subnet_list(client: NeutronClient) -> list[dict]: + return client.list_all("/v2.0/subnets", "subnets") + + +def ike_policy_list(client: NeutronClient) -> list[dict]: + return client.list_all("/v2.0/vpn/ikepolicies", "ikepolicies") + + +def ipsec_policy_list(client: NeutronClient) -> list[dict]: + return client.list_all("/v2.0/vpn/ipsecpolicies", "ipsecpolicies") + + +def endpoint_group_list(client: NeutronClient) -> list[dict]: + return client.list_all("/v2.0/vpn/endpoint-groups", "endpoint_groups") + + +def vpn_service_list(client: NeutronClient) -> list[dict]: + return client.list_all("/v2.0/vpn/vpnservices", "vpnservices") + + +def ipsec_site_connection_list(client: NeutronClient) -> list[dict]: + return client.list_all("/v2.0/vpn/ipsec-site-connections", "ipsec_site_connections") diff --git a/ipsec_migrator/redact.py b/ipsec_migrator/redact.py new file mode 100644 index 0000000..f7857fc --- /dev/null +++ b/ipsec_migrator/redact.py @@ -0,0 +1,24 @@ +"""PSK redaction for safe logging. + +Walks any nested dict/list structure and blanks out the value of any dict +key literally named "psk" (Neutron's and Sprut's shared field name) when +that value is not null. Used only for logging -- the real value is always +used wherever the original object is read for actual API calls/storage. +""" +from __future__ import annotations + +from typing import Any + +_PSK_KEYS = {"psk"} +_REDACTED = "***REDACTED***" + + +def redact_psk(value: Any) -> Any: + if isinstance(value, dict): + return { + key: (_REDACTED if key in _PSK_KEYS and val is not None else redact_psk(val)) + for key, val in value.items() + } + if isinstance(value, list): + return [redact_psk(item) for item in value] + return value diff --git a/ipsec_migrator/sprut_client.py b/ipsec_migrator/sprut_client.py new file mode 100644 index 0000000..8574e56 --- /dev/null +++ b/ipsec_migrator/sprut_client.py @@ -0,0 +1,75 @@ +"""REST client for the Sprut SDN API. + +Mirrors sprut_api_call() from the bash original. Token, dry-run flag and +base URL are fixed at construction (replacing the bash globals $token / +$DRY_RUN / $sprut_api_base read by every call site) and a single +requests.Session is reused across all calls for connection reuse -- a +behaviorally-neutral efficiency improvement over one curl process per call. + +The only deliberate behavioral deviation from the bash original (confirmed +with the user): a 30s timeout is applied to every HTTP call. curl in the +original has no --max-time at all, so a hung Sprut endpoint would hang the +whole script indefinitely; a timeout here is treated exactly like a non-2xx +response (error message to stderr, exit 1), so no new failure mode is +introduced, just an upper bound on how long a stuck call can hang. + +The Keystone token is never refreshed after construction -- this directly +carries over the "token not refreshed across STAGE 2-4" known limitation +from the bash original. Do not add retry-on-401 here. +""" +from __future__ import annotations + +import json +import sys + +import requests + +from .redact import redact_psk + + +class SprutClient: + def __init__(self, base_url: str, token: str, dry_run: bool, timeout: float = 30.0): + self.base_url = base_url.rstrip("/") + self.token = token + self.dry_run = dry_run + self.timeout = timeout + self._session = requests.Session() + self._session.headers.update( + { + "Content-Type": "application/json", + "X-Auth-Token": token, + "X-SDN": "SPRUT", + } + ) + + def call(self, method: str, path: str, data: dict | None = None) -> dict: + url = f"{self.base_url}{path}" + + if self.dry_run and method != "GET": + print(f"[DRY-RUN] Would {method} {url}", file=sys.stderr) + if data: + print(f"[DRY-RUN] Body: {json.dumps(redact_psk(data), indent=2)}", file=sys.stderr) + return {} + + try: + response = self._session.request( + method, url, json=data if data is not None else None, timeout=self.timeout + ) + except requests.exceptions.RequestException as exc: + print(f"Error: Sprut API {method} {url} failed: {exc}", file=sys.stderr) + sys.exit(1) + + if not 200 <= response.status_code < 300: + print(f"Error: Sprut API {method} {url} returned HTTP {response.status_code}", file=sys.stderr) + print(f"Response: {response.text}", file=sys.stderr) + sys.exit(1) + + if not response.text: + return {} + return response.json() + + def get(self, path: str) -> dict: + return self.call("GET", path) + + def post(self, path: str, data: dict) -> dict: + return self.call("POST", path, data) diff --git a/ipsec_migrator/sync.py b/ipsec_migrator/sync.py new file mode 100644 index 0000000..6d6f2bc --- /dev/null +++ b/ipsec_migrator/sync.py @@ -0,0 +1,332 @@ +"""STAGE 2-4: collecting existing Sprut objects, comparing/creating missing +IKE/IPsec policies, endpoint groups and VPN services, and finally creating +IPsec site connections. + +STAGE 3's four compare_and_create_* functions each return a +neutron_id -> sprut_id correspondence dict (replacing the bash originals' +`declare -A neutron_to_sprut_*` globals). STAGE 4 looks these up by the +raw connection's referenced IDs to build its request body. +""" +from __future__ import annotations + +import json + +from . import audit +from .models import MigrationState +from .redact import redact_psk +from .sprut_client import SprutClient +from .tables import print_kv_table + + +def _dump(obj) -> str: + return json.dumps(obj, indent=2) + + +def _jq_r_str(value) -> str: + """Mirrors `jq -r`: a JSON null stringifies to the 4-char string "null" + (not Python None/absence), a JSON boolean stringifies to lowercase + "true"/"false" (not Python's "True"/"False" -- confirmed against a real + tenant, where Neutron's ipsec site connection ".State" is a genuine JSON + boolean), everything else via str(). STAGE 4's request body in the bash + original is built entirely with `jq -n --arg`, which means every field + -- even ones that look numeric/boolean -- is sent to Sprut as a JSON + string. Do not "fix" this to native types.""" + if value is None: + return "null" + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + +# -------------------------------------------------------------------------- +# STAGE 2: collect existing Sprut objects (always real GETs, even in +# dry-run -- dry-run only suppresses non-GET calls). +# -------------------------------------------------------------------------- +def collect_sprut_state(sprut_client: SprutClient) -> dict: + print("Collecting IKE policies from Sprut...") + ike_policies = sprut_client.get("/vpn/ikepolicies") + print("IKE policies collected:") + print(_dump(ike_policies)) + print() + + print("Collecting IPsec policies from Sprut...") + ipsec_policies = sprut_client.get("/vpn/ipsecpolicies") + print("IPsec policies collected:") + print(_dump(ipsec_policies)) + print() + + print("Collecting Endpoint Groups from Sprut...") + endpoint_groups = sprut_client.get("/vpn/endpoint-groups") + print("Endpoint Groups collected:") + print(_dump(endpoint_groups)) + print() + + print("Collecting VPN services from Sprut...") + vpn_services = sprut_client.get("/vpn/vpnservices") + print("VPN services collected:") + print(_dump(vpn_services)) + print() + + print("Collecting IPsec site connections from Sprut...") + ipsec_site_connections = sprut_client.get("/vpn/ipsec-site-connections") + print("IPsec site connections collected:") + print(_dump(redact_psk(ipsec_site_connections))) + print() + + return { + "ike_policies": ike_policies, + "ipsec_policies": ipsec_policies, + "endpoint_groups": endpoint_groups, + "vpn_services": vpn_services, + "ipsec_site_connections": ipsec_site_connections, + } + + +# -------------------------------------------------------------------------- +# STAGE 3: compare & create missing objects. Each function is idempotent by +# whatever key Sprut exposes (name, router_id, or exact endpoints array). +# -------------------------------------------------------------------------- +def compare_and_create_ike_policies( + state: MigrationState, sprut_client: SprutClient, sprut_ike_policies: dict +) -> dict[str, str]: + mapping: dict[str, str] = {} + for neutron_id in state.in_scope_ike_policy_ids: + neutron_policy = state.neutron_ike_policy_by_id[neutron_id] + name = neutron_policy.get("name") + + sprut_id = next( + (p.get("id") for p in sprut_ike_policies.get("ikepolicies", []) if p.get("name") == name), + None, + ) + + if not sprut_id: + print(f"Creating IKE policy '{name}' in Sprut") + request_body = { + "ikepolicy": { + "name": neutron_policy.get("name"), + "phase1_negotiation_mode": neutron_policy.get("phase1_negotiation_mode"), + "auth_algorithm": neutron_policy.get("auth_algorithm"), + "encryption_algorithm": neutron_policy.get("encryption_algorithm"), + "pfs": neutron_policy.get("pfs"), + "lifetime": neutron_policy.get("lifetime"), + "ike_version": neutron_policy.get("ike_version"), + } + } + response = sprut_client.post("/vpn/ikepolicies", request_body) + sprut_id = (response.get("ikepolicy") or {}).get("id") + if sprut_client.dry_run and not sprut_id: + sprut_id = f"DRY-RUN-{neutron_id}" + else: + print(f"IKE policy '{name}' already exists in Sprut") + + mapping[neutron_id] = sprut_id + return mapping + + +def compare_and_create_ipsec_policies( + state: MigrationState, sprut_client: SprutClient, sprut_ipsec_policies: dict +) -> dict[str, str]: + mapping: dict[str, str] = {} + for neutron_id in state.in_scope_ipsec_policy_ids: + neutron_policy = state.neutron_ipsec_policy_by_id[neutron_id] + name = neutron_policy.get("name") + + sprut_id = next( + (p.get("id") for p in sprut_ipsec_policies.get("ipsecpolicies", []) if p.get("name") == name), + None, + ) + + if not sprut_id: + print(f"Creating IPsec policy '{name}' in Sprut") + request_body = { + "ipsecpolicy": { + "name": neutron_policy.get("name"), + "transform_protocol": neutron_policy.get("transform_protocol"), + "auth_algorithm": neutron_policy.get("auth_algorithm"), + "encryption_algorithm": neutron_policy.get("encryption_algorithm"), + "encapsulation_mode": neutron_policy.get("encapsulation_mode"), + "pfs": neutron_policy.get("pfs"), + "lifetime": neutron_policy.get("lifetime"), + } + } + response = sprut_client.post("/vpn/ipsecpolicies", request_body) + sprut_id = (response.get("ipsecpolicy") or {}).get("id") + if sprut_client.dry_run and not sprut_id: + sprut_id = f"DRY-RUN-{neutron_id}" + else: + print(f"IPsec policy '{name}' already exists in Sprut") + + mapping[neutron_id] = sprut_id + return mapping + + +def compare_and_create_endpoint_groups( + state: MigrationState, sprut_client: SprutClient, sprut_endpoint_groups: dict +) -> dict[str, str]: + mapping: dict[str, str] = {} + for neutron_id in state.in_scope_endpoint_group_ids: + neutron_group = state.neutron_endpoint_group_by_id[neutron_id] + name = neutron_group.get("name") + + converted = audit.resolve_endpoint_addresses(neutron_group, state.subnet_id_to_subnet_address) + print(f"Total converted UUIDs in endpoints: {', '.join(converted)}") + + matching = next( + (g for g in sprut_endpoint_groups.get("endpoint_groups", []) if g.get("endpoints") == converted), + None, + ) + sprut_id = matching.get("id") if matching else None + + print(f"Comparing Neutron endpoint group '{name}' with endpoints: {', '.join(converted)}") + if matching: + print(f" -> Found corresponding Sprut endpoint group with endpoints: {matching.get('endpoints')}") + else: + print(" -> No corresponding Sprut endpoint group found for these endpoints") + + if not sprut_id: + print(f"Creating Endpoint Group '{name}' in Sprut") + request_body = {"endpoint_group": {"name": name, "endpoints": converted, "type": "cidr"}} + print(f"Request body: {json.dumps(request_body)}") + response = sprut_client.post("/vpn/endpoint-groups", request_body) + sprut_id = (response.get("endpoint_group") or {}).get("id") + if sprut_client.dry_run and not sprut_id: + sprut_id = f"DRY-RUN-{neutron_id}" + print(f"Created Sprut Endpoint group: {sprut_id}") + else: + print(f"Endpoint Group with matching endpoints already exists in Sprut with id {sprut_id}") + + mapping[neutron_id] = sprut_id + print() + return mapping + + +def compare_and_create_vpn_services( + state: MigrationState, sprut_client: SprutClient, sprut_vpn_services: dict +) -> dict[str, str]: + mapping: dict[str, str] = {} + for router_id, vpn_service_id in state.router_id_to_vpn_service_id.items(): + advanced_router_id = state.neutron_to_adv_router.get(router_id) + + sprut_id = next( + ( + s.get("id") + for s in sprut_vpn_services.get("vpnservices", []) + if s.get("router_id") == advanced_router_id + ), + None, + ) + + if not sprut_id: + print(f"Creating VPN Service for router '{router_id}' in Sprut") + request_body = {"vpnservice": {"router_id": advanced_router_id, "admin_state_up": True}} + response = sprut_client.post("/vpn/vpnservices", request_body) + sprut_id = (response.get("vpnservice") or {}).get("id") + if sprut_client.dry_run and not sprut_id: + sprut_id = f"DRY-RUN-{router_id}" + else: + print(f"VPN Service for router '{router_id}' already exists in Sprut") + + # Keyed by the neutron VPN-service id (not the router id) -- STAGE 4 + # looks this map up by the vpn-service id pulled off each connection. + mapping[vpn_service_id] = sprut_id + return mapping + + +def compare_and_create_all( + state: MigrationState, sprut_client: SprutClient, sprut_state: dict +) -> dict[str, dict[str, str]]: + ike_map = compare_and_create_ike_policies(state, sprut_client, sprut_state["ike_policies"]) + print_kv_table(ike_map, "Neutron IKE Policy ID", "Sprut IKE Policy ID", title="Neutron to Sprut IKE Policies") + + ipsec_map = compare_and_create_ipsec_policies(state, sprut_client, sprut_state["ipsec_policies"]) + print_kv_table( + ipsec_map, "Neutron IPsec Policy ID", "Sprut IPsec Policy ID", title="Neutron to Sprut IPsec Policies" + ) + + endpoint_group_map = compare_and_create_endpoint_groups(state, sprut_client, sprut_state["endpoint_groups"]) + print_kv_table( + endpoint_group_map, + "Neutron Endpoint Group ID", + "Sprut Endpoint Group ID", + title="Neutron to Sprut Endpoint Groups", + ) + + vpn_service_map = compare_and_create_vpn_services(state, sprut_client, sprut_state["vpn_services"]) + print_kv_table( + vpn_service_map, "Neutron VPN Service ID", "Sprut VPN Service ID", title="Neutron to Sprut VPN Services" + ) + + return { + "ike_policy": ike_map, + "ipsec_policy": ipsec_map, + "endpoint_group": endpoint_group_map, + "vpn_service": vpn_service_map, + } + + +# -------------------------------------------------------------------------- +# STAGE 4: create IPsec site connections, idempotent by name. +# -------------------------------------------------------------------------- +def create_ipsec_site_connections( + state: MigrationState, + sprut_client: SprutClient, + sprut_state: dict, + correspondence: dict[str, dict[str, str]], +) -> None: + sprut_connections = sprut_state["ipsec_site_connections"] + + for conn_id in state.in_scope_connection_ids: + details = state.neutron_connection_by_id[conn_id] + name = details.get("name") + + print(f"Processing IPsec site connection ID: {conn_id}") + print("IPsec site connection details:") + print(_dump(redact_psk(details))) + + existing_id = next( + ( + c.get("id") + for c in (sprut_connections.get("ipsec_site_connections") or []) + if c.get("name") == name + ), + None, + ) + if existing_id: + print(f"IPsec site connection '{name}' already exists in Sprut (id {existing_id}) — skipping") + print() + continue + + sprut_ipsecpolicy_id = correspondence["ipsec_policy"].get(details.get("ipsecpolicy_id")) + sprut_ikepolicy_id = correspondence["ike_policy"].get(details.get("ikepolicy_id")) + sprut_local_ep_group_id = correspondence["endpoint_group"].get(details.get("local_ep_group_id")) + sprut_peer_ep_group_id = correspondence["endpoint_group"].get(details.get("peer_ep_group_id")) + sprut_vpn_service_id = correspondence["vpn_service"].get(details.get("vpnservice_id")) + + # Every field below is stringified (jq --arg semantics in the bash + # original) -- do not switch mtu/admin_state_up to native types. + request_body = { + "ipsec_site_connection": { + "psk": _jq_r_str(details.get("psk")), + "initiator": _jq_r_str(details.get("initiator")), + "ipsecpolicy_id": _jq_r_str(sprut_ipsecpolicy_id), + "admin_state_up": _jq_r_str(details.get("admin_state_up")), + "mtu": _jq_r_str(details.get("mtu")), + "peer_ep_group_id": _jq_r_str(sprut_peer_ep_group_id), + "ikepolicy_id": _jq_r_str(sprut_ikepolicy_id), + "vpnservice_id": _jq_r_str(sprut_vpn_service_id), + "local_ep_group_id": _jq_r_str(sprut_local_ep_group_id), + "peer_address": _jq_r_str(details.get("peer_address")), + "peer_id": _jq_r_str(details.get("peer_id")), + "name": _jq_r_str(name), + } + } + + print(f"Creating IPsec site connection '{name}' in Sprut") + print("Executing request with body:") + print(_dump(redact_psk(request_body))) + + response = sprut_client.post("/vpn/ipsec-site-connections", request_body) + + print("API response:") + print(_dump(redact_psk(response))) + print() diff --git a/ipsec_migrator/tables.py b/ipsec_migrator/tables.py new file mode 100644 index 0000000..2aa4db5 --- /dev/null +++ b/ipsec_migrator/tables.py @@ -0,0 +1,43 @@ +"""ASCII key/value table printing. + +Replaces the bash original's ad-hoc `echo`/`printf` table blocks (STEP 1, 7, +8) and the nameref-based print_map_as_table() helper (STAGE 3) with a single +generic function. Python has no need for bash's nameref indirection -- the +dict is just passed directly. + +The bash version used a Unicode combining-overline row as an ersatz bottom +border; that's purely decorative with no functional role, so it's replaced +here with a plain ASCII divider of matching visual weight -- a deliberate, +low-risk cosmetic judgment call, not a fidelity gap. +""" +from __future__ import annotations + +from collections.abc import Iterable + + +def print_kv_table( + rows: dict[str, str] | Iterable[tuple[str, str]], + col1: str, + col2: str, + title: str | None = None, + width1: int = 36, + width2: int = 36, +) -> None: + items = list(rows.items()) if isinstance(rows, dict) else list(rows) + border = "=" * (width1 + width2 + 7) + + if title: + print(border) + print(title) + print(border) + else: + print("_" * (width1 + width2 + 7)) + + print(f"| {col1:<{width1}} | {col2:<{width2}} |") + print("-" * (width1 + width2 + 7)) + for key, value in items: + print(f"| {key:<{width1}} | {value:<{width2}} |") + print("-" * (width1 + width2 + 7)) + + if title: + print()