diff --git a/.gitignore b/.gitignore index 6fdc9bb..a6755c6 100644 --- a/.gitignore +++ b/.gitignore @@ -15,9 +15,15 @@ __pycache__/ crash.log crash.*.log -# Real credentials layered on top of terraform.tfvars (see terraform/prod.auto.tfvars.example) - -# terraform.tfvars itself stays a committed, anonymized template; auto-loaded -# overrides with real secrets must never be committed. +# Real credentials, one file per environment, passed explicitly via +# -var-file (see terraform/*.secrets.tfvars.example) - terraform.tfvars and +# terraform/.tfvars stay committed, anonymized/non-secret templates. +*.secrets.tfvars +*.secrets.tfvars.json + +# Legacy *.auto.tfvars pattern (pre-dates the explicit -var-file convention +# above, kept ignored in case a stray file from before the mvm-s3 environment +# is still present locally). *.auto.tfvars *.auto.tfvars.json terraform.tfvars.local diff --git a/README.md b/README.md index ab8e98c..9a86914 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ New Terraform variables: `router_count`, `private_network_cidrs`, `router_availa `terraform/versions.tf` now has a `provider "vkcs" { ... }` block wiring `auth_url`, `username`, `password`, `project_id`, `region`, `user_domain_name` (previously declared in `variables.tf` but never actually connected to anything). `auth_url`/`user_domain_name`/`region` have sane defaults for a regular account; `username`/`password`/`project_id` still have none, same as before. -`terraform.tfvars` stays a committed, anonymized template - real credentials (e.g. from an `openrc.sh` for a service account) go in a separate, gitignored `*.auto.tfvars` file instead, which Terraform loads automatically on top of `terraform.tfvars`. See `terraform/prod.auto.tfvars.example` for the field mapping from `openrc.sh`'s `OS_*` variables. Never put real credentials in `terraform.tfvars` itself. +`terraform.tfvars` stays a committed, anonymized template - real credentials (e.g. from an `openrc.sh` for a service account) go in a separate, gitignored `*.secrets.tfvars` file per environment instead (see `terraform/prod.secrets.tfvars.example` / `terraform/mvm-s3.secrets.tfvars.example` for the field mapping from `openrc.sh`'s `OS_*` variables), passed explicitly via `-var-file` on every plan/apply - see "External fixed-IP networks and multi-environment workspaces" below for why this is no longer auto-loaded. Never put real credentials in a committed `*.tfvars` file. **Default security group** @@ -52,17 +52,29 @@ Every VK Cloud project auto-creates a `default` security group with a UUID uniqu Terraform's variable precedence means a `terraform.tfvars` value always beats a `TF_VAR_` environment variable, never the other way round - `TF_VAR_*` only takes effect for a variable `terraform.tfvars` leaves unset. This repo's `terraform.tfvars` currently pins real values for its actual deployment (`router_count = 3`, a specific `private_network_cidrs`), so `TF_VAR_router_count`/`TF_VAR_private_network_cidrs` have **no effect** while those stay set - to change scale, edit `terraform.tfvars` directly, or override on the command line with `-var`/`-var-file` (which does beat a tfvars file): ```bash -terraform apply -var="router_count=4" -var='private_network_cidrs=["10.90.0.0/28","10.90.0.16/28","10.90.0.32/28"]' +terraform apply -var-file=terraform.tfvars -var-file=prod.secrets.tfvars \ + -var="router_count=4" -var='private_network_cidrs=["10.90.0.0/28","10.90.0.16/28","10.90.0.32/28"]' ``` If you instead comment `router_count`/`private_network_cidrs` back out of `terraform.tfvars` (e.g. for a fresh, non-PROD deployment), `TF_VAR_router_count`/`TF_VAR_private_network_cidrs` (JSON-encoded list) start working again as described above. +**External fixed-IP networks and multi-environment workspaces** + +Not every private interface has to be a network Terraform creates itself: `router_networks` (default `{}`) is a map of pre-existing networks - typically owned by a *different* VK Cloud project, referenced by UUID only - that each router gets a fixed-IP interface into. Map key = role/interface name (e.g. `"primary"`/`"backup"`); `ip_addresses[i]` is the address for `router(i+1)`, pre-agreed by that other project's admin (Terraform never invents or discovers it). `locals.router_interfaces` in `main.tf` merges both sources (`private_network_cidrs` and `router_networks`) into one role → definition map, so the rest of the pipeline - port creation, the instance's `dynamic "network"` blocks, `network-init.sh.tpl`'s CIDR-based interface matching - stays a single generalized mechanism regardless of which source a role came from. A role's `ip_address` is left `null` (Neutron IPAM auto-assigns, same collision-avoidance rationale as before) when it comes from `private_network_cidrs`, and set explicitly when it comes from `router_networks`. + +The first real deployment on this mechanism is `mvm-s3` (see `docs/changes/2026-09-09-mvm-s3-external-networks-*.md` and `terraform/mvm-s3.tfvars`): a separate VK Cloud project where both private networks already exist, so `private_network_cidrs = []` there (no project-managed networks at all) and both interfaces come from `router_networks`. + +Since this repo's single `terraform/` directory now serves more than one environment (PROD and `mvm-s3`), each with different `router_count`/networks/credentials, two conventions changed to keep them from interfering with each other: + +- **Separate Terraform workspaces** per environment (`terraform workspace new mvm-s3`), so each has its own state and applying one never touches the other's resources. +- **Explicit `-var-file` for everything environment-specific**, including credentials - `*.auto.tfvars` auto-loading was fine for exactly one environment, but with two `*.auto.tfvars` files present at once Terraform would load both and silently merge them. Real credentials now live in a `*.secrets.tfvars` per environment (gitignored, `.gitignore` pattern `*.secrets.tfvars`), never auto-loaded, always passed explicitly - see `docs/QUICKSTART.md` §4/§4а for the exact commands. + **Local delivery integrity tests** `terraform/tests/` contains a local pytest suite that checks the delivery is internally consistent - required files present, `terraform fmt` clean, HCL parses, `router_count`/`private_network_cidrs` actually drive the resource/NIC count instead of being hardcoded, the example CIDRs in `terraform.tfvars` don't overlap and have room for `router_count` routers, and the post-install script template renders to valid bash. It also runs: -- a real `terraform init` + `terraform validate` against the actual `vkcs` provider schema (at several `router_count`/`private_network_cidrs` values), against a project-local filesystem-mirror copy of the provider - no cloud API is ever contacted and no credentials are needed; -- a real `terraform plan` against an isolated, provider-free copy of just `variables.tf`, to prove the `validation { ... }` blocks on `router_count` and `private_network_cidrs` (non-empty, valid CIDR syntax, uniqueness) are actually enforced - `terraform validate` alone does **not** enforce custom variable validations for externally-supplied values, only `plan`/`apply` do. +- a real `terraform init` + `terraform validate` against the actual `vkcs` provider schema (at several `router_count`/`private_network_cidrs`/`router_networks` shapes, including an `mvm-s3`-shaped one), against a project-local filesystem-mirror copy of the provider - no cloud API is ever contacted and no credentials are needed; +- a real `terraform plan` against an isolated, provider-free copy of just `variables.tf`, to prove the `validation { ... }` blocks on `router_count`, `private_network_cidrs` (valid CIDR syntax, uniqueness) and `router_networks` (valid CIDR/IPv4 syntax, each address falling inside its own role's `cidr`) are actually enforced - `terraform validate` alone does **not** enforce custom variable validations for externally-supplied values, only `plan`/`apply` do. ```bash terraform/tests/setup-local-terraform.sh # one-time: provisions venv/ with terraform + the vkcs provider diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 9bb69cc..16c2f1c 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -10,7 +10,9 @@ ## 2. Клонировать репозиторий и задать credentials -`terraform/terraform.tfvars` — это закоммиченный обезличенный шаблон, реальные credentials в него вписывать **не нужно**. Вместо этого скопируйте `terraform/prod.auto.tfvars.example` в `terraform/prod.auto.tfvars` (этот файл в `.gitignore`, никогда не попадёт в git) и впишите туда реальные значения: +Начиная с появления второго окружения (`mvm-s3`, см. `docs/changes/2026-09-09-mvm-s3-external-networks-*.md`), в этом каталоге `terraform/` живут конфиги нескольких VK Cloud проектов одновременно — поэтому credentials **больше не подхватываются автоматически** (`*.auto.tfvars`), а передаются явным `-var-file` для каждого окружения. Так один набор реальных credentials никогда не «протечёт» в apply другого окружения. + +`terraform/terraform.tfvars` — закоммиченный обезличенный шаблон (общие некоторые значения), реальные credentials в него вписывать **не нужно**. Для PROD скопируйте `terraform/prod.secrets.tfvars.example` в `terraform/prod.secrets.tfvars` (этот файл в `.gitignore`, никогда не попадёт в git) и впишите туда реальные значения: ``` auth_url = "https://infra.mail.ru:35357/v3/" @@ -21,7 +23,9 @@ region = "RegionOne" user_domain_name = "users" # или "service-users" для сервисного аккаунта svc-* ``` -Terraform подхватывает `*.auto.tfvars` автоматически, в дополнение к `terraform.tfvars` — ничего больше настраивать не нужно. Если у вас есть `openrc.sh` для сервисного аккаунта — соответствие полей: `OS_AUTH_URL`→`auth_url`, `OS_USERNAME`→`username`, `OS_PASSWORD`→`password`, `OS_PROJECT_ID`→`project_id`, `OS_REGION_NAME`→`region`, `OS_USER_DOMAIN_NAME`→`user_domain_name`. +Если у вас есть `openrc.sh` для сервисного аккаунта — соответствие полей: `OS_AUTH_URL`→`auth_url`, `OS_USERNAME`→`username`, `OS_PASSWORD`→`password`, `OS_PROJECT_ID`→`project_id`, `OS_REGION_NAME`→`region`, `OS_USER_DOMAIN_NAME`→`user_domain_name`. + +> Если у вас остался старый `prod.auto.tfvars` с версии до этого изменения — переименуйте его в `prod.secrets.tfvars` (значения не меняются), он больше не подхватывается автоматически. В самом `terraform.tfvars` дополнительно отредактируйте: @@ -34,7 +38,9 @@ private_network_cidrs = [ ] ``` -`private_network_cidrs` — обязательная переменная без значения по умолчанию: один CIDR-префикс на каждую приватную сеть, к которой будут подключены интерфейсы роутеров (один префикс = одна общая сеть = один приватный интерфейс на роутер). Автоматической нарезки нет — префиксы не должны пересекаться. Берите `/28`, а не `/29`: VKCS сам создаёт служебные порты на каждой сети (замечен `network:dns`), которые тоже расходуют адреса из пула — `/29` (5 адресов) на практике оказался слишком тесным. +`private_network_cidrs` (по умолчанию `[]`) — один CIDR-префикс на каждую **project-managed** приватную сеть (Terraform сам создаёт сеть/подсеть, адрес назначает Neutron IPAM): один префикс = одна общая сеть = один приватный интерфейс на роутер. Автоматической нарезки нет — префиксы не должны пересекаться. Берите `/28`, а не `/29`: VKCS сам создаёт служебные порты на каждой сети (замечен `network:dns`), которые тоже расходуют адреса из пула — `/29` (5 адресов) на практике оказался слишком тесным. + +Если вместо (или в дополнение к) `private_network_cidrs` нужны приватные интерфейсы в уже существующих сетях (созданных заранее, в том числе в другом VK Cloud проекте, с фиксированным IP на каждый роутер) — используйте `router_networks` (по умолчанию `{}`), пример реального использования — окружение `mvm-s3`, см. `terraform/mvm-s3.tfvars` и шаг 4а ниже. UUID системной Security Group `default` (уникален для каждого проекта VK Cloud) вычисляется автоматически через `data.vkcs_networking_secgroup`, вручную задавать не нужно. Переменная `default_security_group_id` — override только на крайний случай (нестандартное имя/SDN группы в проекте), не для обычного использования. @@ -43,20 +49,38 @@ UUID системной Security Group `default` (уникален для каж Число роутеров и приватных сетей меняется правкой `router_count`/`private_network_cidrs` прямо в `terraform.tfvars` (сейчас там уже реальные значения этого PROD-деплоя — 3 роутера). Значение из `terraform.tfvars` **всегда перекрывает** `TF_VAR_*` (у tfvars-файла более высокий приоритет), так что `TF_VAR_router_count` сработает, только если убрать `router_count` из `terraform.tfvars`. Разовый override без правки файла — через `-var` в командной строке (он перекрывает даже tfvars): ```bash -terraform apply -var="router_count=4" -var='private_network_cidrs=["10.90.0.0/28","10.90.0.16/28","10.90.0.32/28"]' +terraform apply -var-file=terraform.tfvars -var-file=prod.secrets.tfvars \ + -var="router_count=4" -var='private_network_cidrs=["10.90.0.0/28","10.90.0.16/28","10.90.0.32/28"]' ``` -## 4. Развернуть +## 4. Развернуть (PROD) ```bash cd terraform terraform init -terraform plan -terraform apply +terraform workspace select default # state PROD-окружения +terraform plan -var-file=terraform.tfvars -var-file=prod.secrets.tfvars +terraform apply -var-file=terraform.tfvars -var-file=prod.secrets.tfvars ``` Каждая ВМ сама донастроит сеть при первой загрузке (`network-init.sh.tpl`) и перезагрузится. +## 4а. Развернуть окружение mvm-s3 (внешние сети с фиксированными IP) + +`mvm-s3` — отдельный VK Cloud проект с собственным Terraform state (workspace), где обе приватные сети роутеров уже созданы администратором заранее (см. `docs/changes/2026-09-09-mvm-s3-external-networks-*.md`) — Terraform их не создаёт, только подключает роутеры к ним по UUID с заранее известным IP на каждый роутер (`terraform/mvm-s3.tfvars`). + +1. Скопируйте `terraform/mvm-s3.secrets.tfvars.example` в `terraform/mvm-s3.secrets.tfvars` и впишите реальные credentials проекта `mvm-s3`, а также реальное `ssh_key_name` в `terraform/mvm-s3.tfvars` (сейчас там плейсхолдер). +2. Разверните в отдельном workspace, чтобы не задеть state PROD: + ```bash + cd terraform + terraform workspace new mvm-s3 # один раз; в дальнейшем - terraform workspace select mvm-s3 + terraform plan -var-file=terraform.tfvars -var-file=mvm-s3.tfvars -var-file=mvm-s3.secrets.tfvars + terraform apply -var-file=terraform.tfvars -var-file=mvm-s3.tfvars -var-file=mvm-s3.secrets.tfvars + ``` +3. Ожидаемый результат: 4 роутера, без project-managed приватных сетей (`private_network_cidrs = []`), с 2 фиксированными IP-адресами каждый (`primary`/`backup`) из `router_networks`. + +Чтобы вернуться к работе с PROD: `terraform workspace select default` + прежние `-var-file` (см. шаг 4) — стейты не пересекаются. + ## 5. Настроить Ansible > ⚠️ Важно: `ansible/inventory.ini` и роли (`base`/`frr_router`/`keepalived`) пока жёстко рассчитаны на **2** роутера с интерфейсами `eth0`/`eth1` (VRRP-схема) — под новую N-роутерную/N-NIC архитектуру ещё не адаптированы. Для дефолтных значений (`router_count=2`, 2 записи в `private_network_cidrs`) впишите реальные `wan_ip`/`lan_ip`/GRE/BGP-параметры роутеров в `inventory.ini` вручную. При масштабировании выше 2 роутеров или интерфейсов Ansible-слой нужно дорабатывать отдельно. diff --git a/docs/changes/2026-09-09-mvm-s3-external-networks-plan.md b/docs/changes/2026-09-09-mvm-s3-external-networks-plan.md new file mode 100644 index 0000000..f4d0d23 --- /dev/null +++ b/docs/changes/2026-09-09-mvm-s3-external-networks-plan.md @@ -0,0 +1,189 @@ +# Внешние сети с фиксированными IP для окружения `mvm-s3` + +## Контекст + +Текущий Terraform (`terraform/`) разворачивает 4 маршрутизатора (`router1..router4`) в PROD-проекте VK Cloud: у каждого 1 WAN-интерфейс (`data.vkcs_networking_network.extnet`) и 2 приватных интерфейса в сетях, которые Terraform сам создаёт (`vkcs_networking_network`/`vkcs_networking_subnet` по `var.private_network_cidrs`), с IP, которые назначает Neutron IPAM автоматически (адрес заранее не известен и не фиксируется — см. комментарий про коллизию с служебным портом `network:dns` в `main.tf`). + +Появилось новое окружение — `mvm-s3`, другой VK Cloud проект, недоступный отсюда напрямую. Администратор этого проекта уже создал там (проверено через OpenStack CLI) две приватные сети/подсети и заранее расписал, какой IP получит каждый из 4 маршрутизаторов в каждой из них (диаграмма пользователя): + +| Роль | UUID сети | UUID подсети | CIDR | router1 | router2 | router3 | router4 | +|---|---|---|---|---|---|---|---| +| **primary** (основной канал связи) | `25532efe-2931-4666-a090-0da3a6f18224` | `1fa5357c-1f11-4d20-89f4-b27a4fb56e6e` | `172.16.252.8/29` | `.11` | `.12` | `.13` | `.14` | +| **backup** (резервный канал связи) | `2b4cc25f-55a1-4d36-a3a2-5b16414af31a` | `5eacbc86-e15d-4c49-8704-547a719acc23` | `172.16.252.0/29` | `.3` | `.4` | `.5` | `.6` | + +Третий (External/WAN) интерфейс у роутеров тоже есть, но он вне рамок этой диаграммы и этой задачи — переиспользуется существующий механизм (`data.vkcs_networking_network.extnet`), без изменений. + +Согласовано с пользователем: +- Для `mvm-s3` эти 2 внешние сети **заменяют** `private_network_cidrs`-механизм (никаких project-managed приватных сетей в этом окружении не создаётся) — но сам механизм из кода не удаляется, поскольку PROD (`10.90.x`, уже развёрнут и работает) продолжает от него зависеть в своём отдельном состоянии. +- `mvm-s3` — **отдельный Terraform workspace** (`terraform workspace new mvm-s3`), отдельный state. PROD (`default`/текущий workspace) не трогается. +- Сейчас `terraform.tfvars` и `prod.auto.tfvars` (creds) авто-загружаются Terraform'ом всегда, без явного `-var-file`. С двумя окружениями это небезопасно — переходим на **явные `-var-file`** для всех переменных, отличающихся между окружениями, для обоих окружений симметрично (в т.ч. переименовываем `prod.auto.tfvars` → `prod.secrets.tfvars`, чтобы он больше не подхватывался неявно и не «протекал» в apply для `mvm-s3`). +- У меня нет доступа к `mvm-s3` проекту — `terraform plan`/`apply` там должен будет выполнить пользователь самостоятельно. Проверка с моей стороны — только офлайн (`terraform validate`, pytest-сьют, `terraform plan` при наличии тестовых кредов/моков). + +**Открытый вопрос к пользователю (нужно закрыть до/во время реализации):** реальные `auth_url`/`username`/`password`/`project_id`/`region`/`user_domain_name` и `ssh_key_name`/`ssh_public_key` для `mvm-s3` — я их не знаю и не должен придумывать. Файл с ними создаётся по аналогии с `prod.auto.tfvars.example` — как шаблон с плейсхолдерами, который пользователь заполнит сам. + +## Реализация + +### 1. `terraform/variables.tf` + +- `private_network_cidrs`: сделать **опциональной** (`default = []`), убрать валидацию «минимум 1 элемент» (для `mvm-s3` список должен быть пустым — 0 project-managed приватных сетей). Остальные валидации (валидный CIDR, уникальность) остаются, тривиально проходят на пустом списке. +- Добавить новую переменную `router_networks` — карта заранее существующих внешних сетей с фиксированным IP на роутер: + +```hcl +variable "router_networks" { + description = "Заранее существующие сети (в другом VK Cloud проекте, только по UUID, не управляются этим Terraform), в которые каждый роутер получает интерфейс с фиксированным IP. Ключ карты — имя роли/интерфейса (например \"primary\"/\"backup\"); ip_addresses[i] соответствует router(i+1)." + type = map(object({ + network_id = string + subnet_id = string + cidr = string + ip_addresses = list(string) + })) + default = {} + + validation { + condition = alltrue([for r in var.router_networks : can(cidrhost(r.cidr, 0))]) + error_message = "router_networks[*].cidr must be a valid IPv4 CIDR." + } + validation { + condition = alltrue([ + for r in var.router_networks : alltrue([ + for ip in r.ip_addresses : can(cidrhost("${ip}/32", 0)) + ]) + ]) + error_message = "router_networks[*].ip_addresses entries must be valid IPv4 addresses." + } + validation { + condition = alltrue([ + for r in var.router_networks : alltrue([ + for ip in r.ip_addresses : cidrhost("${ip}/${split("/", r.cidr)[1]}", 0) == cidrhost(r.cidr, 0) + ]) + ]) + error_message = "Every router_networks[*].ip_addresses entry must fall inside that role's own cidr." + } +} +``` + + Проверку `length(ip_addresses) >= var.router_count` вынести в `lifecycle.precondition` порта (см. ниже) — она версия-независима и даёт понятную ошибку на конкретном роутере/роли, а не общую. + +### 2. `terraform/main.tf` — объединить оба механизма в один набор ресурсов + +Ключевая идея переиспользования: и project-managed сети (CIDR), и внешние сети (UUID) в итоге дают роутеру «роль → {network_id, subnet_id, cidr, опциональный список фиксированных IP}» — собрать это в один `local`, и дальше вести **один** `vkcs_networking_port` + один `dynamic "network"` в инстансе, как сейчас, вместо дублирования ресурсов: + +```hcl +locals { + cidr_roles = { for idx, cidr in var.private_network_cidrs : "priv${idx + 1}" => cidr } + + router_interfaces = merge( + { + for role, cidr in local.cidr_roles : role => { + network_id = vkcs_networking_network.router_priv_net[role].id + subnet_id = vkcs_networking_subnet.router_priv_subnet[role].id + cidr = cidr + fixed_ips = null # IPAM сам назначает адрес, как сейчас + } + }, + { + for role, net in var.router_networks : role => { + network_id = net.network_id + subnet_id = net.subnet_id + cidr = net.cidr + fixed_ips = net.ip_addresses # индекс = count.index + } + } + ) + + router_interface_roles = keys(local.router_interfaces) +} +``` + +`vkcs_networking_network.router_priv_net` / `vkcs_networking_subnet.router_priv_subnet` — без изменений (по `local.cidr_roles`, для `mvm-s3` карта пустая → 0 сетей/подсетей создаётся). + +`vkcs_networking_port.router_priv_port` — переименовать по смыслу (например `router_iface_port`) и обобщить: + +```hcl +resource "vkcs_networking_port" "router_iface_port" { + for_each = { + for pair in setproduct(range(var.router_count), local.router_interface_roles) : + "router${pair[0] + 1}-${pair[1]}" => pair + } + name = "router-${each.key}-port" + network_id = local.router_interfaces[each.value[1]].network_id + admin_state_up = true + port_security_enabled = false + full_security_groups_control = true + security_group_ids = [] + sdn = "sprut" # проверить на первом plan/apply в mvm-s3 — актуально ли SDN "sprut" для чужой сети + + fixed_ip { + subnet_id = local.router_interfaces[each.value[1]].subnet_id + ip_address = local.router_interfaces[each.value[1]].fixed_ips == null ? null : local.router_interfaces[each.value[1]].fixed_ips[each.value[0]] + } + + lifecycle { + precondition { + condition = ( + local.router_interfaces[each.value[1]].fixed_ips == null || + each.value[0] < length(local.router_interfaces[each.value[1]].fixed_ips) + ) + error_message = "router_networks[\"${each.value[1]}\"].ip_addresses must have at least var.router_count entries." + } + } +} +``` + +`vkcs_compute_instance.router`: заменить `local.private_roles` → `local.router_interface_roles` в двух местах (`user_data` шаблон и `dynamic "network"`), ссылку на порт — на `router_iface_port`. Больше никаких изменений в ресурсе инстанса не требуется. + +### 3. `terraform/scripts/network-init.sh.tpl` + +**Без изменений.** Скрипт уже определяет приватные интерфейсы по членству живого IP в ожидаемом CIDR (`ip_in_cidr`), а не по имени роли или способу назначения адреса — механизм одинаково работает и для IPAM-адресов, и для явно заданных статических. + +### 4. Файлы окружения `mvm-s3` + +- `terraform/mvm-s3.tfvars` (новый, **коммитится** — по аналогии с тем, что `router_count`/`ssh_key_name`/`private_network_cidrs` для PROD сейчас открыто лежат в `terraform.tfvars`; UUID сетей и IP из диаграммы не секрет): + ```hcl + router_count = 4 + ssh_key_name = "<уточнить у пользователя>" + private_network_cidrs = [] + + router_networks = { + primary = { + network_id = "25532efe-2931-4666-a090-0da3a6f18224" + subnet_id = "1fa5357c-1f11-4d20-89f4-b27a4fb56e6e" + cidr = "172.16.252.8/29" + ip_addresses = ["172.16.252.11", "172.16.252.12", "172.16.252.13", "172.16.252.14"] + } + backup = { + network_id = "2b4cc25f-55a1-4d36-a3a2-5b16414af31a" + subnet_id = "5eacbc86-e15d-4c49-8704-547a719acc23" + cidr = "172.16.252.0/29" + ip_addresses = ["172.16.252.3", "172.16.252.4", "172.16.252.5", "172.16.252.6"] + } + } + ``` +- `terraform/mvm-s3.secrets.tfvars.example` (новый, коммитится) — шаблон creds для `mvm-s3`, зеркало `prod.auto.tfvars.example` (`auth_url`, `username`, `password`, `project_id`, `region`, `user_domain_name`, `ssh_public_key`). +- `terraform/mvm-s3.secrets.tfvars` — пользователь создаёт сам из примера (**не коммитится**, я его не создаю и не заполняю значениями). +- Переименовать `terraform/prod.auto.tfvars` → `terraform/prod.secrets.tfvars` (и `prod.auto.tfvars.example` → `prod.secrets.tfvars.example`), чтобы оба окружения выбирались строго явными `-var-file`, без риска, что один набор creds «протечёт» в apply другого окружения через авто-загрузку. **Это переименование реального файла с секретами делает пользователь сам** (я не имею доступа/не должен его трогать) — в плане это шаг-инструкция, не мой git-коммит. +- `.gitignore`: добавить `*.secrets.tfvars` / `*.secrets.tfvars.json`; старый паттерн `*.auto.tfvars` оставить (безвреден, ничего под него больше не подпадает). + +### 5. Документация (по правилам проекта — `.claude/CLAUDE.md`) + +- `docs/changes/2026-09-09-mvm-s3-external-networks-plan.md` — копия согласованного плана. +- `docs/changes/2026-09-09-mvm-s3-external-networks-summary.md` — по завершении, с фактическими `terraform plan`/`apply`-результатами (которые предоставит пользователь, т.к. я не могу их выполнить). +- `README.md` — дополнить разделом про `mvm-s3` окружение: workspace, `-var-file`, схема сети (аналогично таблице в `DEPLOYMENT_SUMMARY.md`). +- `docs/QUICKSTART.md` — обновить: явные `-var-file` вместо неявной авто-загрузки, инструкция `terraform workspace new/select`, где взять `mvm-s3.secrets.tfvars`. + +### 6. Тесты (`terraform/tests/test_terraform_delivery.py`) + +Изучить существующие проверки для `private_network_cidrs` (в частности тест на «минимум 1 CIDR», который перестанет быть валидным) и дописать по аналогии: +- `private_network_cidrs = []` + `router_networks` с 2 ролями → план создаёт 0 `router_priv_net`/`subnet`, 8 портов (4 роутера × 2 роли), каждый порт с явным `fixed_ip.ip_address` из диаграммы. +- Смешанный кейс (не боевой, но для полноты обобщённого кода) — `private_network_cidrs` непустой И `router_networks` непустой одновременно не ломается (роли не пересекаются). +- `router_networks` с `ip_addresses` короче `router_count` → `terraform plan` падает на precondition с понятной ошибкой. +- IP вне CIDR роли → падает на validation. + +Прогнать `venv/bin/pytest terraform/tests -v` — офлайн, без обращения к облаку (как в прошлых изменениях). + +## Проверка + +1. `terraform fmt -check` / `terraform validate` в `terraform/`. +2. `venv/bin/pytest terraform/tests -v` — все тесты (старые + новые) зелёные. +3. `terraform workspace new mvm-s3` (если ещё не создан) → `terraform plan -var-file=terraform.tfvars -var-file=mvm-s3.tfvars -var-file=mvm-s3.secrets.tfvars` — **выполняет пользователь** (нет доступа к проекту `mvm-s3` отсюда); ожидаемо: 4 роутера, по 2 порта с точными IP из диаграммы, 0 создаваемых приватных сетей/подсетей. +4. Убедиться, что `terraform workspace select default` (PROD) + прежний `-var-file`/creds-файл по-прежнему даёт **пустой diff** (0 changes) — подтверждает, что PROD не задет. diff --git a/docs/changes/2026-09-09-mvm-s3-external-networks-summary.md b/docs/changes/2026-09-09-mvm-s3-external-networks-summary.md new file mode 100644 index 0000000..5201d77 --- /dev/null +++ b/docs/changes/2026-09-09-mvm-s3-external-networks-summary.md @@ -0,0 +1,50 @@ +# Summary: внешние сети с фиксированными IP для окружения mvm-s3 + +План: [2026-09-09-mvm-s3-external-networks-plan.md](2026-09-09-mvm-s3-external-networks-plan.md) + +## Что сделано + +### `terraform/variables.tf` +- `private_network_cidrs` стала опциональной (`default = []`), убрана валидация «минимум 1 CIDR» — деплой, использующий только `router_networks` (как `mvm-s3`), не обязан её задавать. +- Добавлена `router_networks` (`map(object({network_id, subnet_id, cidr, ip_addresses}))`, `default = {}`) — заранее существующие сети (обычно в другом VK Cloud проекте, только по UUID), в которые каждый роутер получает интерфейс с заранее известным IP. Валидации: корректный CIDR, корректные IPv4-адреса, каждый адрес — внутри своего CIDR. + +### `terraform/main.tf` +- Оба механизма (project-managed сети по `private_network_cidrs` и внешние по `router_networks`) объединены в один `local.router_interfaces` (роль → `{network_id, subnet_id, cidr, fixed_ips}`) — переиспользован существующий паттерн (`setproduct`, один `vkcs_networking_port`, один `dynamic "network"`) вместо дублирования ресурсов. +- `vkcs_networking_port.router_priv_port` переименован в `router_iface_port`, `fixed_ip.ip_address` теперь условный: `null` для project-managed ролей (как раньше — IPAM сам назначает, избегая коллизии с служебным портом `network:dns`), явный статический адрес — для ролей из `router_networks`. +- Добавлен `lifecycle.precondition` на порт: понятная ошибка, если `router_networks[role].ip_addresses` короче `var.router_count`. +- `network-init.sh.tpl` **не изменялся** — он уже сопоставляет интерфейсы по вхождению живого IP в ожидаемый CIDR, независимо от роли и способа назначения адреса. + +### Окружение `mvm-s3` (файлы) +- `terraform/mvm-s3.tfvars` (коммитится) — `router_count = 4`, `private_network_cidrs = []`, `router_networks` с двумя ролями (`primary` = `172.16.252.8/29`, `backup` = `172.16.252.0/29`) и точными IP на роутер из диаграммы пользователя. `ssh_key_name` оставлен плейсхолдером — пользователь должен вписать реальное значение. +- `terraform/mvm-s3.secrets.tfvars.example` (коммитится) — шаблон реальных credentials для `mvm-s3`. +- `terraform/prod.secrets.tfvars.example` (коммитится, заменил `prod.auto.tfvars.example`) — тот же шаблон для PROD по новой конвенции именования. +- `.gitignore` — добавлен паттерн `*.secrets.tfvars` (и `.json`), старый `*.auto.tfvars` оставлен как safety net. + +### Схема окружений +Поскольку теперь в одном каталоге `terraform/` живут 2 окружения с разными credentials/router_count/сетями, отказались от неявной авто-загрузки `*.auto.tfvars` в пользу **явных `-var-file`** + **отдельных Terraform workspace** на окружение (`terraform workspace new mvm-s3`). PROD продолжает работать в workspace `default` со своим state — `mvm-s3` их не затрагивает. + +**Важно (ручной шаг для пользователя, не сделан мной):** переименовать существующий `terraform/prod.auto.tfvars` (реальные PROD-креды) в `terraform/prod.secrets.tfvars` — я не трогал этот файл, так как он содержит настоящие секреты. + +### Документация +- `docs/changes/2026-09-09-mvm-s3-external-networks-plan.md` — план (копия согласованного). +- `README.md` — новый раздел "External fixed-IP networks and multi-environment workspaces", обновлены упоминания `*.auto.tfvars` → `*.secrets.tfvars`, описание тестов. +- `docs/QUICKSTART.md` — новый шаг 4а (деплой `mvm-s3`), обновлён шаг 2 (credentials) и шаг 4 (явные `-var-file`, workspace). + +### Тесты (`terraform/tests/test_terraform_delivery.py`) +73 теста (было 56 по данным `DEPLOYMENT_SUMMARY.md`, часть добавлена и в предыдущих изменениях), все зелёные. Ключевые добавления/правки: +- `test_router_networks_variable`, `test_router_networks_validation_is_enforced` (валидный/невалидный CIDR, IP вне подсети, дефолтный `{}`). +- `test_private_network_cidrs_variable`/`test_private_network_cidrs_validation_is_enforced` обновлены под новый `default = []`. +- `test_router_interfaces_local_merges_both_network_sources`, `test_router_interface_roles_local_is_keys_of_router_interfaces`, `test_router_network_blocks_scale_with_router_interface_roles` — заменили тесты на удалённый `local.private_roles`. +- `test_router_iface_port_ip_address_is_conditional_on_fixed_ips`, `test_router_iface_port_has_length_precondition_for_fixed_ips` — заменили тест на переименованный `router_priv_port`. +- `test_mvm_s3_tfvars_*` — сверяют `mvm-s3.tfvars` с диаграммой (UUID, CIDR, IP), запрет project-managed сетей, покрытие `router_count` числом IP. +- `test_secrets_tfvars_example_is_not_gitignored`, `test_gitignore_excludes_secrets_tfvars_overlay`, `test_committed_tfvars_have_no_auth_credentials` — заменили тесты на удалённый `prod.auto.tfvars.example`. +- `test_terraform_init_and_validate_against_real_provider_schema` — добавлен `mvm-s3`-образный кейс (`private_network_cidrs=[]` + 2 роли `router_networks`) для проверки схемы реального провайдера. + +Прогон: `venv/bin/terraform fmt -check` — чисто; `venv/bin/pytest terraform/tests -v` — **73 passed**. + +## Что НЕ сделано (осознанно, вне зоны моего доступа) + +- `terraform plan`/`apply` в проекте `mvm-s3` — у меня нет доступа к этому VK Cloud проекту; должен выполнить пользователь (команды — в `docs/QUICKSTART.md` §4а). +- Переименование реального `terraform/prod.auto.tfvars` → `prod.secrets.tfvars` — файл содержит настоящие секреты, переименовать должен пользователь сам. +- Заполнение реальных credentials/`ssh_key_name` для `mvm-s3` — плейсхолдеры оставлены намеренно, я не должен их придумывать. +- Проверка актуальности `sdn = "sprut"` на порту в чужую сеть — отмечено в `main.tf` комментарием, требует подтверждения на первом реальном `plan`/`apply`. diff --git a/terraform/main.tf b/terraform/main.tf index 1190b86..9923513 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -91,14 +91,40 @@ resource "vkcs_networking_secgroup_rule" "router_ipsec_nat_t" { # CIDR in private_network_cidrs (no VRRP), each router getting its own # port/IP inside every such network. locals { - private_roles = [for idx in range(length(var.private_network_cidrs)) : "priv${idx + 1}"] - private_network_cidr = { for idx, cidr in var.private_network_cidrs : "priv${idx + 1}" => cidr } } +# Unify both sources of per-router interfaces into one role -> definition map: +# - roles from private_network_cidr are project-managed (network/subnet created +# below), address chosen by Neutron IPAM (fixed_ips left null). +# - roles from var.router_networks are pre-existing networks owned elsewhere, +# referenced by UUID only, with a specific IP per router (fixed_ips set). +locals { + router_interfaces = merge( + { + for role, cidr in local.private_network_cidr : role => { + network_id = vkcs_networking_network.router_priv_net[role].id + subnet_id = vkcs_networking_subnet.router_priv_subnet[role].id + cidr = cidr + fixed_ips = null + } + }, + { + for role, net in var.router_networks : role => { + network_id = net.network_id + subnet_id = net.subnet_id + cidr = net.cidr + fixed_ips = net.ip_addresses + } + } + ) + + router_interface_roles = keys(local.router_interfaces) +} + resource "vkcs_networking_network" "router_priv_net" { for_each = local.private_network_cidr name = "router-${each.key}-net" @@ -121,33 +147,51 @@ resource "vkcs_networking_subnet" "router_priv_subnet" { # to ethN by MAC - no actual DHCP protocol exchange with this subnet ever # happens. (Note: this does NOT avoid IP collisions with VKCS's own # auto-created service ports on the network, e.g. a "network:dns" port - - # see router_priv_port below, which leaves ip_address unset instead.) + # see router_iface_port below, which leaves ip_address unset for these + # project-managed roles instead.) enable_dhcp = false } -resource "vkcs_networking_port" "router_priv_port" { +resource "vkcs_networking_port" "router_iface_port" { for_each = { - for pair in setproduct(range(var.router_count), local.private_roles) : - "router${pair[0] + 1}-${pair[1]}" => pair[1] + for pair in setproduct(range(var.router_count), local.router_interface_roles) : + "router${pair[0] + 1}-${pair[1]}" => pair } name = "router-${each.key}-port" - network_id = vkcs_networking_network.router_priv_net[each.value].id + network_id = local.router_interfaces[each.value[1]].network_id admin_state_up = true port_security_enabled = false full_security_groups_control = true security_group_ids = [] sdn = "sprut" - # No explicit ip_address: let Neutron's IPAM auto-assign one from the - # subnet's pool. VKCS auto-creates its own service ports on this network - # (observed: a "network:dns" port) that can silently consume whichever - # low address a hand-computed offset would have picked, causing - # IpAddressAlreadyAllocated - IPAM guarantees no double-booking, our own - # arithmetic doesn't. network-init.sh.tpl matches interfaces by which - # declared CIDR their live IP falls into, not by an exact expected IP, so - # the assigned address doesn't need to be known in advance. + # For project-managed roles (private_network_cidr), ip_address is left null + # so Neutron's IPAM auto-assigns one from the subnet's pool: VKCS + # auto-creates its own service ports on this network (observed: a + # "network:dns" port) that can silently consume whichever low address a + # hand-computed offset would have picked, causing IpAddressAlreadyAllocated + # - IPAM guarantees no double-booking, our own arithmetic doesn't. + # network-init.sh.tpl matches interfaces by which declared CIDR their live + # IP falls into, not by an exact expected IP, so this works either way. + # For var.router_networks roles, the address is pre-agreed externally + # (another project's admin already carved it out), so it's set explicitly. fixed_ip { - subnet_id = vkcs_networking_subnet.router_priv_subnet[each.value].id + subnet_id = local.router_interfaces[each.value[1]].subnet_id + ip_address = ( + local.router_interfaces[each.value[1]].fixed_ips == null + ? null + : local.router_interfaces[each.value[1]].fixed_ips[each.value[0]] + ) + } + + lifecycle { + precondition { + condition = ( + local.router_interfaces[each.value[1]].fixed_ips == null || + each.value[0] < length(local.router_interfaces[each.value[1]].fixed_ips) + ) + error_message = "router_networks[\"${each.value[1]}\"].ip_addresses must have at least var.router_count entries." + } } } @@ -169,9 +213,9 @@ resource "vkcs_compute_instance" "router" { # detection, private interfaces matched by their expected CIDR. user_data = templatefile("${path.module}/scripts/network-init.sh.tpl", { private_interfaces = [ - for role in local.private_roles : { - name = "eth${index(local.private_roles, role) + 1}" - cidr = local.private_network_cidr[role] + for role in local.router_interface_roles : { + name = "eth${index(local.router_interface_roles, role) + 1}" + cidr = local.router_interfaces[role].cidr } ] }) @@ -181,11 +225,13 @@ resource "vkcs_compute_instance" "router" { uuid = data.vkcs_networking_network.extnet.id } - # Private: one pre-created port per role, into that role's shared network + # Private: one pre-created port per role, into that role's network (either + # project-managed via private_network_cidrs, or a pre-existing externally + # owned one via router_networks) dynamic "network" { - for_each = local.private_roles + for_each = local.router_interface_roles content { - port = vkcs_networking_port.router_priv_port["router${count.index + 1}-${network.value}"].id + port = vkcs_networking_port.router_iface_port["router${count.index + 1}-${network.value}"].id } } diff --git a/terraform/mvm-s3.secrets.tfvars.example b/terraform/mvm-s3.secrets.tfvars.example new file mode 100644 index 0000000..71df2cd --- /dev/null +++ b/terraform/mvm-s3.secrets.tfvars.example @@ -0,0 +1,32 @@ +# Example of the per-environment secrets tfvars for "mvm-s3". +# +# Copy this file to mvm-s3.secrets.tfvars (gitignored - see .gitignore, +# pattern *.secrets.tfvars) and fill in real values. Unlike the old +# *.auto.tfvars convention, this file is NOT auto-loaded by Terraform - pass +# it explicitly with -var-file so it can never accidentally merge with +# another environment's creds: +# +# terraform workspace select mvm-s3 +# terraform apply -var-file=terraform.tfvars -var-file=mvm-s3.tfvars -var-file=mvm-s3.secrets.tfvars +# +# Mapping from an OpenStack-style openrc.sh: +# OS_AUTH_URL -> auth_url +# OS_USERNAME -> username +# OS_PASSWORD -> password +# OS_PROJECT_ID -> project_id +# OS_REGION_NAME -> region +# OS_USER_DOMAIN_NAME -> user_domain_name + +auth_url = "https://infra.mail.ru:35357/v3/" +username = "" +password = "" +project_id = "" +region = "RegionOne" +user_domain_name = "users" + +# Optional: Nova keypairs are per-user, not per-project - a keypair uploaded +# under a different account is invisible to whichever account deploys here. +# Set this to have Terraform register var.ssh_key_name (see mvm-s3.tfvars) +# under the deploying account from this public key. Leave unset if a keypair +# with that name already exists under the deploying account. +# ssh_public_key = "ssh-ed25519 AAAA... or ssh-rsa AAAA... your-public-key-content" diff --git a/terraform/mvm-s3.tfvars b/terraform/mvm-s3.tfvars new file mode 100644 index 0000000..8110da5 --- /dev/null +++ b/terraform/mvm-s3.tfvars @@ -0,0 +1,36 @@ +# Non-secret shape config for the "mvm-s3" environment (a separate VK Cloud +# project from PROD - see docs/changes/2026-09-09-mvm-s3-external-networks-*.md). +# Use with -var-file explicitly (auto-load is intentionally not relied upon +# once more than one environment exists - see mvm-s3.secrets.tfvars.example). +# +# Apply against the "mvm-s3" Terraform workspace (terraform workspace new/select +# mvm-s3), never against the "default" workspace that holds PROD's state: +# terraform workspace select mvm-s3 +# terraform apply -var-file=terraform.tfvars -var-file=mvm-s3.tfvars -var-file=mvm-s3.secrets.tfvars + +router_count = 4 + +# TODO: fill in the real SSH key pair name for this project (Nova keypairs +# are per-user, not per-project - a key uploaded under a different account +# won't be visible here even if it has the same name as PROD's "mcs_ru"). +ssh_key_name = "" + +# No project-managed private networks in this environment - both private +# interfaces come from var.router_networks below (pre-existing networks in +# another project, fixed IP per router). +private_network_cidrs = [] + +router_networks = { + primary = { + network_id = "25532efe-2931-4666-a090-0da3a6f18224" + subnet_id = "1fa5357c-1f11-4d20-89f4-b27a4fb56e6e" + cidr = "172.16.252.8/29" + ip_addresses = ["172.16.252.11", "172.16.252.12", "172.16.252.13", "172.16.252.14"] + } + backup = { + network_id = "2b4cc25f-55a1-4d36-a3a2-5b16414af31a" + subnet_id = "5eacbc86-e15d-4c49-8704-547a719acc23" + cidr = "172.16.252.0/29" + ip_addresses = ["172.16.252.3", "172.16.252.4", "172.16.252.5", "172.16.252.6"] + } +} diff --git a/terraform/prod.auto.tfvars.example b/terraform/prod.auto.tfvars.example deleted file mode 100644 index 70c3a2a..0000000 --- a/terraform/prod.auto.tfvars.example +++ /dev/null @@ -1,29 +0,0 @@ -# Example of the *.auto.tfvars overlay pattern for real credentials. -# -# terraform.tfvars stays a committed, anonymized template. To deploy with -# real credentials (e.g. from an openrc.sh for a service account), copy this -# file to prod.auto.tfvars (gitignored - see .gitignore) and fill in real -# values. Terraform auto-loads *.auto.tfvars in addition to terraform.tfvars, -# so this layers on top of the template without ever modifying/committing it. -# -# Mapping from an OpenStack-style openrc.sh: -# OS_AUTH_URL -> auth_url -# OS_USERNAME -> username -# OS_PASSWORD -> password -# OS_PROJECT_ID -> project_id -# OS_REGION_NAME -> region -# OS_USER_DOMAIN_NAME -> user_domain_name - -auth_url = "https://infra.mail.ru:35357/v3/" -username = "svc--svc-deployer" -password = "" -project_id = "" -region = "RegionOne" -user_domain_name = "service-users" - -# Optional: Nova keypairs are per-user, not per-project - a keypair uploaded -# under a different account (e.g. your personal VK Cloud login) is invisible -# to a service account. Set this to have Terraform register var.ssh_key_name -# under the deploying account from this public key. Leave unset if a keypair -# with that name already exists under the deploying account. -ssh_public_key = "ssh-ed25519 AAAA... or ssh-rsa AAAA... your-public-key-content" diff --git a/terraform/prod.secrets.tfvars.example b/terraform/prod.secrets.tfvars.example new file mode 100644 index 0000000..ef7565a --- /dev/null +++ b/terraform/prod.secrets.tfvars.example @@ -0,0 +1,42 @@ +# Example of the per-environment secrets tfvars for PROD. +# +# terraform.tfvars stays a committed, anonymized template of non-secret +# values (router_count, ssh_key_name, ...). Real credentials go here. +# +# Historical note: this file used to be named prod.auto.tfvars and relied on +# Terraform's automatic *.auto.tfvars loading. Since a second environment +# ("mvm-s3", see docs/changes/2026-09-09-mvm-s3-external-networks-*.md) now +# shares this same terraform/ directory, auto-loading is no longer safe - +# two *.auto.tfvars files present at once would both load and silently merge, +# risking one environment's credentials leaking into another's apply. Copy +# this file to prod.secrets.tfvars (gitignored - see .gitignore, pattern +# *.secrets.tfvars) and pass it explicitly: +# +# terraform workspace select default +# terraform apply -var-file=terraform.tfvars -var-file=prod.secrets.tfvars +# +# If you still have the old prod.auto.tfvars from before this change, rename +# it to prod.secrets.tfvars yourself (its values don't need to change) - +# Claude does not read or move files containing real credentials. +# +# Mapping from an OpenStack-style openrc.sh: +# OS_AUTH_URL -> auth_url +# OS_USERNAME -> username +# OS_PASSWORD -> password +# OS_PROJECT_ID -> project_id +# OS_REGION_NAME -> region +# OS_USER_DOMAIN_NAME -> user_domain_name + +auth_url = "https://infra.mail.ru:35357/v3/" +username = "svc--svc-deployer" +password = "" +project_id = "" +region = "RegionOne" +user_domain_name = "service-users" + +# Optional: Nova keypairs are per-user, not per-project - a keypair uploaded +# under a different account (e.g. your personal VK Cloud login) is invisible +# to a service account. Set this to have Terraform register var.ssh_key_name +# under the deploying account from this public key. Leave unset if a keypair +# with that name already exists under the deploying account. +ssh_public_key = "ssh-ed25519 AAAA... or ssh-rsa AAAA... your-public-key-content" diff --git a/terraform/terraform.tfvars b/terraform/terraform.tfvars index 2cf11bc..8bf53a9 100644 --- a/terraform/terraform.tfvars +++ b/terraform/terraform.tfvars @@ -15,4 +15,4 @@ router_count = 4 private_network_cidrs = [ "10.90.0.0/28", "10.90.0.16/28", -] \ No newline at end of file +] diff --git a/terraform/tests/test_terraform_delivery.py b/terraform/tests/test_terraform_delivery.py index 5a740b9..05bad0e 100644 --- a/terraform/tests/test_terraform_delivery.py +++ b/terraform/tests/test_terraform_delivery.py @@ -55,6 +55,23 @@ CHECKOV_BIN = ( else shutil.which("checkov") ) +# The real mvm-s3 shape (two externally-owned, fixed-IP networks - matches +# terraform/mvm-s3.tfvars), reused by several tests below. +MVM_S3_ROUTER_NETWORKS = { + "primary": { + "network_id": "25532efe-2931-4666-a090-0da3a6f18224", + "subnet_id": "1fa5357c-1f11-4d20-89f4-b27a4fb56e6e", + "cidr": "172.16.252.8/29", + "ip_addresses": ["172.16.252.11", "172.16.252.12", "172.16.252.13", "172.16.252.14"], + }, + "backup": { + "network_id": "2b4cc25f-55a1-4d36-a3a2-5b16414af31a", + "subnet_id": "5eacbc86-e15d-4c49-8704-547a719acc23", + "cidr": "172.16.252.0/29", + "ip_addresses": ["172.16.252.3", "172.16.252.4", "172.16.252.5", "172.16.252.6"], + }, +} + def load_tf(relpath): with open(TERRAFORM_DIR / relpath) as f: @@ -99,7 +116,9 @@ REQUIRED_FILES = [ "variables.tf", "versions.tf", "terraform.tfvars", - "prod.auto.tfvars.example", + "prod.secrets.tfvars.example", + "mvm-s3.tfvars", + "mvm-s3.secrets.tfvars.example", "scripts/network-init.sh.tpl", ] @@ -136,15 +155,18 @@ def test_terraform_fmt_clean(): @pytest.mark.parametrize( - "router_count,private_network_cidrs", + "router_count,private_network_cidrs,router_networks", [ - (None, None), # whatever terraform.tfvars already commits to - (1, ["10.90.0.0/29"]), - (4, ["10.90.0.0/28", "10.90.0.16/28", "10.90.0.32/28"]), + (None, None, None), # whatever terraform.tfvars already commits to + (1, ["10.90.0.0/29"], None), + (4, ["10.90.0.0/28", "10.90.0.16/28", "10.90.0.32/28"], None), + # mvm-s3-shaped: no project-managed private networks, two + # externally-owned fixed-IP ones instead. + (4, [], MVM_S3_ROUTER_NETWORKS), ], ) def test_terraform_init_and_validate_against_real_provider_schema( - router_count, private_network_cidrs + router_count, private_network_cidrs, router_networks ): """Real `terraform init` + `terraform validate` against the actual vkcs provider, using the project-local filesystem-mirror copy of the @@ -157,9 +179,10 @@ def test_terraform_init_and_validate_against_real_provider_schema( does not enforce the variable validation{} blocks - see test_variable_validations_are_enforced_by_plan for that). - Parametrized over router_count/private_network_cidrs (set via TF_VAR_*, - exactly how horizontal scaling is meant to be driven) to prove the - delivery actually resolves at other scales, not just the defaults. + Parametrized over router_count/private_network_cidrs/router_networks + (set via TF_VAR_*, exactly how horizontal scaling and the mvm-s3-style + fixed-IP deployment are meant to be driven) to prove the delivery + actually resolves at other scales/shapes, not just the defaults. """ assert TERRAFORM_BIN, "no terraform binary found (checked venv/bin and PATH)" if not CLI_CONFIG_FILE.is_file(): @@ -174,6 +197,8 @@ def test_terraform_init_and_validate_against_real_provider_schema( env["TF_VAR_router_count"] = str(router_count) if private_network_cidrs is not None: env["TF_VAR_private_network_cidrs"] = json.dumps(private_network_cidrs) + if router_networks is not None: + env["TF_VAR_router_networks"] = json.dumps(router_networks) with tempfile.TemporaryDirectory() as tmp: tmp_path = Path(tmp) @@ -252,7 +277,7 @@ def _plan_variables_only(var_overrides): "cidrs,should_pass", [ (["10.90.0.0/29", "10.90.0.8/29"], True), - ([], False), # must contain at least one CIDR + ([], True), # optional now - a router_networks-only deployment (e.g. mvm-s3) sets none (["not-a-cidr"], False), # must be a valid IPv4 CIDR (["10.90.0.0/29", "10.90.0.0/29"], False), # must be unique ], @@ -262,6 +287,27 @@ def test_private_network_cidrs_validation_is_enforced(cidrs, should_pass): assert ok == should_pass, f"unexpected result for private_network_cidrs={cidrs!r}:\n{output}" +def _router_networks_case(**override): + net = json.loads(json.dumps(MVM_S3_ROUTER_NETWORKS)) # deep copy + net["primary"].update(override) + return net + + +@pytest.mark.parametrize( + "router_networks,should_pass", + [ + (MVM_S3_ROUTER_NETWORKS, True), + ({}, True), # optional - a private_network_cidrs-only deployment (e.g. PROD) sets none + (_router_networks_case(cidr="not-a-cidr"), False), # cidr must be a valid IPv4 CIDR + (_router_networks_case(ip_addresses=["not-an-ip"]), False), # ip must be a valid IPv4 address + (_router_networks_case(ip_addresses=["10.0.0.1"]), False), # ip must fall inside its own cidr + ], +) +def test_router_networks_validation_is_enforced(router_networks, should_pass): + ok, output = _plan_variables_only({"TF_VAR_router_networks": json.dumps(router_networks)}) + assert ok == should_pass, f"unexpected result for router_networks={router_networks!r}:\n{output}" + + @pytest.mark.parametrize("count,should_pass", [(2, True), (0, False), (-1, False)]) def test_router_count_validation_is_enforced(count, should_pass): ok, output = _plan_variables_only({"TF_VAR_router_count": str(count)}) @@ -335,44 +381,53 @@ def test_auth_variable_has_a_default(name): # --------------------------------------------------------------------------- -# Real secrets must never land in the git-tracked terraform.tfvars template - -# they belong in a gitignored *.auto.tfvars overlay instead (see -# prod.auto.tfvars.example) +# Real secrets must never land in a git-tracked tfvars file - each +# environment's creds belong in its own gitignored *.secrets.tfvars, passed +# explicitly via -var-file (see *.secrets.tfvars.example). Two environments +# now share this terraform/ directory (PROD and mvm-s3), so unlike the old +# single-environment *.auto.tfvars convention, nothing here may rely on +# Terraform's automatic tfvars loading for secrets. # --------------------------------------------------------------------------- -def test_gitignore_excludes_auto_tfvars_overlay(): +def test_gitignore_excludes_secrets_tfvars_overlay(): gitignore_text = (REPO_ROOT / ".gitignore").read_text() - assert "*.auto.tfvars" in gitignore_text, ( - "*.auto.tfvars must be gitignored - real credentials are meant to be " - "layered on top of terraform.tfvars via such a file, never committed" + assert "*.secrets.tfvars" in gitignore_text, ( + "*.secrets.tfvars must be gitignored - real per-environment " + "credentials are meant to be passed via such a file with an " + "explicit -var-file, never committed" ) -def test_prod_auto_tfvars_example_is_not_gitignored(): - """The *.example file documents the overlay pattern and must ship in the - repo (unlike the real *.auto.tfvars it documents).""" +@pytest.mark.parametrize( + "relpath", ["prod.secrets.tfvars.example", "mvm-s3.secrets.tfvars.example"] +) +def test_secrets_tfvars_example_is_not_gitignored(relpath): + """Each *.example file documents the -var-file pattern for one + environment and must ship in the repo (unlike the real *.secrets.tfvars + it documents).""" result = subprocess.run( - ["git", "check-ignore", "terraform/prod.auto.tfvars.example"], + ["git", "check-ignore", f"terraform/{relpath}"], cwd=REPO_ROOT, capture_output=True, text=True, ) - assert result.returncode != 0, "prod.auto.tfvars.example must NOT be gitignored" + assert result.returncode != 0, f"{relpath} must NOT be gitignored" -def test_tfvars_template_has_no_auth_credentials(): - """terraform.tfvars is a committed, anonymized template - auth_url/ - user_domain_name/real credentials belong in a gitignored *.auto.tfvars - overlay, not here.""" - tfvars_text = (TERRAFORM_DIR / "terraform.tfvars").read_text() +@pytest.mark.parametrize("relpath", ["terraform.tfvars", "mvm-s3.tfvars"]) +def test_committed_tfvars_have_no_auth_credentials(relpath): + """terraform.tfvars and mvm-s3.tfvars are committed, non-secret shape + templates - auth_url/user_domain_name/real credentials belong in each + environment's gitignored *.secrets.tfvars overlay, not here.""" + tfvars_text = (TERRAFORM_DIR / relpath).read_text() active_lines = [ line for line in tfvars_text.splitlines() if not line.strip().startswith("#") ] for forbidden in ("auth_url", "user_domain_name"): assert not any(re.match(rf"^\s*{forbidden}\s*=", line) for line in active_lines), ( - f"{forbidden} should not be set in the committed terraform.tfvars " - f"template - use a gitignored *.auto.tfvars overlay instead" + f"{forbidden} should not be set in the committed {relpath} " + f"template - use a gitignored *.secrets.tfvars overlay instead" ) @@ -392,12 +447,27 @@ def test_private_network_cidrs_variable(): v = find_variable(load_tf("variables.tf"), "private_network_cidrs") assert v is not None, "variable private_network_cidrs is missing" assert v["type"] == ["${list(string)}"] - assert "default" not in v, ( - "private_network_cidrs must NOT have a default - the admin is " - "required to pass it explicitly" + assert v.get("default") == [[]], ( + "private_network_cidrs must default to [] - a deployment that only " + "uses var.router_networks (e.g. mvm-s3) needs no project-managed " + "private networks at all" + ) + assert len(v.get("validation", [])) >= 2, ( + "expected validations for: valid CIDR syntax, uniqueness" + ) + + +def test_router_networks_variable(): + v = find_variable(load_tf("variables.tf"), "router_networks") + assert v is not None, "variable router_networks is missing" + assert v.get("default") == [{}], ( + "router_networks must default to {} - a deployment that only uses " + "var.private_network_cidrs (e.g. PROD) needs no externally-owned " + "fixed-IP networks at all" ) assert len(v.get("validation", [])) >= 3, ( - "expected validations for: non-empty, valid CIDR syntax, uniqueness" + "expected validations for: valid CIDR syntax, valid IPv4 addresses, " + "each address falling inside its own cidr" ) @@ -419,8 +489,8 @@ def test_router_count_pinned_in_tfvars_is_a_valid_number(): def test_private_network_cidrs_is_set_in_tfvars(): - """Unlike router_count, private_network_cidrs has no default, so - terraform.tfvars must actively set it for a working example deployment.""" + """private_network_cidrs defaults to [], but PROD's terraform.tfvars + still pins its real project-managed networks explicitly.""" tfvars_text = (TERRAFORM_DIR / "terraform.tfvars").read_text() active_lines = [ line for line in tfvars_text.splitlines() if not line.strip().startswith("#") @@ -471,7 +541,7 @@ def test_no_legacy_hardcoded_router_resources(): port_names = {name for name, _ in find_resources(main, "vkcs_networking_port")} assert port_names.isdisjoint({"lan_port1", "lan_port2"}), ( "found legacy hardcoded lan_port1/lan_port2 instead of the " - "for_each-based router_priv_port" + "for_each-based router_iface_port" ) @@ -491,21 +561,40 @@ def test_deployment_is_router_only(): assert secgroup_names == {"router_sg"}, f"expected only router_sg, found {secgroup_names}" -def test_private_roles_local_driven_by_variable(): +def test_router_interfaces_local_merges_both_network_sources(): + """locals.router_interfaces unifies the two ways a router can get a + private interface: project-managed (private_network_cidrs, network/ + subnet created by this Terraform) and externally-owned, fixed-IP + (router_networks, referenced by UUID only - e.g. the mvm-s3 + environment).""" main = load_tf("main.tf") - private_roles = find_local(main, "private_roles") - assert private_roles == [ - '${[for idx in range(length(var.private_network_cidrs)) : "priv${idx + 1}"]}' - ], "locals.private_roles must be generated from length(var.private_network_cidrs)" + router_interfaces = find_local(main, "router_interfaces") + assert router_interfaces is not None, "locals.router_interfaces is missing" + expr = router_interfaces[0] + assert expr.startswith("${merge("), "router_interfaces must be built via merge(...)" + assert "for role , cidr in local.private_network_cidr" in expr, ( + "router_interfaces must include the project-managed private_network_cidrs roles" + ) + assert "for role , net in var.router_networks" in expr, ( + "router_interfaces must include the externally-owned router_networks roles" + ) -def test_router_network_blocks_scale_with_private_roles(): +def test_router_interface_roles_local_is_keys_of_router_interfaces(): + main = load_tf("main.tf") + assert find_local(main, "router_interface_roles") == [ + "${keys(local.router_interfaces)}" + ], "locals.router_interface_roles must be keys(local.router_interfaces)" + + +def test_router_network_blocks_scale_with_router_interface_roles(): main = load_tf("main.tf") router = dict(find_resources(main, "vkcs_compute_instance"))["router"] dynamic_network = router["dynamic"][0]["network"] - assert dynamic_network["for_each"] == ["${local.private_roles}"], ( - "the dynamic private network blocks must iterate local.private_roles " - "so the NIC count scales with the number of private_network_cidrs entries" + assert dynamic_network["for_each"] == ["${local.router_interface_roles}"], ( + "the dynamic private network blocks must iterate " + "local.router_interface_roles so the NIC count scales with both " + "private_network_cidrs and router_networks entries" ) @@ -622,23 +711,41 @@ def test_router_priv_subnet_has_dhcp_disabled(): ) -def test_router_priv_port_leaves_ip_address_unset(): - """Regression guard: a hand-computed fixed_ip.ip_address collided with - VKCS's own auto-created service ports on the network (observed: a - "network:dns" port silently consuming an address) during a real PROD - deployment. Leaving ip_address unset lets Neutron's IPAM auto-assign - one, which is guaranteed collision-free; network-init.sh.tpl matches - interfaces by which declared CIDR their live IP falls into, not by an - exact expected IP, so this doesn't need to be known in advance.""" +def test_router_iface_port_ip_address_is_conditional_on_fixed_ips(): + """Regression guard + new behaviour, in one place: a hand-computed + fixed_ip.ip_address collided with VKCS's own auto-created service ports + on the network (observed: a "network:dns" port silently consuming an + address) during a real PROD deployment - so for project-managed roles + (private_network_cidrs) ip_address must stay null and let Neutron's IPAM + auto-assign. But externally-owned roles (router_networks, e.g. mvm-s3) + have their IP pre-agreed by another project's admin, so those must set + it explicitly. Both cases are driven by the same conditional expression.""" main = load_tf("main.tf") ports = dict(find_resources(main, "vkcs_networking_port")) - assert "router_priv_port" in ports - fixed_ip = ports["router_priv_port"]["fixed_ip"][0] - assert "ip_address" not in fixed_ip, ( - "router_priv_port.fixed_ip must not set ip_address - let Neutron's " - "IPAM auto-assign to avoid colliding with platform-reserved ports" - ) + assert "router_iface_port" in ports + fixed_ip = ports["router_iface_port"]["fixed_ip"][0] assert "subnet_id" in fixed_ip + assert fixed_ip["ip_address"] == [ + "${local.router_interfaces[each.value[1]].fixed_ips == None ? None : " + "local.router_interfaces[each.value[1]].fixed_ips[each.value[0]]}" + ], ( + "router_iface_port.fixed_ip.ip_address must stay null when the " + "role's fixed_ips is null (project-managed roles, IPAM auto-assign) " + "and pick the per-router static IP otherwise (router_networks roles)" + ) + + +def test_router_iface_port_has_length_precondition_for_fixed_ips(): + """router_networks.ip_addresses must cover every router - a precondition + (not just a variable validation{}) gives a clear per-role error at plan + time instead of an out-of-range index crash.""" + main = load_tf("main.tf") + ports = dict(find_resources(main, "vkcs_networking_port")) + port = ports["router_iface_port"] + assert "lifecycle" in port, "router_iface_port must declare a lifecycle.precondition" + precondition = port["lifecycle"][0]["precondition"][0] + assert "fixed_ips" in precondition["condition"][0] + assert "length(" in precondition["condition"][0] # --------------------------------------------------------------------------- @@ -673,7 +780,7 @@ def test_tfvars_private_network_cidrs_do_not_overlap_and_have_room_for_routers() router_count = _configured_router_count() for net in networks: # ip_address is left unset on each port (Neutron IPAM auto-assigns - - # see router_priv_port in main.tf), so there's no fixed per-router + # see router_iface_port in main.tf), so there's no fixed per-router # offset to reserve room for - but VKCS auto-creates its own service # ports on the network (observed: one "network:dns" port consuming # an address), so there must be room for router_count routers plus @@ -684,6 +791,69 @@ def test_tfvars_private_network_cidrs_do_not_overlap_and_have_room_for_routers() ) +# --------------------------------------------------------------------------- +# mvm-s3.tfvars: sanity-check the externally-owned network/IP values shipped +# for this environment (no project-managed private networks at all here) +# --------------------------------------------------------------------------- + + +def _mvm_s3_router_count(): + tfvars_text = (TERRAFORM_DIR / "mvm-s3.tfvars").read_text() + for line in tfvars_text.splitlines(): + if line.strip().startswith("#"): + continue + m = re.match(r"^\s*router_count\s*=\s*(\d+)", line) + if m: + return int(m.group(1)) + return find_variable(load_tf("variables.tf"), "router_count")["default"][0] + + +def test_mvm_s3_tfvars_disables_project_managed_private_networks(): + tfvars = load_tf("mvm-s3.tfvars") + assert tfvars["private_network_cidrs"][0] == [], ( + "mvm-s3 must not create any project-managed private network - both " + "of its private interfaces come from router_networks instead" + ) + + +def test_mvm_s3_tfvars_router_networks_matches_the_diagram(): + """Regression guard: mvm-s3.tfvars must keep shipping exactly the + network/subnet UUIDs and per-router IPs from the admin's diagram.""" + tfvars = load_tf("mvm-s3.tfvars") + assert tfvars["router_networks"][0] == MVM_S3_ROUTER_NETWORKS + + +def test_mvm_s3_tfvars_router_networks_ip_addresses_cover_router_count(): + tfvars = load_tf("mvm-s3.tfvars") + router_networks = tfvars["router_networks"][0] + router_count = _mvm_s3_router_count() + assert router_networks, "mvm-s3.tfvars must set router_networks" + for role, net in router_networks.items(): + assert len(net["ip_addresses"]) >= router_count, ( + f"router_networks[{role!r}].ip_addresses has fewer entries " + f"than router_count={router_count}" + ) + + +def test_mvm_s3_tfvars_router_networks_ips_are_valid_and_dont_overlap(): + tfvars = load_tf("mvm-s3.tfvars") + router_networks = tfvars["router_networks"][0] + networks = [] + for role, net in router_networks.items(): + cidr = ipaddress.ip_network(net["cidr"]) + networks.append(cidr) + for ip in net["ip_addresses"]: + assert ipaddress.ip_address(ip) in cidr, ( + f"router_networks[{role!r}] ip {ip} does not fall inside {cidr}" + ) + assert len(net["ip_addresses"]) == len(set(net["ip_addresses"])), ( + f"router_networks[{role!r}].ip_addresses has duplicate entries" + ) + for i, a in enumerate(networks): + for b in networks[i + 1 :]: + assert not a.overlaps(b), f"{a} overlaps {b} in mvm-s3.tfvars router_networks" + + # --------------------------------------------------------------------------- # network-init.sh.tpl: only the intended Terraform interpolations remain # un-escaped, and the rendered result is syntactically valid bash diff --git a/terraform/variables.tf b/terraform/variables.tf index 523face..9be2d58 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -61,13 +61,9 @@ variable "router_availability_zones" { } variable "private_network_cidrs" { - description = "Explicit CIDR prefix for each private network that router VMs get an interface into. One entry = one shared private network = one private interface per router (list order determines eth1..ethN). Must be supplied explicitly - no auto-carving from a supernet." + description = "Explicit CIDR prefix for each project-managed private network that router VMs get an interface into (Terraform creates the network/subnet, Neutron IPAM assigns the address). One entry = one shared private network = one private interface per router. Optional - leave empty ([]) for a deployment that only uses var.router_networks (pre-existing externally-owned networks) for its private interfaces." type = list(string) - - validation { - condition = length(var.private_network_cidrs) >= 1 - error_message = "private_network_cidrs must contain at least one CIDR." - } + default = [] validation { condition = alltrue([for c in var.private_network_cidrs : can(cidrhost(c, 0))]) @@ -80,6 +76,40 @@ variable "private_network_cidrs" { } } +variable "router_networks" { + description = "Pre-existing private networks (typically owned by a different VK Cloud project, referenced by UUID only - not managed by this Terraform) that each router VM gets a fixed-IP interface into. Map key = role/interface name (e.g. \"primary\"/\"backup\"); ip_addresses[i] is the address for router(i+1). Optional - leave empty ({}) for a deployment that only uses var.private_network_cidrs for its private interfaces." + type = map(object({ + network_id = string + subnet_id = string + cidr = string + ip_addresses = list(string) + })) + default = {} + + validation { + condition = alltrue([for r in var.router_networks : can(cidrhost(r.cidr, 0))]) + error_message = "Every router_networks[*].cidr must be a valid IPv4 CIDR (e.g. \"172.16.252.8/29\")." + } + + validation { + condition = alltrue([ + for r in var.router_networks : alltrue([ + for ip in r.ip_addresses : can(cidrhost("${ip}/32", 0)) + ]) + ]) + error_message = "Every router_networks[*].ip_addresses entry must be a valid IPv4 address." + } + + validation { + condition = alltrue([ + for r in var.router_networks : alltrue([ + for ip in r.ip_addresses : cidrhost("${ip}/${split("/", r.cidr)[1]}", 0) == cidrhost(r.cidr, 0) + ]) + ]) + error_message = "Every router_networks[*].ip_addresses entry must fall inside that role's own cidr." + } +} + variable "default_security_group_id" { description = "Explicit override for the project's 'default' security group UUID. Last resort only - by default it's resolved dynamically via data.vkcs_networking_secgroup (see main.tf), since this UUID is unique per project and must not be hardcoded." type = string