Recover

A cascade stopped in the middle

What happened

A subnet re-addressing was applied. Two of the three subnets went through, the third failed part-way, and none of the workers were touched. The configuration, the state file and the cloud now disagree in three different ways. Converge.

hard10 min

The apply

these 3 ran at the same time

cloud_subnet.private["a"]replaced7s simulated
cloud_subnet.private["b"]replaced7s simulated
cloud_subnet.private["c"]failed while replacing2s simulated

the provider refused: that range overlaps a subnet in another account

these 4 ran at the same time

cloud_instance.worker[0]would have been replacednever attempted
cloud_instance.worker[1]would have been replacednever attempted
cloud_instance.worker[2]would have been replacednever attempted
cloud_lb.publicnothing to donever attempted

cloud_dns_record.apinothing to donever attempted

unavailable for: 9s simulated
never attempted: 5

The state file

serial: 18
lineage: world-orders
resources: 15

1 value is in this file in plain text, masked here and not in the file itself.

cloud_bucket.uploads

arn: "arn:cloud:s3:eu-west-1:000000000000:orders-uploads-eu-west-1"
bucket: "orders-uploads-eu-west-1"
force_destroy: false
id: "orders-uploads-eu-west-1"
object_lock_enabled: false
tags: {environment = "prod", managed_by = "opentofu", team = "orders"}

cloud_db_instance.primary

allocated_storage: 100
arn: "arn:cloud:rds:eu-west-1:000000000000:db:orders-primary"
availability_zone: "eu-west-1a"
backup_retention_period: 7
db_name: "orders"
db_subnet_group_name: "orders-private"
endpoint: "orders-primary.83d691363f86.eu-west-1.cloud-rds.internal:5432"
engine: "postgres"
engine_version: "16.4"
id: "orders-primary"
identifier: "orders-primary"
instance_class: "db.r6g.large"
multi_az: false
password: ••••••••
skip_final_snapshot: false
storage_encrypted: true
tags: {environment = "prod", managed_by = "opentofu", team = "orders"}
username: "app"

cloud_iam_policy.bucket_access

arn: "arn:cloud:iam:eu-west-1:000000000000:policy/orders-bucket-access"
description: "Read and write the orders bucket"
id: "arn:cloud:iam:eu-west-1:000000000000:policy/orders-bucket-access"
name: "orders-bucket-access"
path: "/"
policy: "{\"Version\":\"2012-10-17\",\"Statement\":[]}"
tags: {environment = "prod", managed_by = "opentofu", team = "orders"}

cloud_iam_role.application

arn: "arn:cloud:iam:eu-west-1:000000000000:role/orders-application"
assume_role_policy: "{\"Version\":\"2012-10-17\",\"Statement\":[]}"
description: "The orders service"
id: "orders-application"
max_session_duration: 3600
name: "orders-application"
path: "/"
tags: {environment = "prod", managed_by = "opentofu", team = "orders"}

cloud_vpc.main

arn: "arn:cloud:ec2:eu-west-1:000000000000:vpc/vpc-5ad440a3"
cidr_block: "10.0.0.0/16"
enable_dns_hostnames: true
enable_dns_support: true
id: "vpc-f2bc8340"
instance_tenancy: "default"
tags: {environment = "prod", managed_by = "opentofu", team = "orders", name = "orders-main"}

cloud_security_group.database

arn: "arn:cloud:ec2:eu-west-1:000000000000:security-group/sg-2ff112a6"
description: "Managed by Terraform"
egress: []
id: "sg-9905d588"
ingress: []
name: "orders-database"
tags: {environment = "prod", managed_by = "opentofu", team = "orders"}
vpc_id: "vpc-f2bc8340"

cloud_security_group.web

arn: "arn:cloud:ec2:eu-west-1:000000000000:security-group/sg-4eb879ab"
description: "Managed by Terraform"
egress: []
id: "sg-dc3c10ff"
ingress: []
name: "orders-web"
tags: {environment = "prod", managed_by = "opentofu", team = "orders"}
vpc_id: "vpc-f2bc8340"

cloud_subnet.private["a"]

arn: "arn:cloud:ec2:eu-west-1:000000000000:subnet/subnet-63abdf26"
availability_zone: "eu-west-1a"
cidr_block: "10.0.11.0/24"
id: "subnet-0a8face4"
map_public_ip_on_launch: false
tags: {environment = "prod", managed_by = "opentofu", team = "orders", name = "orders-private-a"}
vpc_id: "vpc-f2bc8340"

cloud_subnet.private["b"]

arn: "arn:cloud:ec2:eu-west-1:000000000000:subnet/subnet-d985d9b4"
availability_zone: "eu-west-1b"
cidr_block: "10.0.12.0/24"
id: "subnet-ce71cabd"
map_public_ip_on_launch: false
tags: {environment = "prod", managed_by = "opentofu", team = "orders", name = "orders-private-b"}
vpc_id: "vpc-f2bc8340"

cloud_subnet.private["c"]an apply died while changing this

arn: "arn:cloud:ec2:eu-west-1:000000000000:subnet/subnet-c206fc26"
availability_zone: "eu-west-1c"
cidr_block: "10.0.3.0/24"
id: "subnet-9486b397"
map_public_ip_on_launch: false
tags: {environment = "prod", managed_by = "opentofu", team = "orders", name = "orders-private-c"}
vpc_id: "vpc-f2bc8340"

cloud_instance.worker[0]

ami: "ami-0a1b2c3d4e5f60718"
arn: "arn:cloud:ec2:eu-west-1:000000000000:instance/i-edc72b4be6c41f445"
availability_zone: "eu-west-1a"
id: "i-740986cf9622122c3"
instance_type: "t3.medium"
key_name: null
monitoring: false
private_ip: "10.0.1.226"
public_ip: "52.195.252.190"
subnet_id: "subnet-c8249f67"
tags: {environment = "prod", managed_by = "opentofu", team = "orders", name = "orders-worker"}
user_data: null
vpc_security_group_ids: ["sg-dc3c10ff"]

cloud_instance.worker[1]

ami: "ami-0a1b2c3d4e5f60718"
arn: "arn:cloud:ec2:eu-west-1:000000000000:instance/i-767b78828f7099885"
availability_zone: "eu-west-1a"
id: "i-6f731e0b1836039e6"
instance_type: "t3.medium"
key_name: null
monitoring: false
private_ip: "10.0.1.16"
public_ip: "52.189.23.140"
subnet_id: "subnet-c8249f67"
tags: {environment = "prod", managed_by = "opentofu", team = "orders", name = "orders-worker"}
user_data: null
vpc_security_group_ids: ["sg-dc3c10ff"]

cloud_instance.worker[2]

ami: "ami-0a1b2c3d4e5f60718"
arn: "arn:cloud:ec2:eu-west-1:000000000000:instance/i-24fd4c9fa06d5994c"
availability_zone: "eu-west-1a"
id: "i-0ef9be0db515c57c0"
instance_type: "t3.medium"
key_name: null
monitoring: false
private_ip: "10.0.1.41"
public_ip: "52.223.134.146"
subnet_id: "subnet-c8249f67"
tags: {environment = "prod", managed_by = "opentofu", team = "orders", name = "orders-worker"}
user_data: null
vpc_security_group_ids: ["sg-dc3c10ff"]

cloud_lb.public

arn: "arn:cloud:elasticloadbalancing:eu-west-1:000000000000:loadbalancer/app/orders-public/dd83d73d5d75ca7d"
dns_name: "orders-public-133879307.eu-west-1.cloud-elb.internal"
enable_deletion_protection: true
id: "orders-public"
internal: false
load_balancer_type: "application"
name: "orders-public"
security_groups: ["sg-dc3c10ff"]
subnets: ["subnet-c8249f67", "subnet-c923d137", "subnet-9486b397"]
tags: {environment = "prod", managed_by = "opentofu", team = "orders"}
zone_id: "Z421B1B24846BA"

cloud_dns_record.api

fqdn: "api.orders.example.com"
id: "api.orders.example.com_CNAME"
name: "api.orders.example.com"
records: ["orders-public-133879307.eu-west-1.cloud-elb.internal"]
ttl: 300
type: "CNAME"
zone_id: "Z0SAMPLEZONEID01"

The configuration that was being applied

variable "environment" {
  type    = string
  default = "prod"
}

locals {
  common_tags = {
    environment = var.environment
    managed_by  = "opentofu"
    team        = "orders"
  }

# Which /24 each zone gets. Written out rather than derived, because
  # deriving it from the key is how three subnets end up sharing a range.
  subnet_numbers = {
    a = 11
    b = 12
    c = 13
  }
}

resource "cloud_vpc" "main" {
  cidr_block           = "10.0.0.0/16"
  enable_dns_hostnames = true
  tags                 = merge(local.common_tags, { name = "orders-main" })
}

resource "cloud_subnet" "private" {
  for_each = toset(["a", "b", "c"])

vpc_id                  = cloud_vpc.main.id
  cidr_block              = cidrsubnet(cloud_vpc.main.cidr_block, 8, lookup(local.subnet_numbers, each.key, 0))
  availability_zone       = "eu-west-1${each.key}"
  map_public_ip_on_launch = false
  tags                    = merge(local.common_tags, { name = "orders-private-${each.key}" })
}

resource "cloud_security_group" "database" {
  name        = "orders-database"
  description = "Managed by Terraform"
  vpc_id      = cloud_vpc.main.id
  tags        = local.common_tags
}

resource "cloud_security_group" "web" {
  name        = "orders-web"
  description = "Managed by Terraform"
  vpc_id      = cloud_vpc.main.id
  tags        = local.common_tags
}

resource "cloud_iam_role" "application" {
  name               = "orders-application"
  assume_role_policy = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"
  description        = "The orders service"
  tags               = local.common_tags
}

resource "cloud_iam_policy" "bucket_access" {
  name        = "orders-bucket-access"
  policy      = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"
  description = "Read and write the orders bucket"
  tags        = local.common_tags
}

resource "cloud_bucket" "uploads" {
  bucket        = "orders-uploads-eu-west-1"
  force_destroy = false
  tags          = local.common_tags
}

resource "cloud_db_instance" "primary" {
  identifier              = "orders-primary"
  instance_class          = "db.r6g.large"
  engine                  = "postgres"
  engine_version          = "16.4"
  username                = "app"
  password                = "correct-horse-battery-staple"
  db_name                 = "orders"
  allocated_storage       = 100
  storage_encrypted       = true
  availability_zone       = "eu-west-1a"
  multi_az                = false
  db_subnet_group_name    = "orders-private"
  backup_retention_period = 7
  skip_final_snapshot     = false
  tags                    = local.common_tags
}

resource "cloud_instance" "worker" {
  count = 3

ami                    = "ami-0a1b2c3d4e5f60718"
  instance_type          = "t3.medium"
  subnet_id              = cloud_subnet.private["a"].id
  availability_zone      = "eu-west-1a"
  vpc_security_group_ids = [cloud_security_group.web.id]
  monitoring             = false
  tags                   = merge(local.common_tags, { name = "orders-worker" })
}

resource "cloud_lb" "public" {
  name                       = "orders-public"
  internal                   = false
  load_balancer_type         = "application"
  subnets                    = [for s in cloud_subnet.private : s.id]
  security_groups            = [cloud_security_group.web.id]
  enable_deletion_protection = true
  tags                       = local.common_tags
}

resource "cloud_dns_record" "api" {
  zone_id = "Z0SAMPLEZONEID01"
  name    = "api.orders.example.com"
  type    = "CNAME"
  ttl     = 300
  records = [cloud_lb.public.dns_name]
}

# __APPEND__

The plan from here

Note: objects have changed outside of OpenTofu
 
  # cloud_subnet.private["c"] has changed
  ~ resource "cloud_subnet" "private" {
      - id                      = "subnet-9486b397" -> null
      - arn                     = "arn:cloud:ec2:eu-west-1:000000000000:subnet/subnet-c206fc26" -> null
      - vpc_id                  = "vpc-f2bc8340" -> null
      - cidr_block              = "10.0.3.0/24" -> null
      - availability_zone       = "eu-west-1c" -> null
      - map_public_ip_on_launch = false -> null
      - tags                    = {environment = "prod", managed_by = "opentofu", team = "orders", name = "orders-private-c"} -> null
    }
 
Unless you have made equivalent changes to your configuration, or ignored the relevant attributes using ignore_changes, the following plan may include actions to undo or respond to these changes.
 
  # cloud_instance.worker[0] must be replaced
-/+ resource "cloud_instance" "worker" {
      ~ id                     = "i-740986cf9622122c3"
      ~ arn                    = "arn:cloud:ec2:eu-west-1:000000000000:instance/i-edc72b4be6c41f445"
      ~ ami                    = "ami-0a1b2c3d4e5f60718"
      ~ instance_type          = "t3.medium"
      ~ subnet_id              = "subnet-c8249f67" -> "subnet-0a8face4" # forces replacement
      ~ availability_zone      = "eu-west-1a"
      ~ key_name               = null
      ~ vpc_security_group_ids = ["sg-dc3c10ff"]
      ~ user_data              = null
      ~ monitoring             = false
      ~ private_ip             = "10.0.1.226"
      ~ public_ip              = "52.195.252.190"
      ~ tags                   = {environment = "prod", managed_by = "opentofu", team = "orders", name = "orders-worker"}
    }
 
  # cloud_instance.worker[1] must be replaced
-/+ resource "cloud_instance" "worker" {
      ~ id                     = "i-6f731e0b1836039e6"
      ~ arn                    = "arn:cloud:ec2:eu-west-1:000000000000:instance/i-767b78828f7099885"
      ~ ami                    = "ami-0a1b2c3d4e5f60718"
      ~ instance_type          = "t3.medium"
      ~ subnet_id              = "subnet-c8249f67" -> "subnet-0a8face4" # forces replacement
      ~ availability_zone      = "eu-west-1a"
      ~ key_name               = null
      ~ vpc_security_group_ids = ["sg-dc3c10ff"]
      ~ user_data              = null
      ~ monitoring             = false
      ~ private_ip             = "10.0.1.16"
      ~ public_ip              = "52.189.23.140"
      ~ tags                   = {environment = "prod", managed_by = "opentofu", team = "orders", name = "orders-worker"}
    }
 
  # cloud_instance.worker[2] must be replaced
-/+ resource "cloud_instance" "worker" {
      ~ id                     = "i-0ef9be0db515c57c0"
      ~ arn                    = "arn:cloud:ec2:eu-west-1:000000000000:instance/i-24fd4c9fa06d5994c"
      ~ ami                    = "ami-0a1b2c3d4e5f60718"
      ~ instance_type          = "t3.medium"
      ~ subnet_id              = "subnet-c8249f67" -> "subnet-0a8face4" # forces replacement
      ~ availability_zone      = "eu-west-1a"
      ~ key_name               = null
      ~ vpc_security_group_ids = ["sg-dc3c10ff"]
      ~ user_data              = null
      ~ monitoring             = false
      ~ private_ip             = "10.0.1.41"
      ~ public_ip              = "52.223.134.146"
      ~ tags                   = {environment = "prod", managed_by = "opentofu", team = "orders", name = "orders-worker"}
    }
 
  ~ resource "cloud_lb" "public" {
      ~ id                         = "orders-public"
      ~ arn                        = "arn:cloud:elasticloadbalancing:eu-west-1:000000000000:loadbalancer/app/orders-public/dd83d73d5d75ca7d"
      ~ name                       = "orders-public"
      ~ internal                   = false
      ~ load_balancer_type         = "application"
      ~ subnets                    = ["subnet-c8249f67", "subnet-c923d137", "subnet-9486b397"] -> ["subnet-0a8face4", "subnet-ce71cabd", "subnet-9486b397"]
      ~ security_groups            = ["sg-dc3c10ff"]
      ~ enable_deletion_protection = true
      ~ dns_name                   = "orders-public-133879307.eu-west-1.cloud-elb.internal"
      ~ zone_id                    = "Z421B1B24846BA"
      ~ tags                       = {environment = "prod", managed_by = "opentofu", team = "orders"}
    }
 
  # cloud_subnet.private["c"] will be created
  + resource "cloud_subnet" "private" {
      + id                      = "subnet-9486b397"
      + arn                     = "arn:cloud:ec2:eu-west-1:000000000000:subnet/subnet-c206fc26"
      + vpc_id                  = "vpc-f2bc8340"
      + cidr_block              = "10.0.13.0/24"
      + availability_zone       = "eu-west-1c"
      + map_public_ip_on_launch = false
      + tags                    = {environment = "prod", managed_by = "opentofu", team = "orders", name = "orders-private-c"}
    }
 
Plan: 4 to add, 1 to change, 3 to destroy.

The plan above is computed from where things actually are, not from where they were meant to be.