From 2b1d33cbb9a12b2e6f4836d200126d3ca41e76e1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 3 Mar 2025 21:09:26 -0800 Subject: [PATCH] IPv6 maybe? --- infra/.terraform.lock.hcl | 135 +++++++++++++++++++------------------- infra/main.tf | 4 +- infra/relay-lb.tf | 23 ++++++- infra/relay.tf | 68 ++++++++++++++++--- infra/relay.yml.tpl | 2 +- 5 files changed, 153 insertions(+), 79 deletions(-) diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl index dc43c89..2044a3a 100644 --- a/infra/.terraform.lock.hcl +++ b/infra/.terraform.lock.hcl @@ -2,100 +2,101 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/google" { - version = "4.74.0" + version = "5.45.2" + constraints = ">= 4.50.0, ~> 5.0, < 6.0.0" hashes = [ - "h1:ghkjuvUrHsIlzjNL5KRsYZcP3R9BoFRfb0q069BXBi4=", - "zh:60904193c367b1ba9a3cb1bd86ca469ffcec2f7237e59adf4b0a34c84b2fa9ff", - "zh:6e5ac12f3fefc23907a94e5f6040118c978af76ab5deb60a5b80110c1c8ade09", - "zh:9fc0ae0f97ab598c27fae0a6b19e82c13fd59d020d7cdfeeebdbe41c4a8216ef", - "zh:aa2346dbd9f22e56d011b1b741e1829bc78633cc3da704d7c4e9636c314541fa", - "zh:ca692e666253cca77c6ff68423bf940c7f249a8b4f4af9b80c1a7079808b8ada", - "zh:cbaf541db823cff4379e294ad696bef2940c3eff863fa7fd340ea978872dfdaf", - "zh:d1f6fba2f64d51a804bf4f4e90b7809e26fa0539a1119e55907cc501add6a5d2", - "zh:d556680c85b8c90557b469d30a8082f056a2b724f812b97da6505a9d78139854", - "zh:ea7cd2d4fa940e3518221a67f24b2ff8d795c325fe8b95fd149ac0cc3e1e944a", - "zh:eca542a0e4caed8ab6a30eb0199755dfc2938083942a233d6a5ef60e204ac67c", + "h1:iy2Q9VcnMu4z/bH3v/NmI/nEpgYY7bXgJmT/hVTAUS4=", + "zh:0d09c8f20b556305192cdbe0efa6d333ceebba963a8ba91f9f1714b5a20c4b7a", + "zh:117143fc91be407874568df416b938a6896f94cb873f26bba279cedab646a804", + "zh:16ccf77d18dd2c5ef9c0625f9cf546ebdf3213c0a452f432204c69feed55081e", + "zh:3e555cf22a570a4bd247964671f421ed7517970cd9765ceb46f335edc2c6f392", + "zh:688bd5b05a75124da7ae6e885b2b92bd29f4261808b2b78bd5f51f525c1052ca", + "zh:6db3ef37a05010d82900bfffb3261c59a0c247e0692049cb3eb8c2ef16c9d7bf", + "zh:70316fde75f6a15d72749f66d994ccbdde5f5ed4311b6d06b99850f698c9bbf9", + "zh:84b8e583771a4f2bd514e519d98ed7fd28dce5efe0634e973170e1cfb5556fb4", + "zh:9d4b8ef0a9b6677935c604d94495042e68ff5489932cfd1ec41052e094a279d3", + "zh:a2089dd9bd825c107b148dd12d6b286f71aa37dfd4ca9c35157f2dcba7bc19d8", + "zh:f03d795c0fd9721e59839255ee7ba7414173017dc530b4ce566daf3802a0d6dd", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:f729b6f01050dd443b98cc5e7911102cdc8209e250cc1c2568aa37ba74b7c894", ] } provider "registry.terraform.io/hashicorp/google-beta" { - version = "5.10.0" + version = "5.45.2" constraints = ">= 4.50.0, < 6.0.0" hashes = [ - "h1:FbQG6/zQoZsAmErPjaDsu6snwopIKc9LqrLipyxgiPk=", - "zh:1004ac3733679254abcc7f5e9d594d9ee079cf071391a92f82b50077e07c70b5", - "zh:1e25af33d20b6ab369860d5b7c746b4a3b3dccc061b14dde91b6ccccfe704cc4", - "zh:2873a614a1dc1c460246edc95a558ad9befedf93490a0204bee8fb95362813cc", - "zh:2f421e13247b3822ef3c2e07e1aee948116a5064c386466a53fb72486daded20", - "zh:517c13cd146d3451789da8f13cbfa5355c3e88456cf762ad3918dada84a5f261", - "zh:56553ae44f4089f5149551714daaf3c97205d4638dd93b0675ed777476d56048", - "zh:6925a07bcb9ab70faa84bf36f87990025e3f9cd6c8cfab5260877f60086c8161", - "zh:72454b65ee4a24896d215f7f7af41e31336865c86d6c20ea4acb63596e75ac0d", - "zh:8b05f8a6ff51999bf65e3127618931647a00bc9abf739f0711151e4145cae3d5", - "zh:a3b7d3b39740088174d121bc7e4e3ce27da0ebf0c87877f8fce9277b0046c75b", + "h1:ME/cVZGNln4h166gyo9r7CuunzZ3FEqlIaNyQ0e9yjE=", + "zh:16b77bac5d1555b7f066ba8014f4fc8a6d0de64e252a1988d3fbb400984a4b19", + "zh:1b13f515c4809343840aed8265915cc4191f138bdab5a8c5e1f542fdfc69989f", + "zh:1dcce4309aeab7c88fd36aea664d57e620d8a413b967ce513a5a866e8de901f2", + "zh:24db65d7929f2a731e9cac1750c569cb4528b312ef182a5e2e8c0cf008d8a71b", + "zh:28c0b9e68d97570f03b2c4770607701580055bcba50069efd145954aa13b23e4", + "zh:3a898a1ad1569f6486a2bc20014087284c8cab919bc8f155833de5128ccd12eb", + "zh:4eed99cfb9daada70f813f2cedcf490d3097de1ccb9b391fc451ecc46509c067", + "zh:888c4cb1f13b23674ba1091835dd3f1bff5d8e7729ef302183d8d01233819e54", + "zh:8baae3b949f6e9505425f5fa4785de786e9cedc4c3f3ad906d8ed560bd2e39c6", + "zh:cf2c8928b764592fa2cd14a9f109d01cd0a92049a4fca9d0a74cf2fe588364e2", + "zh:edff09394f5bd0b278a4adc800a31b7f150249a1ea92ca273ccf4acd25be3f63", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:fe2af4fcda1b45d73ef8b8c728c150e00d1a4d5c0323b30d7d43c6f24ed78bcb", ] } provider "registry.terraform.io/hashicorp/random" { - version = "3.6.0" + version = "3.7.1" constraints = ">= 2.1.0" hashes = [ - "h1:I8MBeauYA8J8yheLJ8oSMWqB0kovn16dF/wKZ1QTdkk=", - "zh:03360ed3ecd31e8c5dac9c95fe0858be50f3e9a0d0c654b5e504109c2159287d", - "zh:1c67ac51254ba2a2bb53a25e8ae7e4d076103483f55f39b426ec55e47d1fe211", - "zh:24a17bba7f6d679538ff51b3a2f378cedadede97af8a1db7dad4fd8d6d50f829", - "zh:30ffb297ffd1633175d6545d37c2217e2cef9545a6e03946e514c59c0859b77d", - "zh:454ce4b3dbc73e6775f2f6605d45cee6e16c3872a2e66a2c97993d6e5cbd7055", + "h1:t152MY0tQH4a8fLzTtEWx70ITd3azVOrFDn/pQblbto=", + "zh:3193b89b43bf5805493e290374cdda5132578de6535f8009547c8b5d7a351585", + "zh:3218320de4be943e5812ed3de995946056db86eb8d03aa3f074e0c7316599bef", + "zh:419861805a37fa443e7d63b69fb3279926ccf98a79d256c422d5d82f0f387d1d", + "zh:4df9bd9d839b8fc11a3b8098a604b9b46e2235eb65ef15f4432bde0e175f9ca6", + "zh:5814be3f9c9cc39d2955d6f083bae793050d75c572e70ca11ccceb5517ced6b1", + "zh:63c6548a06de1231c8ee5570e42ca09c4b3db336578ded39b938f2156f06dd2e", + "zh:697e434c6bdee0502cc3deb098263b8dcd63948e8a96d61722811628dce2eba1", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:91df0a9fab329aff2ff4cf26797592eb7a3a90b4a0c04d64ce186654e0cc6e17", - "zh:aa57384b85622a9f7bfb5d4512ca88e61f22a9cea9f30febaa4c98c68ff0dc21", - "zh:c4a3e329ba786ffb6f2b694e1fd41d413a7010f3a53c20b432325a94fa71e839", - "zh:e2699bc9116447f96c53d55f2a00570f982e6f9935038c3810603572693712d0", - "zh:e747c0fd5d7684e5bfad8aa0ca441903f15ae7a98a737ff6aca24ba223207e2c", - "zh:f1ca75f417ce490368f047b63ec09fd003711ae48487fba90b4aba2ccf71920e", + "zh:a0b8e44927e6327852bbfdc9d408d802569367f1e22a95bcdd7181b1c3b07601", + "zh:b7d3af018683ef22794eea9c218bc72d7c35a2b3ede9233b69653b3c782ee436", + "zh:d63b911d618a6fe446c65bfc21e793a7663e934b2fef833d42d3ccd38dd8d68d", + "zh:fa985cd0b11e6d651f47cff3055f0a9fd085ec190b6dbe99bf5448174434cdea", ] } provider "registry.terraform.io/hashicorp/tls" { - version = "4.0.5" + version = "4.0.6" hashes = [ - "h1:zeG5RmggBZW/8JWIVrdaeSJa0OG62uFX5HY1eE8SjzY=", - "zh:01cfb11cb74654c003f6d4e32bbef8f5969ee2856394a96d127da4949c65153e", - "zh:0472ea1574026aa1e8ca82bb6df2c40cd0478e9336b7a8a64e652119a2fa4f32", - "zh:1a8ddba2b1550c5d02003ea5d6cdda2eef6870ece86c5619f33edd699c9dc14b", - "zh:1e3bb505c000adb12cdf60af5b08f0ed68bc3955b0d4d4a126db5ca4d429eb4a", - "zh:6636401b2463c25e03e68a6b786acf91a311c78444b1dc4f97c539f9f78de22a", - "zh:76858f9d8b460e7b2a338c477671d07286b0d287fd2d2e3214030ae8f61dd56e", - "zh:a13b69fb43cb8746793b3069c4d897bb18f454290b496f19d03c3387d1c9a2dc", - "zh:a90ca81bb9bb509063b736842250ecff0f886a91baae8de65c8430168001dad9", - "zh:c4de401395936e41234f1956ebadbd2ed9f414e6908f27d578614aaa529870d4", - "zh:c657e121af8fde19964482997f0de2d5173217274f6997e16389e7707ed8ece8", - "zh:d68b07a67fbd604c38ec9733069fbf23441436fecf554de6c75c032f82e1ef19", + "h1:n3M50qfWfRSpQV9Pwcvuse03pEizqrmYEryxKky4so4=", + "zh:10de0d8af02f2e578101688fd334da3849f56ea91b0d9bd5b1f7a243417fdda8", + "zh:37fc01f8b2bc9d5b055dc3e78bfd1beb7c42cfb776a4c81106e19c8911366297", + "zh:4578ca03d1dd0b7f572d96bd03f744be24c726bfd282173d54b100fd221608bb", + "zh:6c475491d1250050765a91a493ef330adc24689e8837a0f07da5a0e1269e11c1", + "zh:81bde94d53cdababa5b376bbc6947668be4c45ab655de7aa2e8e4736dfd52509", + "zh:abdce260840b7b050c4e401d4f75c7a199fafe58a8b213947a258f75ac18b3e8", + "zh:b754cebfc5184873840f16a642a7c9ef78c34dc246a8ae29e056c79939963c7a", + "zh:c928b66086078f9917aef0eec15982f2e337914c5c4dbc31dd4741403db7eb18", + "zh:cded27bee5f24de6f2ee0cfd1df46a7f88e84aaffc2ecbf3ff7094160f193d50", + "zh:d65eb3867e8f69aaf1b8bb53bd637c99c6b649ba3db16ded50fa9a01076d1a27", + "zh:ecb0c8b528c7a619fa71852bb3fb5c151d47576c5aab2bf3af4db52588722eeb", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } provider "registry.terraform.io/vancluever/acme" { - version = "2.19.0" + version = "2.30.1" constraints = "~> 2.0" hashes = [ - "h1:up+gI3vTyo/jrqODp7L3/r+1WT1RcHF2iTHKeL4c5x0=", - "zh:0362a3cd06e5180387f68f6a2b354014057b3efe3c032614654f6303e9295ce9", - "zh:1fca8dd9711f2ac7c62d84e96bd08a365bd33de1c3329c35fda8e57590e0305b", - "zh:22dfd6003158b48fe346f706f254781197331cc8378b5e6c76a70ecaee12e19b", - "zh:27d8b7fc3bad6dbf1b85cc4f0c0c09119603efaede8123a82d4e5b49d31e5205", - "zh:89f8ede926346968e75f48860a964bd4453800546fc46a4fdd5c5a43069b9b99", - "zh:912615105939c6ab65f4c633049b4528cc9fc3316aed2c78e1e8f620554bc40a", - "zh:9eb8091ea8f373a1644733550630f2b2a7d13c48b6868b4344a9508ce199b3b9", - "zh:ba24e6c3bade5a0d601e61950795067d23842c05df0b4618aed3a08a41e6fb0a", - "zh:d3df98345e0c33ca3e81a552e07558dc0107248f10217faae90e95a2d641f5d8", - "zh:e03ffe66fb9ecef8cc5f14729837ec801319f1cd4d5f80560563217b9111ec13", - "zh:e78b4dad910bd791b55f12058cf9f0bda122062cede330c3a2f7e50881e285cb", - "zh:ee153c54ef182aeadae9db3221771a8a3c3bf7f967863ea6d8bfa4f592081062", - "zh:f570a2b13da78a22764c9e626a8b8ce58615f07934d000b567ab12590ea155a1", + "h1:qaRhQGSgO3h8ElHnSU4i3s2hqomQ2Dd5LnOg25shKtU=", + "zh:02ee2fddb4a8afd90b7cb5fa44ee0ac40674d319dd31f754eab627c82b52fe42", + "zh:1e1b513f065bef7d242d75995c74b21274061352b90f137f047ff924b8307e1f", + "zh:3f6b95245b1c970d3bb33e4570c621d94ce37edb2ce11a9991eba6a9b7449909", + "zh:420f3957ba7e375340478c2c82c476fbfde2153ebd346119237ff94309c1deb7", + "zh:43a3a0a80d526b487fd30daaf1a6bc29feaf77426d77694263196c97e805705f", + "zh:560b861269018ecf9e195a3687a5635002cbca930f0e7386a1f02bbec8a828d7", + "zh:72c27405a7cd6970de410cb2d59df5848dd29d10ee7905372954cf9d0e5885d9", + "zh:794c7ef1d08716b59c1c87a468cfd7389c85119aa64abd2b6f6bd9b47c87841e", + "zh:c6105129827f2322e28c7aebf0a91eb7b08156c480ece93adfa647718ba06cd4", + "zh:d5b4dc8b69115aec777d6c21a0794b422feb7dc75477607b79e8f5fe27b3c099", + "zh:df22d4320930fe07c234337364994edfea7fb281fa7d39dfe27a7324534f0cc3", + "zh:e4ad5c74eefe4915dd08d5e9ff49f45a979153bef0a031c2e28c5567e57e53cb", + "zh:f749cc57a7c0381cfd42262a7d0820f911418198dac01be0ee5358935febcf31", ] } diff --git a/infra/main.tf b/infra/main.tf index 6bd4a1d..90205bc 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = "4.74.0" + version = "~> 5.0" } acme = { @@ -16,7 +16,7 @@ terraform { prefix = "terraform/state" } - required_version = ">= 0.14" + required_version = ">= 1.5" } provider "google" { diff --git a/infra/relay-lb.tf b/infra/relay-lb.tf index bb4f172..237fe42 100644 --- a/infra/relay-lb.tf +++ b/infra/relay-lb.tf @@ -1,4 +1,4 @@ -# Global Geo DNS, routing to the closest region. +# Global Geo DNS for IPv4, routing to the closest region. resource "google_dns_record_set" "relay_global" { name = "relay.${var.domain}." type = "A" @@ -19,6 +19,27 @@ resource "google_dns_record_set" "relay_global" { } } +# Global Geo DNS for IPv6, routing to the closest region. +resource "google_dns_record_set" "relay_global_ipv6" { + name = "relay.${var.domain}." + type = "AAAA" + ttl = 60 + managed_zone = google_dns_managed_zone.public.name + + routing_policy { + dynamic "geo" { + for_each = local.relays + + content { + location = geo.value.region + rrdatas = [ + google_compute_address.relay_ipv6[geo.key].address + ] + } + } + } +} + # Unfortunately GCP doesn't support global UDP load balancing despite their marketing. # oof there goes a few hours; here's my progress for posterity: # TODO We could still use this for regional load balancing. diff --git a/infra/relay.tf b/infra/relay.tf index ebb30d1..4580a46 100644 --- a/infra/relay.tf +++ b/infra/relay.tf @@ -1,3 +1,19 @@ +resource "google_compute_network" "relay" { + name = "relay" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "relay" { + for_each = local.relays + + name = "relay-${each.key}" + ip_cidr_range = "10.${index(keys(local.relays), each.key) + 1}.0.0/24" + region = each.value.region + network = google_compute_network.relay.id + stack_type = "IPV4_IPV6" + ipv6_access_type = "EXTERNAL" +} + resource "google_compute_instance" "relay" { for_each = local.relays @@ -19,10 +35,20 @@ resource "google_compute_instance" "relay" { } network_interface { - network = "default" + network = google_compute_network.relay.id + subnetwork = google_compute_subnetwork.relay[each.key].id + stack_type = "IPV4_IPV6" access_config { - nat_ip = google_compute_address.relay[each.key].address + nat_ip = google_compute_address.relay[each.key].address + network_tier = "PREMIUM" + public_ptr_domain_name = "relay.${each.key}.${var.domain}." + } + + ipv6_access_config { + network_tier = "PREMIUM" + public_ptr_domain_name = "relay.${each.key}.${var.domain}." + external_ipv6 = google_compute_address.relay_ipv6[each.key].address } } @@ -56,11 +82,6 @@ resource "google_compute_instance" "relay" { # For the firewall tags = ["relay"] - lifecycle { - # There seems to be a terraform bug causing this to be recreated on every apply - # ignore_changes = [boot_disk] - } - allow_stopping_for_update = true } @@ -69,6 +90,22 @@ resource "google_compute_address" "relay" { name = "relay-${each.key}" region = each.value.region + + address_type = "EXTERNAL" + ip_version = "IPV4" + network_tier = "PREMIUM" +} + +resource "google_compute_address" "relay_ipv6" { + for_each = local.relays + + name = "relay-${each.key}-ipv6" + region = each.value.region + address_type = "EXTERNAL" + ip_version = "IPV6" + ipv6_endpoint_type = "VM" + network_tier = "PREMIUM" + subnetwork = google_compute_subnetwork.relay[each.key].id } # Create a DNS entry for each node. @@ -85,7 +122,7 @@ resource "google_dns_record_set" "relay" { # Allow UDP 443 resource "google_compute_firewall" "relay" { name = "relay" - network = "default" + network = google_compute_network.relay.id allow { protocol = "udp" @@ -96,6 +133,21 @@ resource "google_compute_firewall" "relay" { target_tags = ["relay"] } +# Allow UDP 443 for IPv6 +resource "google_compute_firewall" "relay_ipv6" { + name = "relay-ipv6" + network = google_compute_network.relay.id + + allow { + protocol = "udp" + ports = ["443"] + } + + source_ranges = ["::/0"] + target_tags = ["relay"] +} + + # We must use a legacy health check for the UDP load balancer resource "google_compute_http_health_check" "relay" { name = "relay" diff --git a/infra/relay.yml.tpl b/infra/relay.yml.tpl index be8f501..11e3dfb 100644 --- a/infra/relay.yml.tpl +++ b/infra/relay.yml.tpl @@ -57,7 +57,7 @@ write_files: --cap-add=SYS_PTRACE \ -v "/etc/cert:/etc/cert:ro" \ -e RUST_LOG=debug -e RUST_BACKTRACE=1 \ - ${docker}/moq-relay --bind 0.0.0.0:443 \ + ${docker}/moq-relay --bind [::]:443 \ --tls-cert "/etc/cert/${cluster_node}.crt" --tls-key "/etc/cert/${cluster_node}.key" \ --tls-cert "/etc/cert/${public_host}.crt" --tls-key "/etc/cert/${public_host}.key" \ --tls-root "/etc/cert/internal.ca" \