fix: firewall rules for autopilot clusters are ineffective. add cluster_network_tag to autopilot cluster network_tags if firewalls are toggled on (#1817)

GorginZ · apeabody · web-flow · commit e7b20cd19318 · 2024-06-20T09:27:42.000-07:00
Co-authored-by: Andrew Peabody &lt;andrewpeabody@google.com&gt;
diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl
@@ -258,10 +258,10 @@ resource "google_container_cluster" "primary" {
   }
 {% if autopilot_cluster %}
   dynamic "node_pool_auto_config" {
-    for_each = length(var.network_tags) > 0 ? [1] : []
+    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : []
     content {
       network_tags {
-        tags = var.network_tags
+        tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags
       }
     }
   }
diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
@@ -454,6 +454,27 @@ steps:
     - verify simple-autopilot-private-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestSimpleAutopilotPrivate --stage teardown --verbose']
+- id: init autopilot-private-firewalls
+  waitFor:
+    - create-all
+    - destroy simple-autopilot-private-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestAutopilotPrivateFirewalls --stage init --verbose']
+- id: apply autopilot-private-firewalls
+  waitFor:
+    - init autopilot-private-firewalls
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestAutopilotPrivateFirewalls --stage apply --verbose']
+- id: verify autopilot-private-firewalls
+  waitFor:
+    - apply autopilot-private-firewalls
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestAutopilotPrivateFirewalls --stage verify --verbose']
+- id: destroy autopilot-private-firewalls
+  waitFor:
+    - verify autopilot-private-firewalls
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestAutopilotPrivateFirewalls --stage destroy --verbose']
 - id: init simple-autopilot-public-local
   waitFor:
     - create-all
diff --git a/examples/autopilot_private_firewalls/README.md b/examples/autopilot_private_firewalls/README.md
@@ -0,0 +1,42 @@
+# Private Regional Autopilot Cluster With Firewall Rules
+
+This example creates a regional autopilot cluster with beta features
+
+It will:
+- Create a private autopilot cluster
+- Variables ```add_cluster_firewall_rules```, ```add_master_webhook_firewall_rules``` and ```add_shadow_firewal_rules``` are toggled on. In a environment with stringent firewall rules, these cluster firewall rules may be required to allow intra cluster communication
+- Adds an example network tag. This example network tag aligns with a firewall rule target tag from the [terraform-example-foundation](https://github.com/terraform-google-modules/terraform-example-foundation) configuration to allow google api access
+
+>note: this example does **not** create a network with stringent firewall rules. This example shows what cluster configuration may be required for a networking configuration comparable to the [terraform-example-foundation](https://github.com/terraform-google-modules/terraform-example-foundation)'s base shared VPCs, or any VPC with firewall rules to deny 0.0.0.0/0 egress and an allow rule for private google api egress access which targets tags "allow-google-apis". By toggling on the firewall rules variables and adding network_tags to allow private google api access, a private cluster can come up healthy with no internet egress. This configuration shows how to ensure those firewalls that explicitly allow intra cluster ingress and egress are created and appropriate network tags are attached to the cluster.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| project\_id | The project ID to host the cluster in | `any` | n/a | yes |
+| region | The region the cluster in | `string` | `"us-central1"` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| ca\_certificate | The cluster ca certificate (base64 encoded) |
+| cluster\_name | Cluster name |
+| kubernetes\_endpoint | The cluster endpoint |
+| location | n/a |
+| master\_kubernetes\_version | Kubernetes version of the master |
+| network\_name | The name of the VPC being created |
+| project\_id | The project ID the cluster is in |
+| region | The region in which the cluster resides |
+| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| subnet\_names | The names of the subnet being created |
+| zones | List of zones in which the cluster resides |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+To provision this example, run the following from within this directory:
+- `terraform init` to get the plugins
+- `terraform plan` to see the infrastructure plan
+- `terraform apply` to apply the infrastructure build
+- `terraform destroy` to destroy the built infrastructure
diff --git a/examples/autopilot_private_firewalls/main.tf b/examples/autopilot_private_firewalls/main.tf
@@ -0,0 +1,63 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  cluster_type           = "autopilot-private-firewalls"
+  network_name           = "${local.cluster_type}-network"
+  subnet_name            = "${local.cluster_type}-subnet"
+  master_auth_subnetwork = "${local.cluster_type}-master-subnet"
+  pods_range_name        = "ip-range-pods-${local.cluster_type}"
+  svc_range_name         = "ip-range-svc-${local.cluster_type}"
+  subnet_names           = [for subnet_self_link in module.gcp-network.subnets_self_links : split("/", subnet_self_link)[length(split("/", subnet_self_link)) - 1]]
+}
+
+data "google_client_config" "default" {}
+
+provider "kubernetes" {
+  host                   = "https://${module.gke.endpoint}"
+  token                  = data.google_client_config.default.access_token
+  cluster_ca_certificate = base64decode(module.gke.ca_certificate)
+}
+
+module "gke" {
+  source                            = "../../modules/beta-autopilot-private-cluster/"
+  project_id                        = var.project_id
+  name                              = "${local.cluster_type}-cluster"
+  regional                          = true
+  region                            = var.region
+  network                           = module.gcp-network.network_name
+  subnetwork                        = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)]
+  ip_range_pods                     = local.pods_range_name
+  ip_range_services                 = local.svc_range_name
+  release_channel                   = "REGULAR"
+  enable_vertical_pod_autoscaling   = true
+  enable_private_endpoint           = true
+  enable_private_nodes              = true
+  master_ipv4_cidr_block            = "172.16.0.0/28"
+  add_cluster_firewall_rules        = true
+  add_master_webhook_firewall_rules = true
+  add_shadow_firewall_rules         = true
+  network_tags                      = ["allow-google-apis"]
+  deletion_protection               = false
+
+  master_authorized_networks = [
+    {
+      cidr_block   = "10.60.0.0/17"
+      display_name = "VPC"
+    },
+  ]
+}
+
diff --git a/examples/autopilot_private_firewalls/network.tf b/examples/autopilot_private_firewalls/network.tf
@@ -0,0 +1,49 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "gcp-network" {
+  source  = "terraform-google-modules/network/google"
+  version = ">= 9.0.0"
+
+  project_id   = var.project_id
+  network_name = local.network_name
+
+  subnets = [
+    {
+      subnet_name   = local.subnet_name
+      subnet_ip     = "10.0.0.0/17"
+      subnet_region = var.region
+    },
+    {
+      subnet_name   = local.master_auth_subnetwork
+      subnet_ip     = "10.60.0.0/17"
+      subnet_region = var.region
+    },
+  ]
+
+  secondary_ranges = {
+    (local.subnet_name) = [
+      {
+        range_name    = local.pods_range_name
+        ip_cidr_range = "192.168.0.0/18"
+      },
+      {
+        range_name    = local.svc_range_name
+        ip_cidr_range = "192.168.64.0/18"
+      },
+    ]
+  }
+}
diff --git a/examples/autopilot_private_firewalls/outputs.tf b/examples/autopilot_private_firewalls/outputs.tf
@@ -0,0 +1,71 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "kubernetes_endpoint" {
+  description = "The cluster endpoint"
+  sensitive   = true
+  value       = module.gke.endpoint
+}
+
+output "cluster_name" {
+  description = "Cluster name"
+  value       = module.gke.name
+}
+
+output "location" {
+  value = module.gke.location
+}
+
+output "master_kubernetes_version" {
+  description = "Kubernetes version of the master"
+  value       = module.gke.master_version
+}
+
+output "ca_certificate" {
+  sensitive   = true
+  description = "The cluster ca certificate (base64 encoded)"
+  value       = module.gke.ca_certificate
+}
+
+output "service_account" {
+  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  value       = module.gke.service_account
+}
+
+output "network_name" {
+  description = "The name of the VPC being created"
+  value       = module.gcp-network.network_name
+}
+
+output "subnet_names" {
+  description = "The names of the subnet being created"
+  value       = module.gcp-network.subnets_names
+}
+
+output "region" {
+  description = "The region in which the cluster resides"
+  value       = module.gke.region
+}
+
+output "zones" {
+  description = "List of zones in which the cluster resides"
+  value       = module.gke.zones
+}
+
+output "project_id" {
+  description = "The project ID the cluster is in"
+  value       = var.project_id
+}
diff --git a/examples/autopilot_private_firewalls/variables.tf b/examples/autopilot_private_firewalls/variables.tf
@@ -0,0 +1,24 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "The project ID to host the cluster in"
+}
+
+variable "region" {
+  description = "The region the cluster in"
+  default     = "us-central1"
+}
diff --git a/examples/autopilot_private_firewalls/versions.tf b/examples/autopilot_private_firewalls/versions.tf
@@ -0,0 +1,27 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  required_providers {
+    google = {
+      source = "hashicorp/google"
+    }
+    kubernetes = {
+      source = "hashicorp/kubernetes"
+    }
+  }
+  required_version = ">= 0.13"
+}
diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf
@@ -99,10 +99,10 @@ resource "google_container_cluster" "primary" {
     }
   }
   dynamic "node_pool_auto_config" {
-    for_each = length(var.network_tags) > 0 ? [1] : []
+    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : []
     content {
       network_tags {
-        tags = var.network_tags
+        tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags
       }
     }
   }
diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf
@@ -99,10 +99,10 @@ resource "google_container_cluster" "primary" {
     }
   }
   dynamic "node_pool_auto_config" {
-    for_each = length(var.network_tags) > 0 ? [1] : []
+    for_each = length(var.network_tags) > 0 || var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? [1] : []
     content {
       network_tags {
-        tags = var.network_tags
+        tags = var.add_cluster_firewall_rules || var.add_master_webhook_firewall_rules || var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags
       }
     }
   }
diff --git a/test/fixtures/autopilot_private_firewalls/example.tf b/test/fixtures/autopilot_private_firewalls/example.tf
@@ -0,0 +1,22 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "example" {
+  source = "../../../examples/autopilot_private_firewalls"
+
+  project_id = var.project_ids[0]
+  region     = var.region
+}
diff --git a/test/fixtures/autopilot_private_firewalls/outputs.tf b/test/fixtures/autopilot_private_firewalls/outputs.tf
diff --git a/test/fixtures/autopilot_private_firewalls/variables.tf b/test/fixtures/autopilot_private_firewalls/variables.tf
diff --git a/test/integration/autopilot_private_firewalls/autopilot_private_firewalls_test.go b/test/integration/autopilot_private_firewalls/autopilot_private_firewalls_test.go
diff --git a/test/integration/autopilot_private_firewalls/testdata/TestAutopilotPrivateFirewalls.json b/test/integration/autopilot_private_firewalls/testdata/TestAutopilotPrivateFirewalls.json

Original file line number	Diff line number	Diff line change
`@@ -258,10 +258,10 @@ resource "google_container_cluster" "primary" {`
`258`	`258`	`}`
`259`	`259`	`{% if autopilot_cluster %}`
`260`	`260`	`dynamic "node_pool_auto_config" {`
`261`		`- for_each = length(var.network_tags) > 0 ? [1] : []`
	`261`	`+ for_each = length(var.network_tags) > 0 \|\| var.add_cluster_firewall_rules \|\| var.add_master_webhook_firewall_rules \|\| var.add_shadow_firewall_rules ? [1] : []`
`262`	`262`	`content {`
`263`	`263`	`network_tags {`
`264`		`- tags = var.network_tags`
	`264`	`+ tags = var.add_cluster_firewall_rules \|\| var.add_master_webhook_firewall_rules \|\| var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags`
`265`	`265`	`}`
`266`	`266`	`}`
`267`	`267`	`}`
Original file line number	Diff line number	Diff line change
`@@ -99,10 +99,10 @@ resource "google_container_cluster" "primary" {`
`99`	`99`	`}`
`100`	`100`	`}`
`101`	`101`	`dynamic "node_pool_auto_config" {`
`102`		`- for_each = length(var.network_tags) > 0 ? [1] : []`
	`102`	`+ for_each = length(var.network_tags) > 0 \|\| var.add_cluster_firewall_rules \|\| var.add_master_webhook_firewall_rules \|\| var.add_shadow_firewall_rules ? [1] : []`
`103`	`103`	`content {`
`104`	`104`	`network_tags {`
`105`		`- tags = var.network_tags`
	`105`	`+ tags = var.add_cluster_firewall_rules \|\| var.add_master_webhook_firewall_rules \|\| var.add_shadow_firewall_rules ? concat(var.network_tags, [local.cluster_network_tag]) : var.network_tags`
`106`	`106`	`}`
`107`	`107`	`}`
`108`	`108`	`}`