feat: ACM - Wait for gatekeeper & Hub: expose module_depends_on (#689)

* wait for gatekeeper if policy controller is enabled

* destroy command

* forgot the var

* use deployment ready

* add context

* more wait

* webhook

* depends_on

* concat modules

* depends on

* terraform format

* update docs

* shell check

Author: cloud-pharaoh

Diff for: modules/acm/README.md

@@ -68,5 +68,6 @@ By default, this module will attempt to download the ACM operator from Google di
 | Name | Description |
 |------|-------------|
 | git\_creds\_public | Public key of SSH keypair to allow the Anthos Config Management Operator to authenticate to your Git repository. |
+| wait | An output to use when you want to depend on cmd finishing |
 
  <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

Diff for: modules/acm/outputs.tf

@@ -18,3 +18,8 @@ output "git_creds_public" {
   description = "Public key of SSH keypair to allow the Anthos Config Management Operator to authenticate to your Git repository."
   value       = module.acm_operator.git_creds_public
 }
+
+output "wait" {
+  description = "An output to use when you want to depend on cmd finishing"
+  value       = module.acm_operator.wait
+}

Diff for: modules/hub/README.md

@@ -40,6 +40,7 @@ To deploy this config:
 | gke\_hub\_membership\_name | Memebership name that uniquely represents the cluster being registered on the Hub | string | `"gke-hub-membership"` | no |
 | gke\_hub\_sa\_name | Name for the GKE Hub SA stored as a secret `creds-gcp` in the `gke-connect` namespace. | string | `"gke-hub-sa"` | no |
 | location | The location (zone or region) this cluster has been created in. | string | n/a | yes |
+| module\_depends\_on | List of modules or resources this module depends on. | list | `<list>` | no |
 | project\_id | The project in which the resource belongs. | string | n/a | yes |
 | sa\_private\_key | Private key for service account base64 encoded. Required only if `use_existing_sa` is set to `true`. | string | `"null"` | no |
 | skip\_gcloud\_download | Whether to skip downloading gcloud (assumes gcloud and kubectl already available outside the module) | bool | `"true"` | no |

Diff for: modules/hub/main.tf

@@ -49,7 +49,7 @@ module "gke_hub_registration" {
   skip_download                     = var.skip_gcloud_download
   upgrade                           = true
   use_tf_google_credentials_env_var = var.use_tf_google_credentials_env_var
-  module_depends_on                 = [var.cluster_endpoint]
+  module_depends_on                 = concat([var.cluster_endpoint], var.module_depends_on)
 
   create_cmd_entrypoint  = "${path.module}/scripts/gke_hub_registration.sh"
   create_cmd_body        = "${var.gke_hub_membership_name} ${var.location} ${var.cluster_name} ${local.gke_hub_sa_key} ${var.project_id}"

Diff for: modules/hub/variables.tf

@@ -81,3 +81,9 @@ variable "sa_private_key" {
   type        = string
   default     = null
 }
+
+variable "module_depends_on" {
+  description = "List of modules or resources this module depends on."
+  type        = list
+  default     = []
+}

Diff for: modules/k8s-operator-crd-support/main.tf

@@ -113,3 +113,19 @@ module "k8sop_config" {
   kubectl_create_command  = "kubectl apply -f ${local_file.operator_cr.filename}"
   kubectl_destroy_command = "kubectl delete -f ${local_file.operator_cr.filename}"
 }
+
+module "wait_for_gatekeeper" {
+  source                   = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper"
+  version                  = "~> 2.0.2"
+  enabled                  = var.enable_policy_controller ? true : false
+  module_depends_on        = [module.k8sop_config.wait]
+  skip_download            = var.skip_gcloud_download
+  cluster_name             = var.cluster_name
+  cluster_location         = var.location
+  project_id               = var.project_id
+  create_cmd_triggers      = { script_sha1 = sha1(file("${path.module}/scripts/wait_for_gatekeeper.sh")) }
+  service_account_key_file = var.service_account_key_file
+
+  kubectl_create_command  = "${path.module}/scripts/wait_for_gatekeeper.sh ${var.project_id} ${var.cluster_name} ${var.location}"
+  kubectl_destroy_command = ""
+}

Diff for: modules/k8s-operator-crd-support/outputs.tf

@@ -19,5 +19,10 @@ output "git_creds_public" {
   value       = var.create_ssh_key ? tls_private_key.k8sop_creds.*.public_key_openssh : null
 }
 
+output "wait" {
+  description = "An output to use when you want to depend on cmd finishing"
+  value       = var.enable_policy_controller ? module.wait_for_gatekeeper.wait : module.k8sop_config.wait
+}
+
 
 

Diff for: modules/k8s-operator-crd-support/scripts/wait_for_gatekeeper.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+is_deployment_ready() {
+    kubectl --context "$1" -n "$2" get deploy "$3" &> /dev/null
+    export exit_code=$?
+    while [ ! " ${exit_code} " -eq 0 ]
+        do
+            sleep 5
+            echo -e "Waiting for deployment $3 in cluster $1 to be created..."
+            kubectl --context "$1" -n "$2" get deploy "$3" &> /dev/null
+            export exit_code=$?
+        done
+    echo -e "Deployment $3 in cluster $1 created."
+
+    # Once deployment is created, check for deployment status.availableReplicas is greater than 0
+    availableReplicas=$(kubectl --context "$1" -n "$2" get deploy "$3" -o json | jq -r '.status.availableReplicas')
+    while [[ " ${availableReplicas} " == " null " ]]
+        do
+            sleep 5
+            echo -e "Waiting for deployment $3 in cluster $1 to become ready..."
+            availableReplicas=$(kubectl --context "$1" -n "$2" get deploy "$3" -o json | jq -r '.status.availableReplicas')
+        done
+
+    echo -e "$3 in cluster $1 is ready with replicas ${availableReplicas}."
+    return "${availableReplicas}"
+}
+
+is_service_ready() {
+    kubectl --context "$1" -n "$2" get service "$3" &> /dev/null
+    export exit_code=$?
+    while [ ! " ${exit_code} " -eq 0 ]
+        do
+            sleep 5
+            echo -e "Waiting for service $3 in cluster $1 to be created..."
+            kubectl --context "$1" -n "$2" get service "$3" &> /dev/null
+            export exit_code=$?
+        done
+    echo -e "Service $3 in cluster $1 created."
+
+    # Once service is created, check endpoints is greater than 0
+    kubectl --context "$1" -n "$2" get endpoints "$3"
+    export exit_code=$?
+
+    while [ ! " ${exit_code} " -eq 0 ]
+        do
+            sleep 5
+            echo -e "Waiting for endpoints for service $3 in cluster $1 to become ready..."
+            kubectl --context "$1" -n "$2" get endpoints "$3"
+            export exit_code=$?
+        done
+
+    echo -e "Service $3 in cluster $1 is ready with endpoints."
+    return
+}
+
+if [ "$#" -lt 3 ]; then
+    >&2 echo "Not all expected arguments set."
+    exit 1
+fi
+
+PROJECT_ID=$1
+CLUSTER_NAME=$2
+CLUSTER_LOCATION=$3
+
+# Gatekeeper causes issues if not ready
+is_deployment_ready gke_"${PROJECT_ID}"_"${CLUSTER_LOCATION}"_"${CLUSTER_NAME}" gatekeeper-system gatekeeper-controller-manager
+is_service_ready gke_"${PROJECT_ID}"_"${CLUSTER_LOCATION}"_"${CLUSTER_NAME}" gatekeeper-system gatekeeper-webhook-service