diff --git a/buildbot/google/config.sh b/buildbot/google/config.sh old mode 100644 new mode 100755 diff --git a/buildbot/google/terraform/.gitignore b/buildbot/google/terraform/.gitignore new file mode 100644 --- /dev/null +++ b/buildbot/google/terraform/.gitignore @@ -0,0 +1,3 @@ +.terraform/ +.terraform.tfstate.lock.info +terraform.tfstate.backup diff --git a/buildbot/google/terraform/main.tf b/buildbot/google/terraform/main.tf --- a/buildbot/google/terraform/main.tf +++ b/buildbot/google/terraform/main.tf @@ -5,11 +5,25 @@ region = var.gcp_config.region } +# create a network for the cluster, required for Kubernetes on Windows +# FIXME: rename to "buildbot-vpc-network", causes destruction of the cluster! +resource "google_compute_network" "vpc_network" { + name = "vpc-network" +} + # Create the cluster runningn all Kubernetes services resource "google_container_cluster" "primary" { name = "buildbot-cluster" # maybe have a regional cluster for Kubernetes, as we depend on this... location = var.gcp_config.zone_a + + # configure local network, required for Kubernetes on Windows + network = google_compute_network.vpc_network.name + # enable alias IP addresses, required for Kubernetes for Windows + ip_allocation_policy {} + + # use newer Kubernetes version, otherwise Windows node pools can't be created + min_master_version = "1.16" # one node is enough (at the moment) initial_node_count = 1 @@ -21,6 +35,7 @@ # use preemptible, as this saves costs preemptible = true } + } # Create machines for mlir-nvidia @@ -65,3 +80,140 @@ } } } + +# node pool for windows machines +resource "google_container_node_pool" "windows_16core_pool_nodes" { + name = "windows-16core-pool" + # specify a zone here (e.g. "-a") to avoid a redundant deployment + location = var.gcp_config.zone_a + cluster = google_container_cluster.primary.name + + # use autoscaling to only create a machine when there is a deployment + autoscaling { + min_node_count = 0 + max_node_count = 1 + } + + node_config { + # use preemptible, as this saves costs + preemptible = true + machine_type = "n1-highcpu-16" + # Windows deployment tend to require more disk space, so using 200GB here. + disk_size_gb = 200 + # FIXME: test if SSDs are actually faster than HDDs for our use case + disk_type = "pd-ssd" + + # Configure Windows image. As Windows is picky about the combination of + # host and container OS versions, this must be compatible with the version + # in your container. Recommondation: Use LTSC for long-term stability. + # For details see + # https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility + # https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#choose_your_windows_server_node_image + image_type = "WINDOWS_LTSC" + + # set the premissions required for the deployment later + oauth_scopes = [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/devstorage.read_only", + ] + + # add a label to all machines of this type, so we can select them + # during deployment + labels = { + pool = "win-16core-pool" + } + } +} + +# deployment for the buildbot windows10_vs2019 running on Windows rather than +# Linux. +# Note: Deploying this takes significantly longer (~35 min) than on Linux +# as the images thend to be larger (~18GB) and IO performance is lower. +resource "kubernetes_deployment" "windows10_vs2019" { + metadata { + name = "windows10-vs2019" + labels = { + app = "windows10_vs2019" + } + } + + spec { + # create one instance of this container + replicas = 1 + + selector { + match_labels = { + app = "windows10_vs2019" + } + } + strategy{ + rolling_update{ + # do not deploy more replicas, as the buildbot server + # can't handle multiple workers with the same credentials + max_surge = 0 + # Allow to have 0 replicas during updates. + max_unavailable = 1 + } + type = "RollingUpdate" + } + template { + metadata { + labels = { + app = "windows10_vs2019" + } + } + + spec { + container { + image = "${var.gcp_config.gcr_prefix}/buildbot-windows10-vs2019" + name = "windows10-vs2019" + + # reserve "-1" for this image, kubernetes also + # needs <1 core for management tools + resources { + limits { + cpu = "15" + memory = "10Gi" + } + requests { + cpu = "15" + memory = "10Gi" + } + } + + # mount the secrets into a folder + volume_mount { + mount_path = "c:\\secrets" + name = "buildbot-token" + } + } + # select which node pool to deploy to + node_selector = { + pool = "win-16core-pool" + } + # restart in case of any crashes + restart_policy = "Always" + + # select the secret to be mounted + volume { + name = "buildbot-token" + secret { + optional = false + secret_name = "password-windows10-vs2019" + } + + } + # Windows nodes from the node pool are marked with the taint + # "node.kubernetes.io/os=windows". So we need to "tolerate" this to + # deploy to such nodes. + toleration { + effect = "NoSchedule" + key = "node.kubernetes.io/os" + operator = "Equal" + value = "windows" + } + } + } + } +} \ No newline at end of file diff --git a/buildbot/google/terraform/terraform.tfstate b/buildbot/google/terraform/terraform.tfstate new file mode 100644 --- /dev/null +++ b/buildbot/google/terraform/terraform.tfstate @@ -0,0 +1,570 @@ +{ + "version": 4, + "terraform_version": "0.12.26", + "serial": 34, + "lineage": "3e0972f3-f59f-78f6-d0ce-7a95d7fbed85", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "google_compute_network", + "name": "vpc_network", + "provider": "provider.google", + "instances": [ + { + "schema_version": 0, + "attributes": { + "auto_create_subnetworks": true, + "delete_default_routes_on_create": false, + "description": "", + "gateway_ipv4": "", + "id": "projects/sanitizer-bots/global/networks/vpc-network", + "ipv4_range": null, + "name": "vpc-network", + "project": "sanitizer-bots", + "routing_mode": "REGIONAL", + "self_link": "https://www.googleapis.com/compute/v1/projects/sanitizer-bots/global/networks/vpc-network", + "timeouts": null + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoyNDAwMDAwMDAwMDAsImRlbGV0ZSI6MjQwMDAwMDAwMDAwLCJ1cGRhdGUiOjI0MDAwMDAwMDAwMH19" + } + ] + }, + { + "mode": "managed", + "type": "google_container_cluster", + "name": "primary", + "provider": "provider.google", + "instances": [ + { + "schema_version": 1, + "attributes": { + "additional_zones": [], + "addons_config": [ + { + "cloudrun_config": [], + "horizontal_pod_autoscaling": [], + "http_load_balancing": [], + "kubernetes_dashboard": [], + "network_policy_config": [ + { + "disabled": true + } + ] + } + ], + "authenticator_groups_config": [], + "cluster_autoscaling": [ + { + "auto_provisioning_defaults": [], + "enabled": false, + "resource_limits": [] + } + ], + "cluster_ipv4_cidr": "10.20.0.0/14", + "default_max_pods_per_node": 110, + "description": "", + "enable_binary_authorization": false, + "enable_intranode_visibility": null, + "enable_kubernetes_alpha": false, + "enable_legacy_abac": false, + "enable_shielded_nodes": false, + "enable_tpu": null, + "endpoint": "34.67.203.67", + "id": "projects/sanitizer-bots/locations/us-central1-a/clusters/buildbot-cluster", + "initial_node_count": 1, + "instance_group_urls": [ + "https://www.googleapis.com/compute/v1/projects/sanitizer-bots/zones/us-central1-a/instanceGroups/gke-buildbot-cluster-default-pool-4c37e6b4-grp", + "https://www.googleapis.com/compute/v1/projects/sanitizer-bots/zones/us-central1-a/instanceGroups/gke-buildbot-cluster-nvidia-16core-po-4c3868a6-grp", + "https://www.googleapis.com/compute/v1/projects/sanitizer-bots/zones/us-central1-a/instanceGroups/gke-buildbot-cluster-windows-16core-p-9e75e9f9-grp" + ], + "ip_allocation_policy": [ + { + "cluster_ipv4_cidr_block": "10.20.0.0/14", + "cluster_secondary_range_name": "gke-buildbot-cluster-pods-36a01b55", + "node_ipv4_cidr_block": "", + "services_ipv4_cidr_block": "10.24.0.0/20", + "services_secondary_range_name": "gke-buildbot-cluster-services-36a01b55", + "subnetwork_name": "" + } + ], + "label_fingerprint": "a9dc16a7", + "location": "us-central1-a", + "logging_service": "logging.googleapis.com/kubernetes", + "maintenance_policy": [], + "master_auth": [ + { + "client_certificate": "", + "client_certificate_config": [ + { + "issue_client_certificate": false + } + ], + "client_key": "", + "cluster_ca_certificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURDekNDQWZPZ0F3SUJBZ0lRS1B5VXRCRG5TMTU1QzdCS3hLUGk1REFOQmdrcWhraUc5dzBCQVFzRkFEQXYKTVMwd0t3WURWUVFERXlSaE16ZzRNbVk1WWkwM1lXSTNMVFJpWWpRdE9EZGhOeTFpWmpSak5ETmxNR1ptWmpJdwpIaGNOTWpBd05qSTVNVEUwTURBM1doY05NalV3TmpJNE1USTBNREEzV2pBdk1TMHdLd1lEVlFRREV5UmhNemc0Ck1tWTVZaTAzWVdJM0xUUmlZalF0T0RkaE55MWlaalJqTkRObE1HWm1aakl3Z2dFaU1BMEdDU3FHU0liM0RRRUIKQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUUREWjJzdHFCSzJCMjl2dGpTVGRDclFiRzJRUUI5anpKSTVTQk53SVFxcgowS01neGJZY0tFbW15Wk51VW5aY1Rsdnp1clhjelg2T3VoMGY3bFpDOGlyYnp1MUdENzJ3QmdJNmhoRlVPV1FZCm03MjZHQW9ZT0JFd2ZTTk55UlBRL2VOOGJPUXZhVi9VNTFsMit4VUVOcWRKZUJqMGhpNzVqQ05iRHNuTWt1czcKYnhMVXVCcVpTU1ZFL3RuQ3FsYjdlUytqSHczSEd3SG1sQmVheFZBNzFCT0IvejZ6ekNDSUV5cldHa3EvK0VIQwp5aDd4YU03R1NGS0FqY0hJU2kyYUJUck5uN2lNdEwzc2hDaTZ3WXN2NXZMZEZTNElpcFpGQ3NNV2NYYjZEWDRJCmpYK2Z4MFc3QmNlVlpzN3ROS3p4Rk4vWnR5RlVWbWdjTmVmdGtBNW5YRy9iQWdNQkFBR2pJekFoTUE0R0ExVWQKRHdFQi93UUVBd0lDQkRBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFCYQo3R3VpWms3WTRKS0hYOG02RHorU0lzd25uc2tzYk90UldYTncxaFAxMVAxTEQvWFJzSnVEL3hBUy9laEtFdXltCjVMRG9zYnV3Rks2T2VzOGVVeENlNUxCdW9LUHMwc3hINXpXRThxWnB2MXFnNC9Gb0Rwc3RINkNCVDJVcXVtM04KaHRDMTZ1OWZzUmxadkdmWXBDQktma0VqaFVvK0MvUHpVa1ZEZzRZV3YwdHJ4eUtwTzdCUmhEVHFFNXRYL1EvOQp2SDkrNWNteEJDQTMyR3N2WUdPbERqc2krc1JXS01VaW95V0hnVHJqNlVrRFhZUzNsRkxERkZqVVNIb09Wdk9DCnJkZVYzZlBhQTJ1TDNSK21RekVmWXdKclVib1pvRld4UGlySkEyUXp6UkZpYjNLZG40a1dlRHJSWGRzZkROcmcKM1NDMTR3b3diRnkrcGN1cW1lY0sKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=", + "password": "", + "username": "" + } + ], + "master_authorized_networks_config": [], + "master_version": "1.16.9-gke.6", + "min_master_version": "1.16", + "monitoring_service": "monitoring.googleapis.com/kubernetes", + "name": "buildbot-cluster", + "network": "projects/sanitizer-bots/global/networks/vpc-network", + "network_policy": [ + { + "enabled": false, + "provider": "PROVIDER_UNSPECIFIED" + } + ], + "node_config": [ + { + "disk_size_gb": 100, + "disk_type": "pd-standard", + "guest_accelerator": [], + "image_type": "COS", + "labels": {}, + "local_ssd_count": 0, + "machine_type": "n1-standard-2", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "min_cpu_platform": "", + "oauth_scopes": [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring" + ], + "preemptible": true, + "sandbox_config": [], + "service_account": "default", + "shielded_instance_config": [ + { + "enable_integrity_monitoring": true, + "enable_secure_boot": false + } + ], + "tags": [], + "taint": [], + "workload_metadata_config": [] + } + ], + "node_locations": [], + "node_pool": [ + { + "autoscaling": [], + "initial_node_count": 1, + "instance_group_urls": [ + "https://www.googleapis.com/compute/v1/projects/sanitizer-bots/zones/us-central1-a/instanceGroupManagers/gke-buildbot-cluster-default-pool-4c37e6b4-grp" + ], + "management": [ + { + "auto_repair": true, + "auto_upgrade": true + } + ], + "max_pods_per_node": 110, + "name": "default-pool", + "name_prefix": "", + "node_config": [ + { + "disk_size_gb": 100, + "disk_type": "pd-standard", + "guest_accelerator": [], + "image_type": "COS", + "labels": {}, + "local_ssd_count": 0, + "machine_type": "n1-standard-2", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "min_cpu_platform": "", + "oauth_scopes": [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring" + ], + "preemptible": true, + "sandbox_config": [], + "service_account": "default", + "shielded_instance_config": [ + { + "enable_integrity_monitoring": true, + "enable_secure_boot": false + } + ], + "tags": [], + "taint": [], + "workload_metadata_config": [] + } + ], + "node_count": 1, + "node_locations": [ + "us-central1-a" + ], + "upgrade_settings": [ + { + "max_surge": 1, + "max_unavailable": 0 + } + ], + "version": "1.16.9-gke.6" + }, + { + "autoscaling": [ + { + "max_node_count": 1, + "min_node_count": 0 + } + ], + "initial_node_count": 1, + "instance_group_urls": [ + "https://www.googleapis.com/compute/v1/projects/sanitizer-bots/zones/us-central1-a/instanceGroupManagers/gke-buildbot-cluster-nvidia-16core-po-4c3868a6-grp" + ], + "management": [ + { + "auto_repair": true, + "auto_upgrade": true + } + ], + "max_pods_per_node": 110, + "name": "nvidia-16core-pool", + "name_prefix": "", + "node_config": [ + { + "disk_size_gb": 100, + "disk_type": "pd-ssd", + "guest_accelerator": [ + { + "count": 1, + "type": "nvidia-tesla-t4" + } + ], + "image_type": "COS", + "labels": { + "pool": "nvidia-16core-pool" + }, + "local_ssd_count": 0, + "machine_type": "n1-highcpu-16", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "min_cpu_platform": "", + "oauth_scopes": [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring" + ], + "preemptible": true, + "sandbox_config": [], + "service_account": "default", + "shielded_instance_config": [ + { + "enable_integrity_monitoring": true, + "enable_secure_boot": false + } + ], + "tags": [], + "taint": [ + { + "effect": "NO_SCHEDULE", + "key": "nvidia.com/gpu", + "value": "present" + } + ], + "workload_metadata_config": [] + } + ], + "node_count": 1, + "node_locations": [ + "us-central1-a" + ], + "upgrade_settings": [ + { + "max_surge": 1, + "max_unavailable": 0 + } + ], + "version": "1.16.9-gke.6" + }, + { + "autoscaling": [ + { + "max_node_count": 1, + "min_node_count": 0 + } + ], + "initial_node_count": 0, + "instance_group_urls": [ + "https://www.googleapis.com/compute/v1/projects/sanitizer-bots/zones/us-central1-a/instanceGroupManagers/gke-buildbot-cluster-windows-16core-p-9e75e9f9-grp" + ], + "management": [ + { + "auto_repair": true, + "auto_upgrade": true + } + ], + "max_pods_per_node": 110, + "name": "windows-16core-pool", + "name_prefix": "", + "node_config": [ + { + "disk_size_gb": 200, + "disk_type": "pd-ssd", + "guest_accelerator": [], + "image_type": "WINDOWS_LTSC", + "labels": { + "pool": "win-16core-pool" + }, + "local_ssd_count": 0, + "machine_type": "n1-highcpu-16", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "min_cpu_platform": "", + "oauth_scopes": [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring" + ], + "preemptible": true, + "sandbox_config": [], + "service_account": "default", + "shielded_instance_config": [], + "tags": [], + "taint": [ + { + "effect": "NO_SCHEDULE", + "key": "node.kubernetes.io/os", + "value": "windows" + } + ], + "workload_metadata_config": [] + } + ], + "node_count": 1, + "node_locations": [ + "us-central1-a" + ], + "upgrade_settings": [ + { + "max_surge": 1, + "max_unavailable": 0 + } + ], + "version": "1.16.9-gke.6" + } + ], + "node_version": "1.16.9-gke.6", + "operation": null, + "pod_security_policy_config": [], + "private_cluster_config": [], + "project": "sanitizer-bots", + "region": null, + "remove_default_node_pool": null, + "resource_labels": {}, + "resource_usage_export_config": [], + "services_ipv4_cidr": "10.24.0.0/20", + "subnetwork": "projects/sanitizer-bots/regions/us-central1/subnetworks/vpc-network", + "timeouts": null, + "vertical_pod_autoscaling": [], + "workload_identity_config": [], + "zone": null + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoyNDAwMDAwMDAwMDAwLCJkZWxldGUiOjI0MDAwMDAwMDAwMDAsInJlYWQiOjI0MDAwMDAwMDAwMDAsInVwZGF0ZSI6MzYwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "google_compute_network.vpc_network" + ] + } + ] + }, + { + "mode": "managed", + "type": "google_container_node_pool", + "name": "nvidia_16core_pool_nodes", + "provider": "provider.google", + "instances": [ + { + "schema_version": 1, + "attributes": { + "autoscaling": [ + { + "max_node_count": 1, + "min_node_count": 0 + } + ], + "cluster": "buildbot-cluster", + "id": "projects/sanitizer-bots/locations/us-central1-a/clusters/buildbot-cluster/nodePools/nvidia-16core-pool", + "initial_node_count": 1, + "instance_group_urls": [ + "https://www.googleapis.com/compute/v1/projects/sanitizer-bots/zones/us-central1-a/instanceGroupManagers/gke-buildbot-cluster-nvidia-16core-po-4c3868a6-grp" + ], + "location": "us-central1-a", + "management": [ + { + "auto_repair": true, + "auto_upgrade": true + } + ], + "max_pods_per_node": 110, + "name": "nvidia-16core-pool", + "name_prefix": "", + "node_config": [ + { + "disk_size_gb": 100, + "disk_type": "pd-ssd", + "guest_accelerator": [ + { + "count": 1, + "type": "nvidia-tesla-t4" + } + ], + "image_type": "COS", + "labels": { + "pool": "nvidia-16core-pool" + }, + "local_ssd_count": 0, + "machine_type": "n1-highcpu-16", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "min_cpu_platform": "", + "oauth_scopes": [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring" + ], + "preemptible": true, + "sandbox_config": [], + "service_account": "default", + "shielded_instance_config": [ + { + "enable_integrity_monitoring": true, + "enable_secure_boot": false + } + ], + "tags": [], + "taint": [ + { + "effect": "NO_SCHEDULE", + "key": "nvidia.com/gpu", + "value": "present" + } + ], + "workload_metadata_config": [] + } + ], + "node_count": 1, + "node_locations": [ + "us-central1-a" + ], + "project": "sanitizer-bots", + "region": null, + "timeouts": null, + "upgrade_settings": [ + { + "max_surge": 1, + "max_unavailable": 0 + } + ], + "version": "1.16.9-gke.6", + "zone": null + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6MTgwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "google_compute_network.vpc_network", + "google_container_cluster.primary" + ] + } + ] + }, + { + "mode": "managed", + "type": "google_container_node_pool", + "name": "windows_16core_pool_nodes", + "provider": "provider.google", + "instances": [ + { + "schema_version": 1, + "attributes": { + "autoscaling": [ + { + "max_node_count": 1, + "min_node_count": 0 + } + ], + "cluster": "buildbot-cluster", + "id": "projects/sanitizer-bots/locations/us-central1-a/clusters/buildbot-cluster/nodePools/windows-16core-pool", + "initial_node_count": 0, + "instance_group_urls": [ + "https://www.googleapis.com/compute/v1/projects/sanitizer-bots/zones/us-central1-a/instanceGroupManagers/gke-buildbot-cluster-windows-16core-p-9e75e9f9-grp" + ], + "location": "us-central1-a", + "management": [ + { + "auto_repair": true, + "auto_upgrade": true + } + ], + "max_pods_per_node": 110, + "name": "windows-16core-pool", + "name_prefix": "", + "node_config": [ + { + "disk_size_gb": 200, + "disk_type": "pd-ssd", + "guest_accelerator": [], + "image_type": "WINDOWS_LTSC", + "labels": { + "pool": "win-16core-pool" + }, + "local_ssd_count": 0, + "machine_type": "n1-highcpu-16", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "min_cpu_platform": "", + "oauth_scopes": [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring" + ], + "preemptible": true, + "sandbox_config": [], + "service_account": "default", + "shielded_instance_config": [], + "tags": [], + "taint": [ + { + "effect": "NO_SCHEDULE", + "key": "node.kubernetes.io/os", + "value": "windows" + } + ], + "workload_metadata_config": [] + } + ], + "node_count": 1, + "node_locations": [ + "us-central1-a" + ], + "project": "sanitizer-bots", + "region": null, + "timeouts": null, + "upgrade_settings": [ + { + "max_surge": 1, + "max_unavailable": 0 + } + ], + "version": "1.16.9-gke.6", + "zone": null + }, + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6MTgwMDAwMDAwMDAwMH0sInNjaGVtYV92ZXJzaW9uIjoiMSJ9", + "dependencies": [ + "google_container_cluster.primary" + ] + } + ] + } + ] +}