From 99355df630fdad454e79adc4c6d24e5342caa2f1 Mon Sep 17 00:00:00 2001 From: Nan Jiang Date: Thu, 1 Nov 2018 17:18:37 -0400 Subject: [PATCH] GKE to use nodePool instead of nodeConfig to get autoscaling and management of pool working --- .../templates/gke/examples/gke_regional.yaml | 27 +- .../gke/examples/gke_regional_private.yaml | 83 +++-- .../templates/gke/examples/gke_zonal.yaml | 23 +- .../cloud-foundation/templates/gke/gke.py | 10 +- .../templates/gke/gke.py.schema | 336 ++++++++++-------- .../templates/gke/tests/integration/gke.yaml | 21 +- 6 files changed, 297 insertions(+), 203 deletions(-) diff --git a/community/cloud-foundation/templates/gke/examples/gke_regional.yaml b/community/cloud-foundation/templates/gke/examples/gke_regional.yaml index e3fa95d28..ba614e945 100644 --- a/community/cloud-foundation/templates/gke/examples/gke_regional.yaml +++ b/community/cloud-foundation/templates/gke/examples/gke_regional.yaml @@ -20,12 +20,27 @@ resources: description: my awesome k8s cluster network: subnetwork: - nodeConfig: - oauthScopes: - - https://www.googleapis.com/auth/compute - - https://www.googleapis.com/auth/devstorage.read_only - - https://www.googleapis.com/auth/logging.write - - https://www.googleapis.com/auth/monitoring + nodePools: + - name: uniq1 + initialNodeCount: 1 + config: + localSsdCount: 1 + oauthScopes: + - https://www.googleapis.com/auth/compute + - https://www.googleapis.com/auth/devstorage.read_only + - https://www.googleapis.com/auth/logging.write + - https://www.googleapis.com/auth/monitoring + taints: + - key: mykey1 + value: value1 + effect: NO_SCHEDULE + autoscaling: + enabled: True + minNodeCount: 1 + maxNodeCount: 3 + management: + autoUpgrade: True + autoRepair: True locations: - us-east1-c - us-east1-b diff --git a/community/cloud-foundation/templates/gke/examples/gke_regional_private.yaml b/community/cloud-foundation/templates/gke/examples/gke_regional_private.yaml index c534380c7..0d9401de9 100644 --- a/community/cloud-foundation/templates/gke/examples/gke_regional_private.yaml +++ b/community/cloud-foundation/templates/gke/examples/gke_regional_private.yaml @@ -15,45 +15,70 @@ imports: name: gke.py resources: - - name: myk8sregional + - name: myk8regprivate type: gke.py properties: clusterLocationType: Regional region: us-east1 cluster: - name: myk8sregional - description: my awesome k8s cluster + name: k8v1private + description: k8regional 2 pool network: - subnetwork: - intialNodeCount: 1 - initialClusterVersion: 1.10.6-gke.3 - nodeConfig: - localSsdCount: 1 - oauthScopes: - - https://www.googleapis.com/auth/compute - - https://www.googleapis.com/auth/devstorage.read_only - - https://www.googleapis.com/auth/logging.write - - https://www.googleapis.com/auth/monitoring - taints: - - key: mykey1 - value: value1 - effect: NO_SCHEDULE - - key: mykey2 - value: value2 - effect: NO_EXECUTE + subnetwork: + initialClusterVersion: 1.10.6-gke.6 + nodePools: + - name: uniq1 + initialNodeCount: 1 + config: + localSsdCount: 1 + oauthScopes: + - https://www.googleapis.com/auth/compute + - https://www.googleapis.com/auth/devstorage.read_only + - https://www.googleapis.com/auth/logging.write + - https://www.googleapis.com/auth/monitoring + taints: + - key: mykey1 + value: value1 + effect: NO_SCHEDULE + - key: mykey2 + value: value2 + effect: NO_EXECUTE + autoscaling: + enabled: True + minNodeCount: 1 + maxNodeCount: 3 + management: + autoUpgrade: True + autoRepair: True + - name: uniq2 + initialNodeCount: 1 + config: + localSsdCount: 1 + oauthScopes: + - https://www.googleapis.com/auth/compute + - https://www.googleapis.com/auth/devstorage.read_only + - https://www.googleapis.com/auth/logging.write + - https://www.googleapis.com/auth/monitoring + taints: + - key: mykey3 + value: value3 + effect: NO_SCHEDULE + - key: mykey4 + value: value4 + effect: NO_EXECUTE + autoscaling: + enabled: True + minNodeCount: 1 + maxNodeCount: 2 + management: + autoUpgrade: True + autoRepair: True locations: - us-east1-c - us-east1-b - autoScaling: - enabled: True - minNodeCount: 1 - maxNodeCount: 2 - management: - autoUpgrade: True - autoRepair: True masterAuth: - username: - password: + username: + password: loggingService: logging.googleapis.com monitoringService: monitoring.googleapis.com privateCluster: True diff --git a/community/cloud-foundation/templates/gke/examples/gke_zonal.yaml b/community/cloud-foundation/templates/gke/examples/gke_zonal.yaml index 928e15414..6ace931be 100644 --- a/community/cloud-foundation/templates/gke/examples/gke_zonal.yaml +++ b/community/cloud-foundation/templates/gke/examples/gke_zonal.yaml @@ -21,9 +21,20 @@ resources: description: my awesome k8s cluster network: subnetwork: - nodeConfig: - oauthScopes: - - https://www.googleapis.com/auth/compute - - https://www.googleapis.com/auth/devstorage.read_only - - https://www.googleapis.com/auth/logging.write - - https://www.googleapis.com/auth/monitoring + nodePools: + - name: uniq1 + initialNodeCount: 1 + config: + localSsdCount: 1 + oauthScopes: + - https://www.googleapis.com/auth/compute + - https://www.googleapis.com/auth/devstorage.read_only + - https://www.googleapis.com/auth/logging.write + - https://www.googleapis.com/auth/monitoring + autoscaling: + enabled: True + minNodeCount: 1 + maxNodeCount: 2 + management: + autoUpgrade: True + autoRepair: True \ No newline at end of file diff --git a/community/cloud-foundation/templates/gke/gke.py b/community/cloud-foundation/templates/gke/gke.py index 94958e5cf..50887c5b4 100644 --- a/community/cloud-foundation/templates/gke/gke.py +++ b/community/cloud-foundation/templates/gke/gke.py @@ -33,8 +33,6 @@ def generate_config(context): { 'name': name + '-cluster', - 'initialNodeCount': - propc.get('initialNodeCount'), 'initialClusterVersion': propc.get('initialClusterVersion') } @@ -42,6 +40,7 @@ def generate_config(context): } if cluster_type == 'Regional': + #TODO: container-v1 was released and this will need to be updated provider = 'gcp-types/container-v1beta1:projects.locations.clusters' if not properties.get('region'): raise KeyError( @@ -69,7 +68,7 @@ def generate_config(context): optional_props = [ 'description', - 'nodeConfig', + 'nodePools', 'masterAuth', 'loggingService', 'monitoringService', @@ -128,8 +127,9 @@ def generate_config(context): output_obj['value'] = '$(ref.' + name + \ '.masterAuth.' + outprop + ')' elif outprop == 'instanceGroupUrls': - output_obj['value'] = '$(ref.' + name + \ - '.nodePools[0].' + outprop + ')' + for index, _ in enumerate(propc['nodePools']): + output_obj['value'] = '$(ref.' + name + \ + '.nodePools[' + str(index) + '].' + outprop + ')' else: output_obj['value'] = '$(ref.' + name + '.' + outprop + ')' diff --git a/community/cloud-foundation/templates/gke/gke.py.schema b/community/cloud-foundation/templates/gke/gke.py.schema index 83141d7c4..d5eedb27b 100644 --- a/community/cloud-foundation/templates/gke/gke.py.schema +++ b/community/cloud-foundation/templates/gke/gke.py.schema @@ -57,167 +57,209 @@ properties: description: type: string description: An optional description of the cluster. - initialNodeCount: - type: number - default: 1 - description: | - The number of nodes to create in this cluster. You must ensure that - your Compute Engine resource quota is sufficient for this number of - instances. You must also have available firewall and routes quota. - minimum: 1 - nodeConfig: - type: object - description: Parameters used in creating the cluster's nodes. - required: - - oauthScopes - properties: - machineType: - type: string - default: n1-standard-1 - description: | - The name of the Google Compute Engine machine type. - diskSizeGb: - type: number - default: 100 - minimum: 10 - description: | - Size of the disk attached to each node, specified in GB. - The smallest allowed disk size is 10GB. - imageType: - type: string - default: cos - description: The image type to use for the node. - enum: - - cos - - Ubuntu - oauthScopes: - type: array - description: | - The set of Google API scopes to be made available on all - of the node VMs under the "default" service account. - E.g., scopes - https://www.googleapis.com/auth/compute - https://www.googleapis.com/auth/devstorage.read_only - https://www.googleapis.com/auth/logging.write - https://www.googleapis.com/auth/monitoring - items: - type: string - serviceAccount: - type: string - description: | - The GCP Service Account to be used by the node VMs. - metadata: - type: object - pattern: "[a-zA-Z0-9-_]+" - description: | - The metadata key/value pairs assigned to instances in the - cluster. Keys must conform to the regexp [a-zA-Z0-9-_]+ and be - less than 128 bytes in length. Additionally, to avoid ambiguity, - keys must neiter conflict with any other metadata keys for the - project nor be one of the reserved keys "cluster-location", - "cluster-name", "cluster-uid", "configure-sh", - "gci-update-strategy", "gci-ensure-gke-docker", - "instance-template", "kube-env", "startup-script", or - "user-data". The total size of all keys and values must be less - than 512 KB. - labels: - type: object - description: | - The map of Kubernetes labels (key/value pairs) to be applied to each - node. These are added to the default label(s) that - Kubernetes may apply to the nodes. - localSsdCount: - type: number - description: The number of local SSD disks to be attached to the node. - tags: - type: array - description: | - A list of instance tags applied to all nodes. Tags are used to - identify valid sources or targets for network firewalls, and are - specified by the client during the cluster or node pool creation. - All tags must comply with RFC1035. - items: + nodePools: + type: array + description: node pool configuration of the cluster + items: + type: object + description: single node pool configuration + required: + - initialNodeCount + properties: + initialNodeCount: + type: number + default: 1 + description: | + The number of nodes to create in this cluster. You must ensure that + your Compute Engine resource quota is sufficient for this number of + instances. You must also have available firewall and routes quota. + minimum: 1 + name: type: string - preemptible: - type: boolean - default: False - description: | - Defines whether the nodes are created as preemptible VM instances. - https://cloud.google.com/compute/docs/instances/preemptible - accelerators: - type: array - description: | - A list of hardware accelerators to be attached to each node. - See https://cloud.google.com/compute/docs/gpus for more - information about support for GPUs. - items: + description: The name parameter of node pool. + autoscaling: type: object - description: The Hardware Accelerator request object. - required: - - acceleratorCount - - acceleratorType + description: | + Node level scaling configuration. properties: - acceleratorCount: - type: string + enabled: + type: boolean description: | - The number of the accelerator cards exposed to an instance. - acceleratorType: - type: string + Enables auto scaling at node level. + minNodeCount: + type: number description: | - The accelerator type resource name. The list of supported - accelerator types can be found here - https://cloud.google.com/compute/docs/gpus/#Introduction - minCpuPlatform: - type: string - description: | - The minimum CPU platform to be used by the instance. - The instance may be scheduled on the specified or newer CPU - platform. Applicable values are the friendly names of CPU - platforms, such as "Intel Haswell" or "Intel Sandy Bridge". - workloadMetadataConfig: - type: object - description: The workload metadata configuration for the node. - items: + Minimum number of nodes in the cluster. + maxNodeCount: + type: number + description: | + Maximum number of nodes in the cluster. + management: + type: object + description: | + Cluster level management properties. + properties: + autoUpgrade: + type: boolean + description: | + Enables or disables cluster auto upgrade. + autoRepair: + type: boolean + description: | + Enables or disable cluster auto repair. + config: type: object + description: Parameters used in creating the cluster's nodes. required: - - nodeMetadata + - oauthScopes properties: - nodeMetadata: + machineType: + type: string + default: n1-standard-1 + description: | + The name of the Google Compute Engine machine type. + diskSizeGb: + type: number + default: 100 + minimum: 10 + description: | + Size of the disk attached to each node, specified in GB. + The smallest allowed disk size is 10GB. + imageType: + type: string + default: cos + description: The image type to use for the node. + enum: + - cos + - Ubuntu + oauthScopes: type: array description: | - Configuration that defines how to expose the node - metadata to the workload running on the node. + The set of Google API scopes to be made available on all + of the node VMs under the "default" service account. + E.g., scopes + https://www.googleapis.com/auth/compute + https://www.googleapis.com/auth/devstorage.read_only + https://www.googleapis.com/auth/logging.write + https://www.googleapis.com/auth/monitoring items: type: string - enum: - - UNSPECIFIED - - SECURE - - EXPOSE - taints: - type: array - description: | - A list of Kubernetes taints to be applied to each node. - items: - type: object - description: The taint object's key, value, and effect. - required: - - key - - value - - effect - properties: - key: - type: string - description: The taint object's key. - value: + serviceAccount: type: string - description: The taint object's value. - effect: + description: | + The GCP Service Account to be used by the node VMs. + metadata: + type: object + pattern: "[a-zA-Z0-9-_]+" + description: | + The metadata key/value pairs assigned to instances in the + cluster. Keys must conform to the regexp [a-zA-Z0-9-_]+ and be + less than 128 bytes in length. Additionally, to avoid ambiguity, + keys must neiter conflict with any other metadata keys for the + project nor be one of the reserved keys "cluster-location", + "cluster-name", "cluster-uid", "configure-sh", + "gci-update-strategy", "gci-ensure-gke-docker", + "instance-template", "kube-env", "startup-script", or + "user-data". The total size of all keys and values must be less + than 512 KB. + labels: + type: object + description: | + The map of Kubernetes labels (key/value pairs) to be applied to each + node. These are added to the default label(s) that + Kubernetes may apply to the nodes. + localSsdCount: + type: number + description: The number of local SSD disks to be attached to the node. + tags: + type: array + description: | + A list of instance tags applied to all nodes. Tags are used to + identify valid sources or targets for network firewalls, and are + specified by the client during the cluster or node pool creation. + All tags must comply with RFC1035. + items: + type: string + preemptible: + type: boolean + default: False + description: | + Defines whether the nodes are created as preemptible VM instances. + https://cloud.google.com/compute/docs/instances/preemptible + accelerators: + type: array + description: | + A list of hardware accelerators to be attached to each node. + See https://cloud.google.com/compute/docs/gpus for more + information about support for GPUs. + items: + type: object + description: The Hardware Accelerator request object. + required: + - acceleratorCount + - acceleratorType + properties: + acceleratorCount: + type: string + description: | + The number of the accelerator cards exposed to an instance. + acceleratorType: + type: string + description: | + The accelerator type resource name. The list of supported + accelerator types can be found here + https://cloud.google.com/compute/docs/gpus/#Introduction + minCpuPlatform: type: string - enum: - - EFFECT_UNSPECIFIED - - NO_SCHEDULE - - PREFER_NO_SCHEDULE - - NO_EXECUTE + description: | + The minimum CPU platform to be used by the instance. + The instance may be scheduled on the specified or newer CPU + platform. Applicable values are the friendly names of CPU + platforms, such as "Intel Haswell" or "Intel Sandy Bridge". + workloadMetadataConfig: + type: object + description: The workload metadata configuration for the node. + items: + type: object + required: + - nodeMetadata + properties: + nodeMetadata: + type: array + description: | + Configuration that defines how to expose the node + metadata to the workload running on the node. + items: + type: string + enum: + - UNSPECIFIED + - SECURE + - EXPOSE + taints: + type: array + description: | + A list of Kubernetes taints to be applied to each node. + items: + type: object + description: The taint object's key, value, and effect. + required: + - key + - value + - effect + properties: + key: + type: string + description: The taint object's key. + value: + type: string + description: The taint object's value. + effect: + type: string + enum: + - EFFECT_UNSPECIFIED + - NO_SCHEDULE + - PREFER_NO_SCHEDULE + - NO_EXECUTE masterAuth: type: object description: | diff --git a/community/cloud-foundation/templates/gke/tests/integration/gke.yaml b/community/cloud-foundation/templates/gke/tests/integration/gke.yaml index cc44fb429..199d4d850 100644 --- a/community/cloud-foundation/templates/gke/tests/integration/gke.yaml +++ b/community/cloud-foundation/templates/gke/tests/integration/gke.yaml @@ -16,15 +16,16 @@ resources: network: ${NETWORK_NAME} subnetwork: ${SUBNET_NAME} initialClusterVersion: ${CLUSTER_VERSION} - initialNodeCount: ${NODE_COUNT} - nodeConfig: - machineType: ${MACHINE_TYPE} - oauthScopes: - - https://www.googleapis.com/auth/compute - - https://www.googleapis.com/auth/devstorage.read_only - - https://www.googleapis.com/auth/logging.write - - https://www.googleapis.com/auth/monitoring - localSsdCount: ${LOCALSSD_COUNT} + nodePools: + - name: uniq1 + initialNodeCount: ${NODE_COUNT} + config: + localSsdCount: ${LOCALSSD_COUNT} + oauthScopes: + - https://www.googleapis.com/auth/compute + - https://www.googleapis.com/auth/devstorage.read_only + - https://www.googleapis.com/auth/logging.write + - https://www.googleapis.com/auth/monitoring locations: - us-east1-b - us-east1-d @@ -34,4 +35,4 @@ resources: clusterIpv4Cidr: ${CLUSTERIPV4_CIDR} ipAllocationPolicy: useIpAliases: True - servicesIpv4CidrBlock: ${SERVICESIPV4_CIDRBLOCK} + servicesIpv4CidrBlock: ${SERVICESIPV4_CIDRBLOCK} \ No newline at end of file