diff --git a/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-external-and-internal-lb.yaml b/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-external-and-internal-lb.yaml new file mode 100644 index 000000000..95048c8bc --- /dev/null +++ b/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-external-and-internal-lb.yaml @@ -0,0 +1,173 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + labels: + cni: "${CLUSTER_NAME}-crs-cni" + ccm: "${CLUSTER_NAME}-crs-ccm" +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: GCPCluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + kind: KubeadmControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: GCPCluster +metadata: + name: "${CLUSTER_NAME}" +spec: + project: "${GCP_PROJECT}" + region: "${GCP_REGION}" + network: + name: "${GCP_NETWORK_NAME}" + subnets: + - name: control-plane-subnet + cidrBlock: "10.0.0.0/17" + purpose: PRIVATE + region: us-east4 + loadBalancer: + loadBalancerType: InternalExternal +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: GCPMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + name: '{{ ds.meta_data.local_hostname.split(".")[0] }}' + kubeletExtraArgs: + cloud-provider: external + clusterConfiguration: + apiServer: + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + cloud-provider: external + allocate-node-cidrs: "false" + kubernetesVersion: "${KUBERNETES_VERSION}" + files: + - content: | + [Global] + + project-id = "${GCP_PROJECT}" + network-name = "${GCP_NETWORK_NAME}" + multizone = true + owner: root:root + path: /etc/kubernetes/cloud.config + permissions: "0744" + joinConfiguration: + nodeRegistration: + name: '{{ ds.meta_data.local_hostname.split(".")[0] }}' + kubeletExtraArgs: + cloud-provider: external + version: "${KUBERNETES_VERSION}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: GCPMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + instanceType: "${GCP_CONTROL_PLANE_MACHINE_TYPE}" + image: "${IMAGE_ID}" +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: GCPMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: GCPMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + instanceType: "${GCP_NODE_MACHINE_TYPE}" + image: "${IMAGE_ID}" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + name: '{{ ds.meta_data.local_hostname.split(".")[0] }}' + kubeletExtraArgs: + cloud-provider: external +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: "${CLUSTER_NAME}-crs-cni" +data: ${CNI_RESOURCES} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: "${CLUSTER_NAME}-crs-cni" +spec: + strategy: ApplyOnce + clusterSelector: + matchLabels: + cni: "${CLUSTER_NAME}-crs-cni" + resources: + - name: "${CLUSTER_NAME}-crs-cni" + kind: ConfigMap +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: "${CLUSTER_NAME}-crs-ccm" +data: ${CCM_RESOURCES} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: "${CLUSTER_NAME}-crs-ccm" +spec: + strategy: ApplyOnce + clusterSelector: + matchLabels: + ccm: "${CLUSTER_NAME}-crs-ccm" + resources: + - name: "${CLUSTER_NAME}-crs-ccm" + kind: ConfigMap diff --git a/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-internal-lb.yaml b/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-internal-lb.yaml index 95048c8bc..6377a022b 100644 --- a/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-internal-lb.yaml +++ b/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-internal-lb.yaml @@ -8,6 +8,7 @@ metadata: ccm: "${CLUSTER_NAME}-crs-ccm" spec: clusterNetwork: + apiServerPort: 6443 pods: cidrBlocks: ["192.168.0.0/16"] infrastructureRef: @@ -33,8 +34,10 @@ spec: cidrBlock: "10.0.0.0/17" purpose: PRIVATE region: us-east4 + controlPlaneEndpoint: + port: 6443 loadBalancer: - loadBalancerType: InternalExternal + loadBalancerType: Internal --- apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 3e1823a27..1fec178ad 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -188,8 +188,38 @@ var _ = Describe("Workload cluster creation", func() { }) }) + Context("Creating a control-plane cluster with an external and an internal load balancer", func() { + It("Should create a cluster with 1 control-plane and 1 worker node with an external and an internal load balancer", func() { + clusterName := fmt.Sprintf("%s-internal-lb", clusterNamePrefix) + By("Creating a cluster with an internal load balancer and an external load balancer") + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: clusterctlLogFolder, + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "ci-with-external-and-internal-lb", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.MustGetVariable(KubernetesVersion), + ControlPlaneMachineCount: ptr.To[int64](1), + WorkerMachineCount: ptr.To[int64](1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + }) + }) + + // Skipping the test case for now. The internal load balancer test case requires the management cluster + // to have access to network that the cluster being created is in. + // An option to reach that is to use a GKE cluster as the management cluster. Context("Creating a control-plane cluster with an internal load balancer", func() { It("Should create a cluster with 1 control-plane and 1 worker node with an internal load balancer", func() { + Skip("This test requires a bootstrap cluster that has access to the network where the cluster is being created.") + clusterName := fmt.Sprintf("%s-internal-lb", clusterNamePrefix) By("Creating a cluster with internal load balancer") clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ @@ -213,6 +243,36 @@ var _ = Describe("Workload cluster creation", func() { }) }) + // Skipping the test case for now. The internal load balancer test case requires the management cluster + // to have access to network that the cluster being created is in. + // An option to reach that is to use a GKE cluster as the management cluster. + Context("Creating a control-plane cluster with three control plane nodes and an internal load balancer", func() { + It("Should create a cluster with 3 control-plane and 1 worker node with an internal load balancer", func() { + Skip("This test requires a bootstrap cluster that has access to the network where the cluster is being created.") + + clusterName := fmt.Sprintf("%s-internal-lb", clusterNamePrefix) + By("Creating a cluster with internal load balancer from GKE bootstrap cluster") + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: clusterctlLogFolder, + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "ci-with-internal-lb", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.MustGetVariable(KubernetesVersion), + ControlPlaneMachineCount: ptr.To[int64](3), + WorkerMachineCount: ptr.To[int64](1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + }) + }) + Context("Creating a cluster using a cluster class", func() { It("Should create a cluster class and then a cluster based on it", func() { clusterName := fmt.Sprintf("%s-topology", clusterNamePrefix)