Add CI test for disabling the default CNI in Microk8s CAPI provider

KU-1226

Author: Maciej Golaszewski

README.md

@@ -9,12 +9,12 @@ This project offers a cluster API bootstrap provider controller that manages the
 ### Prerequisites
 
   * Install clusterctl following the [upstream instructions](https://cluster-api.sigs.k8s.io/user/quick-start.html#install-clusterctl)
-```
+```bash
 curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.3/clusterctl-linux-amd64 -o clusterctl
 ```
 
   * Install a bootstrap Kubernetes cluster. To use MicroK8s as a bootstrap cluster:
-```
+```bash
 sudo snap install microk8s --classic
 sudo microk8s.config  > ~/.kube/config
 sudo microk8s enable dns
@@ -24,7 +24,7 @@ sudo microk8s enable dns
 
 To to configure clusterctl with the two MicroK8s providers edit `~/.cluster-api/clusterctl.yaml`
 and add the following:
-```
+```yaml
 providers:
   - name: "microk8s"
     url: "https://github.com/canonical/cluster-api-bootstrap-provider-microk8s/releases/latest/bootstrap-components.yaml"
@@ -44,21 +44,21 @@ Alternatively, you can build the providers manually as described in the followin
 
 
 
-### Building from source
+### Building from sourcebssdfsdfsdfsdfsdfsdsdfsdfsfdsfdsfsdfsdfsd
 
   * Install the cluster provider of your choice. Have a look at the [cluster API book](https://cluster-api.sigs.k8s.io/user/quick-start.html#initialization-for-common-providers) for your options at this step. You should deploy only the infrastructure controller leaving the bootstrap and control plane ones empty. For example assuming we want to provision a MicroK8s cluster on AWS:
-```
+```bash
 clusterctl init --infrastructure aws --bootstrap "-" --control-plane "-"
 ```
 
   * Clone the two cluster API MicroK8s specific repositories and start the controllers on two separate terminals:
-```
+```bash
 cd $GOPATH/src/github.com/canonical/cluster-api-bootstrap-provider-microk8s/
 make install
 make run
 ```
 And:
-```
+```bash
 cd $GOPATH/src/github.com/canonical/cluster-api-control-plane-provider-microk8s/
 make install
 make run

integration/README.md

@@ -4,36 +4,43 @@
 
 The integration/e2e tests have the following prerequisites:
 
-  * an environment variable `CLUSTER_MANIFEST_FILE` pointing to the cluster manifest. Cluster manifests can be produced with the help of the templates found under `templates`. For example:
+  * an environment variable `CLUSTER_MANIFEST_FILE` pointing to the cluster manifest. Cluster manifests can be produced with the help of the templates found under [`templates`](../templates). For example:
+  ```bash 
+export AWS_REGION=us-east-1
+export AWS_SSH_KEY_NAME=capi
+export CONTROL_PLANE_MACHINE_COUNT=3
+export WORKER_MACHINE_COUNT=3
+export AWS_CREATE_BASTION=false
+export AWS_PUBLIC_IP=false
+export AWS_CONTROL_PLANE_MACHINE_FLAVOR=t3.large
+export AWS_NODE_MACHINE_FLAVOR=t3.large
+export CLUSTER_NAME=test-ci-cluster
+clusterctl generate cluster ${CLUSTER_NAME} --from "templates/cluster-template-aws.yaml" --kubernetes-version 1.27.0 > cluster.yaml
+export CLUSTER_MANIFEST_FILE=$PWD/cluster.yaml
   ```
-    export AWS_REGION=us-east-1
-    export AWS_SSH_KEY_NAME=capi
-    export CONTROL_PLANE_MACHINE_COUNT=3
-    export WORKER_MACHINE_COUNT=3
-    export AWS_CREATE_BASTION=false
-    export AWS_PUBLIC_IP=false
-    export AWS_CONTROL_PLANE_MACHINE_FLAVOR=t3.large
-    export AWS_NODE_MACHINE_FLAVOR=t3.large
-    export CLUSTER_NAME=test-ci-cluster
-    clusterctl generate cluster ${CLUSTER_NAME} --from "templates/cluster-template-aws.yaml" --kubernetes-version 1.25.0 > cluster.yaml
-    export CLUSTER_MANIFEST_FILE=$PWD/cluster.yaml
-  ```
-
+> NOTE: AWS_SSH_KEY_NAME is name of ssh key in AWS that you are plan to use, if you don't have one yet refer
+> to capi on [aws prerequisites documentation](https://cluster-api-aws.sigs.k8s.io/topics/using-clusterawsadm-to-fulfill-prerequisites#ssh-key-pair)
   *  Additional environment variables when testing cluster upgrades:
-  ```
-    export CAPI_UPGRADE_VERSION=v1.26.0
-    export CAPI_UPGRADE_MD_NAME=${CLUSTER_NAME}-md-0
-    export CAPI_UPGRADE_MD_TYPE=machinedeployments.cluster.x-k8s.io
-    export CAPI_UPGRADE_CP_NAME=${CLUSTER_NAME}-control-plane
-    export CAPI_UPGRADE_CP_TYPE=microk8scontrolplanes.controlplane.cluster.x-k8s.io
-
-    # Change the control plane and worker machine count to desired values for in-place upgrades tests and create a new cluster manifest.
-    CONTROL_PLANE_MACHINE_COUNT=1
-    WORKER_MACHINE_COUNT=1
-    clusterctl generate cluster ${CLUSTER_NAME} --from "templates/cluster-template-aws.yaml" --kubernetes-version 1.25.0 > cluster-inplace.yaml
-    export CLUSTER_INPLACE_MANIFEST_FILE=$PWD/cluster-inplace.yaml
+  ```bash
+export CAPI_UPGRADE_VERSION=v1.28.0
+export CAPI_UPGRADE_MD_NAME=${CLUSTER_NAME}-md-0
+export CAPI_UPGRADE_MD_TYPE=machinedeployments.cluster.x-k8s.io
+export CAPI_UPGRADE_CP_NAME=${CLUSTER_NAME}-control-plane
+export CAPI_UPGRADE_CP_TYPE=microk8scontrolplanes.controlplane.cluster.x-k8s.io
+# Change the control plane and worker machine count to desired values for in-place upgrades tests and create a new cluster manifest.
+CONTROL_PLANE_MACHINE_COUNT=1
+WORKER_MACHINE_COUNT=1
+clusterctl generate cluster ${CLUSTER_NAME} --from "templates/cluster-template-aws.yaml" --kubernetes-version 1.27.0 > cluster-inplace.yaml
+export CLUSTER_INPLACE_MANIFEST_FILE=$PWD/cluster-inplace.yaml
+```
 
-  ```
+  *  Additional environment variables when testing disable default CNI flag:
+  ```bash
+export DISABLE_DEFAULT_CNI=true
+export POST_RUN_COMMANDS='["helm install cilium cilium/cilium  --namespace kube-system --set cni.confPath=/var/snap/microk8s/current/args/cni-network --set cni.binPath=/var/snap/microk8s/current/opt/cni/bin --set daemon.runPath=/var/snap/microk8s/current/var/run/cilium --set operator.replicas=1 --set ipam.operator.clusterPoolIPv4PodCIDRList=\"10.1.0.0/16\" --set nodePort.enabled=true"]' # install Calico in place default CNI
+clusterctl generate cluster ${CLUSTER_NAME} --from "templates/cluster-template-aws.yaml" --kubernetes-version 1.27.0 > cluster_disable_default_cni.yaml
+export CLUSTER_DISABLE_DEFAULT_CNI_MANIFEST_FILE=$PWD/cluster_disable_default_cni.yaml
+```
 
   * `clusterctl` available in the PATH
 
@@ -67,10 +74,10 @@ microk8s config > ~/.kube/config
 
 #### Initialize infrastructure provider
 
-Visit [here](https://cluster-api.sigs.k8s.io/user/quick-start.html#initialization-for-common-providers) for a list of common infrasturture providers.
+Visit [here](https://cluster-api.sigs.k8s.io/user/quick-start.html#initialization-for-common-providers) for a list of common infrastructure providers.
 
 ```bash
-  clusterctl init --infrastructure <infra> --bootstrap - --control-plane -
+clusterctl init --infrastructure <infra> --bootstrap - --control-plane -
 ```
 
 #### Build Docker images and release manifests from the checked out source code
@@ -83,7 +90,7 @@ docker push <username>/capi-bootstrap-provider-microk8s:<tag>
 sed "s,docker.io/cdkbot/capi-bootstrap-provider-microk8s:latest,docker.io/<username>/capi-bootstrap-provider-microk8s:<tag>," -i bootstrap-components.yaml
 ```
 
-Similarly for control-plane provider
+Similarly, for control-plane provider
 ```bash
 cd control-plane
 docker build -t <username>/capi-control-plane-provider-microk8s:<tag> .

integration/e2e_test.go

@@ -45,7 +45,7 @@ func init() {
 
 // TestBasic waits for the target cluster to deploy and start a 30 pod deployment.
 // The CLUSTER_MANIFEST_FILE environment variable should point to a manifest with the target cluster
-// kubectl and clusterctl have to be avaibale in the caller's path.
+// kubectl and clusterctl have to be available in the caller's path.
 // kubectl should be setup so it uses the kubeconfig of the management cluster by default.
 func TestBasic(t *testing.T) {
 	cluster_manifest_file := os.Getenv("CLUSTER_MANIFEST_FILE")
@@ -88,7 +88,29 @@ func TestInPlaceUpgrade(t *testing.T) {
 	// Important: the cluster is deleted in the Cleanup function
 	// which is called after all subtests are finished.
 	t.Logf("Deleting the cluster")
+}
+
+// TestDisableDefaultCNI deploys cluster disabled  defalut CNI  .
+// The CLUSTER_DISABLE_DEFAULT_CNI_MANIFEST_FILE environment variable should point to a manifest with the target cluster
+// With post actions calico will be installed.
+func TestDisableDefaultCNI(t *testing.T) {
+	cluster_manifest_file := os.Getenv("CLUSTER_DISABLE_DEFAULT_CNI_MANIFEST_FILE")
+	if cluster_manifest_file == "" {
+		t.Fatalf("Environment variable CLUSTER_DISABLE_DEFAULT_CNI_MANIFEST_FILE is not set. " +
+			"CLUSTER_DISABLE_DEFAULT_CNI_MANIFEST_FILE is expected to hold the PATH to a cluster manifest.")
+	}
+	t.Logf("Cluster to setup is in %s", cluster_manifest_file)
 
+	setupCheck(t)
+	t.Cleanup(teardownCluster)
+
+	t.Run("DeployCluster", func(t *testing.T) { deployCluster(t, os.Getenv("CLUSTER_DISABLE_DEFAULT_CNI_MANIFEST_FILE")) })
+	t.Run("ValidateCalico", func(t *testing.T) { validateCalico(t) })
+	t.Run("DeployMicrobot", func(t *testing.T) { deployMicrobot(t) })
+	t.Run("UpgradeClusterRollout", func(t *testing.T) { upgradeCluster(t, "RollingUpgrade") })
+	// Important: the cluster is deleted in the Cleanup function
+	// which is called after all subtests are finished.
+	t.Logf("Deleting the cluster")
 }
 
 // setupCheck checks that the environment is ready to run the tests.
@@ -149,7 +171,7 @@ func teardownCluster() {
 
 // deployCluster deploys a cluster using the manifest in CLUSTER_MANIFEST_FILE.
 func deployCluster(t testing.TB, cluster_manifest_file string) {
-	t.Log("Setting up the cluster")
+	t.Logf("Setting up the cluster using %s", cluster_manifest_file)
 	command := []string{"kubectl", "apply", "-f", cluster_manifest_file}
 	cmd := exec.Command(command[0], command[1:]...)
 	outputBytes, err := cmd.CombinedOutput()
@@ -182,7 +204,7 @@ func deployCluster(t testing.TB, cluster_manifest_file string) {
 				t.Fatal(err)
 			} else {
 				attempt++
-				t.Log("Failed to get the target's kubeconfig, retrying.")
+				t.Logf("Failed to get the target's kubeconfig for %s, retrying.", cluster)
 				time.Sleep(20 * time.Second)
 			}
 		} else {
@@ -307,6 +329,32 @@ func deployMicrobot(t testing.TB) {
 	command = []string{"kubectl", "--kubeconfig=" + KUBECONFIG, "wait", "deploy/bot", "--for=jsonpath={.status.readyReplicas}=30"}
 	for {
 		cmd = exec.Command(command[0], command[1:]...)
+		outputBytes, err = cmd.CombinedOutput()
+		if err != nil {
+			t.Log(string(outputBytes))
+			if attempt >= maxAttempts {
+				t.Fatal(err)
+			} else {
+				attempt++
+				t.Log("Retrying")
+				time.Sleep(10 * time.Second)
+			}
+		} else {
+			break
+		}
+	}
+}
+
+// validateCalico checks a deployment of calico demonset.
+func validateCalico(t testing.TB) {
+	t.Log("Validate Calico")
+	// Make sure we have as many nodes as machines
+	attempt := 0
+	maxAttempts := 60
+	t.Log("Waiting for the deployment to complete")
+	command := []string{"kubectl", "--kubeconfig=" + KUBECONFIG, "-n", "kube-system", "wait", "ds/calico-node", "--for=jsonpath={.status.numberAvailable}=6"}
+	for {
+		cmd := exec.Command(command[0], command[1:]...)
 		outputBytes, err := cmd.CombinedOutput()
 		if err != nil {
 			t.Log(string(outputBytes))

templates/cluster-template-aws.yaml

@@ -108,3 +108,5 @@ spec:
       initConfiguration:
         riskLevel: "${SNAP_RISKLEVEL:=}"
         confinement: "${SNAP_CONFINEMENT:=}"
+        disableDefaultCNI: ${DISABLE_DEFAULT_CNI:=false}
+        postRunCommands: ${POST_RUN_COMMANDS:=[]}