Add capi-auth-token file to control plane machines (#115)

Author: HomayoonAlimohammadi

Dockerfile

@@ -15,6 +15,7 @@ RUN go mod download
 COPY main.go main.go
 COPY apis/ apis/
 COPY controllers/ controllers/
+COPY pkg/ pkg/
 
 # Build
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=$arch go build -a -ldflags '-s -w' -o manager main.go

controllers/cloudinit/cloudinit.go

@@ -19,9 +19,14 @@ package cloudinit
 import (
 	"bytes"
 	"fmt"
+	"path/filepath"
 	"text/template"
 )
 
+var (
+	CAPIAuthTokenPath = filepath.Join("/capi", "etc", "token")
+)
+
 // File is a file that cloud-init will create.
 type File struct {
 	// Content of the file to create.

controllers/cloudinit/controlplane_init.go

@@ -27,6 +27,8 @@ import (
 
 // ControlPlaneInitInput defines the context needed to generate a controlplane instance to init a cluster.
 type ControlPlaneInitInput struct {
+	// AuthToken will be used for authenticating CAPI-only requests to the cluster-agent.
+	AuthToken string
 	// CAKey is the PEM-encoded key of the cluster CA certificate.
 	CAKey string
 	// CACert is the PEM-encoded cert of the cluster CA certificate.
@@ -131,6 +133,7 @@ func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) {
 		cloudConfig.WriteFiles,
 		File{Content: input.CAKey, Path: filepath.Join("/var", "tmp", "ca.key"), Permissions: "0600", Owner: "root:root"},
 		File{Content: input.CACert, Path: filepath.Join("/var", "tmp", "ca.crt"), Permissions: "0600", Owner: "root:root"},
+		File{Content: input.AuthToken, Path: CAPIAuthTokenPath, Permissions: "0600", Owner: "root:root"},
 	)
 	cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, input.ExtraWriteFiles...)
 

controllers/cloudinit/controlplane_init_test.go

@@ -28,7 +28,9 @@ func TestControlPlaneInit(t *testing.T) {
 	t.Run("Simple", func(t *testing.T) {
 		g := NewWithT(t)
 
+		authToken := "capi-auth-token"
 		cloudConfig, err := cloudinit.NewInitControlPlane(&cloudinit.ControlPlaneInitInput{
+			AuthToken:            authToken,
 			CAKey:                `CA KEY DATA`,
 			CACert:               `CA CERT DATA`,
 			ControlPlaneEndpoint: "k8s.my-domain.com",
@@ -75,6 +77,12 @@ func TestControlPlaneInit(t *testing.T) {
 				Permissions: "0600",
 				Owner:       "root:root",
 			},
+			cloudinit.File{
+				Content:     authToken,
+				Path:        cloudinit.CAPIAuthTokenPath,
+				Permissions: "0600",
+				Owner:       "root:root",
+			},
 		))
 
 		_, err = cloudinit.GenerateCloudConfig(cloudConfig)

controllers/cloudinit/controlplane_join.go

@@ -27,6 +27,8 @@ import (
 
 // ControlPlaneJoinInput defines the context needed to generate a controlplane instance to join a cluster.
 type ControlPlaneJoinInput struct {
+	// AuthToken will be used for authenticating CAPI-only requests to the cluster-agent.
+	AuthToken string
 	// ControlPlaneEndpoint is the control plane endpoint of the cluster.
 	ControlPlaneEndpoint string
 	// Token is the token that will be used for joining other nodes to the cluster.
@@ -109,6 +111,12 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) {
 	}
 
 	cloudConfig := NewBaseCloudConfig()
+	cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, File{
+		Content:     input.AuthToken,
+		Path:        CAPIAuthTokenPath,
+		Permissions: "0600",
+		Owner:       "root:root",
+	})
 	cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, input.ExtraWriteFiles...)
 	if args := input.ExtraKubeletArgs; len(args) > 0 {
 		cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, File{

controllers/cloudinit/controlplane_join_test.go

@@ -28,7 +28,9 @@ func TestControlPlaneJoin(t *testing.T) {
 	t.Run("Simple", func(t *testing.T) {
 		g := NewWithT(t)
 
+		authToken := "capi-auth-token"
 		cloudConfig, err := cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{
+			AuthToken:            authToken,
 			ControlPlaneEndpoint: "k8s.my-domain.com",
 			KubernetesVersion:    "v1.25.2",
 			ClusterAgentPort:     "30000",
@@ -60,6 +62,15 @@ func TestControlPlaneJoin(t *testing.T) {
 			`microk8s add-node --token-ttl 10000 --token "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`,
 		}))
 
+		g.Expect(cloudConfig.WriteFiles).To(ContainElements(
+			cloudinit.File{
+				Content:     authToken,
+				Path:        cloudinit.CAPIAuthTokenPath,
+				Permissions: "0600",
+				Owner:       "root:root",
+			},
+		))
+
 		_, err = cloudinit.GenerateCloudConfig(cloudConfig)
 		g.Expect(err).ToNot(HaveOccurred())
 	})

controllers/microk8sconfig_controller.go

@@ -59,6 +59,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	tokenpkg "github.com/canonical/cluster-api-bootstrap-provider-microk8s/pkg/token"
 )
 
 type InitLocker interface {
@@ -213,6 +215,10 @@ func (r *MicroK8sConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 		return ctrl.Result{}, nil
 	}
 
+	if err = tokenpkg.Reconcile(ctx, r.Client, util.ObjectKey(scope.Cluster)); err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to reconcile token: %w", err)
+	}
+
 	// Note: can't use IsFalse here because we need to handle the absence of the condition as well as false.
 	if !conditions.IsTrue(cluster, clusterv1.ControlPlaneInitializedCondition) {
 		log.Info("Cluster control plane is not initialized, waiting")
@@ -296,7 +302,13 @@ func (r *MicroK8sConfigReconciler) handleClusterNotInitialized(ctx context.Conte
 		portOfDqlite = remappedDqlitePort
 	}
 
+	authToken, err := tokenpkg.Lookup(ctx, r.Client, util.ObjectKey(scope.Cluster))
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to lookup auth token: %w", err)
+	}
+
 	controlPlaneInput := &cloudinit.ControlPlaneInitInput{
+		AuthToken:            authToken,
 		CACert:               *cert,
 		CAKey:                *key,
 		ControlPlaneEndpoint: scope.Cluster.Spec.ControlPlaneEndpoint.Host,
@@ -403,7 +415,13 @@ func (r *MicroK8sConfigReconciler) handleJoiningControlPlaneNode(ctx context.Con
 		return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
 	}
 
+	authToken, err := tokenpkg.Lookup(ctx, r.Client, util.ObjectKey(scope.Cluster))
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to lookup auth token: %w", err)
+	}
+
 	controlPlaneInput := &cloudinit.ControlPlaneJoinInput{
+		AuthToken:            authToken,
 		ControlPlaneEndpoint: scope.Cluster.Spec.ControlPlaneEndpoint.Host,
 		Token:                token,
 		TokenTTL:             microk8sConfig.Spec.InitConfiguration.JoinTokenTTLInSecs,

pkg/token/token.go

@@ -0,0 +1,101 @@
+package token
+
+import (
+	"context"
+	cryptorand "crypto/rand"
+	"encoding/base64"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	AuthTokenNameSuffix = "capi-auth-token"
+)
+
+// Reconcile ensures that a token secret exists for the given cluster.
+func Reconcile(ctx context.Context, c client.Client, clusterKey client.ObjectKey) error {
+	if _, err := getSecret(ctx, c, clusterKey); err != nil {
+		if apierrors.IsNotFound(err) {
+			if _, err := generateAndStore(ctx, c, clusterKey); err != nil {
+				return fmt.Errorf("failed to generate and store token: %w", err)
+			}
+			return nil
+		}
+	}
+
+	return nil
+}
+
+// Lookup retrieves the token for the given cluster.
+func Lookup(ctx context.Context, c client.Client, clusterKey client.ObjectKey) (string, error) {
+	secret, err := getSecret(ctx, c, clusterKey)
+	if err != nil {
+		return "", fmt.Errorf("failed to get secret: %w", err)
+	}
+
+	v, ok := secret.Data["token"]
+	if !ok {
+		return "", fmt.Errorf("token not found in secret")
+	}
+
+	return string(v), nil
+}
+
+// authTokenName returns the name of the auth-token secret, computed by convention using the name of the cluster.
+func authTokenName(clusterName string) string {
+	return fmt.Sprintf("%s-%s", clusterName, AuthTokenNameSuffix)
+}
+
+// getSecret retrieves the token secret for the given cluster.
+func getSecret(ctx context.Context, c client.Client, clusterKey client.ObjectKey) (*corev1.Secret, error) {
+	s := &corev1.Secret{}
+	key := client.ObjectKey{
+		Name:      authTokenName(clusterKey.Name),
+		Namespace: clusterKey.Namespace,
+	}
+	if err := c.Get(ctx, key, s); err != nil {
+		return nil, fmt.Errorf("failed to get secret: %w", err)
+	}
+
+	return s, nil
+}
+
+// generateAndStore generates a new token and stores it in a secret.
+func generateAndStore(ctx context.Context, c client.Client, clusterKey client.ObjectKey) (*corev1.Secret, error) {
+	token, err := randomB64(16)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate token: %w", err)
+	}
+
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: clusterKey.Namespace,
+			Name:      authTokenName(clusterKey.Name),
+		},
+		Data: map[string][]byte{
+			"token": []byte(token),
+		},
+		Type: clusterv1.ClusterSecretType,
+	}
+
+	if err := c.Create(ctx, secret); err != nil {
+		return nil, fmt.Errorf("failed to create secret: %w", err)
+	}
+
+	return secret, nil
+}
+
+// randomB64 generates a random base64 string of n bytes.
+func randomB64(n int) (string, error) {
+	b := make([]byte, n)
+	_, err := cryptorand.Read(b)
+	if err != nil {
+		return "", fmt.Errorf("failed to read random bytes: %w", err)
+	}
+	return base64.StdEncoding.EncodeToString(b), nil
+}

pkg/token/token_test.go

@@ -0,0 +1,86 @@
+package token_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	"github.com/canonical/cluster-api-bootstrap-provider-microk8s/pkg/token"
+)
+
+func TestReconcile(t *testing.T) {
+	t.Run("SecretAvailableSucceeds", func(t *testing.T) {
+		namespace := "test-namespace"
+		clusterName := "test-cluster"
+		secret := &corev1.Secret{
+			ObjectMeta: v1.ObjectMeta{
+				Name:      fmt.Sprintf("%s-%s", clusterName, token.AuthTokenNameSuffix),
+				Namespace: namespace,
+			},
+		}
+		c := fake.NewClientBuilder().WithObjects(secret).Build()
+
+		g := NewWithT(t)
+
+		g.Expect(token.Reconcile(context.Background(), c, client.ObjectKey{Name: clusterName, Namespace: namespace})).To(Succeed())
+	})
+
+	t.Run("SecretNotFoundGenerates", func(t *testing.T) {
+		namespace := "test-namespace"
+		clusterName := "test-cluster"
+		c := fake.NewClientBuilder().Build()
+
+		g := NewWithT(t)
+
+		g.Expect(token.Reconcile(context.Background(), c, client.ObjectKey{Name: clusterName, Namespace: namespace})).To(Succeed())
+
+		s := &corev1.Secret{}
+		key := client.ObjectKey{
+			Name:      fmt.Sprintf("%s-%s", clusterName, token.AuthTokenNameSuffix),
+			Namespace: namespace,
+		}
+		g.Expect(c.Get(context.Background(), key, s)).To(Succeed())
+		g.Expect(s.ObjectMeta.Name).To(Equal(fmt.Sprintf("%s-%s", clusterName, token.AuthTokenNameSuffix)))
+		g.Expect(s.ObjectMeta.Namespace).To(Equal(namespace))
+		g.Expect(string(s.Data["token"])).ToNot(BeEmpty())
+	})
+
+	t.Run("LookupFailsIfNoSecret", func(t *testing.T) {
+		namespace := "test-namespace"
+		clusterName := "test-cluster"
+		c := fake.NewClientBuilder().Build()
+
+		g := NewWithT(t)
+
+		_, err := token.Lookup(context.Background(), c, client.ObjectKey{Name: clusterName, Namespace: namespace})
+		g.Expect(err).To(HaveOccurred())
+	})
+
+	t.Run("LookupSucceedsIfSecretExists", func(t *testing.T) {
+		namespace := "test-namespace"
+		clusterName := "test-cluster"
+		expToken := "test-token"
+		secret := &corev1.Secret{
+			ObjectMeta: v1.ObjectMeta{
+				Name:      fmt.Sprintf("%s-%s", clusterName, token.AuthTokenNameSuffix),
+				Namespace: namespace,
+			},
+			Data: map[string][]byte{
+				"token": []byte(expToken),
+			},
+		}
+		c := fake.NewClientBuilder().WithObjects(secret).Build()
+
+		g := NewWithT(t)
+
+		token, err := token.Lookup(context.Background(), c, client.ObjectKey{Name: clusterName, Namespace: namespace})
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(token).To(Equal(expToken))
+	})
+}