Apply additional kube manifests from configs to cluster (#817)

* Apply kube resources to the cluster

Signed-off-by: Kimmo Lehto <klehto@mirantis.com>

Author: kke

action/apply.go

@@ -90,6 +90,7 @@ func NewApply(opts ApplyOptions) *Apply {
 			&phase.ResetWorkers{NoDrain: opts.NoDrain},
 			&phase.ResetControllers{NoDrain: opts.NoDrain},
 			&phase.RunHooks{Stage: "after", Action: "apply"},
+			&phase.ApplyManifests{},
 			unlockPhase,
 			&phase.Disconnect{},
 		},

cmd/flags.go

@@ -8,6 +8,7 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/a8m/envsubst"
@@ -239,6 +240,23 @@ func readConfig(ctx *cli.Context) (*v1beta1.Cluster, error) {
 			cfg.Spec.K0s.Config.Merge(k0s)
 		}
 	}
+	otherConfigs := mr.FilterResources(func(rd *manifest.ResourceDefinition) bool {
+		if strings.EqualFold(rd.APIVersion, v1beta1.APIVersion) && strings.EqualFold(rd.Kind, "cluster") {
+			return false
+		}
+		if strings.EqualFold(rd.APIVersion, "k0s.k0sproject.io/v1beta1") && strings.EqualFold(rd.Kind, "clusterconfig") {
+			return false
+		}
+		return true
+	})
+	if len(otherConfigs) > 0 {
+		cfg.Metadata.Manifests = make(map[string][]byte)
+		log.Debugf("found %d additional resources in the configuration", len(otherConfigs))
+		for _, otherConfig := range otherConfigs {
+			log.Debugf("found resource: %s (%d bytes)", otherConfig.Filename(), len(otherConfig.Raw))
+			cfg.Metadata.Manifests[otherConfig.Filename()] = otherConfig.Raw
+		}
+	}
 
 	if err := cfg.Validate(); err != nil {
 		return nil, fmt.Errorf("cluster config validation failed: %w", err)

phase/apply_manifests.go

@@ -0,0 +1,69 @@
+package phase
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+
+	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1"
+	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster"
+	"github.com/k0sproject/rig/exec"
+	log "github.com/sirupsen/logrus"
+)
+
+// ApplyManifests is a phase that applies additional manifests to the cluster
+type ApplyManifests struct {
+	GenericPhase
+	leader *cluster.Host
+}
+
+// Title for the phase
+func (p *ApplyManifests) Title() string {
+	return "Apply additional manifests"
+}
+
+// Prepare the phase
+func (p *ApplyManifests) Prepare(config *v1beta1.Cluster) error {
+	p.Config = config
+	p.leader = p.Config.Spec.K0sLeader()
+
+	return nil
+}
+
+// ShouldRun is true when there are additional manifests to apply
+func (p *ApplyManifests) ShouldRun() bool {
+	return len(p.Config.Metadata.Manifests) > 0
+}
+
+// Run the phase
+func (p *ApplyManifests) Run() error {
+	for name, content := range p.Config.Metadata.Manifests {
+		if err := p.apply(name, content); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (p *ApplyManifests) apply(name string, content []byte) error {
+	if !p.IsWet() {
+		p.DryMsgf(p.leader, "apply manifest %s (%d bytes)", name, len(content))
+		return nil
+	}
+
+	log.Infof("%s: apply manifest %s (%d bytes)", p.leader, name, len(content))
+	kubectlCmd := p.leader.Configurer.KubectlCmdf(p.leader, p.leader.K0sDataDir(), "apply -f -")
+	var stdout, stderr bytes.Buffer
+
+	cmd, err := p.leader.ExecStreams(kubectlCmd, io.NopCloser(bytes.NewReader(content)), &stdout, &stderr, exec.Sudo(p.leader))
+	if err != nil {
+		return fmt.Errorf("failed to run apply for manifest %s: %w", name, err)
+	}
+	if err := cmd.Wait(); err != nil {
+		log.Errorf("%s: kubectl apply failed for manifest %s", p.leader, name)
+		log.Errorf("%s: kubectl apply stderr: %s", p.leader, stderr.String())
+	}
+	log.Infof("%s: kubectl apply: %s", p.leader, stdout.String())
+	return nil
+}

pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster.go

@@ -18,6 +18,7 @@ type ClusterMetadata struct {
 	User        string            `yaml:"user" default:"admin"`
 	Kubeconfig  string            `yaml:"-"`
 	EtcdMembers []string          `yaml:"-"`
+	Manifests   map[string][]byte `yaml:"-"`
 }
 
 // Cluster describes launchpad.yaml configuration

smoke-test/smoke-multidoc.sh

@@ -21,6 +21,39 @@ echo "* Starting apply"
 ../k0sctl apply --config multidoc/ --kubeconfig-out applykubeconfig --debug
 echo "* Apply OK"
 
+echo "* Downloading kubectl for local test"
+downloadKubectl
+    
+export KUBECONFIG=applykubeconfig 
+
+echo "*Waiting until the test pod is running"
+./kubectl wait --for=condition=Ready pod/hello --timeout=120s
+
+retries=10
+delay=2
+nginx_ready=false
+i=1
+
+while [ "$i" -le "$retries" ]; do
+    echo "* Attempt $i: Checking if nginx is ready..."
+    ./kubectl exec pod/hello -- curl http://localhost/ | grep -q "Welcome to nginx!"
+    if kubectl exec pod/hello -- curl -s http://localhost/ | grep -q "Welcome to nginx!"; then
+        echo "nginx is ready!"
+        nginx_ready=true
+        break
+    fi
+    echo "  - nginx is not ready"
+    sleep $delay
+    i=$((i + 1))
+done
+
+if [ "$nginx_ready" = false ]; then
+    echo "nginx failed to become ready after $retries attempts."
+    exit 1
+fi
+
+echo " - nginx is ready"
+
 remoteCommand root@manager0 "cat /etc/k0s/k0s.yaml" > k0syaml
 echo Resulting k0s.yaml:
 cat k0syaml