Merge pull request #83 from gianlucam76/stop-matching

Introduce StopMatchingBehavior

Author: gianlucam76

Makefile

@@ -181,7 +181,7 @@ kind-test: test create-cluster fv ## Build docker image; start kind cluster; loa
 
 .PHONY: fv
 fv: $(GINKGO) ## Run Sveltos Controller tests using existing cluster
-	cd test/fv; ../../$(GINKGO) -nodes $(NUM_NODES) --label-filter='FV1' --v --trace --randomize-all
+	cd test/fv; ../../$(GINKGO) -nodes $(NUM_NODES) --label-filter='FV' --v --trace --randomize-all
 
 .PHONY: create-cluster
 create-cluster: $(KIND) $(CLUSTERCTL) $(KUBECTL) $(ENVSUBST) ## Create a new kind cluster designed for development
@@ -224,10 +224,13 @@ delete-cluster: $(KIND) ## Deletes the kind cluster $(CONTROL_CLUSTER_NAME)
 #
 # add this target. It needs to be run only when changing cluster-api version. create-cluster target uses the output of this command which is stored within repo
 # It requires control cluster to exist. So first "make create-control-cluster" then run this target.
+# Once generated, remove
+#      enforce: "{{ .podSecurityStandard.enforce }}"
+#      enforce-version: "latest"
 create-clusterapi-kind-cluster-yaml: $(CLUSTERCTL) 
-	KUBERNETES_VERSION=$(K8S_VERSION) SERVICE_CIDR=["10.225.0.0/16"] POD_CIDR=["10.220.0.0/16"] $(CLUSTERCTL) generate cluster $(WORKLOAD_CLUSTER_NAME) --flavor development \
+	ENABLE_POD_SECURITY_STANDARD="true" KUBERNETES_VERSION=$(K8S_VERSION) SERVICE_CIDR=["10.225.0.0/16"] POD_CIDR=["10.220.0.0/16"] $(CLUSTERCTL) generate cluster $(WORKLOAD_CLUSTER_NAME) --flavor development \
 		--control-plane-machine-count=1 \
-  		--worker-machine-count=1 > $(KIND_CLUSTER_YAML)
+  		--worker-machine-count=2 > $(KIND_CLUSTER_YAML)
 
 create-control-cluster:
 	sed -e "s/K8S_VERSION/$(K8S_VERSION)/g"  test/$(KIND_CONFIG) > test/$(KIND_CONFIG).tmp

api/v1alpha1/clusterprofile_types.go

@@ -115,6 +115,18 @@ type HelmChart struct {
 	HelmChartAction HelmChartAction `json:"helmChartAction,omitempty"`
 }
 
+// StopMatchingBehavior indicates what will happen when Cluster stops matching
+// a ClusterProfile. By default, withdrawpolicies, deployed Helm charts and Kubernetes
+// resources will be removed from Cluster. LeavePolicy instead leaves Helm charts
+// and Kubernetes policies in the Cluster.
+type StopMatchingBehavior string
+
+// Define the StopMatchingBehavior constants.
+const (
+	WithdrawPolicies StopMatchingBehavior = "WithdrawPolicies"
+	LeavePolicies    StopMatchingBehavior = "LeavePolicies"
+)
+
 // ClusterProfileSpec defines the desired state of ClusterProfile
 type ClusterProfileSpec struct {
 	// ClusterSelector identifies ClusterAPI clusters to associate to.
@@ -127,10 +139,21 @@ type ClusterProfileSpec struct {
 	// - Continuous means first time a workload cluster matches the ClusterProfile,
 	// features will be deployed in such a cluster. Any subsequent feature configuration
 	// change will be applied into the matching workload clusters.
+	// - DryRun means no change will be propagated to any matching cluster. A report
+	// instead will be generated summarizing what would happen in any matching cluster
+	// because of the changes made to ClusterProfile while in DryRun mode.
 	// +kubebuilder:default:=Continuous
 	// +optional
 	SyncMode SyncMode `json:"syncMode,omitempty"`
 
+	// StopMatchingBehavior indicates what behavior should be when a Cluster stop matching
+	// the ClusterProfile. By default all deployed Helm charts and Kubernetes resources will
+	// be withdrawn from Cluster. Setting StopMatchingBehavior to LeavePolicies will instead
+	// leave ClusterProfile deployed policies in the Cluster.
+	// +kubebuilder:default:=WithdrawPolicies
+	// +optional
+	StopMatchingBehavior StopMatchingBehavior `json:"stopMatchingBehavior,omitempty"`
+
 	// PolicyRefs references all the ConfigMaps containing kubernetes resources
 	// that need to be deployed in the matching CAPI clusters.
 	// +optional

config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml

@@ -116,6 +116,14 @@ spec:
                   - namespace
                   type: object
                 type: array
+              stopMatchingBehavior:
+                default: WithdrawPolicies
+                description: StopMatchingBehavior indicates what behavior should be
+                  when a Cluster stop matching the ClusterProfile. By default all
+                  deployed Helm charts and Kubernetes resources will be withdrawn
+                  from Cluster. Setting StopMatchingBehavior to LeavePolicies will
+                  instead leave ClusterProfile deployed policies in the Cluster.
+                type: string
               syncMode:
                 default: Continuous
                 description: SyncMode specifies how features are synced in a matching
@@ -125,7 +133,11 @@ spec:
                   the matching workload clusters; - Continuous means first time a
                   workload cluster matches the ClusterProfile, features will be deployed
                   in such a cluster. Any subsequent feature configuration change will
-                  be applied into the matching workload clusters.
+                  be applied into the matching workload clusters. - DryRun means no
+                  change will be propagated to any matching cluster. A report instead
+                  will be generated summarizing what would happen in any matching
+                  cluster because of the changes made to ClusterProfile while in DryRun
+                  mode.
                 enum:
                 - OneTime
                 - Continuous

config/crd/bases/config.projectsveltos.io_clustersummaries.yaml

@@ -129,6 +129,14 @@ spec:
                       - namespace
                       type: object
                     type: array
+                  stopMatchingBehavior:
+                    default: WithdrawPolicies
+                    description: StopMatchingBehavior indicates what behavior should
+                      be when a Cluster stop matching the ClusterProfile. By default
+                      all deployed Helm charts and Kubernetes resources will be withdrawn
+                      from Cluster. Setting StopMatchingBehavior to LeavePolicies
+                      will instead leave ClusterProfile deployed policies in the Cluster.
+                    type: string
                   syncMode:
                     default: Continuous
                     description: SyncMode specifies how features are synced in a matching
@@ -139,6 +147,10 @@ spec:
                       first time a workload cluster matches the ClusterProfile, features
                       will be deployed in such a cluster. Any subsequent feature configuration
                       change will be applied into the matching workload clusters.
+                      - DryRun means no change will be propagated to any matching
+                      cluster. A report instead will be generated summarizing what
+                      would happen in any matching cluster because of the changes
+                      made to ClusterProfile while in DryRun mode.
                     enum:
                     - OneTime
                     - Continuous

config/manager/manager.yaml

@@ -59,8 +59,6 @@ spec:
             port: 8081
           initialDelaySeconds: 5
           periodSeconds: 10
-        # TODO(user): Configure the resources accordingly based on the project requirements.
-        # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
         resources:
           limits:
             cpu: 500m

controllers/export_test.go

@@ -69,6 +69,8 @@ var (
 	GetPolicyInfo                = getPolicyInfo
 	UndeployStaleResources       = undeployStaleResources
 	GetDeployedGroupVersionKinds = getDeployedGroupVersionKinds
+	CanDelete                    = canDelete
+	HandleResourceDelete         = handleResourceDelete
 
 	ResourcesHash   = resourcesHash
 	GetResourceRefs = getResourceRefs

controllers/handlers_helm.go

@@ -135,47 +135,12 @@ func undeployHelmCharts(ctx context.Context, c client.Client,
 	}
 	defer os.Remove(kubeconfig)
 
-	chartManager, err := chartmanager.GetChartManagerInstance(ctx, c)
+	var releaseReports []configv1alpha1.ReleaseReport
+	releaseReports, err = uninstallHelmCharts(ctx, c, clusterSummary, kubeconfig, logger)
 	if err != nil {
 		return err
 	}
 
-	releaseReports := make([]configv1alpha1.ReleaseReport, 0)
-	for i := range clusterSummary.Spec.ClusterProfileSpec.HelmCharts {
-		currentChart := &clusterSummary.Spec.ClusterProfileSpec.HelmCharts[i]
-		if chartManager.CanManageChart(clusterSummary, currentChart) {
-			logger.V(logs.LogInfo).Info(fmt.Sprintf("Uninstalling chart %s from repo %s %s)",
-				currentChart.ChartName,
-				currentChart.RepositoryURL,
-				currentChart.RepositoryName))
-
-			// If another ClusterSummary is queued to manage this chart in this cluster, do not uninstall.
-			// Let the other ClusterSummary take it over.
-			if chartManager.GetNumberOfRegisteredClusterSummaries(clusterSummary.Spec.ClusterNamespace,
-				clusterSummary.Spec.ClusterName, currentChart) > 1 {
-				// Immediately unregister so next inline ClusterSummary can take this over
-				chartManager.UnregisterClusterSummaryForChart(clusterSummary, currentChart)
-			} else {
-				err = doUninstallRelease(clusterSummary, currentChart, kubeconfig, logger)
-				if err != nil {
-					if !errors.Is(err, driver.ErrReleaseNotFound) {
-						return err
-					}
-				}
-			}
-
-			releaseReports = append(releaseReports, configv1alpha1.ReleaseReport{
-				ReleaseNamespace: currentChart.ReleaseNamespace, ReleaseName: currentChart.ReleaseName,
-				Action: string(configv1alpha1.UninstallHelmAction),
-			})
-		} else {
-			releaseReports = append(releaseReports, configv1alpha1.ReleaseReport{
-				ReleaseNamespace: currentChart.ReleaseNamespace, ReleaseName: currentChart.ReleaseName,
-				Action: string(configv1alpha1.NoHelmAction), Message: "Currently managed by another ClusterProfile",
-			})
-		}
-	}
-
 	// First get the helm releases currently managed and uninstall all the ones
 	// not referenced anymore. Only if this operation succeeds, removes all stale
 	// helm release registration for this clusterSummary.
@@ -201,6 +166,11 @@ func undeployHelmCharts(ctx context.Context, c client.Client,
 		return err
 	}
 
+	chartManager, err := chartmanager.GetChartManagerInstance(ctx, c)
+	if err != nil {
+		return err
+	}
+
 	// In dry-run mode nothing gets deployed/undeployed. So if this instance used to manage
 	// an helm release and it is now not referencing anymore, do not unsubscribe.
 	if clusterSummary.Spec.ClusterProfileSpec.SyncMode != configv1alpha1.SyncModeDryRun {
@@ -211,6 +181,60 @@ func undeployHelmCharts(ctx context.Context, c client.Client,
 	return &configv1alpha1.DryRunReconciliationError{}
 }
 
+func uninstallHelmCharts(ctx context.Context, c client.Client, clusterSummary *configv1alpha1.ClusterSummary,
+	kubeconfig string, logger logr.Logger) ([]configv1alpha1.ReleaseReport, error) {
+
+	chartManager, err := chartmanager.GetChartManagerInstance(ctx, c)
+	if err != nil {
+		return nil, err
+	}
+
+	releaseReports := make([]configv1alpha1.ReleaseReport, 0)
+	for i := range clusterSummary.Spec.ClusterProfileSpec.HelmCharts {
+		currentChart := &clusterSummary.Spec.ClusterProfileSpec.HelmCharts[i]
+		if chartManager.CanManageChart(clusterSummary, currentChart) {
+			logger.V(logs.LogInfo).Info(fmt.Sprintf("Uninstalling chart %s from repo %s %s)",
+				currentChart.ChartName,
+				currentChart.RepositoryURL,
+				currentChart.RepositoryName))
+
+			// If another ClusterSummary is queued to manage this chart in this cluster, do not uninstall.
+			// Let the other ClusterSummary take it over.
+			if chartManager.GetNumberOfRegisteredClusterSummaries(clusterSummary.Spec.ClusterNamespace,
+				clusterSummary.Spec.ClusterName, currentChart) > 1 {
+				// Immediately unregister so next inline ClusterSummary can take this over
+				chartManager.UnregisterClusterSummaryForChart(clusterSummary, currentChart)
+			} else {
+				// If StopMatchingBehavior is LeavePolicies, do not uninstall helm charts
+				if !clusterSummary.DeletionTimestamp.IsZero() &&
+					clusterSummary.Spec.ClusterProfileSpec.StopMatchingBehavior == configv1alpha1.LeavePolicies {
+
+					logger.V(logs.LogInfo).Info("ClusterProfile StopMatchingBehavior set to LeavePolicies")
+				} else {
+					err = doUninstallRelease(clusterSummary, currentChart, kubeconfig, logger)
+					if err != nil {
+						if !errors.Is(err, driver.ErrReleaseNotFound) {
+							return nil, err
+						}
+					}
+				}
+			}
+
+			releaseReports = append(releaseReports, configv1alpha1.ReleaseReport{
+				ReleaseNamespace: currentChart.ReleaseNamespace, ReleaseName: currentChart.ReleaseName,
+				Action: string(configv1alpha1.UninstallHelmAction),
+			})
+		} else {
+			releaseReports = append(releaseReports, configv1alpha1.ReleaseReport{
+				ReleaseNamespace: currentChart.ReleaseNamespace, ReleaseName: currentChart.ReleaseName,
+				Action: string(configv1alpha1.NoHelmAction), Message: "Currently managed by another ClusterProfile",
+			})
+		}
+	}
+
+	return releaseReports, nil
+}
+
 func helmHash(ctx context.Context, c client.Client, clusterSummaryScope *scope.ClusterSummaryScope,
 	logger logr.Logger) ([]byte, error) {
 
@@ -364,8 +388,13 @@ func handleUpgrade(ctx context.Context, clusterSummary *configv1alpha1.ClusterSu
 	if err != nil {
 		return nil, err
 	}
-	message := fmt.Sprintf("Current version: %s. Would move to version: %s",
-		currentRelease.ChartVersion, currentChart.ChartVersion)
+	var message string
+	if currentRelease.ChartVersion != currentChart.ChartVersion {
+		message = fmt.Sprintf("Current version: %s. Would move to version: %s",
+			currentRelease.ChartVersion, currentChart.ChartVersion)
+	} else {
+		message = fmt.Sprintf("No op, already at version: %s", currentRelease.ChartVersion)
+	}
 	report = &configv1alpha1.ReleaseReport{
 		ReleaseNamespace: currentChart.ReleaseNamespace, ReleaseName: currentChart.ReleaseName,
 		ChartVersion: currentChart.ChartVersion, Action: string(configv1alpha1.UpgradeHelmAction),

controllers/handlers_utils.go

@@ -489,7 +489,9 @@ func undeployStaleResources(ctx context.Context, remoteConfig *rest.Config, c, r
 			// If this ClusterSummary is the only OwnerReference and it is not deploying this policy anymore,
 			// policy would be withdrawn
 			if clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1alpha1.SyncModeDryRun {
-				if canDelete(&r, currentPolicies) && isOnlyhOwnerReference(&r, clusterProfile) {
+				if canDelete(&r, currentPolicies) && isOnlyhOwnerReference(&r, clusterProfile) &&
+					!isLeavePolicies(clusterSummary, logger) {
+
 					undeployed = append(undeployed, configv1alpha1.ResourceReport{
 						Resource: configv1alpha1.Resource{
 							Kind: r.GetObjectKind().GroupVersionKind().Kind, Namespace: r.GetNamespace(), Name: r.GetName(),
@@ -509,7 +511,7 @@ func undeployStaleResources(ctx context.Context, remoteConfig *rest.Config, c, r
 				}
 
 				if canDelete(&r, currentPolicies) {
-					err = remoteClient.Delete(ctx, &r)
+					err = handleResourceDelete(ctx, remoteClient, &r, clusterSummary, logger)
 					if err != nil {
 						return nil, err
 					}
@@ -521,6 +523,23 @@ func undeployStaleResources(ctx context.Context, remoteConfig *rest.Config, c, r
 	return undeployed, nil
 }
 
+func handleResourceDelete(ctx context.Context, remoteClient client.Client, policy client.Object,
+	clusterSummary *configv1alpha1.ClusterSummary, logger logr.Logger) error {
+
+	// If mode is set to LeavePolicies, leave policies in the workload cluster.
+	// Remove all labels added by Sveltos.
+	if isLeavePolicies(clusterSummary, logger) {
+		labels := policy.GetLabels()
+		delete(labels, ReferenceLabelKind)
+		delete(labels, ReferenceLabelName)
+		delete(labels, ReferenceLabelNamespace)
+		policy.SetLabels(labels)
+		return remoteClient.Update(ctx, policy)
+	}
+
+	return remoteClient.Delete(ctx, policy)
+}
+
 // canDelete returns true if a policy can be deleted. For a policy to be deleted:
 // - policy is not part of currentReferencedPolicies
 func canDelete(policy client.Object, currentReferencedPolicies map[string]configv1alpha1.Resource) bool {
@@ -533,9 +552,23 @@ func canDelete(policy client.Object, currentReferencedPolicies map[string]config
 	if _, ok := currentReferencedPolicies[name]; ok {
 		return false
 	}
+
 	return true
 }
 
+// isLeavePolicies returns true if:
+// - ClusterSummary is marked for deletion
+// - StopMatchingBehavior is set to LeavePolicies
+func isLeavePolicies(clusterSummary *configv1alpha1.ClusterSummary, logger logr.Logger) bool {
+	if !clusterSummary.DeletionTimestamp.IsZero() &&
+		clusterSummary.Spec.ClusterProfileSpec.StopMatchingBehavior == configv1alpha1.LeavePolicies {
+
+		logger.V(logs.LogInfo).Info("ClusterProfile StopMatchingBehavior set to LeavePolicies")
+		return true
+	}
+	return false
+}
+
 // hasLabel search if key is one of the label.
 // If value is empty, returns true if key is present.
 // If value is not empty, returns true if key is present and value is a match.