Fix TestKubeletConfigRemediation issue

Author: Anna-Koudelkova

tests/e2e/framework/common.go

@@ -894,6 +894,88 @@ func (f *Framework) restoreNodeLabelsForPool(n string) error {
 	return nil
 }
 
+// WaitForMachineConfigPoolUpdated waits for a MachineConfigPool to complete an update cycle.
+// It first waits for the Updated condition to become False (indicating update/reboot started),
+// then waits for it to become True again (indicating update/reboot completed).
+func (f *Framework) WaitForMachineConfigPoolUpdated(poolName string) error {
+	if f.Platform == "rosa" {
+		log.Printf("Bypassing MachineConfigPool update check on %s", f.Platform)
+		return nil
+	}
+
+	// First, get the initial state
+	pool := mcfgv1.MachineConfigPool{}
+	err := f.Client.Get(context.TODO(), types.NamespacedName{Name: poolName}, &pool)
+	if err != nil {
+		return fmt.Errorf("failed to find Machine Config Pool %s: %w", poolName, err)
+	}
+
+	// Check initial state of MachineConfigPoolUpdated condition
+	initialUpdated := false
+	for _, c := range pool.Status.Conditions {
+		if c.Type == mcfgv1.MachineConfigPoolUpdated {
+			initialUpdated = (c.Status == core.ConditionTrue)
+			break
+		}
+	}
+
+	// If the pool is already updated (True), wait for it to start updating (False)
+	if initialUpdated {
+		log.Printf("Machine Config Pool %s is currently updated. Waiting for update to start...\n", poolName)
+		err = wait.PollImmediate(machineOperationRetryInterval, machineOperationTimeout, func() (bool, error) {
+			pool := mcfgv1.MachineConfigPool{}
+			err := f.Client.Get(context.TODO(), types.NamespacedName{Name: poolName}, &pool)
+			if err != nil {
+				log.Printf("failed to find Machine Config Pool %s\n", poolName)
+				return false, err
+			}
+
+			for _, c := range pool.Status.Conditions {
+				if c.Type == mcfgv1.MachineConfigPoolUpdated {
+					if c.Status == core.ConditionFalse {
+						log.Printf("Machine Config Pool %s update started (Updated=False)\n", poolName)
+						return true, nil
+					}
+					break
+				}
+			}
+
+			log.Printf("Machine Config Pool %s still updated, waiting for update to start...\n", poolName)
+			return false, nil
+		})
+		if err != nil {
+			return fmt.Errorf("failed waiting for Machine Config Pool %s to start updating: %w", poolName, err)
+		}
+	} else {
+		log.Printf("Machine Config Pool %s is already updating (Updated=False). Waiting for update to complete...\n", poolName)
+	}
+
+	// Now wait for the update to complete (Updated=True)
+	err = wait.PollImmediate(machineOperationRetryInterval, machineOperationTimeout, func() (bool, error) {
+		pool := mcfgv1.MachineConfigPool{}
+		err := f.Client.Get(context.TODO(), types.NamespacedName{Name: poolName}, &pool)
+		if err != nil {
+			log.Printf("failed to find Machine Config Pool %s\n", poolName)
+			return false, err
+		}
+
+		for _, c := range pool.Status.Conditions {
+			if c.Type == mcfgv1.MachineConfigPoolUpdated && c.Status == core.ConditionTrue {
+				return true, nil
+			}
+		}
+
+		log.Printf("Machine Config Pool %s has not finished updating yet... retrying\n", poolName)
+		return false, nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed waiting for Machine Config Pool %s to be updated: %w", poolName, err)
+	}
+
+	log.Printf("Machine Config Pool %s is updated\n", poolName)
+	return nil
+}
+
 func (f *Framework) getNodesForPool(p *mcfgv1.MachineConfigPool) (core.NodeList, error) {
 	var nodeList core.NodeList
 	opts := &dynclient.ListOptions{
@@ -1048,6 +1130,25 @@ func (f *Framework) WaitForScanSettingBindingStatus(namespace, name string, targ
 	return nil
 }
 
+func (f *Framework) WaitForComplianceSuiteDeletion(name, namespace string) error {
+	suiteKey := types.NamespacedName{Name: name, Namespace: namespace}
+	suite := &compv1alpha1.ComplianceSuite{}
+	err := wait.Poll(RetryInterval, 30*time.Second, func() (bool, error) {
+		err := f.Client.Get(context.TODO(), suiteKey, suite)
+		if apierrors.IsNotFound(err) {
+			return true, nil
+		}
+		if err != nil {
+			return false, err
+		}
+		return false, nil
+	})
+	if err != nil {
+		return fmt.Errorf("ComplianceSuite %s may not have been fully cleaned up: %w", name, err)
+	}
+	return nil
+}
+
 func (f *Framework) AssertProfileInRuleAnnotation(r *compv1alpha1.Rule, expectedProfileId string) bool {
 	if r.Annotations == nil {
 		return false

tests/e2e/framework/main_entry.go

@@ -168,6 +168,11 @@ func (f *Framework) TearDown() error {
 	if err != nil {
 		return err
 	}
+	// Wait for worker pool to be updated after nodes are unlabeled
+	err = f.WaitForMachineConfigPoolUpdated(workerPoolName)
+	if err != nil {
+		return err
+	}
 	err = f.cleanUpMachineConfigPool("e2e")
 	if err != nil {
 		return err

tests/e2e/serial/main_test.go

@@ -1569,47 +1569,32 @@ func TestVariableTemplate(t *testing.T) {
 
 func TestKubeletConfigRemediation(t *testing.T) {
 	f := framework.Global
-	var baselineImage = fmt.Sprintf("%s:%s", brokenContentImagePath, "new_kubeletconfig")
-	const requiredRule = "kubelet-enable-streaming-connections"
-	pbName := framework.GetObjNameFromTest(t)
-	prefixName := func(profName, ruleBaseName string) string { return profName + "-" + ruleBaseName }
-
-	ocpPb, err := f.CreateProfileBundle(pbName, baselineImage, framework.OcpContentFile)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer f.Client.Delete(context.TODO(), ocpPb)
-	if err := f.WaitForProfileBundleStatus(pbName, compv1alpha1.DataStreamValid); err != nil {
-		t.Fatal(err)
-	}
-
-	// Check that if the rule we are going to test is there
-	requiredRuleName := prefixName(pbName, requiredRule)
-	requiredVersionRuleName := prefixName(pbName, "version-detect-in-ocp")
-	requiredVariableName := prefixName(pbName, "var-streaming-connection-timeouts")
-	suiteName := "kubelet-remediation-test-suite-node"
+	suiteName := framework.GetObjNameFromTest(t) + "-kubelet-remediation"
 
 	tp := &compv1alpha1.TailoredProfile{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      suiteName,
 			Namespace: f.OperatorNamespace,
+			Annotations: map[string]string{
+				compv1alpha1.ProductTypeAnnotation: "Node",
+			},
 		},
 		Spec: compv1alpha1.TailoredProfileSpec{
 			Title:       "kubelet-remediation-test-node",
 			Description: "A test tailored profile to test kubelet remediation",
 			EnableRules: []compv1alpha1.RuleReferenceSpec{
 				{
-					Name:      requiredRuleName,
+					Name:      "ocp4-kubelet-enable-streaming-connections",
 					Rationale: "To be tested",
 				},
 				{
-					Name:      requiredVersionRuleName,
+					Name:      "ocp4-version-detect-in-ocp",
 					Rationale: "To be tested",
 				},
 			},
 			SetValues: []compv1alpha1.VariableValueSpec{
 				{
-					Name:      requiredVariableName,
+					Name:      "ocp4-var-streaming-connection-timeouts",
 					Rationale: "Value to be set",
 					Value:     "8h0m0s",
 				},
@@ -1641,11 +1626,21 @@ func TestKubeletConfigRemediation(t *testing.T) {
 		},
 	}
 
-	err = f.Client.Create(context.TODO(), ssb, nil)
+	err := f.Client.Create(context.TODO(), ssb, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer f.Client.Delete(context.TODO(), ssb)
+	// Cleanup: Delete ScanSettingBinding first, then wait for ComplianceSuite to be cascade-deleted
+	// This ensures proper cleanup before the next test runs
+	defer func() {
+		if err := f.Client.Delete(context.TODO(), ssb); err != nil {
+			t.Logf("Failed to delete ScanSettingBinding: %v", err)
+			return
+		}
+		if err := f.WaitForComplianceSuiteDeletion(suiteName, f.OperatorNamespace); err != nil {
+			t.Logf("ComplianceSuite cleanup warning: %v", err)
+		}
+	}()
 
 	// Ensure that all the scans in the suite have finished and are marked as Done
 	err = f.WaitForSuiteScansStatus(f.OperatorNamespace, suiteName, compv1alpha1.PhaseDone, compv1alpha1.ResultNonCompliant)
@@ -1664,6 +1659,21 @@ func TestKubeletConfigRemediation(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	// Cleanup: Unapply the remediation before test ends to prevent leaving nodes in a modified state
+	// This ensures the cluster is reset after the test completes
+	defer func() {
+		// Finally clean up by removing the remediation and waiting for the nodes to reboot one more time
+		err = f.UnApplyRemediationAndCheck(f.OperatorNamespace, remName, "worker")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = f.WaitForNodesToBeReady()
+		if err != nil {
+			t.Fatalf("failed waiting for nodes to reboot after unapplying MachineConfig: %s", err)
+		}
+	}()
+
 	err = f.ReRunScan(scanName, f.OperatorNamespace)
 	if err != nil {
 		t.Fatal(err)