From 9ae0b4590b1d8e395ac95fcef338ffb9b3b64a80 Mon Sep 17 00:00:00 2001 From: Abner-1 Date: Fri, 18 Apr 2025 11:26:40 +0800 Subject: [PATCH 1/2] wait for status eventually consistent Signed-off-by: Abner-1 --- test/e2e/apps/inplace_vpa.go | 43 +++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/test/e2e/apps/inplace_vpa.go b/test/e2e/apps/inplace_vpa.go index 485bde160d..3c98d33ba9 100644 --- a/test/e2e/apps/inplace_vpa.go +++ b/test/e2e/apps/inplace_vpa.go @@ -429,6 +429,23 @@ var _ = SIGDescribe("InplaceVPA", func() { return cs.Status.UpdatedAvailableReplicas }, 600*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) + // add this to avoid the situation where the pod status keeps changing + // TODO: Remove this for status change optimization, maybe in Kubernetes 1.33. + ginkgo.By("Wait for the pod status to be consistent") + gomega.Eventually(func() string { + pods, err = tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + a1, b1, c1 := getResourcesInfo(pods[0]) + if a1 != a || b1 != b || c1 != c { + framework.Logf("updateSpec %v", a1) + framework.Logf("spec %v", b1) + framework.Logf("container status %v ", c1) + a, b, c = a1, b1, c1 + } + SkipTestWhenCgroupError(pods[0]) + return pods[0].Status.ContainerStatuses[0].ContainerID + }, 120*time.Second, 3*time.Second).Should(gomega.Not(gomega.Equal(oldContainerStatus.ContainerID))) + ginkgo.By("Verify the containerID changed and restartCount should not be 0") pods, err = tester.ListPodsForCloneSet(cs.Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -588,6 +605,23 @@ var _ = SIGDescribe("InplaceVPA", func() { return cs.Status.UpdatedAvailableReplicas }, 600*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) + // add this to avoid the situation where the pod status keeps changing + // TODO: Remove this for status change optimization, maybe in Kubernetes 1.33. + ginkgo.By("Wait for the pod status to be consistent") + gomega.Eventually(func() string { + pods, err = tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + a1, b1, c1 := getResourcesInfo(pods[0]) + if a1 != a || b1 != b || c1 != c { + framework.Logf("updateSpec %v", a1) + framework.Logf("spec %v", b1) + framework.Logf("container status %v ", c1) + a, b, c = a1, b1, c1 + } + SkipTestWhenCgroupError(pods[0]) + return pods[0].Status.ContainerStatuses[0].ContainerID + }, 120*time.Second, 3*time.Second).Should(gomega.Not(gomega.Equal(oldContainerStatus.ContainerID))) + ginkgo.By("Verify the containerID changed and restartCount should be 1") pods, err = tester.ListPodsForCloneSet(cs.Name) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -899,7 +933,14 @@ var _ = SIGDescribe("InplaceVPA", func() { gomega.Expect(redisContainerStatus.RestartCount).Should(gomega.Equal(int32(1))) ginkgo.By("Verify nginx should be stopped after new redis has started") - gomega.Expect(nginxContainerStatus.LastTerminationState.Terminated.FinishedAt.After(redisContainerStatus.State.Running.StartedAt.Time.Add(time.Second*10))). + var t time.Time + if nginxContainerStatus.LastTerminationState.Terminated != nil { + t = nginxContainerStatus.LastTerminationState.Terminated.FinishedAt.Time + } else { + // fix https://github.com/openkruise/kruise/issues/1925 + t = nginxContainerStatus.State.Running.StartedAt.Time + } + gomega.Expect(t.After(redisContainerStatus.State.Running.StartedAt.Time.Add(time.Second*10))). Should(gomega.Equal(true), fmt.Sprintf("nginx finish at %v is not after redis start %v + 10s", nginxContainerStatus.LastTerminationState.Terminated.FinishedAt, redisContainerStatus.State.Running.StartedAt)) From 8187610ef153eeee3430965ba226c29a9fdd8aa0 Mon Sep 17 00:00:00 2001 From: Abner-1 Date: Fri, 18 Apr 2025 15:07:51 +0800 Subject: [PATCH 2/2] refactor inplace e2e Signed-off-by: Abner-1 --- test/e2e/apps/inplace_vpa.go | 1147 +++++++++++++++------------------- 1 file changed, 498 insertions(+), 649 deletions(-) diff --git a/test/e2e/apps/inplace_vpa.go b/test/e2e/apps/inplace_vpa.go index 3c98d33ba9..c7f0a73732 100644 --- a/test/e2e/apps/inplace_vpa.go +++ b/test/e2e/apps/inplace_vpa.go @@ -31,13 +31,13 @@ import ( "github.com/openkruise/kruise/test/e2e/framework" ) +var tester *framework.CloneSetTester + var _ = SIGDescribe("InplaceVPA", func() { f := framework.NewDefaultFramework("inplace-vpa") var ns string var c clientset.Interface var kc kruiseclientset.Interface - var tester *framework.CloneSetTester - var randStr string IsKubernetesVersionLessThan127 := func() bool { if v, err := c.Discovery().ServerVersion(); err != nil { framework.Logf("Failed to discovery server version: %v", err) @@ -52,33 +52,12 @@ var _ = SIGDescribe("InplaceVPA", func() { kc = f.KruiseClientSet ns = f.Namespace.Name tester = framework.NewCloneSetTester(c, kc, ns) - randStr = rand.String(10) if IsKubernetesVersionLessThan127() { ginkgo.Skip("skip this e2e case, it can only run on K8s >= 1.27") } }) - oldResource := v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("200m"), - v1.ResourceMemory: resource.MustParse("200Mi"), - }, - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("1Gi"), - }, - } - newResource := v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("100m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - }, - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("800m"), - v1.ResourceMemory: resource.MustParse("800Mi"), - }, - } // TODO(Abner-1)update only inplace resources may fail in kind e2e. // I will resolve it in another PR //framework.KruiseDescribe("CloneSet Updating with only inplace resource", func() { @@ -304,199 +283,6 @@ var _ = SIGDescribe("InplaceVPA", func() { //}) framework.KruiseDescribe("CloneSet failed to inplace update resource", func() { - var err error - largeResource := v1.ResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("100"), - v1.ResourceMemory: resource.MustParse("1000Gi"), - }, - Limits: map[v1.ResourceName]resource.Quantity{ - v1.ResourceCPU: resource.MustParse("800"), - v1.ResourceMemory: resource.MustParse("8000Gi"), - }, - } - testResizePolicyFailed := func(resizePolicy []v1.ContainerResizePolicy) { - testUpdateResource := func(fn func(pod *v1.PodTemplateSpec), resizePolicy []v1.ContainerResizePolicy) { - j, _ := json.Marshal(resizePolicy) - ginkgo.By(fmt.Sprintf("resize policy %v", string(j))) - cs := tester.NewCloneSet("clone-"+randStr, 1, appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType}) - cs.Spec.Template.Spec.Containers[0].ResizePolicy = resizePolicy - cs.Spec.Template.Spec.Containers[0].Image = NginxImage - cs.Spec.Template.ObjectMeta.Labels["test-env"] = "foo" - cs.Spec.Template.Spec.Containers[0].Env = append(cs.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{ - Name: "TEST_ENV", - ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.labels['test-env']"}}, - }) - cs, err = tester.CreateCloneSet(cs) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(cs.Spec.UpdateStrategy.Type).To(gomega.Equal(appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType)) - - ginkgo.By("Wait for replicas satisfied") - gomega.Eventually(func() int32 { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return cs.Status.Replicas - }, 3*time.Second, time.Second).Should(gomega.Equal(int32(1))) - - ginkgo.By("Wait for all pods ready") - gomega.Eventually(func() int32 { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return cs.Status.ReadyReplicas - }, 60*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) - - pods, err := tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(pods)).Should(gomega.Equal(1)) - oldPodUID := pods[0].UID - oldContainerStatus := pods[0].Status.ContainerStatuses[0] - oldPodResource := getPodResource(pods[0]) - - ginkgo.By("Update CloneSet with large resource") - err = tester.UpdateCloneSet(cs.Name, func(cs *appsv1alpha1.CloneSet) { - if cs.Annotations == nil { - cs.Annotations = map[string]string{} - } - fn(&cs.Spec.Template) - cs.Spec.Template.Spec.Containers[0].Resources = largeResource - }) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Wait for CloneSet generation consistent") - gomega.Eventually(func() bool { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return cs.Generation == cs.Status.ObservedGeneration - }, 10*time.Second, 3*time.Second).Should(gomega.Equal(true)) - - updatedVersion := cs.Status.UpdateRevision - ginkgo.By("Wait for one pods updated and rejected") - gomega.Eventually(func() int32 { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("Cloneset updatedReplicas %v updatedReady %v updatedAvailableReplicas %v ", - cs.Status.UpdatedReplicas, cs.Status.UpdatedReadyReplicas, cs.Status.UpdatedAvailableReplicas) - - pods, err = tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - for _, pod := range pods { - revision := pod.Labels[apps.ControllerRevisionHashLabelKey] - if strings.Contains(updatedVersion, revision) { - if pod.Status.Resize == v1.PodResizeStatusInfeasible { - return 1 - } - } - } - - SkipTestWhenCgroupError(pods[0]) - return 0 - }, 120*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) - - ginkgo.By("Update CloneSet with input resource") - err = tester.UpdateCloneSet(cs.Name, func(cs *appsv1alpha1.CloneSet) { - if cs.Annotations == nil { - cs.Annotations = map[string]string{} - } - fn(&cs.Spec.Template) - }) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Wait for CloneSet generation consistent") - gomega.Eventually(func() bool { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return cs.Generation == cs.Status.ObservedGeneration - }, 10*time.Second, 3*time.Second).Should(gomega.Equal(true)) - - a, b, c := getResourcesInfo(pods[0]) - ginkgo.By("Wait for all pods updated and ready") - gomega.Eventually(func() int32 { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("Cloneset updatedReplicas %v updatedReady %v updatedAvailableReplicas %v ", - cs.Status.UpdatedReplicas, cs.Status.UpdatedReadyReplicas, cs.Status.UpdatedAvailableReplicas) - - pods, err = tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - a1, b1, c1 := getResourcesInfo(pods[0]) - if a1 != a || b1 != b || c1 != c { - framework.Logf("updateSpec %v", a1) - framework.Logf("spec %v", b1) - framework.Logf("container status %v ", c1) - a, b, c = a1, b1, c1 - } - SkipTestWhenCgroupError(pods[0]) - return cs.Status.UpdatedAvailableReplicas - }, 600*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) - - // add this to avoid the situation where the pod status keeps changing - // TODO: Remove this for status change optimization, maybe in Kubernetes 1.33. - ginkgo.By("Wait for the pod status to be consistent") - gomega.Eventually(func() string { - pods, err = tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - a1, b1, c1 := getResourcesInfo(pods[0]) - if a1 != a || b1 != b || c1 != c { - framework.Logf("updateSpec %v", a1) - framework.Logf("spec %v", b1) - framework.Logf("container status %v ", c1) - a, b, c = a1, b1, c1 - } - SkipTestWhenCgroupError(pods[0]) - return pods[0].Status.ContainerStatuses[0].ContainerID - }, 120*time.Second, 3*time.Second).Should(gomega.Not(gomega.Equal(oldContainerStatus.ContainerID))) - - ginkgo.By("Verify the containerID changed and restartCount should not be 0") - pods, err = tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(pods)).Should(gomega.Equal(1)) - newPodUID := pods[0].UID - newContainerStatus := pods[0].Status.ContainerStatuses[0] - - gomega.Expect(oldPodUID).Should(gomega.Equal(newPodUID)) - gomega.Expect(newContainerStatus.ContainerID).NotTo(gomega.Equal(oldContainerStatus.ContainerID)) - gomega.Expect(newContainerStatus.RestartCount).ShouldNot(gomega.Equal(int32(0))) - - pods, err = tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(checkPodResource(pods, oldPodResource, []string{"redis"})).Should(gomega.Equal(true)) - } - // This can't be Conformance yet. - ginkgo.It("in-place update image and resource", func() { - fn := func(pod *v1.PodTemplateSpec) { - spec := &pod.Spec - ginkgo.By("in-place update image and resource") - spec.Containers[0].Image = NewNginxImage - spec.Containers[0].Resources = newResource - } - testUpdateResource(fn, resizePolicy) - }) - - // This can't be Conformance yet. - ginkgo.FIt("in-place update resource and env from label", func() { - fn := func(pod *v1.PodTemplateSpec) { - spec := &pod.Spec - ginkgo.By("in-place update resource and env from label") - pod.Labels["test-env"] = "bar" - spec.Containers[0].Resources = newResource - } - testUpdateResource(fn, resizePolicy) - }) - - // This can't be Conformance yet. - ginkgo.It("in-place update image, resource and env from label", func() { - fn := func(pod *v1.PodTemplateSpec) { - spec := &pod.Spec - ginkgo.By("in-place update image, resource and env from label") - spec.Containers[0].Image = NewNginxImage - pod.Labels["test-env"] = "bar" - spec.Containers[0].Resources = newResource - } - testUpdateResource(fn, resizePolicy) - }) - - } - ginkgo.By("inplace update resources with RestartContainer policy") testResizePolicyFailed([]v1.ContainerResizePolicy{ { @@ -530,432 +316,6 @@ var _ = SIGDescribe("InplaceVPA", func() { }) framework.KruiseDescribe("CloneSet Updating with inplace resource", func() { - var err error - testWithResizePolicy := func(resizePolicy []v1.ContainerResizePolicy) { - testUpdateResource := func(fn func(pod *v1.PodTemplateSpec), resizePolicy []v1.ContainerResizePolicy) { - j, _ := json.Marshal(resizePolicy) - ginkgo.By(fmt.Sprintf("resize policy %v", string(j))) - cs := tester.NewCloneSet("clone-"+randStr, 1, appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType}) - cs.Spec.Template.Spec.Containers[0].ResizePolicy = resizePolicy - cs.Spec.Template.Spec.Containers[0].Image = NginxImage - cs.Spec.Template.ObjectMeta.Labels["test-env"] = "foo" - cs.Spec.Template.Spec.Containers[0].Env = append(cs.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{ - Name: "TEST_ENV", - ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.labels['test-env']"}}, - }) - cs, err = tester.CreateCloneSet(cs) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(cs.Spec.UpdateStrategy.Type).To(gomega.Equal(appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType)) - - ginkgo.By("Wait for replicas satisfied") - gomega.Eventually(func() int32 { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return cs.Status.Replicas - }, 3*time.Second, time.Second).Should(gomega.Equal(int32(1))) - - ginkgo.By("Wait for all pods ready") - gomega.Eventually(func() int32 { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return cs.Status.ReadyReplicas - }, 60*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) - - pods, err := tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(pods)).Should(gomega.Equal(1)) - oldPodUID := pods[0].UID - oldContainerStatus := pods[0].Status.ContainerStatuses[0] - oldPodResource := getPodResource(pods[0]) - - ginkgo.By("Update CloneSet") - err = tester.UpdateCloneSet(cs.Name, func(cs *appsv1alpha1.CloneSet) { - if cs.Annotations == nil { - cs.Annotations = map[string]string{} - } - fn(&cs.Spec.Template) - }) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Wait for CloneSet generation consistent") - gomega.Eventually(func() bool { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return cs.Generation == cs.Status.ObservedGeneration - }, 10*time.Second, 3*time.Second).Should(gomega.Equal(true)) - - a, b, c := getResourcesInfo(pods[0]) - ginkgo.By("Wait for all pods updated and ready") - gomega.Eventually(func() int32 { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - framework.Logf("Cloneset updatedReplicas %v updatedReady %v updatedAvailableReplicas %v ", - cs.Status.UpdatedReplicas, cs.Status.UpdatedReadyReplicas, cs.Status.UpdatedAvailableReplicas) - - pods, err = tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - a1, b1, c1 := getResourcesInfo(pods[0]) - if a1 != a || b1 != b || c1 != c { - framework.Logf("updateSpec %v", a1) - framework.Logf("spec %v", b1) - framework.Logf("container status %v ", c1) - a, b, c = a1, b1, c1 - } - SkipTestWhenCgroupError(pods[0]) - return cs.Status.UpdatedAvailableReplicas - }, 600*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) - - // add this to avoid the situation where the pod status keeps changing - // TODO: Remove this for status change optimization, maybe in Kubernetes 1.33. - ginkgo.By("Wait for the pod status to be consistent") - gomega.Eventually(func() string { - pods, err = tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - a1, b1, c1 := getResourcesInfo(pods[0]) - if a1 != a || b1 != b || c1 != c { - framework.Logf("updateSpec %v", a1) - framework.Logf("spec %v", b1) - framework.Logf("container status %v ", c1) - a, b, c = a1, b1, c1 - } - SkipTestWhenCgroupError(pods[0]) - return pods[0].Status.ContainerStatuses[0].ContainerID - }, 120*time.Second, 3*time.Second).Should(gomega.Not(gomega.Equal(oldContainerStatus.ContainerID))) - - ginkgo.By("Verify the containerID changed and restartCount should be 1") - pods, err = tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(pods)).Should(gomega.Equal(1)) - newPodUID := pods[0].UID - newContainerStatus := pods[0].Status.ContainerStatuses[0] - - gomega.Expect(oldPodUID).Should(gomega.Equal(newPodUID)) - gomega.Expect(newContainerStatus.ContainerID).NotTo(gomega.Equal(oldContainerStatus.ContainerID)) - gomega.Expect(newContainerStatus.RestartCount).Should(gomega.Equal(int32(1))) - - pods, err = tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(checkPodResource(pods, oldPodResource, []string{"redis"})).Should(gomega.Equal(true)) - } - // This can't be Conformance yet. - ginkgo.It("in-place update image and resource", func() { - fn := func(pod *v1.PodTemplateSpec) { - spec := &pod.Spec - ginkgo.By("in-place update image and resource") - spec.Containers[0].Image = NewNginxImage - spec.Containers[0].Resources = newResource - } - testUpdateResource(fn, resizePolicy) - }) - - // This can't be Conformance yet. - ginkgo.FIt("in-place update resource and env from label", func() { - fn := func(pod *v1.PodTemplateSpec) { - spec := &pod.Spec - ginkgo.By("in-place update resource and env from label") - pod.Labels["test-env"] = "bar" - spec.Containers[0].Resources = newResource - } - testUpdateResource(fn, resizePolicy) - }) - - // This can't be Conformance yet. - ginkgo.It("in-place update image, resource and env from label", func() { - fn := func(pod *v1.PodTemplateSpec) { - spec := &pod.Spec - ginkgo.By("in-place update image, resource and env from label") - spec.Containers[0].Image = NewNginxImage - pod.Labels["test-env"] = "bar" - spec.Containers[0].Resources = newResource - } - testUpdateResource(fn, resizePolicy) - }) - - framework.ConformanceIt("in-place update two container image, resource with priorities successfully", func() { - cs := tester.NewCloneSet("clone-"+randStr, 1, appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType}) - cs.Spec.Template.Spec.Containers[0].ResizePolicy = resizePolicy - cs.Spec.Template.Spec.Containers = append(cs.Spec.Template.Spec.Containers, v1.Container{ - Name: "redis", - Image: RedisImage, - Command: []string{"sleep", "999"}, - Env: []v1.EnvVar{{Name: appspub.ContainerLaunchPriorityEnvName, Value: "10"}}, - Lifecycle: &v1.Lifecycle{PostStart: &v1.LifecycleHandler{Exec: &v1.ExecAction{Command: []string{"sleep", "10"}}}}, - }) - cs.Spec.Template.Spec.TerminationGracePeriodSeconds = utilpointer.Int64(3) - cs, err = tester.CreateCloneSet(cs) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(cs.Spec.UpdateStrategy.Type).To(gomega.Equal(appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType)) - - ginkgo.By("Wait for replicas satisfied") - gomega.Eventually(func() int32 { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return cs.Status.Replicas - }, 3*time.Second, time.Second).Should(gomega.Equal(int32(1))) - - ginkgo.By("Wait for all pods ready") - gomega.Eventually(func() int32 { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return cs.Status.ReadyReplicas - }, 60*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) - - pods, err := tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(pods)).Should(gomega.Equal(1)) - oldPodResource := getPodResource(pods[0]) - - ginkgo.By("Update images of nginx and redis") - err = tester.UpdateCloneSet(cs.Name, func(cs *appsv1alpha1.CloneSet) { - cs.Spec.Template.Spec.Containers[0].Image = NewNginxImage - cs.Spec.Template.Spec.Containers[1].Image = imageutils.GetE2EImage(imageutils.BusyBox) - cs.Spec.Template.Spec.Containers[0].Resources = newResource - }) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Wait for CloneSet generation consistent") - gomega.Eventually(func() bool { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return cs.Generation == cs.Status.ObservedGeneration - }, 10*time.Second, 3*time.Second).Should(gomega.Equal(true)) - - ginkgo.By("Wait for all pods updated and ready") - a, b, c := getResourcesInfo(pods[0]) - gomega.Eventually(func() int32 { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - pods, err = tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - a1, b1, c1 := getResourcesInfo(pods[0]) - if a1 != a || b1 != b || c1 != c { - framework.Logf("updateSpec %v", a1) - framework.Logf("spec %v", b1) - framework.Logf("container status %v ", c1) - a, b, c = a1, b1, c1 - } - SkipTestWhenCgroupError(pods[0]) - - return cs.Status.UpdatedAvailableReplicas - }, 600*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) - - ginkgo.By("Verify two containers have all updated in-place") - pods, err = tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(pods)).Should(gomega.Equal(1)) - - pod := pods[0] - nginxContainerStatus := util.GetContainerStatus("nginx", pod) - redisContainerStatus := util.GetContainerStatus("redis", pod) - gomega.Expect(nginxContainerStatus.RestartCount).Should(gomega.Equal(int32(1))) - gomega.Expect(redisContainerStatus.RestartCount).Should(gomega.Equal(int32(1))) - - ginkgo.By("Verify nginx should be stopped after new redis has started 10s") - gomega.Expect(nginxContainerStatus.LastTerminationState.Terminated.FinishedAt.After(redisContainerStatus.State.Running.StartedAt.Time.Add(time.Second*10))). - Should(gomega.Equal(true), fmt.Sprintf("nginx finish at %v is not after redis start %v + 10s", - nginxContainerStatus.LastTerminationState.Terminated.FinishedAt, - redisContainerStatus.State.Running.StartedAt)) - - ginkgo.By("Verify in-place update state in two batches") - inPlaceUpdateState := appspub.InPlaceUpdateState{} - gomega.Expect(pod.Annotations[appspub.InPlaceUpdateStateKey]).ShouldNot(gomega.BeEmpty()) - err = json.Unmarshal([]byte(pod.Annotations[appspub.InPlaceUpdateStateKey]), &inPlaceUpdateState) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(inPlaceUpdateState.ContainerBatchesRecord)).Should(gomega.Equal(2)) - gomega.Expect(inPlaceUpdateState.ContainerBatchesRecord[0].Containers).Should(gomega.Equal([]string{"redis"})) - gomega.Expect(inPlaceUpdateState.ContainerBatchesRecord[1].Containers).Should(gomega.Equal([]string{"nginx"})) - gomega.Expect(checkPodResource(pods, oldPodResource, []string{"redis"})).Should(gomega.Equal(true)) - }) - - framework.ConformanceIt("in-place update two container image, resource with priorities, should not update the next when the previous one failed", func() { - cs := tester.NewCloneSet("clone-"+randStr, 1, appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType}) - cs.Spec.Template.Spec.Containers = append(cs.Spec.Template.Spec.Containers, v1.Container{ - Name: "redis", - Image: RedisImage, - Env: []v1.EnvVar{{Name: appspub.ContainerLaunchPriorityEnvName, Value: "10"}}, - Lifecycle: &v1.Lifecycle{PostStart: &v1.LifecycleHandler{Exec: &v1.ExecAction{Command: []string{"sleep", "10"}}}}, - }) - cs.Spec.Template.Spec.TerminationGracePeriodSeconds = utilpointer.Int64(3) - cs, err = tester.CreateCloneSet(cs) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(cs.Spec.UpdateStrategy.Type).To(gomega.Equal(appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType)) - - ginkgo.By("Wait for replicas satisfied") - gomega.Eventually(func() int32 { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return cs.Status.Replicas - }, 3*time.Second, time.Second).Should(gomega.Equal(int32(1))) - - ginkgo.By("Wait for all pods ready") - gomega.Eventually(func() int32 { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return cs.Status.ReadyReplicas - }, 60*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) - - pods, err := tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(pods)).Should(gomega.Equal(1)) - oldPodResource := getPodResource(pods[0]) - - ginkgo.By("Update images of nginx and redis") - err = tester.UpdateCloneSet(cs.Name, func(cs *appsv1alpha1.CloneSet) { - cs.Spec.Template.Spec.Containers[0].Image = NewNginxImage - cs.Spec.Template.Spec.Containers[1].Image = imageutils.GetE2EImage(imageutils.BusyBox) - cs.Spec.Template.Spec.Containers[0].Resources = newResource - }) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Wait for CloneSet generation consistent") - gomega.Eventually(func() bool { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return cs.Generation == cs.Status.ObservedGeneration - }, 10*time.Second, 3*time.Second).Should(gomega.Equal(true)) - - ginkgo.By("Wait for redis failed to start") - var pod *v1.Pod - gomega.Eventually(func() *v1.ContainerStateTerminated { - pods, err = tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(pods)).Should(gomega.Equal(1)) - pod = pods[0] - redisContainerStatus := util.GetContainerStatus("redis", pod) - return redisContainerStatus.LastTerminationState.Terminated - }, 60*time.Second, time.Second).ShouldNot(gomega.BeNil()) - - gomega.Eventually(func() *v1.ContainerStateWaiting { - pods, err = tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(pods)).Should(gomega.Equal(1)) - pod = pods[0] - redisContainerStatus := util.GetContainerStatus("redis", pod) - return redisContainerStatus.State.Waiting - }, 60*time.Second, time.Second).ShouldNot(gomega.BeNil()) - - nginxContainerStatus := util.GetContainerStatus("nginx", pod) - gomega.Expect(nginxContainerStatus.RestartCount).Should(gomega.Equal(int32(0))) - - ginkgo.By("Verify in-place update state only one batch and remain next") - inPlaceUpdateState := appspub.InPlaceUpdateState{} - gomega.Expect(pod.Annotations[appspub.InPlaceUpdateStateKey]).ShouldNot(gomega.BeEmpty()) - err = json.Unmarshal([]byte(pod.Annotations[appspub.InPlaceUpdateStateKey]), &inPlaceUpdateState) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(inPlaceUpdateState.ContainerBatchesRecord)).Should(gomega.Equal(1)) - gomega.Expect(inPlaceUpdateState.ContainerBatchesRecord[0].Containers).Should(gomega.Equal([]string{"redis"})) - gomega.Expect(inPlaceUpdateState.NextContainerImages).Should(gomega.Equal(map[string]string{"nginx": NewNginxImage})) - gomega.Expect(checkPodResource(pods, oldPodResource, []string{"redis"})).Should(gomega.Equal(false)) - }) - - //This can't be Conformance yet. - ginkgo.It("in-place update two container image, resource and env from metadata with priorities", func() { - cs := tester.NewCloneSet("clone-"+randStr, 1, appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType}) - cs.Spec.Template.Spec.Containers[0].ResizePolicy = resizePolicy - cs.Spec.Template.Annotations = map[string]string{"config": "foo"} - cs.Spec.Template.Spec.Containers = append(cs.Spec.Template.Spec.Containers, v1.Container{ - Name: "redis", - Image: RedisImage, - Env: []v1.EnvVar{ - {Name: appspub.ContainerLaunchPriorityEnvName, Value: "10"}, - {Name: "CONFIG", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.annotations['config']"}}}, - }, - Lifecycle: &v1.Lifecycle{PostStart: &v1.LifecycleHandler{Exec: &v1.ExecAction{Command: []string{"sleep", "10"}}}}, - }) - cs, err = tester.CreateCloneSet(cs) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(cs.Spec.UpdateStrategy.Type).To(gomega.Equal(appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType)) - - ginkgo.By("Wait for replicas satisfied") - gomega.Eventually(func() int32 { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return cs.Status.Replicas - }, 3*time.Second, time.Second).Should(gomega.Equal(int32(1))) - - ginkgo.By("Wait for all pods ready") - gomega.Eventually(func() int32 { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return cs.Status.ReadyReplicas - }, 60*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) - - pods, err := tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(pods)).Should(gomega.Equal(1)) - oldPodResource := getPodResource(pods[0]) - - ginkgo.By("Update nginx image and config annotation") - err = tester.UpdateCloneSet(cs.Name, func(cs *appsv1alpha1.CloneSet) { - cs.Spec.Template.Spec.Containers[0].Image = NewNginxImage - cs.Spec.Template.Annotations["config"] = "bar" - cs.Spec.Template.Spec.Containers[0].Resources = newResource - }) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - ginkgo.By("Wait for CloneSet generation consistent") - gomega.Eventually(func() bool { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - return cs.Generation == cs.Status.ObservedGeneration - }, 10*time.Second, 3*time.Second).Should(gomega.Equal(true)) - - ginkgo.By("Wait for all pods updated and ready") - a, b, c := getResourcesInfo(pods[0]) - gomega.Eventually(func() int32 { - cs, err = tester.GetCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - pods, err = tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - a1, b1, c1 := getResourcesInfo(pods[0]) - if a1 != a || b1 != b || c1 != c { - framework.Logf("updateSpec %v", a1) - framework.Logf("spec %v", b1) - framework.Logf("container status %v ", c1) - a, b, c = a1, b1, c1 - } - SkipTestWhenCgroupError(pods[0]) - - return cs.Status.UpdatedAvailableReplicas - }, 600*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) - - ginkgo.By("Verify two containers have all updated in-place") - pods, err = tester.ListPodsForCloneSet(cs.Name) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(pods)).Should(gomega.Equal(1)) - - pod := pods[0] - nginxContainerStatus := util.GetContainerStatus("nginx", pod) - redisContainerStatus := util.GetContainerStatus("redis", pod) - gomega.Expect(nginxContainerStatus.RestartCount).Should(gomega.Equal(int32(1))) - gomega.Expect(redisContainerStatus.RestartCount).Should(gomega.Equal(int32(1))) - - ginkgo.By("Verify nginx should be stopped after new redis has started") - var t time.Time - if nginxContainerStatus.LastTerminationState.Terminated != nil { - t = nginxContainerStatus.LastTerminationState.Terminated.FinishedAt.Time - } else { - // fix https://github.com/openkruise/kruise/issues/1925 - t = nginxContainerStatus.State.Running.StartedAt.Time - } - gomega.Expect(t.After(redisContainerStatus.State.Running.StartedAt.Time.Add(time.Second*10))). - Should(gomega.Equal(true), fmt.Sprintf("nginx finish at %v is not after redis start %v + 10s", - nginxContainerStatus.LastTerminationState.Terminated.FinishedAt, - redisContainerStatus.State.Running.StartedAt)) - - ginkgo.By("Verify in-place update state in two batches") - inPlaceUpdateState := appspub.InPlaceUpdateState{} - gomega.Expect(pod.Annotations[appspub.InPlaceUpdateStateKey]).ShouldNot(gomega.BeEmpty()) - err = json.Unmarshal([]byte(pod.Annotations[appspub.InPlaceUpdateStateKey]), &inPlaceUpdateState) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Expect(len(inPlaceUpdateState.ContainerBatchesRecord)).Should(gomega.Equal(2)) - gomega.Expect(inPlaceUpdateState.ContainerBatchesRecord[0].Containers).Should(gomega.Equal([]string{"redis"})) - gomega.Expect(inPlaceUpdateState.ContainerBatchesRecord[1].Containers).Should(gomega.Equal([]string{"nginx"})) - gomega.Expect(checkPodResource(pods, oldPodResource, []string{"redis"})).Should(gomega.Equal(true)) - }) - } ginkgo.By("inplace update resources with RestartContainer policy") testWithResizePolicy([]v1.ContainerResizePolicy{ @@ -1147,13 +507,8 @@ var _ = SIGDescribe("InplaceVPA", func() { oldPodList, err = tester.ListDaemonPods(label) pod := &oldPodList.Items[lastId] gomega.Expect(err).NotTo(gomega.HaveOccurred()) - a1, b1, c1 := getResourcesInfo(pod) - if a1 != a || b1 != b || c1 != c { - framework.Logf("updateSpec %v", a1) - framework.Logf("spec %v", b1) - framework.Logf("container status %v ", c1) - a, b, c = a1, b1, c1 - } + getNewestResourcesInfo(pod, &a, &b, &c) + SkipTestWhenCgroupError(pod) return updatedAvailable }, 600*time.Second, 3*time.Second).Should(gomega.Equal(int32(len(oldPodList.Items)))) @@ -1260,6 +615,7 @@ func baseValidFn(pods []v1.Pod, image string, revision string) { revision)) } } + func rollbackWithUpdateFnTest(c clientset.Interface, kc kruiseclientset.Interface, ns string, ss *appsv1beta1.StatefulSet, updateFn, rollbackFn func(update *appsv1beta1.StatefulSet), validateFn1, validateFn2 func([]v1.Pod)) { sst := framework.NewStatefulSetTester(c, kc) @@ -1350,6 +706,20 @@ func SkipTestWhenCgroupError(pod *v1.Pod) { } } +func getNewestResourcesInfo(po *v1.Pod, a, b, c *string) { + if po == nil { + return + } + a1, b1, c1 := getResourcesInfo(po) + + if a1 != *a || b1 != *b || c1 != *c { + framework.Logf("updateSpec %v", a1) + framework.Logf("spec %v", b1) + framework.Logf("container status %v ", c1) + a, b, c = &a1, &b1, &c1 + } +} + func getResourcesInfo(po *v1.Pod) (string, string, string) { if po == nil { return "", "", "" @@ -1365,6 +735,7 @@ func getResourcesInfo(po *v1.Pod) (string, string, string) { containerStatusJson, _ := json.Marshal(containerStatus) return lastState, string(specResourcesJson), string(containerStatusJson) } + func IsPodCreateError(pod *v1.Pod) bool { if pod == nil { return false @@ -1383,3 +754,481 @@ func IsPodCreateError(pod *v1.Pod) bool { } return false } + +var ( + largeResource = v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("100"), + v1.ResourceMemory: resource.MustParse("1000Gi"), + }, + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("800"), + v1.ResourceMemory: resource.MustParse("8000Gi"), + }, + } + oldResource = v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse("200Mi"), + }, + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("1Gi"), + }, + } + newResource = v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("800m"), + v1.ResourceMemory: resource.MustParse("800Mi"), + }, + } +) + +func updateClone(tester *framework.CloneSetTester, fn func(pod *v1.PodTemplateSpec), cs *appsv1alpha1.CloneSet, injectFailedResource bool) { + err := tester.UpdateCloneSet(cs.Name, func(cs *appsv1alpha1.CloneSet) { + if cs.Annotations == nil { + cs.Annotations = map[string]string{} + } + fn(&cs.Spec.Template) + if injectFailedResource { + cs.Spec.Template.Spec.Containers[0].Resources = largeResource + } + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("Wait for CloneSet generation consistent") + gomega.Eventually(func() bool { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Generation == cs.Status.ObservedGeneration + }, 10*time.Second, 3*time.Second).Should(gomega.Equal(true)) +} + +// add this to avoid the situation where the pod status keeps changing +// TODO: Remove this for status change optimization, maybe in Kubernetes 1.33. +func waitUtilAllConsistent(oldContainerId string, fn func() (pods []*v1.Pod, err error), a, b, c *string) { + ginkgo.By("Wait for the pod status to be consistent") + gomega.Eventually(func() string { + pods, err := fn() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + getNewestResourcesInfo(pods[0], a, b, c) + + SkipTestWhenCgroupError(pods[0]) + return pods[0].Status.ContainerStatuses[0].ContainerID + }, 120*time.Second, 3*time.Second).Should(gomega.Not(gomega.Equal(oldContainerId))) +} + +func waitCloneSetUpdatedAndReady(tester *framework.CloneSetTester, cs *appsv1alpha1.CloneSet, oldContainerId string, pods []*v1.Pod) { + var err error + a, b, c := getResourcesInfo(pods[0]) + ginkgo.By("Wait for all pods updated and ready") + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("Cloneset updatedReplicas %v updatedReady %v updatedAvailableReplicas %v ", + cs.Status.UpdatedReplicas, cs.Status.UpdatedReadyReplicas, cs.Status.UpdatedAvailableReplicas) + + pods, err = tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + getNewestResourcesInfo(pods[0], &a, &b, &c) + SkipTestWhenCgroupError(pods[0]) + return cs.Status.UpdatedAvailableReplicas + }, 600*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) + + waitUtilAllConsistent(oldContainerId, func() (pods []*v1.Pod, err error) { + return tester.ListPodsForCloneSet(cs.Name) + }, &a, &b, &c) +} + +func testUpdateResource(tester *framework.CloneSetTester, fn func(pod *v1.PodTemplateSpec), resizePolicy []v1.ContainerResizePolicy, injectFailedResource bool) { + var err error + randStr := rand.String(10) + j, _ := json.Marshal(resizePolicy) + ginkgo.By(fmt.Sprintf("resize policy %v", string(j))) + cs := tester.NewCloneSet("clone-"+randStr, 1, appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType}) + cs.Spec.Template.Spec.Containers[0].ResizePolicy = resizePolicy + cs.Spec.Template.Spec.Containers[0].Image = NginxImage + cs.Spec.Template.ObjectMeta.Labels["test-env"] = "foo" + cs.Spec.Template.Spec.Containers[0].Env = append(cs.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{ + Name: "TEST_ENV", + ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.labels['test-env']"}}, + }) + cs, err = tester.CreateCloneSet(cs) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(cs.Spec.UpdateStrategy.Type).To(gomega.Equal(appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType)) + + ginkgo.By("Wait for replicas satisfied") + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.Replicas + }, 3*time.Second, time.Second).Should(gomega.Equal(int32(1))) + + ginkgo.By("Wait for all pods ready") + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.ReadyReplicas + }, 60*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) + + pods, err := tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(pods)).Should(gomega.Equal(1)) + oldPodUID := pods[0].UID + oldContainerStatus := pods[0].Status.ContainerStatuses[0] + oldPodResource := getPodResource(pods[0]) + + if injectFailedResource { + ginkgo.By("Update with large resource") + updateClone(tester, fn, cs, true) + + updatedVersion := cs.Status.UpdateRevision + ginkgo.By("Wait for one pods updated and rejected") + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + framework.Logf("Cloneset updatedReplicas %v updatedReady %v updatedAvailableReplicas %v ", + cs.Status.UpdatedReplicas, cs.Status.UpdatedReadyReplicas, cs.Status.UpdatedAvailableReplicas) + + pods, err = tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + for _, pod := range pods { + revision := pod.Labels[apps.ControllerRevisionHashLabelKey] + if strings.Contains(updatedVersion, revision) { + if pod.Status.Resize == v1.PodResizeStatusInfeasible { + return 1 + } + } + } + + SkipTestWhenCgroupError(pods[0]) + return 0 + }, 120*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) + } + + ginkgo.By("Update with input resource") + updateClone(tester, fn, cs, false) + + waitCloneSetUpdatedAndReady(tester, cs, oldContainerStatus.ContainerID, pods) + + ginkgo.By("Verify the containerID changed and restartCount should not be 0") + pods, err = tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(pods)).Should(gomega.Equal(1)) + newPodUID := pods[0].UID + newContainerStatus := pods[0].Status.ContainerStatuses[0] + + gomega.Expect(oldPodUID).Should(gomega.Equal(newPodUID)) + gomega.Expect(newContainerStatus.ContainerID).NotTo(gomega.Equal(oldContainerStatus.ContainerID)) + gomega.Expect(newContainerStatus.RestartCount).ShouldNot(gomega.Equal(int32(0))) + + pods, err = tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(checkPodResource(pods, oldPodResource, []string{"redis"})).Should(gomega.Equal(true)) +} + +func testResizePolicyFailed(resizePolicy []v1.ContainerResizePolicy) { + injectFailedResource := true + // This can't be Conformance yet. + ginkgo.It("in-place update image and resource", func() { + fn := func(pod *v1.PodTemplateSpec) { + spec := &pod.Spec + ginkgo.By("in-place update image and resource") + spec.Containers[0].Image = NewNginxImage + spec.Containers[0].Resources = newResource + } + testUpdateResource(tester, fn, resizePolicy, injectFailedResource) + }) + + // This can't be Conformance yet. + ginkgo.FIt("in-place update resource and env from label", func() { + fn := func(pod *v1.PodTemplateSpec) { + spec := &pod.Spec + ginkgo.By("in-place update resource and env from label") + pod.Labels["test-env"] = "bar" + spec.Containers[0].Resources = newResource + } + testUpdateResource(tester, fn, resizePolicy, injectFailedResource) + }) + + // This can't be Conformance yet. + ginkgo.It("in-place update image, resource and env from label", func() { + fn := func(pod *v1.PodTemplateSpec) { + spec := &pod.Spec + ginkgo.By("in-place update image, resource and env from label") + spec.Containers[0].Image = NewNginxImage + pod.Labels["test-env"] = "bar" + spec.Containers[0].Resources = newResource + } + testUpdateResource(tester, fn, resizePolicy, injectFailedResource) + }) +} + +func testWithResizePolicy(resizePolicy []v1.ContainerResizePolicy) { + var err error + randStr := rand.String(10) + injectFailedResource := false + + // This can't be Conformance yet. + ginkgo.It("in-place update image and resource", func() { + fn := func(pod *v1.PodTemplateSpec) { + spec := &pod.Spec + ginkgo.By("in-place update image and resource") + spec.Containers[0].Image = NewNginxImage + spec.Containers[0].Resources = newResource + } + testUpdateResource(tester, fn, resizePolicy, injectFailedResource) + }) + + // This can't be Conformance yet. + ginkgo.FIt("in-place update resource and env from label", func() { + fn := func(pod *v1.PodTemplateSpec) { + spec := &pod.Spec + ginkgo.By("in-place update resource and env from label") + pod.Labels["test-env"] = "bar" + spec.Containers[0].Resources = newResource + } + testUpdateResource(tester, fn, resizePolicy, injectFailedResource) + }) + + // This can't be Conformance yet. + ginkgo.It("in-place update image, resource and env from label", func() { + fn := func(pod *v1.PodTemplateSpec) { + spec := &pod.Spec + ginkgo.By("in-place update image, resource and env from label") + spec.Containers[0].Image = NewNginxImage + pod.Labels["test-env"] = "bar" + spec.Containers[0].Resources = newResource + } + testUpdateResource(tester, fn, resizePolicy, injectFailedResource) + }) + + framework.ConformanceIt("in-place update two container image, resource with priorities successfully", func() { + cs := tester.NewCloneSet("clone-"+randStr, 1, appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType}) + cs.Spec.Template.Spec.Containers[0].ResizePolicy = resizePolicy + cs.Spec.Template.Spec.Containers = append(cs.Spec.Template.Spec.Containers, v1.Container{ + Name: "redis", + Image: RedisImage, + Command: []string{"sleep", "999"}, + Env: []v1.EnvVar{{Name: appspub.ContainerLaunchPriorityEnvName, Value: "10"}}, + Lifecycle: &v1.Lifecycle{PostStart: &v1.LifecycleHandler{Exec: &v1.ExecAction{Command: []string{"sleep", "10"}}}}, + }) + cs.Spec.Template.Spec.TerminationGracePeriodSeconds = utilpointer.Int64(3) + cs, err = tester.CreateCloneSet(cs) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(cs.Spec.UpdateStrategy.Type).To(gomega.Equal(appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType)) + + ginkgo.By("Wait for replicas satisfied") + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.Replicas + }, 3*time.Second, time.Second).Should(gomega.Equal(int32(1))) + + ginkgo.By("Wait for all pods ready") + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.ReadyReplicas + }, 60*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) + + pods, err := tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(pods)).Should(gomega.Equal(1)) + oldPodResource := getPodResource(pods[0]) + + updateClone(tester, func(pod *v1.PodTemplateSpec) { + pod.Spec.Containers[0].Image = NewNginxImage + pod.Spec.Containers[1].Image = imageutils.GetE2EImage(imageutils.BusyBox) + pod.Spec.Containers[0].Resources = newResource + }, cs, false) + + waitCloneSetUpdatedAndReady(tester, cs, pods[0].Status.ContainerStatuses[0].ContainerID, pods) + + ginkgo.By("Verify two containers have all updated in-place") + pods, err = tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(pods)).Should(gomega.Equal(1)) + + pod := pods[0] + nginxContainerStatus := util.GetContainerStatus("nginx", pod) + redisContainerStatus := util.GetContainerStatus("redis", pod) + gomega.Expect(nginxContainerStatus.RestartCount).Should(gomega.Equal(int32(1))) + gomega.Expect(redisContainerStatus.RestartCount).Should(gomega.Equal(int32(1))) + + ginkgo.By("Verify nginx should be stopped after new redis has started 10s") + gomega.Expect(nginxContainerStatus.LastTerminationState.Terminated.FinishedAt.After(redisContainerStatus.State.Running.StartedAt.Time.Add(time.Second*10))). + Should(gomega.Equal(true), fmt.Sprintf("nginx finish at %v is not after redis start %v + 10s", + nginxContainerStatus.LastTerminationState.Terminated.FinishedAt, + redisContainerStatus.State.Running.StartedAt)) + + ginkgo.By("Verify in-place update state in two batches") + inPlaceUpdateState := appspub.InPlaceUpdateState{} + gomega.Expect(pod.Annotations[appspub.InPlaceUpdateStateKey]).ShouldNot(gomega.BeEmpty()) + err = json.Unmarshal([]byte(pod.Annotations[appspub.InPlaceUpdateStateKey]), &inPlaceUpdateState) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(inPlaceUpdateState.ContainerBatchesRecord)).Should(gomega.Equal(2)) + gomega.Expect(inPlaceUpdateState.ContainerBatchesRecord[0].Containers).Should(gomega.Equal([]string{"redis"})) + gomega.Expect(inPlaceUpdateState.ContainerBatchesRecord[1].Containers).Should(gomega.Equal([]string{"nginx"})) + gomega.Expect(checkPodResource(pods, oldPodResource, []string{"redis"})).Should(gomega.Equal(true)) + }) + + framework.ConformanceIt("in-place update two container image, resource with priorities, should not update the next when the previous one failed", func() { + cs := tester.NewCloneSet("clone-"+randStr, 1, appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType}) + cs.Spec.Template.Spec.Containers = append(cs.Spec.Template.Spec.Containers, v1.Container{ + Name: "redis", + Image: RedisImage, + Env: []v1.EnvVar{{Name: appspub.ContainerLaunchPriorityEnvName, Value: "10"}}, + Lifecycle: &v1.Lifecycle{PostStart: &v1.LifecycleHandler{Exec: &v1.ExecAction{Command: []string{"sleep", "10"}}}}, + }) + cs.Spec.Template.Spec.TerminationGracePeriodSeconds = utilpointer.Int64(3) + cs, err = tester.CreateCloneSet(cs) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(cs.Spec.UpdateStrategy.Type).To(gomega.Equal(appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType)) + + ginkgo.By("Wait for replicas satisfied") + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.Replicas + }, 3*time.Second, time.Second).Should(gomega.Equal(int32(1))) + + ginkgo.By("Wait for all pods ready") + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.ReadyReplicas + }, 60*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) + + pods, err := tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(pods)).Should(gomega.Equal(1)) + oldPodResource := getPodResource(pods[0]) + + ginkgo.By("Update images of nginx and redis") + updateClone(tester, func(pod *v1.PodTemplateSpec) { + pod.Spec.Containers[0].Image = NewNginxImage + pod.Spec.Containers[1].Image = imageutils.GetE2EImage(imageutils.BusyBox) + pod.Spec.Containers[0].Resources = newResource + }, cs, false) + + ginkgo.By("Wait for redis failed to start") + var pod *v1.Pod + gomega.Eventually(func() *v1.ContainerStateTerminated { + pods, err = tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(pods)).Should(gomega.Equal(1)) + pod = pods[0] + redisContainerStatus := util.GetContainerStatus("redis", pod) + return redisContainerStatus.LastTerminationState.Terminated + }, 60*time.Second, time.Second).ShouldNot(gomega.BeNil()) + + gomega.Eventually(func() *v1.ContainerStateWaiting { + pods, err = tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(pods)).Should(gomega.Equal(1)) + pod = pods[0] + redisContainerStatus := util.GetContainerStatus("redis", pod) + return redisContainerStatus.State.Waiting + }, 60*time.Second, time.Second).ShouldNot(gomega.BeNil()) + + nginxContainerStatus := util.GetContainerStatus("nginx", pod) + gomega.Expect(nginxContainerStatus.RestartCount).Should(gomega.Equal(int32(0))) + + ginkgo.By("Verify in-place update state only one batch and remain next") + inPlaceUpdateState := appspub.InPlaceUpdateState{} + gomega.Expect(pod.Annotations[appspub.InPlaceUpdateStateKey]).ShouldNot(gomega.BeEmpty()) + err = json.Unmarshal([]byte(pod.Annotations[appspub.InPlaceUpdateStateKey]), &inPlaceUpdateState) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(inPlaceUpdateState.ContainerBatchesRecord)).Should(gomega.Equal(1)) + gomega.Expect(inPlaceUpdateState.ContainerBatchesRecord[0].Containers).Should(gomega.Equal([]string{"redis"})) + gomega.Expect(inPlaceUpdateState.NextContainerImages).Should(gomega.Equal(map[string]string{"nginx": NewNginxImage})) + gomega.Expect(checkPodResource(pods, oldPodResource, []string{"redis"})).Should(gomega.Equal(false)) + }) + + //This can't be Conformance yet. + ginkgo.It("in-place update two container image, resource and env from metadata with priorities", func() { + cs := tester.NewCloneSet("clone-"+randStr, 1, appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType}) + cs.Spec.Template.Spec.Containers[0].ResizePolicy = resizePolicy + cs.Spec.Template.Annotations = map[string]string{"config": "foo"} + cs.Spec.Template.Spec.Containers = append(cs.Spec.Template.Spec.Containers, v1.Container{ + Name: "redis", + Image: RedisImage, + Env: []v1.EnvVar{ + {Name: appspub.ContainerLaunchPriorityEnvName, Value: "10"}, + {Name: "CONFIG", ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "metadata.annotations['config']"}}}, + }, + Lifecycle: &v1.Lifecycle{PostStart: &v1.LifecycleHandler{Exec: &v1.ExecAction{Command: []string{"sleep", "10"}}}}, + }) + cs, err = tester.CreateCloneSet(cs) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(cs.Spec.UpdateStrategy.Type).To(gomega.Equal(appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType)) + + ginkgo.By("Wait for replicas satisfied") + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.Replicas + }, 3*time.Second, time.Second).Should(gomega.Equal(int32(1))) + + ginkgo.By("Wait for all pods ready") + gomega.Eventually(func() int32 { + cs, err = tester.GetCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cs.Status.ReadyReplicas + }, 60*time.Second, 3*time.Second).Should(gomega.Equal(int32(1))) + + pods, err := tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(pods)).Should(gomega.Equal(1)) + oldPodResource := getPodResource(pods[0]) + + ginkgo.By("Update nginx image and config annotation") + + updateClone(tester, func(pod *v1.PodTemplateSpec) { + pod.Spec.Containers[0].Image = NewNginxImage + pod.Annotations["config"] = "bar" + pod.Spec.Containers[0].Resources = newResource + }, cs, false) + + waitCloneSetUpdatedAndReady(tester, cs, pods[0].Status.ContainerStatuses[0].ContainerID, pods) + + ginkgo.By("Verify two containers have all updated in-place") + pods, err = tester.ListPodsForCloneSet(cs.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(pods)).Should(gomega.Equal(1)) + + pod := pods[0] + nginxContainerStatus := util.GetContainerStatus("nginx", pod) + redisContainerStatus := util.GetContainerStatus("redis", pod) + gomega.Expect(nginxContainerStatus.RestartCount).Should(gomega.Equal(int32(1))) + gomega.Expect(redisContainerStatus.RestartCount).Should(gomega.Equal(int32(1))) + + ginkgo.By("Verify nginx should be stopped after new redis has started") + var t time.Time + if nginxContainerStatus.LastTerminationState.Terminated != nil { + t = nginxContainerStatus.LastTerminationState.Terminated.FinishedAt.Time + } else { + // fix https://github.com/openkruise/kruise/issues/1925 + t = nginxContainerStatus.State.Running.StartedAt.Time + } + gomega.Expect(t.After(redisContainerStatus.State.Running.StartedAt.Time.Add(time.Second*10))). + Should(gomega.Equal(true), fmt.Sprintf("nginx finish at %v is not after redis start %v + 10s", + nginxContainerStatus.LastTerminationState.Terminated.FinishedAt, + redisContainerStatus.State.Running.StartedAt)) + + ginkgo.By("Verify in-place update state in two batches") + inPlaceUpdateState := appspub.InPlaceUpdateState{} + gomega.Expect(pod.Annotations[appspub.InPlaceUpdateStateKey]).ShouldNot(gomega.BeEmpty()) + err = json.Unmarshal([]byte(pod.Annotations[appspub.InPlaceUpdateStateKey]), &inPlaceUpdateState) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(inPlaceUpdateState.ContainerBatchesRecord)).Should(gomega.Equal(2)) + gomega.Expect(inPlaceUpdateState.ContainerBatchesRecord[0].Containers).Should(gomega.Equal([]string{"redis"})) + gomega.Expect(inPlaceUpdateState.ContainerBatchesRecord[1].Containers).Should(gomega.Equal([]string{"nginx"})) + gomega.Expect(checkPodResource(pods, oldPodResource, []string{"redis"})).Should(gomega.Equal(true)) + }) +}