fix(instance): fix and refactor ready condition (#6153)

Author: liubog2008

api/core/v1alpha1/common_types.go

@@ -31,7 +31,19 @@ const (
 const (
 	// Ready means all managed resources are ready.
 	// NOTE: It does not mean all managed resources are up to date.
+	//
+	// condition
 	CondReady = "Ready"
+	// reason for both
+	ReasonReady   = "Ready"
+	ReasonUnready = "Unready"
+	// reason for group
+	ReasonNotAllInstancesReady = "NotAllInstancesReady"
+	// reason for instance
+	ReasonPodNotCreated      = "PodNotCreated"
+	ReasonPodNotReady        = "PodNotReady"
+	ReasonPodTerminating     = "PodTerminating"
+	ReasonInstanceNotHealthy = "InstanceNotHealthy"
 
 	// Synced means all specs of managed resources are as expected and
 	// nothing need to do in controller but only status updation.

pkg/apicall/instance.go

@@ -41,6 +41,10 @@ func GetPod[
 		return nil, err
 	}
 
+	if len(pl.Items) == 0 {
+		return nil, nil
+	}
+
 	if len(pl.Items) != 1 {
 		return nil, fmt.Errorf("expected only 1 pod, but now %d", len(pl.Items))
 	}

pkg/apiutil/core/v1alpha1/condition.go

@@ -15,6 +15,8 @@
 package coreutil
 
 import (
+	"fmt"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/pingcap/tidb-operator/api/v2/core/v1alpha1"
@@ -51,17 +53,34 @@ func Ready() *metav1.Condition {
 	return &metav1.Condition{
 		Type:    v1alpha1.CondReady,
 		Status:  metav1.ConditionTrue,
-		Reason:  "Ready",
+		Reason:  v1alpha1.CondReady,
 		Message: "all subreources are ready",
 	}
 }
 
-func Unready() *metav1.Condition {
+func Unready(reason string) *metav1.Condition {
+	var msg string
+	switch reason {
+	case v1alpha1.ReasonNotAllInstancesReady:
+		msg = "not all instances are ready"
+	case v1alpha1.ReasonPodNotCreated:
+		msg = "pod of the instance does not exist"
+	case v1alpha1.ReasonPodNotReady:
+		msg = "pod of the instance is not ready"
+	case v1alpha1.ReasonPodTerminating:
+		msg = "pod of the instance is terminating"
+	case v1alpha1.ReasonInstanceNotHealthy:
+		msg = "instance is not probed as healthy"
+	default:
+		msg = fmt.Sprintf("unready because of unknown reason: %s", reason)
+		reason = v1alpha1.ReasonUnready
+	}
+
 	return &metav1.Condition{
 		Type:    v1alpha1.CondReady,
 		Status:  metav1.ConditionFalse,
-		Reason:  "Unready",
-		Message: "not all instances are ready",
+		Reason:  reason,
+		Message: msg,
 	}
 }
 

pkg/controllers/common/interfaces.go

@@ -246,13 +246,38 @@ type (
 	}
 	PodState interface {
 		Pod() *corev1.Pod
+		IsPodTerminating() bool
+	}
+	PodStateUpdater interface {
+		// It will be called after get or update api calls to k8s
+		SetPod(pod *corev1.Pod)
+		// Pod cannot be updated when call DELETE API, so we have to mark pod is deleting after calling DELETE api
+		DeletePod(pod *corev1.Pod)
 	}
 )
 
 type StoreState interface {
 	GetStoreState() string
-	SetStoreState(string)
 	// IsStoreUp returns true if the store state is `Preparing` or `Serving`,
 	// which means the store is in the state of providing services.
 	IsStoreUp() bool
 }
+
+type StoreStateUpdater interface {
+	SetStoreState(string)
+}
+
+type HealthyState interface {
+	// It means the instance is healthy to serve.
+	// Normally, it's from PD or api exposed by the instance.
+	// Now the operator checks it just like a liveness/readiness prober.
+	// It may be removed if all components support /ready api in the future.
+	// And then we can use pod's ready condition directly to check whether the instance
+	// is ready.
+	// But now, we still probe health from the operator.
+	IsHealthy() bool
+}
+
+type HealthyStateUpdater interface {
+	SetHealthy()
+}

pkg/controllers/common/interfaces_test.go

@@ -91,6 +91,10 @@ func (f *fakePodState) PodInitializer() PodInitializer {
 	return f.s.Initializer()
 }
 
+func (f *fakePodState) IsPodTerminating() bool {
+	return false
+}
+
 type fakePDSliceState struct {
 	s *fakeSliceState[v1alpha1.PD]
 }

pkg/controllers/common/mock_generated.go

@@ -21,6 +21,7 @@ import (
 	"slices"
 	"strings"
 
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	kuberuntime "k8s.io/apimachinery/pkg/runtime"
@@ -126,11 +127,6 @@ func TaskContextTiCDC(state TiCDCStateInitializer, c client.Client) task.Task {
 	return taskContextResource("TiCDC", w, c, false)
 }
 
-func TaskContextPod(state PodStateInitializer, c client.Client) task.Task {
-	w := state.PodInitializer()
-	return taskContextResource("Pod", w, c, false)
-}
-
 func TaskContextPDGroup(state PDGroupStateInitializer, c client.Client) task.Task {
 	w := state.PDGroupInitializer()
 	return taskContextResource("PDGroup", w, c, false)
@@ -232,3 +228,29 @@ func TaskContextCluster[
 		return task.Complete().With("cluster is set")
 	})
 }
+
+type ContextPodNewer[
+	F client.Object,
+] interface {
+	Object() F
+	SetPod(pod *corev1.Pod)
+}
+
+func TaskContextPod[
+	S scope.Instance[F, T],
+	F client.Object,
+	T runtime.Instance,
+](state ContextPodNewer[F], c client.Client) task.Task {
+	return task.NameTaskFunc("ContextPod", func(ctx context.Context) task.Result {
+		pod, err := apicall.GetPod[S](ctx, c, state.Object())
+		if err != nil {
+			return task.Fail().With("cannot get pod: %v", err)
+		}
+		if pod == nil {
+			return task.Complete().With("pod doesn't exist")
+		}
+
+		state.SetPod(pod)
+		return task.Complete().With("pod is set")
+	})
+}