Skip to content

Commit 2a9af0c

Browse files
liubog2008liubog2008
authored
feat(updater): defer scale in after rolling update is done (#6052)
1 parent 3d2a1b5 commit 2a9af0c

File tree

13 files changed

+212
-64
lines changed

13 files changed

+212
-64
lines changed

‎apis/core/v1alpha1/common_types.go

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,15 @@ const (
3535
const (
3636
// Finalizer is the finalizer used by all resources managed by TiDB Operator.
3737
Finalizer = "core.pingcap.com/finalizer"
38+
)
3839

40+
const (
41+
KeyPrefix = "pingcap.com/"
42+
)
43+
44+
const (
3945
// LabelKeyPrefix defines key prefix of well known labels
40-
LabelKeyPrefix = "pingcap.com/"
46+
LabelKeyPrefix = KeyPrefix
4147

4248
// LabelKeyManagedBy means resources are managed by tidb operator
4349
LabelKeyManagedBy = LabelKeyPrefix + "managed-by"
@@ -83,6 +89,17 @@ const (
8389
LabelKeyStoreID = "tidb.pingcap.com/store-id"
8490
)
8591

92+
const (
93+
// AnnoKeyPrefix defines key prefix of well known annotations
94+
AnnoKeyPrefix = KeyPrefix
95+
96+
// all bool anno will use this val as default
97+
AnnoValTrue = "true"
98+
99+
// means the instance is marked as deleted and will be deleted later
100+
AnnoKeyDeferDelete = AnnoKeyPrefix + "defer-delete"
101+
)
102+
86103
const (
87104
// NamePrefix for "names" in k8s resources
88105
// Users may overlay some fields in managed resource such as pods. Names with this

‎pkg/client/alias.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,3 +44,7 @@ var ObjectKeyFromObject = client.ObjectKeyFromObject
4444
var IgnoreNotFound = client.IgnoreNotFound
4545

4646
type GracePeriodSeconds = client.GracePeriodSeconds
47+
48+
type MergeFromOption = client.MergeFromOption
49+
50+
var RawPatch = client.RawPatch

‎pkg/controllers/pdgroup/tasks/updater_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ func TestTaskUpdater(t *testing.T) {
5555
},
5656
},
5757

58-
expectedStatus: task.SComplete,
58+
expectedStatus: task.SWait,
5959
expectedPDNum: 1,
6060
},
6161
{
@@ -103,7 +103,7 @@ func TestTaskUpdater(t *testing.T) {
103103
},
104104
},
105105

106-
expectedStatus: task.SComplete,
106+
expectedStatus: task.SWait,
107107
expectedPDNum: 2,
108108
},
109109
{
@@ -236,7 +236,7 @@ func TestTaskUpdater(t *testing.T) {
236236
},
237237
},
238238

239-
expectedStatus: task.SComplete,
239+
expectedStatus: task.SWait,
240240
expectedPDNum: 3,
241241
},
242242
}

‎pkg/controllers/tidbgroup/tasks/updater_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ func TestTaskUpdater(t *testing.T) {
5555
},
5656
},
5757

58-
expectedStatus: task.SComplete,
58+
expectedStatus: task.SWait,
5959
expectedTiDBNum: 1,
6060
},
6161
{
@@ -103,7 +103,7 @@ func TestTaskUpdater(t *testing.T) {
103103
},
104104
},
105105

106-
expectedStatus: task.SComplete,
106+
expectedStatus: task.SWait,
107107
expectedTiDBNum: 2,
108108
},
109109
{
@@ -236,7 +236,7 @@ func TestTaskUpdater(t *testing.T) {
236236
},
237237
},
238238

239-
expectedStatus: task.SComplete,
239+
expectedStatus: task.SWait,
240240
expectedTiDBNum: 3,
241241
},
242242
}

‎pkg/controllers/tiflashgroup/tasks/updater_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ func TestTaskUpdater(t *testing.T) {
5555
},
5656
},
5757

58-
expectedStatus: task.SComplete,
58+
expectedStatus: task.SWait,
5959
expectedTiFlashNum: 1,
6060
},
6161
{
@@ -103,7 +103,7 @@ func TestTaskUpdater(t *testing.T) {
103103
},
104104
},
105105

106-
expectedStatus: task.SComplete,
106+
expectedStatus: task.SWait,
107107
expectedTiFlashNum: 2,
108108
},
109109
{
@@ -236,7 +236,7 @@ func TestTaskUpdater(t *testing.T) {
236236
},
237237
},
238238

239-
expectedStatus: task.SComplete,
239+
expectedStatus: task.SWait,
240240
expectedTiFlashNum: 3,
241241
},
242242
}

‎pkg/controllers/tikvgroup/tasks/updater_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ func TestTaskUpdater(t *testing.T) {
5555
},
5656
},
5757

58-
expectedStatus: task.SComplete,
58+
expectedStatus: task.SWait,
5959
expectedTiKVNum: 1,
6060
},
6161
{
@@ -103,7 +103,7 @@ func TestTaskUpdater(t *testing.T) {
103103
},
104104
},
105105

106-
expectedStatus: task.SComplete,
106+
expectedStatus: task.SWait,
107107
expectedTiKVNum: 2,
108108
},
109109
{
@@ -236,7 +236,7 @@ func TestTaskUpdater(t *testing.T) {
236236
},
237237
},
238238

239-
expectedStatus: task.SComplete,
239+
expectedStatus: task.SWait,
240240
expectedTiKVNum: 3,
241241
},
242242
}

‎pkg/updater/actor.go

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,12 @@ package updater
1616

1717
import (
1818
"context"
19+
"encoding/json"
1920
"fmt"
2021

22+
"k8s.io/apimachinery/pkg/types"
23+
24+
"github.com/pingcap/tidb-operator/apis/core/v1alpha1"
2125
"github.com/pingcap/tidb-operator/pkg/client"
2226
"github.com/pingcap/tidb-operator/pkg/runtime"
2327
)
@@ -41,6 +45,8 @@ type actor[T runtime.Tuple[O, R], O client.Object, R runtime.Instance] struct {
4145

4246
update State[R]
4347
outdated State[R]
48+
// deleted set records all instances that are marked by defer delete annotation
49+
deleted State[R]
4450

4551
addHooks []AddHook[R]
4652
updateHooks []UpdateHook[R]
@@ -112,7 +118,7 @@ func (act *actor[T, O, R]) ScaleInOutdated(ctx context.Context) (bool, error) {
112118
obj := act.outdated.Del(name)
113119
isUnavailable := !obj.IsHealthy() || !obj.IsUpToDate()
114120

115-
if err := act.c.Delete(ctx, act.converter.To(obj)); err != nil {
121+
if err := act.deferDelete(ctx, obj); err != nil {
116122
return false, err
117123
}
118124

@@ -123,6 +129,40 @@ func (act *actor[T, O, R]) ScaleInOutdated(ctx context.Context) (bool, error) {
123129
return isUnavailable, nil
124130
}
125131

132+
type Patch struct {
133+
Metadata Metadata `json:"metadata"`
134+
}
135+
136+
type Metadata struct {
137+
ResourceVersion string `json:"resourceVersion"`
138+
Annotations map[string]string `json:"annotations"`
139+
}
140+
141+
func (act *actor[T, O, R]) deferDelete(ctx context.Context, obj R) error {
142+
o := act.converter.To(obj)
143+
p := Patch{
144+
Metadata: Metadata{
145+
ResourceVersion: o.GetResourceVersion(),
146+
Annotations: map[string]string{
147+
v1alpha1.AnnoKeyDeferDelete: v1alpha1.AnnoValTrue,
148+
},
149+
},
150+
}
151+
152+
data, err := json.Marshal(&p)
153+
if err != nil {
154+
return fmt.Errorf("invaid patch: %w", err)
155+
}
156+
157+
if err := act.c.Patch(ctx, o, client.RawPatch(types.MergePatchType, data)); err != nil {
158+
return fmt.Errorf("cannot mark obj %s/%s as defer delete: %w", obj.GetNamespace(), obj.GetName(), err)
159+
}
160+
161+
act.deleted.Add(obj)
162+
163+
return nil
164+
}
165+
126166
func (act *actor[T, O, R]) Update(ctx context.Context) error {
127167
name, err := act.chooseToUpdate(act.outdated.List())
128168
if err != nil {
@@ -143,3 +183,13 @@ func (act *actor[T, O, R]) Update(ctx context.Context) error {
143183

144184
return nil
145185
}
186+
187+
func (act *actor[T, O, R]) Cleanup(ctx context.Context) error {
188+
for _, item := range act.deleted.List() {
189+
if err := act.c.Delete(ctx, act.converter.To(item)); err != nil {
190+
return err
191+
}
192+
}
193+
194+
return nil
195+
}

‎pkg/updater/builder.go

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
package updater
1616

1717
import (
18+
"github.com/pingcap/tidb-operator/apis/core/v1alpha1"
1819
"github.com/pingcap/tidb-operator/pkg/client"
1920
"github.com/pingcap/tidb-operator/pkg/runtime"
2021
)
@@ -55,7 +56,7 @@ type builder[T runtime.Tuple[O, R], O client.Object, R runtime.Instance] struct
5556
}
5657

5758
func (b *builder[T, O, R]) Build() Executor {
58-
update, outdated := split(b.instances, b.rev)
59+
update, outdated, deleted := split(b.instances, b.rev)
5960

6061
updatePolicies := b.updatePreferPolicies
6162
updatePolicies = append(updatePolicies, PreferUnavailable[R]())
@@ -65,6 +66,7 @@ func (b *builder[T, O, R]) Build() Executor {
6566

6667
update: NewState(update),
6768
outdated: NewState(outdated),
69+
deleted: NewState(deleted),
6870

6971
addHooks: b.addHooks,
7072
updateHooks: append(b.updateHooks, KeepName[R](), KeepTopology[R]()),
@@ -141,7 +143,7 @@ func (b *builder[T, O, R]) WithUpdatePreferPolicy(ps ...PreferPolicy[R]) Builder
141143
return b
142144
}
143145

144-
func split[R runtime.Instance](all []R, rev string) (update, outdated []R) {
146+
func split[R runtime.Instance](all []R, rev string) (update, outdated, deleted []R) {
145147
for _, instance := range all {
146148
// if instance is deleting, just ignore it
147149
// TODO(liubo02): make sure it's ok for PD
@@ -150,12 +152,14 @@ func split[R runtime.Instance](all []R, rev string) (update, outdated []R) {
150152
}
151153
if instance.GetUpdateRevision() == rev {
152154
update = append(update, instance)
155+
} else if _, ok := instance.GetAnnotations()[v1alpha1.AnnoKeyDeferDelete]; ok {
156+
deleted = append(deleted, instance)
153157
} else {
154158
outdated = append(outdated, instance)
155159
}
156160
}
157161

158-
return update, outdated
162+
return update, outdated, deleted
159163
}
160164

161165
func countUnavailable[R runtime.Instance](all []R) int {

‎pkg/updater/builder_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,12 @@ func TestBuilder(t *testing.T) {
5050
{
5151
desc: "scale out",
5252
desired: 3,
53-
expectedWait: false,
53+
expectedWait: true,
5454
},
5555
{
5656
desc: "scale in",
5757
desired: 1,
58-
expectedWait: false,
58+
expectedWait: true,
5959
},
6060
}
6161

‎pkg/updater/executor.go

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@ type Actor interface {
2323
Update(ctx context.Context) error
2424
ScaleInUpdate(ctx context.Context) (unavailable bool, _ error)
2525
ScaleInOutdated(ctx context.Context) (unavailable bool, _ error)
26+
27+
// delete all instances marked as defer deletion
28+
Cleanup(ctx context.Context) error
2629
}
2730

2831
// TODO: return instance list after Do
@@ -150,7 +153,7 @@ func (ex *executor) Do(ctx context.Context) (bool, error) {
150153
checkAvail = true
151154
}
152155
} else {
153-
// ex.update + ex.outdated > ex.desired + min(ex.maxSurge, ex.outdated) and ex.update >= ex.desired
156+
// ex.update + ex.outdated > ex.desired + min(ex.maxSurge, ex.outdated) and ex.update <= ex.desired
154157
// => ex.outdated > min(ex.maxSurge, ex.outdated)
155158
// => ex.outdated > 0
156159
unavailable, err := ex.act.ScaleInOutdated(ctx)
@@ -173,5 +176,14 @@ func (ex *executor) Do(ctx context.Context) (bool, error) {
173176
}
174177
}
175178

179+
if ex.unavailableUpdate > 0 {
180+
// wait until update are all available
181+
return true, nil
182+
}
183+
184+
if err := ex.act.Cleanup(ctx); err != nil {
185+
return false, err
186+
}
187+
176188
return false, nil
177189
}

0 commit comments

Comments
 (0)