Skip to content

Commit 469992a

Browse files
liubog2008liubog2008
authored
fix(updater): fix topology policy (#6054)
* fix(updater): fix topology policy Signed-off-by: liubo02 <liubo02@pingcap.com> * fix lint Signed-off-by: liubo02 <liubo02@pingcap.com> * fix e2e Signed-off-by: liubo02 <liubo02@pingcap.com> --------- Signed-off-by: liubo02 <liubo02@pingcap.com>
1 parent 7d238e5 commit 469992a

File tree

12 files changed

+241
-19
lines changed

12 files changed

+241
-19
lines changed

‎.golangci.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,11 @@ linters-settings:
4444
- '3'
4545
ignored-functions:
4646
- strings.SplitN
47+
ignored-files:
48+
- 'tests/e2e/cluster/.+\.go$'
49+
- 'tests/e2e/pd/.+\.go$'
50+
- 'tests/e2e/tidb/.+\.go$'
51+
- 'tests/e2e/tikv/.+\.go$'
4752
govet:
4853
enable:
4954
- nilness

‎hack/lib/kind.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,15 @@ networking:
5656
ipFamily: dual
5757
nodes:
5858
- role: control-plane
59+
- role: worker
60+
labels:
61+
zone: zone-a
62+
- role: worker
63+
labels:
64+
zone: zone-b
65+
- role: worker
66+
labels:
67+
zone: zone-c
5968
EOF
6069

6170

‎pkg/updater/policy/topology.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ func NewTopologyPolicy[R runtime.Instance](ts []v1alpha1.ScheduleTopology, rs ..
4040
scheduler: s,
4141
}
4242
for _, r := range rs {
43-
p.Add(r)
43+
p.scheduler.Add(r.GetName(), r.GetTopology())
4444
}
4545
return p, nil
4646
}

‎tests/e2e/cluster/cluster.go

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -537,7 +537,6 @@ var _ = Describe("TiDB Cluster", func() {
537537
Eventually(func(g Gomega) {
538538
podList, err = clientSet.CoreV1().Pods(tc.Namespace).List(ctx, metav1.ListOptions{})
539539
g.Expect(err).To(BeNil())
540-
//nolint:mnd // easy to understand
541540
g.Expect(len(podList.Items)).To(Equal(4))
542541
}).WithTimeout(time.Minute).WithPolling(createClusterPolling).Should(Succeed())
543542

@@ -676,7 +675,6 @@ var _ = Describe("TiDB Cluster", func() {
676675
var kvgGet v1alpha1.TiKVGroup
677676
Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: tc.Namespace, Name: kvg.Name}, &kvgGet)).To(Succeed())
678677
Expect(len(kvgGet.Spec.Template.Spec.Volumes)).To(Equal(1))
679-
//nolint:mnd // easy to understand
680678
kvgGet.Spec.Template.Spec.Volumes[0].Storage = data.StorageSizeGi2quantity(5)
681679
Expect(k8sClient.Update(ctx, &kvgGet)).To(Succeed())
682680

@@ -692,7 +690,6 @@ var _ = Describe("TiDB Cluster", func() {
692690
pvcList, err := clientSet.CoreV1().PersistentVolumeClaims(tc.Namespace).List(ctx, listOpts)
693691
Expect(err).To(BeNil())
694692
Expect(len(pvcList.Items)).To(Equal(1))
695-
//nolint:mnd // easy to understand
696693
Expect(pvcList.Items[0].Status.Capacity.Storage()).To(Equal(data.StorageSizeGi2quantity(5)))
697694
}).WithTimeout(createClusterTimeout).WithPolling(createClusterPolling).Should(Succeed())
698695
})
@@ -908,7 +905,6 @@ var _ = Describe("TiDB Cluster", func() {
908905
GinkgoWriter.Printf("%v(%v) created at %s, deleted at %s\n", info.name, info.uid, info.creationTime, info.deletionTime)
909906
}
910907
}
911-
//nolint:mnd // easy to understand
912908
Expect(len(infos)).To(Equal(6))
913909
Expect(infos[0].name).To(Equal(infos[1].name))
914910
Expect(infos[2].name).To(Equal(infos[3].name))
@@ -1127,7 +1123,6 @@ var _ = Describe("TiDB Cluster", func() {
11271123
GinkgoWriter.Printf("%v(%v) created at %s, deleted at %s\n", info.name, info.uid, info.creationTime, info.deletionTime)
11281124
}
11291125
}
1130-
//nolint:mnd // easy to understand
11311126
Expect(len(infos)).To(Equal(6))
11321127
Expect(infos[0].name).To(Equal(infos[1].name))
11331128
Expect(infos[2].name).To(Equal(infos[3].name))
@@ -1307,8 +1302,7 @@ var _ = Describe("TiDB Cluster", func() {
13071302
VolumeSource: corev1.VolumeSource{
13081303
Secret: &corev1.SecretVolumeSource{
13091304
// TODO(liubo02): extract to a namer pkg
1310-
SecretName: groupName + "-" + componentName + "-cluster-secret",
1311-
//nolint:mnd // easy to understand
1305+
SecretName: groupName + "-" + componentName + "-cluster-secret",
13121306
DefaultMode: ptr.To(int32(420)),
13131307
},
13141308
},
@@ -1327,8 +1321,7 @@ var _ = Describe("TiDB Cluster", func() {
13271321
VolumeSource: corev1.VolumeSource{
13281322
Secret: &corev1.SecretVolumeSource{
13291323
// TODO(liubo02): extract to a namer pkg
1330-
SecretName: dbg.Name + "-tidb-server-secret",
1331-
//nolint:mnd // easy to understand
1324+
SecretName: dbg.Name + "-tidb-server-secret",
13321325
DefaultMode: ptr.To(int32(420)),
13331326
},
13341327
},
@@ -1446,7 +1439,7 @@ location-labels = ["region", "zone", "host"]`
14461439
g.Expect(v).To(ContainSubstring(`zone=`))
14471440
g.Expect(v).To(ContainSubstring(`host=`))
14481441
}
1449-
}).WithTimeout(time.Minute).WithPolling(5 * time.Second).Should(Succeed()) //nolint:mnd // easy to understand
1442+
}).WithTimeout(time.Minute).WithPolling(5 * time.Second).Should(Succeed())
14501443
})
14511444

14521445
It("should enable readiness probe for PD, TiKV and TiFlash", func() {
@@ -1489,8 +1482,8 @@ location-labels = ["region", "zone", "host"]`
14891482
Expect(pod.Spec.Containers[0].ReadinessProbe.TimeoutSeconds).To(Equal(int32(1)))
14901483
Expect(pod.Spec.Containers[0].ReadinessProbe.FailureThreshold).To(Equal(int32(3)))
14911484
Expect(pod.Spec.Containers[0].ReadinessProbe.SuccessThreshold).To(Equal(int32(1)))
1492-
Expect(pod.Spec.Containers[0].ReadinessProbe.InitialDelaySeconds).To(Equal(int32(10))) //nolint:mnd // default value in Operator
1493-
Expect(pod.Spec.Containers[0].ReadinessProbe.PeriodSeconds).To(Equal(int32(10))) //nolint:mnd // easy to understand
1485+
Expect(pod.Spec.Containers[0].ReadinessProbe.InitialDelaySeconds).To(Equal(int32(10)))
1486+
Expect(pod.Spec.Containers[0].ReadinessProbe.PeriodSeconds).To(Equal(int32(10)))
14941487
Expect(pod.Spec.Containers[0].ReadinessProbe.TCPSocket).NotTo(BeNil())
14951488
}
14961489
}

‎tests/e2e/data/tikv.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,3 +53,30 @@ func NewTiKVGroup(ns string, patches ...GroupPatch[*runtime.TiKVGroup]) *v1alpha
5353

5454
return runtime.ToTiKVGroup(kvg)
5555
}
56+
57+
func WithEvenlySpreadPolicy() GroupPatch[*runtime.TiKVGroup] {
58+
return func(obj *runtime.TiKVGroup) {
59+
obj.Spec.SchedulePolicies = append(obj.Spec.SchedulePolicies, v1alpha1.SchedulePolicy{
60+
Type: v1alpha1.SchedulePolicyTypeEvenlySpread,
61+
EvenlySpread: &v1alpha1.SchedulePolicyEvenlySpread{
62+
Topologies: []v1alpha1.ScheduleTopology{
63+
{
64+
Topology: v1alpha1.Topology{
65+
"zone": "zone-a",
66+
},
67+
},
68+
{
69+
Topology: v1alpha1.Topology{
70+
"zone": "zone-b",
71+
},
72+
},
73+
{
74+
Topology: v1alpha1.Topology{
75+
"zone": "zone-c",
76+
},
77+
},
78+
},
79+
},
80+
})
81+
}
82+
}

‎tests/e2e/e2e_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
_ "github.com/pingcap/tidb-operator/tests/e2e/cluster"
2727
_ "github.com/pingcap/tidb-operator/tests/e2e/pd"
2828
_ "github.com/pingcap/tidb-operator/tests/e2e/tidb"
29+
_ "github.com/pingcap/tidb-operator/tests/e2e/tikv"
2930
)
3031

3132
func TestE2E(t *testing.T) {

‎tests/e2e/framework/tikv.go

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,15 @@ package framework
1616

1717
import (
1818
"context"
19+
"math"
20+
"strings"
1921

2022
"github.com/onsi/ginkgo/v2"
2123

2224
"github.com/pingcap/tidb-operator/apis/core/v1alpha1"
25+
"github.com/pingcap/tidb-operator/pkg/client"
2326
"github.com/pingcap/tidb-operator/pkg/runtime"
27+
"github.com/pingcap/tidb-operator/pkg/utils/topology"
2428
"github.com/pingcap/tidb-operator/tests/e2e/utils/waiter"
2529
)
2630

@@ -30,3 +34,54 @@ func (f *Framework) WaitForTiKVGroupReady(ctx context.Context, kvg *v1alpha1.TiK
3034
f.Must(waiter.WaitForTiKVsHealthy(ctx, f.Client, kvg, waiter.LongTaskTimeout))
3135
f.Must(waiter.WaitForPodsReady(ctx, f.Client, runtime.FromTiKVGroup(kvg), waiter.LongTaskTimeout))
3236
}
37+
38+
func (f *Framework) MustEvenlySpreadTiKV(ctx context.Context, kvg *v1alpha1.TiKVGroup) {
39+
list := v1alpha1.TiKVList{}
40+
f.Must(f.Client.List(ctx, &list, client.InNamespace(kvg.GetNamespace()), client.MatchingLabels{
41+
v1alpha1.LabelKeyCluster: kvg.Spec.Cluster.Name,
42+
v1alpha1.LabelKeyGroup: kvg.GetName(),
43+
v1alpha1.LabelKeyComponent: v1alpha1.LabelValComponentTiKV,
44+
}))
45+
46+
encoder := topology.NewEncoder()
47+
topo := map[string]int{}
48+
49+
detail := strings.Builder{}
50+
for i := range list.Items {
51+
item := &list.Items[i]
52+
53+
key := encoder.Encode(item.Spec.Topology)
54+
val, ok := topo[key]
55+
if !ok {
56+
val = 0
57+
}
58+
val += 1
59+
topo[key] = val
60+
61+
detail.WriteString(item.Name)
62+
detail.WriteString(":\n")
63+
for k, v := range item.Spec.Topology {
64+
detail.WriteString(" ")
65+
detail.WriteString(k)
66+
detail.WriteString(":")
67+
detail.WriteString(v)
68+
detail.WriteString(":\n")
69+
}
70+
}
71+
72+
minimum, maximum := math.MaxInt, 0
73+
for _, val := range topo {
74+
if val < minimum {
75+
minimum = val
76+
}
77+
if val > maximum {
78+
maximum = val
79+
}
80+
}
81+
82+
if maximum-minimum > 1 {
83+
ginkgo.AddReportEntry("TopologyInfo", detail.String())
84+
}
85+
86+
f.True(maximum-minimum <= 1)
87+
}

‎tests/e2e/label/well_known.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,7 @@ var (
3333
Update = ginkgo.Label("op:Update")
3434
Scale = ginkgo.Label("op:Scale")
3535
Suspend = ginkgo.Label("op:Suspend")
36+
37+
// Env
38+
MultipleAZ = ginkgo.Label("env:MultipleAZ")
3639
)

‎tests/e2e/pd/pd.go

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ var _ = ginkgo.Describe("PD", label.PD, func() {
7171
f.WaitForPDGroupReady(ctx, pdg)
7272

7373
patch := client.MergeFrom(pdg.DeepCopy())
74-
pdg.Spec.Replicas = ptr.To[int32](5) //nolint:mnd // easy for test
74+
pdg.Spec.Replicas = ptr.To[int32](5)
7575

7676
ginkgo.By("Change replica of the PDGroup")
7777
f.Must(f.Client.Patch(ctx, pdg, patch))
@@ -81,7 +81,6 @@ var _ = ginkgo.Describe("PD", label.PD, func() {
8181
ginkgo.It("support scale PD form 5 to 3", label.Scale, func(ctx context.Context) {
8282
pdg := data.NewPDGroup(
8383
f.Namespace.Name,
84-
//nolint:mnd // easy for test
8584
data.WithReplicas[*runtime.PDGroup](5),
8685
)
8786

@@ -130,7 +129,6 @@ var _ = ginkgo.Describe("PD", label.PD, func() {
130129
ginkgo.It("support scale PD form 5 to 3 and rolling update at same time", label.Scale, label.Update, func(ctx context.Context) {
131130
pdg := data.NewPDGroup(
132131
f.Namespace.Name,
133-
//nolint:mnd // easy for test
134132
data.WithReplicas[*runtime.PDGroup](5),
135133
)
136134

‎tests/e2e/tikv/tikv.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
// Copyright 2024 PingCAP, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package tikv

‎tests/e2e/tikv/topology.go

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
// Copyright 2024 PingCAP, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package tikv
16+
17+
import (
18+
"context"
19+
"time"
20+
21+
"github.com/onsi/ginkgo/v2"
22+
"k8s.io/utils/ptr"
23+
"sigs.k8s.io/controller-runtime/pkg/client"
24+
25+
"github.com/pingcap/tidb-operator/pkg/runtime"
26+
"github.com/pingcap/tidb-operator/tests/e2e/data"
27+
"github.com/pingcap/tidb-operator/tests/e2e/framework"
28+
"github.com/pingcap/tidb-operator/tests/e2e/label"
29+
"github.com/pingcap/tidb-operator/tests/e2e/utils/waiter"
30+
)
31+
32+
var _ = ginkgo.Describe("Topology", label.TiKV, label.MultipleAZ, label.P0, func() {
33+
f := framework.New()
34+
f.Setup()
35+
f.SetupCluster()
36+
37+
ginkgo.It("Create tikv evenly spread in multiple azs", func(ctx context.Context) {
38+
ginkgo.By("Creating cluster")
39+
pdg := f.MustCreatePD(ctx)
40+
kvg := f.MustCreateTiKV(ctx,
41+
data.WithReplicas[*runtime.TiKVGroup](6),
42+
data.WithEvenlySpreadPolicy(),
43+
)
44+
45+
f.WaitForPDGroupReady(ctx, pdg)
46+
f.WaitForTiKVGroupReady(ctx, kvg)
47+
48+
f.MustEvenlySpreadTiKV(ctx, kvg)
49+
})
50+
51+
ginkgo.It("support scale from 3 to 6 and rolling update at same time", label.Scale, label.Update, func(ctx context.Context) {
52+
ginkgo.By("Creating cluster")
53+
pdg := f.MustCreatePD(ctx)
54+
kvg := f.MustCreateTiKV(ctx,
55+
data.WithReplicas[*runtime.TiKVGroup](3),
56+
data.WithEvenlySpreadPolicy(),
57+
)
58+
59+
f.WaitForPDGroupReady(ctx, pdg)
60+
f.WaitForTiKVGroupReady(ctx, kvg)
61+
62+
f.MustEvenlySpreadTiKV(ctx, kvg)
63+
64+
patch := client.MergeFrom(kvg.DeepCopy())
65+
kvg.Spec.Replicas = ptr.To[int32](6)
66+
kvg.Spec.Template.Spec.Config = `log.level = 'warn'`
67+
68+
nctx, cancel := context.WithCancel(ctx)
69+
ch := make(chan struct{})
70+
go func() {
71+
defer close(ch)
72+
defer ginkgo.GinkgoRecover()
73+
f.Must(waiter.WaitPodsRollingUpdateOnce(nctx, f.Client, runtime.FromTiKVGroup(kvg), 3, waiter.LongTaskTimeout))
74+
}()
75+
76+
maxTime, err := waiter.MaxPodsCreateTimestamp(ctx, f.Client, runtime.FromTiKVGroup(kvg))
77+
f.Must(err)
78+
changeTime := maxTime.Add(time.Second)
79+
80+
ginkgo.By("Change config and replicas of the TiKVGroup")
81+
f.Must(f.Client.Patch(ctx, kvg, patch))
82+
f.Must(waiter.WaitForPodsRecreated(ctx, f.Client, runtime.FromTiKVGroup(kvg), changeTime, waiter.LongTaskTimeout))
83+
f.WaitForTiKVGroupReady(ctx, kvg)
84+
cancel()
85+
<-ch
86+
87+
f.MustEvenlySpreadTiKV(ctx, kvg)
88+
})
89+
})

0 commit comments

Comments
 (0)