Skip to content

Commit 92d20f3

Browse files
fgksgffgksgf
authored
Feat: add tiproxy controllers (#6214)
1 parent e239850 commit 92d20f3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+5459
-77
lines changed

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,5 @@ output
33
.idea/
44
coverage.txt
55
.vscode/
6+
.cursorignore
7+
.DS_Store

‎.golangci.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ linters:
4242
paths:
4343
- ".*/br/.*/_test.go"
4444
- ".*/br/.*/testutils/.*"
45-
- "tests/e2e/br/.*"
45+
- "tests/e2e/.*"
4646
- "third_party/.*"
4747
settings:
4848
dupl:

‎api/core/v1alpha1/common_types.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,7 @@ const (
271271
)
272272

273273
type SchedulePolicyEvenlySpread struct {
274-
// All instances of a group will evenly spread in differnet topologies
274+
// All instances of a group will evenly spread in different topologies
275275
Topologies []ScheduleTopology `json:"topologies"`
276276
}
277277

‎api/core/v1alpha1/names.go

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,12 @@ const (
3838
VolumeNameClusterClientTLS = meta.NamePrefix + "cluster-client-tls"
3939
// VolumeNameMySQLTLS is the volume name for the TLS secret used by TLS communication between TiDB server and MySQL client.
4040
VolumeNameMySQLTLS = meta.NamePrefix + "tidb-sql-tls"
41+
// VolumeNameTiProxyMySQLTLS is the volume name for the TLS secret used by TLS communication between TiProxy and MySQL client.
42+
VolumeNameTiProxyMySQLTLS = meta.NamePrefix + "tiproxy-sql-tls"
43+
// VolumeNameTiProxyHTTPTLS is the volume name for the TLS secret used by TLS communication between TiProxy HTTP server and HTTP client.
44+
VolumeNameTiProxyHTTPTLS = meta.NamePrefix + "tiproxy-http-tls"
45+
// VolumeNameTiProxyTiDBTLS is the volume name for the TLS secret used by TLS communication between TiProxy and TiDB server.
46+
VolumeNameTiProxyTiDBTLS = meta.NamePrefix + "tiproxy-tidb-tls"
4147
)
4248

4349
// All container names
@@ -51,6 +57,7 @@ const (
5157
ContainerNameTiCDC = "ticdc"
5258
ContainerNameTSO = "tso"
5359
ContainerNameScheduler = "scheduler"
60+
ContainerNameTiProxy = "tiproxy"
5461

5562
// An init container to copy pre stop checker cmd to main container
5663
ContainerNamePrestopChecker = meta.NamePrefix + "prestop-checker"
@@ -78,6 +85,7 @@ const (
7885
DirPathConfigTiCDC = "/etc/ticdc"
7986
DirPathConfigTSO = "/etc/tso"
8087
DirPathConfigScheduler = "/etc/scheduler"
88+
DirPathConfigTiProxy = "/etc/tiproxy"
8189

8290
// DirPathPrestop defines dir path of pre stop checker cmd
8391
DirPathPrestop = "/prestop"
@@ -99,8 +107,15 @@ const (
99107
DirPathClusterTLSTiCDC = "/var/lib/ticdc-tls"
100108
DirPathClusterTLSTSO = "/var/lib/tso-tls"
101109
DirPathClusterTLSScheduler = "/var/lib/scheduler-tls"
102-
// Dir path of tls file for tidb and mysql client
110+
DirPathClusterTLSTiProxy = "/var/lib/tiproxy-tls"
111+
// DirPathMySQLTLS is the dir path of tls file for tidb and mysql client
103112
DirPathMySQLTLS = "/var/lib/tidb-sql-tls"
113+
// DirPathTiProxyMySQLTLS is the dir path of tls file for tiproxy and mysql client
114+
DirPathTiProxyMySQLTLS = "/var/lib/tiproxy-sql-tls"
115+
// DirPathTiProxyHTTPTLS is the dir path of tls file for tiproxy http server
116+
DirPathTiProxyHTTPTLS = "/var/lib/tiproxy-http-tls"
117+
// DirPathTiProxyTiDBTLS is the dir path of tls file for tiproxy and tidb
118+
DirPathTiProxyTiDBTLS = "/var/lib/tiproxy-tidb-tls"
104119
)
105120

106121
// All file names

‎api/core/v1alpha1/tiproxy_types.go

Lines changed: 31 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,18 @@ import (
1919
)
2020

2121
const (
22-
TiProxyPortNameClient = "mysql-client"
23-
TiProxyPortNameStatus = "status"
22+
TiProxyPortNameClient = "mysql-client"
23+
TiProxyPortNameAPI = "api"
24+
TiProxyPortNamePeer = "peer"
25+
2426
DefaultTiProxyPortClient = 6000
25-
DefaultTiProxyPortStatus = 3080
27+
DefaultTiProxyPortAPI = 3080
28+
DefaultTiProxyPortPeer = 3081
29+
)
30+
31+
const (
32+
TiProxyGroupCondAvailable = "Available"
33+
TiProxyGroupAvailableReason = "TiProxyGroupAvailable"
2634
)
2735

2836
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -142,8 +150,6 @@ type TiProxyTemplateSpec struct {
142150
type TiProxyPreStop struct {
143151
// SleepSeconds is the seconds to sleep before sending the SIGTERM to the TiProxy container.
144152
// It's useful to achieve a graceful shutdown of the TiProxy container.
145-
// Operator will calculate the TiProxy pod's `terminationGracePeriod` based on this field:
146-
// `terminationGracePeriod` = `preStopHookSleepSeconds` + 15(gracefulCloseConnectionsTimeout) + 5(buffer)
147153
// Default is 10 seconds.
148154
SleepSeconds int32 `json:"sleepSeconds,omitempty"`
149155
}
@@ -156,13 +162,22 @@ type TiProxySecurity struct {
156162
type TiProxyServer struct {
157163
// Port defines all ports listened by TiProxy.
158164
Ports TiProxyPorts `json:"ports,omitempty"`
165+
166+
// Labels defines the server labels of the TiProxy.
167+
// TiDB Operator will ignore `labels` in TiProxy's config file and use this field instead.
168+
// Note these label keys are managed by TiDB Operator, it will be set automatically and you can not modify them:
169+
// - zone
170+
// +kubebuilder:validation:XValidation:rule="!('zone' in self)",message="labels cannot contain 'zone', it's managed by TiDB Operator"
171+
Labels map[string]string `json:"labels,omitempty"`
159172
}
160173

161174
type TiProxyPorts struct {
162175
// Client defines port for TiProxy's SQL service.
163176
Client *Port `json:"client,omitempty"`
164-
// Status defines port for TiProxy status API.
165-
Status *Port `json:"status,omitempty"`
177+
// API defines port for TiProxy API service.
178+
API *Port `json:"api,omitempty"`
179+
// Peer defines port for TiProxy's peer service.
180+
Peer *Port `json:"peer,omitempty"`
166181
}
167182

168183
type TiProxyProbes struct {
@@ -173,29 +188,27 @@ type TiProxyProbes struct {
173188

174189
type TiProxyProb struct {
175190
// "tcp" will use TCP socket to connect component port.
176-
// "command" will probe the status api of TiProxy.
191+
// "command" will probe the HTTP API of TiProxy.
177192
// +kubebuilder:validation:Enum=tcp;command
178193
Type *string `json:"type,omitempty"`
179194
}
180195

181196
type TiProxyTLS struct {
182-
// When enabled, TiProxy will accept TLS encrypted connections from MySQL clients.
197+
// MySQL defines the TLS configuration for connections between TiProxy and MySQL clients.
183198
// The steps to enable this feature:
184-
// 1. Generate a TiProxy server-side certificate and a client-side certificate for the TiProxy cluster.
199+
// 1. Generate a TiProxy server-side certificate for the TiProxy cluster.
185200
// There are multiple ways to generate certificates:
186201
// - user-provided certificates: https://docs.pingcap.com/TiProxy/stable/generate-self-signed-certificates
187202
// - use the K8s built-in certificate signing system signed certificates: https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/
188203
// - or use cert-manager signed certificates: https://cert-manager.io/
189204
// 2. Create a K8s Secret object which contains the TiProxy server-side certificate created above.
190205
// The name of this Secret must be: <groupName>-tiproxy-server-secret.
191206
// kubectl create secret generic <groupName>-tiproxy-server-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
192-
// 3. Create a K8s Secret object which contains the TiProxy client-side certificate created above which will be used by TiProxy Operator.
193-
// The name of this Secret must be: <groupName>-tiproxy-client-secret.
194-
// kubectl create secret generic <groupName>-tiproxy-client-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
195-
// 4. Set Enabled to `true`.
207+
// 3. Set Enabled to `true`.
196208
MySQL *TLS `json:"mysql,omitempty"`
197209

198-
// Backend defines the TLS configuration for connections between TiProxy and TiDB.
210+
// Backend defines the TLS configuration for connections between TiProxy and TiDB servers.
211+
// To enable this feature, the corresponding TiDB server must be configured with TLS enabled.
199212
Backend *TLS `json:"backend,omitempty"`
200213
}
201214

@@ -209,13 +222,13 @@ type TiProxySpec struct {
209222
Cluster ClusterReference `json:"cluster"`
210223

211224
// Topology defines the topology domain of this TiProxy instance.
212-
// It will be translated into a node affnity config.
225+
// It will be translated into a node affinity config.
213226
// Topology cannot be changed.
214227
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="topology is immutable"
215228
Topology Topology `json:"topology,omitempty"`
216229

217-
// Subdomain means the subdomain of the exported pd dns.
218-
// A same pd cluster will use a same subdomain
230+
// Subdomain means the subdomain of the exported tiproxy dns.
231+
// A same tiproxy cluster will use a same subdomain
219232
Subdomain string `json:"subdomain"`
220233

221234
// TiProxyTemplateSpec embeded some fields managed by TiProxyGroup.

‎api/core/v1alpha1/zz_generated.deepcopy.go

Lines changed: 14 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎manifests/crd/core.pingcap.com_pdgroups.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ spec:
106106
properties:
107107
topologies:
108108
description: All instances of a group will evenly spread
109-
in differnet topologies
109+
in different topologies
110110
items:
111111
properties:
112112
topology:

‎manifests/crd/core.pingcap.com_schedulergroups.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ spec:
9898
properties:
9999
topologies:
100100
description: All instances of a group will evenly spread
101-
in differnet topologies
101+
in different topologies
102102
items:
103103
properties:
104104
topology:

‎manifests/crd/core.pingcap.com_ticdcgroups.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ spec:
9595
properties:
9696
topologies:
9797
description: All instances of a group will evenly spread
98-
in differnet topologies
98+
in different topologies
9999
items:
100100
properties:
101101
topology:

‎manifests/crd/core.pingcap.com_tidbgroups.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ spec:
9898
properties:
9999
topologies:
100100
description: All instances of a group will evenly spread
101-
in differnet topologies
101+
in different topologies
102102
items:
103103
properties:
104104
topology:

‎manifests/crd/core.pingcap.com_tiflashgroups.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ spec:
9797
properties:
9898
topologies:
9999
description: All instances of a group will evenly spread
100-
in differnet topologies
100+
in different topologies
101101
items:
102102
properties:
103103
topology:

‎manifests/crd/core.pingcap.com_tikvgroups.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ spec:
9898
properties:
9999
topologies:
100100
description: All instances of a group will evenly spread
101-
in differnet topologies
101+
in different topologies
102102
items:
103103
properties:
104104
topology:

‎manifests/crd/core.pingcap.com_tiproxies.yaml

Lines changed: 34 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8360,8 +8360,6 @@ spec:
83608360
description: |-
83618361
SleepSeconds is the seconds to sleep before sending the SIGTERM to the TiProxy container.
83628362
It's useful to achieve a graceful shutdown of the TiProxy container.
8363-
Operator will calculate the TiProxy pod's `terminationGracePeriod` based on this field:
8364-
`terminationGracePeriod` = `preStopHookSleepSeconds` + 15(gracefulCloseConnectionsTimeout) + 5(buffer)
83658363
Default is 10 seconds.
83668364
format: int32
83678365
type: integer
@@ -8377,7 +8375,7 @@ spec:
83778375
type:
83788376
description: |-
83798377
"tcp" will use TCP socket to connect component port.
8380-
"command" will probe the status api of TiProxy.
8378+
"command" will probe the HTTP API of TiProxy.
83818379
enum:
83828380
- tcp
83838381
- command
@@ -8406,28 +8404,26 @@ spec:
84068404
description: Whether enable the TLS connection.
84078405
properties:
84088406
backend:
8409-
description: Backend defines the TLS configuration for connections
8410-
between TiProxy and TiDB.
8407+
description: |-
8408+
Backend defines the TLS configuration for connections between TiProxy and TiDB servers.
8409+
To enable this feature, the corresponding TiDB server must be configured with TLS enabled.
84118410
properties:
84128411
enabled:
84138412
type: boolean
84148413
type: object
84158414
mysql:
84168415
description: |-
8417-
When enabled, TiProxy will accept TLS encrypted connections from MySQL clients.
8416+
MySQL defines the TLS configuration for connections between TiProxy and MySQL clients.
84188417
The steps to enable this feature:
8419-
1. Generate a TiProxy server-side certificate and a client-side certificate for the TiProxy cluster.
8418+
1. Generate a TiProxy server-side certificate for the TiProxy cluster.
84208419
There are multiple ways to generate certificates:
84218420
- user-provided certificates: https://docs.pingcap.com/TiProxy/stable/generate-self-signed-certificates
84228421
- use the K8s built-in certificate signing system signed certificates: https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/
84238422
- or use cert-manager signed certificates: https://cert-manager.io/
84248423
2. Create a K8s Secret object which contains the TiProxy server-side certificate created above.
84258424
The name of this Secret must be: <groupName>-tiproxy-server-secret.
84268425
kubectl create secret generic <groupName>-tiproxy-server-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
8427-
3. Create a K8s Secret object which contains the TiProxy client-side certificate created above which will be used by TiProxy Operator.
8428-
The name of this Secret must be: <groupName>-tiproxy-client-secret.
8429-
kubectl create secret generic <groupName>-tiproxy-client-secret --namespace=<namespace> --from-file=tls.crt=<path/to/tls.crt> --from-file=tls.key=<path/to/tls.key> --from-file=ca.crt=<path/to/ca.crt>
8430-
4. Set Enabled to `true`.
8426+
3. Set Enabled to `true`.
84318427
properties:
84328428
enabled:
84338429
type: boolean
@@ -8437,9 +8433,31 @@ spec:
84378433
server:
84388434
description: Server defines the server configuration of TiProxy.
84398435
properties:
8436+
labels:
8437+
additionalProperties:
8438+
type: string
8439+
description: |-
8440+
Labels defines the server labels of the TiProxy.
8441+
TiDB Operator will ignore `labels` in TiProxy's config file and use this field instead.
8442+
Note these label keys are managed by TiDB Operator, it will be set automatically and you can not modify them:
8443+
- zone
8444+
type: object
8445+
x-kubernetes-validations:
8446+
- message: labels cannot contain 'zone', it's managed by TiDB
8447+
Operator
8448+
rule: '!(''zone'' in self)'
84408449
ports:
84418450
description: Port defines all ports listened by TiProxy.
84428451
properties:
8452+
api:
8453+
description: API defines port for TiProxy API service.
8454+
properties:
8455+
port:
8456+
format: int32
8457+
type: integer
8458+
required:
8459+
- port
8460+
type: object
84438461
client:
84448462
description: Client defines port for TiProxy's SQL service.
84458463
properties:
@@ -8449,8 +8467,8 @@ spec:
84498467
required:
84508468
- port
84518469
type: object
8452-
status:
8453-
description: Status defines port for TiProxy status API.
8470+
peer:
8471+
description: Peer defines port for TiProxy's peer service.
84548472
properties:
84558473
port:
84568474
format: int32
@@ -8462,15 +8480,15 @@ spec:
84628480
type: object
84638481
subdomain:
84648482
description: |-
8465-
Subdomain means the subdomain of the exported pd dns.
8466-
A same pd cluster will use a same subdomain
8483+
Subdomain means the subdomain of the exported tiproxy dns.
8484+
A same tiproxy cluster will use a same subdomain
84678485
type: string
84688486
topology:
84698487
additionalProperties:
84708488
type: string
84718489
description: |-
84728490
Topology defines the topology domain of this TiProxy instance.
8473-
It will be translated into a node affnity config.
8491+
It will be translated into a node affinity config.
84748492
Topology cannot be changed.
84758493
minProperties: 1
84768494
type: object

0 commit comments

Comments
 (0)