Skip to content

Commit 3dfd18a

Browse files
neoaggelosneoaggelos
authored
Merge pull request #90 from canonical/KU-454/machine-addresses
ensure we use InternalIP or ExternalIP machine addresses
2 parents d31d053 + e4bd4d5 commit 3dfd18a

File tree

6 files changed

+64
-45
lines changed

6 files changed

+64
-45
lines changed

‎controllers/cloudinit/controlplane_join.go

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ type ControlPlaneJoinInput struct {
4848
// IPinIP defines whether Calico will use IPinIP mode for cluster networking.
4949
IPinIP bool
5050
// JoinNodeIPs is the IP addresses of the nodes to join.
51-
JoinNodeIPs [2]string
51+
JoinNodeIPs []string
5252
// Confinement specifies a classic or strict deployment of microk8s snap.
5353
Confinement string
5454
// RiskLevel specifies the risk level (strict, candidate, beta, edge) for the snap channels.
@@ -111,8 +111,10 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) {
111111
})
112112
}
113113

114-
joinStr := fmt.Sprintf("%s:%s/%s", input.JoinNodeIPs[0], input.ClusterAgentPort, input.Token)
115-
joinStrAlt := fmt.Sprintf("%s:%s/%s", input.JoinNodeIPs[1], input.ClusterAgentPort, input.Token)
114+
joinURLs := make([]string, 0, len(input.JoinNodeIPs))
115+
for _, nodeIP := range input.JoinNodeIPs {
116+
joinURLs = append(joinURLs, fmt.Sprintf("%q", fmt.Sprintf("%s:%s/%s", nodeIP, input.ClusterAgentPort, input.Token)))
117+
}
116118

117119
cloudConfig.BootCommands = append(cloudConfig.BootCommands, input.BootCommands...)
118120

@@ -130,7 +132,7 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) {
130132
fmt.Sprintf("%s %q", scriptPath(configureDqlitePortScript), input.DqlitePort),
131133
"microk8s status --wait-ready",
132134
fmt.Sprintf("%s %q %q", scriptPath(configureCertLB), endpointType, input.ControlPlaneEndpoint),
133-
fmt.Sprintf("%s no %q %q", scriptPath(microk8sJoinScript), joinStr, joinStrAlt),
135+
fmt.Sprintf("%s no %s", scriptPath(microk8sJoinScript), strings.Join(joinURLs, " ")),
134136
scriptPath(configureAPIServerScript),
135137
fmt.Sprintf("microk8s add-node --token-ttl %v --token %q", input.TokenTTL, input.Token),
136138
)

‎controllers/cloudinit/controlplane_join_test.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ func TestControlPlaneJoin(t *testing.T) {
2828
t.Run("Simple", func(t *testing.T) {
2929
g := NewWithT(t)
3030

31-
joins := [2]string{"10.0.3.39", "10.0.3.40"}
3231
cloudConfig, err := cloudinit.NewJoinControlPlane(&cloudinit.ControlPlaneJoinInput{
3332
ControlPlaneEndpoint: "k8s.my-domain.com",
3433
KubernetesVersion: "v1.25.2",
@@ -37,7 +36,7 @@ func TestControlPlaneJoin(t *testing.T) {
3736
IPinIP: true,
3837
Token: strings.Repeat("a", 32),
3938
TokenTTL: 10000,
40-
JoinNodeIPs: joins,
39+
JoinNodeIPs: []string{"10.0.3.39", "10.0.3.40", "10.0.3.41"},
4140
})
4241
g.Expect(err).NotTo(HaveOccurred())
4342

@@ -55,7 +54,7 @@ func TestControlPlaneJoin(t *testing.T) {
5554
`/capi-scripts/10-configure-dqlite-port.sh "2379"`,
5655
`microk8s status --wait-ready`,
5756
`/capi-scripts/10-configure-cert-for-lb.sh "DNS" "k8s.my-domain.com"`,
58-
`/capi-scripts/20-microk8s-join.sh no "10.0.3.39:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "10.0.3.40:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`,
57+
`/capi-scripts/20-microk8s-join.sh no "10.0.3.39:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "10.0.3.40:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "10.0.3.41:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`,
5958
`/capi-scripts/10-configure-apiserver.sh`,
6059
`microk8s add-node --token-ttl 10000 --token "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`,
6160
}))

‎controllers/cloudinit/scripts/20-microk8s-join.sh

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,33 @@
11
#!/bin/bash -xe
22

33
# Usage:
4-
# $0 $worker_yes_no $join_string $alternative_join_string
4+
# $0 $worker_yes_no $join_string_1 $join_string_2 ... $join_string_N
55
#
66
# Assumptions:
77
# - microk8s is installed
88
# - microk8s node is ready to join the cluster
99

10-
join="${2}"
11-
join_alt="${3}"
12-
10+
join_args=""
1311
if [ ${1} == "yes" ]; then
14-
join+=" --worker"
15-
join_alt+=" --worker"
12+
join_args="--worker"
1613
fi
1714

18-
while ! microk8s join ${join}; do
19-
echo "Failed to join MicroK8s cluster, retring alternative join string"
20-
if ! microk8s join ${join_alt} ; then
21-
break
22-
fi
23-
echo "Failed to join MicroK8s cluster, will retry"
24-
sleep 5
15+
shift
16+
17+
# Loop over the given join addresses until microk8s join command succeeds.
18+
joined="false"
19+
while [ "$joined" = "false" ]; do
20+
21+
for url in "${@}"; do
22+
if microk8s join "${url}" $join_args; then
23+
joined="true"
24+
break
25+
fi
26+
27+
echo "Failed to join MicroK8s cluster, will retry"
28+
sleep 5
29+
done
30+
2531
done
2632

2733
# What is this hack? Why do we call snap set here?

‎controllers/cloudinit/worker_join.go

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ type WorkerInput struct {
4141
// ContainerdNoProxy is no_proxy configuration for containerd.
4242
ContainerdNoProxy string
4343
// JoinNodeIPs is the IP addresses of the nodes to join.
44-
JoinNodeIPs [2]string
44+
JoinNodeIPs []string
4545
// Confinement specifies a classic or strict deployment of microk8s snap.
4646
Confinement string
4747
// RiskLevel specifies the risk level (strict, candidate, beta, edge) for the snap channels.
@@ -100,8 +100,10 @@ func NewJoinWorker(input *WorkerInput) (*CloudConfig, error) {
100100
})
101101
}
102102

103-
joinStr := fmt.Sprintf("%s:%s/%s", input.JoinNodeIPs[0], input.ClusterAgentPort, input.Token)
104-
joinStrAlt := fmt.Sprintf("%s:%s/%s", input.JoinNodeIPs[1], input.ClusterAgentPort, input.Token)
103+
joinURLs := make([]string, 0, len(input.JoinNodeIPs))
104+
for _, nodeIP := range input.JoinNodeIPs {
105+
joinURLs = append(joinURLs, fmt.Sprintf("%q", fmt.Sprintf("%s:%s/%s", nodeIP, input.ClusterAgentPort, input.Token)))
106+
}
105107

106108
cloudConfig.BootCommands = append(cloudConfig.BootCommands, input.BootCommands...)
107109

@@ -115,7 +117,7 @@ func NewJoinWorker(input *WorkerInput) (*CloudConfig, error) {
115117
scriptPath(configureKubeletScript),
116118
"microk8s status --wait-ready",
117119
fmt.Sprintf("%s %q", scriptPath(configureClusterAgentPortScript), input.ClusterAgentPort),
118-
fmt.Sprintf("%s yes %q %q", scriptPath(microk8sJoinScript), joinStr, joinStrAlt),
120+
fmt.Sprintf("%s yes %s", scriptPath(microk8sJoinScript), strings.Join(joinURLs, " ")),
119121
fmt.Sprintf("%s %s 6443 %s", scriptPath(configureTraefikScript), input.ControlPlaneEndpoint, stopApiServerProxyRefreshes),
120122
)
121123
cloudConfig.RunCommands = append(cloudConfig.RunCommands, input.PostRunCommands...)

‎controllers/cloudinit/worker_join_test.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,12 @@ func TestWorkerJoin(t *testing.T) {
2828
t.Run("Simple", func(t *testing.T) {
2929
g := NewWithT(t)
3030

31-
joins := [2]string{"10.0.3.194", "10.0.3.195"}
3231
cloudConfig, err := cloudinit.NewJoinWorker(&cloudinit.WorkerInput{
3332
ControlPlaneEndpoint: "capi-aws-apiserver-1647391446.us-east-1.elb.amazonaws.com",
3433
KubernetesVersion: "v1.24.3",
3534
ClusterAgentPort: "30000",
3635
Token: strings.Repeat("a", 32),
37-
JoinNodeIPs: joins,
36+
JoinNodeIPs: []string{"10.0.3.194", "10.0.3.195"},
3837
})
3938
g.Expect(err).NotTo(HaveOccurred())
4039

‎controllers/microk8sconfig_controller.go

Lines changed: 30 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,14 @@ const (
9191
remappedClusterAgentPort string = "30000"
9292
)
9393

94+
var (
95+
// preferMachineAddressTypeList is the order of preference of machine addresses to use for joining microk8s nodes to a cluster.
96+
preferMachineAddressTypeList = []clusterv1.MachineAddressType{
97+
clusterv1.MachineInternalIP,
98+
clusterv1.MachineExternalIP,
99+
}
100+
)
101+
94102
//+kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=microk8sconfigs,verbs=get;list;watch;create;update;patch;delete
95103
//+kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=microk8sconfigs/status,verbs=get;update;patch
96104
//+kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=microk8sconfigs/finalizers,verbs=update
@@ -366,7 +374,7 @@ func (r *MicroK8sConfigReconciler) handleJoiningControlPlaneNode(ctx context.Con
366374

367375
scope.Info("Creating BootstrapData for the join control plane")
368376
ipsOfNodesToConnectTo, err := r.getControlPlaneNodesToJoin(ctx, scope)
369-
if err != nil || ipsOfNodesToConnectTo[0] == "" {
377+
if err != nil || len(ipsOfNodesToConnectTo) == 0 {
370378
scope.Info("Failed to discover a control plane IP, requeueing.")
371379
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
372380
}
@@ -482,7 +490,7 @@ func (r *MicroK8sConfigReconciler) handleJoiningWorkerNode(ctx context.Context,
482490
}
483491

484492
ipOfNodesToConnectTo, err := r.getControlPlaneNodesToJoin(ctx, scope)
485-
if err != nil || ipOfNodesToConnectTo[0] == "" {
493+
if err != nil || len(ipOfNodesToConnectTo) == 0 {
486494
scope.Info("Failed to discover a control plane IP, requeueing.")
487495
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
488496
}
@@ -539,34 +547,37 @@ func (r *MicroK8sConfigReconciler) handleJoiningWorkerNode(ctx context.Context,
539547
return ctrl.Result{}, nil
540548
}
541549

542-
func (r *MicroK8sConfigReconciler) getControlPlaneNodesToJoin(ctx context.Context, scope *Scope) ([2]string, error) {
543-
var ipOfNodesToConnectTo [2]string
550+
func (r *MicroK8sConfigReconciler) getControlPlaneNodesToJoin(ctx context.Context, scope *Scope) ([]string, error) {
544551
nodes, err := r.getControlPlaneMachinesForCluster(ctx, util.ObjectKey(scope.Cluster))
545552
if err != nil {
546553
scope.Error(err, "Lookup control plane nodes")
547-
return ipOfNodesToConnectTo, err
554+
return nil, err
548555
}
549556

550-
n := 0
557+
addressesByType := make(map[clusterv1.MachineAddressType][]string)
551558
for _, node := range nodes {
552-
if n >= 2 {
553-
break
559+
if node.Spec.ProviderID == nil || node.Status.Phase != "Running" {
560+
continue
554561
}
555-
if node.Spec.ProviderID != nil && node.Status.Phase == "Running" {
556-
for _, address := range node.Status.Addresses {
557-
if address.Address != "" {
558-
ipOfNodesToConnectTo[n] = address.Address
559-
n += 1
560-
if n == 1 {
561-
ipOfNodesToConnectTo[n] = address.Address
562-
}
563-
break
564-
}
562+
for _, address := range node.Status.Addresses {
563+
if address.Address == "" {
564+
continue
565565
}
566+
addressesByType[address.Type] = append(addressesByType[address.Type], address.Address)
566567
}
567568
}
568569

569-
return ipOfNodesToConnectTo, nil
570+
for _, addressType := range preferMachineAddressTypeList {
571+
if addresses, ok := addressesByType[addressType]; ok && len(addresses) > 0 {
572+
return addresses, nil
573+
}
574+
}
575+
576+
// if we are here, no addresses were found
577+
err = fmt.Errorf("machines do not have any addresses with types %v", preferMachineAddressTypeList)
578+
scope.Error(err, "Lookup control plane node IPs", "addressesByType", addressesByType)
579+
580+
return nil, err
570581
}
571582

572583
func (r *MicroK8sConfigReconciler) storeBootstrapData(ctx context.Context, scope *Scope, data []byte) error {

0 commit comments

Comments
 (0)