Skip to content

Commit 999ff57

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #778 from cprivitere/cprivitere/issue767
cluster-template-development
2 parents 7367492 + d954ed0 commit 999ff57

File tree

4 files changed

+397
-148
lines changed

4 files changed

+397
-148
lines changed

‎Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,7 @@ generate: ## Generate code
286286

287287
.PHONY: generate-templates
288288
generate-templates: $(KUSTOMIZE) ## Generate cluster templates
289+
$(KUSTOMIZE) build templates/development --load-restrictor LoadRestrictionsNone > templates/cluster-template-development.yaml
289290
$(KUSTOMIZE) build templates/experimental-emlb --load-restrictor LoadRestrictionsNone > templates/cluster-template-emlb.yaml
290291
$(KUSTOMIZE) build templates/experimental-emlb-crs-cni --load-restrictor LoadRestrictionsNone > templates/cluster-template-emlb-crs-cni.yaml
291292
$(KUSTOMIZE) build templates/experimental-kube-vip-crs-cni --load-restrictor LoadRestrictionsNone > templates/cluster-template-kube-vip-crs-cni.yaml

‎templates/cluster-template-development.yaml

Lines changed: 243 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,243 @@
1+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
2+
kind: KubeadmConfigTemplate
3+
metadata:
4+
name: ${CLUSTER_NAME}-worker-a
5+
spec:
6+
template:
7+
spec:
8+
joinConfiguration:
9+
nodeRegistration:
10+
kubeletExtraArgs:
11+
cloud-provider: external
12+
provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
13+
preKubeadmCommands:
14+
- |
15+
sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab
16+
swapoff -a
17+
mount -a
18+
cat <<EOF > /etc/modules-load.d/containerd.conf
19+
overlay
20+
br_netfilter
21+
EOF
22+
modprobe overlay
23+
modprobe br_netfilter
24+
cat <<EOF > /etc/sysctl.d/99-kubernetes-cri.conf
25+
net.bridge.bridge-nf-call-iptables = 1
26+
net.ipv4.ip_forward = 1
27+
net.bridge.bridge-nf-call-ip6tables = 1
28+
EOF
29+
sysctl --system
30+
export DEBIAN_FRONTEND=noninteractive
31+
apt-get update -y
32+
apt-get remove -y docker docker-engine containerd runc
33+
apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
34+
install -m 0755 -d /etc/apt/keyrings
35+
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
36+
MINOR_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | cut -d. -f1-2 )
37+
curl -fsSL https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
38+
chmod a+r /etc/apt/keyrings/docker.gpg
39+
chmod a+r /etc/apt/keyrings/kubernetes-archive-keyring.gpg
40+
echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" > /etc/apt/sources.list.d/docker.list
41+
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/ /" > /etc/apt/sources.list.d/kubernetes.list
42+
apt-get update -y
43+
TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
44+
RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
45+
apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
46+
cat <<EOF > /etc/crictl.yaml
47+
runtime-endpoint: unix:///run/containerd/containerd.sock
48+
image-endpoint: unix:///run/containerd/containerd.sock
49+
EOF
50+
containerd config default > /etc/containerd/config.toml
51+
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
52+
sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
53+
systemctl restart containerd
54+
---
55+
apiVersion: cluster.x-k8s.io/v1beta1
56+
kind: Cluster
57+
metadata:
58+
name: ${CLUSTER_NAME}
59+
spec:
60+
clusterNetwork:
61+
pods:
62+
cidrBlocks:
63+
- ${POD_CIDR:=192.168.0.0/16}
64+
services:
65+
cidrBlocks:
66+
- ${SERVICE_CIDR:=172.26.0.0/16}
67+
controlPlaneRef:
68+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
69+
kind: KubeadmControlPlane
70+
name: ${CLUSTER_NAME}-control-plane
71+
infrastructureRef:
72+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
73+
kind: PacketCluster
74+
name: ${CLUSTER_NAME}
75+
---
76+
apiVersion: cluster.x-k8s.io/v1beta1
77+
kind: MachineDeployment
78+
metadata:
79+
labels:
80+
cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
81+
pool: worker-a
82+
name: ${CLUSTER_NAME}-worker-a
83+
spec:
84+
clusterName: ${CLUSTER_NAME}
85+
replicas: ${WORKER_MACHINE_COUNT}
86+
selector:
87+
matchLabels:
88+
cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
89+
pool: worker-a
90+
template:
91+
metadata:
92+
labels:
93+
cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
94+
pool: worker-a
95+
spec:
96+
bootstrap:
97+
configRef:
98+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
99+
kind: KubeadmConfigTemplate
100+
name: ${CLUSTER_NAME}-worker-a
101+
clusterName: ${CLUSTER_NAME}
102+
infrastructureRef:
103+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
104+
kind: PacketMachineTemplate
105+
name: ${CLUSTER_NAME}-worker-a
106+
version: ${KUBERNETES_VERSION}
107+
---
108+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
109+
kind: KubeadmControlPlane
110+
metadata:
111+
name: ${CLUSTER_NAME}-control-plane
112+
spec:
113+
kubeadmConfigSpec:
114+
clusterConfiguration:
115+
apiServer:
116+
extraArgs:
117+
cloud-provider: external
118+
controllerManager:
119+
extraArgs:
120+
cloud-provider: external
121+
initConfiguration:
122+
nodeRegistration:
123+
kubeletExtraArgs:
124+
cloud-provider: external
125+
provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
126+
joinConfiguration:
127+
nodeRegistration:
128+
ignorePreflightErrors:
129+
- DirAvailable--etc-kubernetes-manifests
130+
kubeletExtraArgs:
131+
cloud-provider: external
132+
provider-id: equinixmetal://{{ `{{ v1.instance_id }}` }}
133+
postKubeadmCommands:
134+
- |
135+
cat <<EOF >> /etc/network/interfaces
136+
auto lo:0
137+
iface lo:0 inet static
138+
address {{ .controlPlaneEndpoint }}
139+
netmask 255.255.255.255
140+
EOF
141+
systemctl restart networking
142+
mkdir -p $HOME/.kube
143+
cp /etc/kubernetes/admin.conf $HOME/.kube/config
144+
echo "source <(kubectl completion bash)" >> $HOME/.bashrc
145+
echo "alias k=kubectl" >> $HOME/.bashrc
146+
echo "complete -o default -F __start_kubectl k" >> $HOME/.bashrc
147+
if [ -f "/run/kubeadm/kubeadm.yaml" ]; then
148+
export KUBECONFIG=/etc/kubernetes/admin.conf
149+
export CPEM_YAML=https://github.com/equinix/cloud-provider-equinix-metal/releases/download/${CPEM_VERSION:=v3.7.0}/deployment.yaml
150+
export SECRET_DATA='cloud-sa.json=''{"apiKey": "{{ .apiKey }}","projectID": "${PROJECT_ID}", "eipTag": "cluster-api-provider-packet:cluster-id:${CLUSTER_NAME}", "eipHealthCheckUseHostIP": true}'''
151+
kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}" || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}") || (sleep 1 && kubectl create secret generic -n kube-system metal-cloud-config --from-literal="$${SECRET_DATA}")
152+
kubectl apply -f $${CPEM_YAML} || (sleep 1 && kubectl apply -f $${CPEM_YAML}) || (sleep 1 && kubectl apply -f $${CPEM_YAML})
153+
fi
154+
preKubeadmCommands:
155+
- |
156+
sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab
157+
swapoff -a
158+
mount -a
159+
cat <<EOF > /etc/modules-load.d/containerd.conf
160+
overlay
161+
br_netfilter
162+
EOF
163+
modprobe overlay
164+
modprobe br_netfilter
165+
cat <<EOF > /etc/sysctl.d/99-kubernetes-cri.conf
166+
net.bridge.bridge-nf-call-iptables = 1
167+
net.ipv4.ip_forward = 1
168+
net.bridge.bridge-nf-call-ip6tables = 1
169+
EOF
170+
sysctl --system
171+
export DEBIAN_FRONTEND=noninteractive
172+
apt-get update -y
173+
apt-get remove -y docker docker-engine containerd runc
174+
apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release linux-generic jq
175+
major_vers=$(lsb_release -r | awk '{ print $2 }' | cut -d. -f1)
176+
if [ "$major_vers" -ge 20 ]; then
177+
apt-get install -y kubetail
178+
fi
179+
install -m 0755 -d /etc/apt/keyrings
180+
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
181+
MINOR_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | cut -d. -f1-2 )
182+
curl -fsSL https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
183+
chmod a+r /etc/apt/keyrings/docker.gpg
184+
chmod a+r /etc/apt/keyrings/kubernetes-archive-keyring.gpg
185+
echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" > /etc/apt/sources.list.d/docker.list
186+
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${MINOR_KUBERNETES_VERSION}/deb/ /" > /etc/apt/sources.list.d/kubernetes.list
187+
apt-get update -y
188+
TRIMMED_KUBERNETES_VERSION=$(echo {{ .kubernetesVersion }} | sed 's/\./\\\\./g' | sed 's/^v//')
189+
RESOLVED_KUBERNETES_VERSION=$(apt-cache madison kubelet | awk -v VERSION=$${TRIMMED_KUBERNETES_VERSION} '$3~ VERSION { print $3 }' | head -n1)
190+
apt-get install -y containerd.io kubelet=$${RESOLVED_KUBERNETES_VERSION} kubeadm=$${RESOLVED_KUBERNETES_VERSION} kubectl=$${RESOLVED_KUBERNETES_VERSION}
191+
containerd config default > /etc/containerd/config.toml
192+
cat <<EOF > /etc/crictl.yaml
193+
runtime-endpoint: unix:///run/containerd/containerd.sock
194+
image-endpoint: unix:///run/containerd/containerd.sock
195+
EOF
196+
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
197+
sed -i "s,sandbox_image.*$,sandbox_image = \"$(kubeadm config images list | grep pause | sort -r | head -n1)\"," /etc/containerd/config.toml
198+
systemctl restart containerd
199+
ping -c 3 -q {{ .controlPlaneEndpoint }} && echo OK || ip addr add {{ .controlPlaneEndpoint }} dev lo
200+
machineTemplate:
201+
infrastructureRef:
202+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
203+
kind: PacketMachineTemplate
204+
name: ${CLUSTER_NAME}-control-plane
205+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
206+
version: ${KUBERNETES_VERSION}
207+
---
208+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
209+
kind: PacketCluster
210+
metadata:
211+
name: ${CLUSTER_NAME}
212+
spec:
213+
metro: ${METRO}
214+
projectID: ${PROJECT_ID}
215+
vipManager: CPEM
216+
---
217+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
218+
kind: PacketMachineTemplate
219+
metadata:
220+
name: ${CLUSTER_NAME}-control-plane
221+
spec:
222+
template:
223+
spec:
224+
billingCycle: hourly
225+
machineType: ${CONTROLPLANE_NODE_TYPE}
226+
os: ${NODE_OS:=ubuntu_20_04}
227+
sshKeys:
228+
- ${SSH_KEY}
229+
tags: []
230+
---
231+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
232+
kind: PacketMachineTemplate
233+
metadata:
234+
name: ${CLUSTER_NAME}-worker-a
235+
spec:
236+
template:
237+
spec:
238+
billingCycle: hourly
239+
machineType: ${WORKER_NODE_TYPE}
240+
os: ${NODE_OS:=ubuntu_20_04}
241+
sshKeys:
242+
- ${SSH_KEY}
243+
tags: []

0 commit comments

Comments
 (0)