Skip to content
This repository was archived by the owner on Sep 8, 2025. It is now read-only.

Commit a068fd0

Browse files
harguthargut
authored
feat(build): enhance container build times (#438)
This draft pr is intended to further discussing #424. It only comprises the build for controlplane and provides a draft layout that could be enrolled to other containers. - based on Debian trixie slim - using rustup via apt to install stable rust - separated build and runtime container - runtime container based on plain debian-slim (including apt, shell, ...) - requires a workspace volume mount - optionally supports a workspace/target volume mount to enable re-using of compiled elements - in case not mounted a full fledged build is executed - all source code artefacts required for the build are copied into the container - this is currently the most time intense operation during re-usage of pre-compiled elements - CARGO_HOME is redefined to save downloaded dependencies into target/ ``` $ time docker build -f build/Containerfile.controlplane -v "$(pwd):/workspace" -v "$(pwd)/target:/build/target/" --build-arg BUILD_TIMESTAMP=$(date +%s%3N) [1/2] STEP 1/7: FROM debian:trixie-slim AS builder [1/2] STEP 2/7: RUN apt update && apt install -y build-essential rustup && rustup install stable --> Using cache 5dc77c2dd41693beebc66a306cf73312f5c9355dfcca1bbfdfa45a960c03b7e1 --> 5dc77c2dd416 [1/2] STEP 3/7: ARG BUILD_TIMESTAMP --> Using cache 52db6776fc06faad520ef09bb1bdeb59125489af46afc0c35c6c9c7977e61eee --> 52db6776fc06 [1/2] STEP 4/7: WORKDIR /build --> Using cache 2862ae0bb1b14024eebbda34b432496c94bbe41a359932b351f0a4bd0470d514 --> 2862ae0bb1b1 [1/2] STEP 5/7: ENV CARGO_HOME=/build/target --> Using cache 61f2d25c7adb3077b1b50f68ea3cf38a031cb31f17672f3257143f5dbc7f28b6 --> 61f2d25c7adb [1/2] STEP 6/7: RUN echo "${BUILD_TIMESTAMP}" && date +%s && cp -a /workspace/Cargo.toml /workspace/Cargo.lock /workspace/controlplane/ /workspace/dataplane/ /workspace/tests-integration/ /workspace/tools/ /workspace/xtask/ /build 1755529247016 1755529248 --> ae88babc1118 [1/2] STEP 7/7: RUN date +%s && cargo build -p controlplane && date +%s && mkdir -p results/ && cp target/debug/controller results/ 1755529254 Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.21s 1755529254 --> 7452142d2300 [2/2] STEP 1/6: FROM debian:trixie-slim [2/2] STEP 2/6: LABEL org.opencontainers.image.source=https://github.com/kubernetes-sigs/blixt --> Using cache fd399516894249f40dc62e585e7f2816bf54cdac0987414cec291507bf59abf7 --> fd3995168942 [2/2] STEP 3/6: WORKDIR / --> Using cache d63dbdec7772ab7fcd3423962f7bfe8b221a6b81d03236efd9307c3424607143 --> d63dbdec7772 [2/2] STEP 4/6: COPY --from=builder /build/results/controller /controller --> Using cache 41a90ea11d0b008100a60b8c6aff04c335414efb09322e3f51bc4122bd8f45fd --> 41a90ea11d0b [2/2] STEP 5/6: USER 1000:1000 --> Using cache e7d41a49b6da537fa698e3ddc31a8ebaa41c2a6f12c7719d99219e9c80e128b0 --> e7d41a49b6da [2/2] STEP 6/6: ENTRYPOINT [ "/controller" ] --> Using cache a21a4a52596ea6a6adfa0c1e8731a73f83a882d46d1d4ffbe5d38e7dbf6342d7 --> a21a4a52596e a21a4a52596ea6a6adfa0c1e8731a73f83a882d46d1d4ffbe5d38e7dbf6342d7 real 0m14.483s user 0m1.076s sys 0m0.129s $ ``` fixes: #424 Signed-off-by: Harald Gutmann <harald@gutmann.one>
1 parent 8bd6c56 commit a068fd0

File tree

6 files changed

+250
-153
lines changed

6 files changed

+250
-153
lines changed

‎.github/workflows/build.yaml‎

Lines changed: 37 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -58,28 +58,42 @@ jobs:
5858
- 'build/Containerfile.*'
5959
- name: Build controlplane Container Image
6060
if: steps.filter.outputs.sources
61-
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
62-
with:
63-
push: false
64-
context: .
65-
file: build/Containerfile.controlplane
66-
tags: localhost/blixt-controlplane:pr-${{ github.event.pull_request.number }}-${{ github.sha }}
61+
run: |
62+
mkdir -p target/ &&
63+
podman build \
64+
--userns=host \
65+
--file build/Containerfile.controlplane \
66+
--volume "$(pwd):/workspace" \
67+
--volume "$(pwd)/target:$(pwd)/target/" \
68+
--build-arg BUILD_TIMESTAMP="$(date +%s%3N)" \
69+
--build-arg UID="0" \
70+
--build-arg GID="0" \
71+
--build-arg WORK_DIR="$(pwd)" \
72+
--tag localhost/blixt-controlplane:pr-${{ github.event.pull_request.number }}-${{ github.sha }}
6773
- name: Build dataplane Container Image
68-
if: steps.filter.outputs.sources
69-
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
70-
with:
71-
push: false
72-
context: .
73-
file: build/Containerfile.dataplane
74-
tags: localhost/blixt-dataplane:pr-${{ github.event.pull_request.number }}-${{ github.sha }}
74+
run: |
75+
podman build \
76+
--userns=host \
77+
--file build/Containerfile.dataplane \
78+
--volume "$(pwd):/workspace" \
79+
--volume "$(pwd)/target:$(pwd)/target/" \
80+
--build-arg BUILD_TIMESTAMP="$(date +%s%3N)" \
81+
--build-arg UID="0" \
82+
--build-arg GID="0" \
83+
--build-arg WORK_DIR="$(pwd)" \
84+
--tag localhost/blixt-dataplane:pr-${{ github.event.pull_request.number }}-${{ github.sha }}
7585
- name: Build udp-test-server Container Image
76-
if: steps.filter.outputs.sources
77-
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
78-
with:
79-
push: false
80-
context: .
81-
file: build/Containerfile.udp-test-server
82-
tags: localhost/blixt-udp-test-server:pr-${{ github.event.pull_request.number }}-${{ github.sha }}
86+
run: |
87+
podman build \
88+
--userns=host \
89+
--file build/Containerfile.udp-test-server \
90+
--volume "$(pwd):/workspace" \
91+
--volume "$(pwd)/target:$(pwd)/target/" \
92+
--build-arg BUILD_TIMESTAMP="$(date +%s%3N)" \
93+
--build-arg UID="0" \
94+
--build-arg GID="0" \
95+
--build-arg WORK_DIR="$(pwd)" \
96+
--tag localhost/blixt-udp-test-server:pr-${{ github.event.pull_request.number }}-${{ github.sha }}
8397
- name: Install kind and kubectl
8498
uses: helm/kind-action@b72c923563e6e80ea66e8e8c810798cc73e97e5e # current main, includes cloud-provider-kind support
8599
if: steps.filter.outputs.sources
@@ -95,6 +109,8 @@ jobs:
95109
- name: Run Integration Tests
96110
if: steps.filter.outputs.sources
97111
run: |
112+
# kind load broken for podman https://github.com/kubernetes-sigs/kind/issues/3945
98113
export REGISTRY="localhost"
99114
export TAG="pr-${{ github.event.pull_request.number }}-${{ github.sha }}"
100-
make test.integration
115+
sudo chown -R "$(id -u):$(id -g)" target/
116+
make test.integration.reuse

‎Makefile‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,14 @@ clean: ## clean repo
6060
rm $(TEST_CERTS_PATH)/{*.pem,*.csr}
6161

6262
.PHONY: build
63-
build: ## Build dataplane
63+
build:
6464
cargo xtask build-ebpf
6565
cargo build
6666

67+
build.reuse:
68+
CARGO_HOME="$(WORK_DIR)/target" RUSTUP_HOME="$(WORK_DIR)/target" cargo xtask build-ebpf
69+
CARGO_HOME="$(WORK_DIR)/target" RUSTUP_HOME="$(WORK_DIR)/target" cargo build
70+
6771
.PHONY: build.release
6872
build.release: ## Build dataplane release
6973
cargo xtask build-ebpf --release
@@ -88,17 +92,26 @@ CONTROLPLANE_CONTAINERFILE ?= build/Containerfile.controlplane
8892
DATAPLANE_CONTAINERFILE ?= build/Containerfile.dataplane
8993
UDP_SERVER_CONTAINERFILE ?= build/Containerfile.udp-test-server
9094

95+
UID = $(shell id -u)
96+
GID = $(shell id -g)
97+
WORK_DIR= $(shell pwd)
98+
BUILD_TIMESTAMP = $(shell date +%s%3N)
99+
CONTAINER_BUILD_ARGS = --userns=host --volume "$(WORK_DIR):/workspace" --volume "$(WORK_DIR)/target:$(WORK_DIR)/target/" \
100+
--build-arg BUILD_TIMESTAMP="$(BUILD_TIMESTAMP)" --build-arg UID="$(UID)" --build-arg GID="$(GID)" --build-arg WORK_DIR="$(WORK_DIR)" $(BUILD_ARGS)
101+
91102
.PHONY: build.image.controlplane
92103
build.image.controlplane:
93-
$(CONTAINER_RUNTIME) build $(BUILD_ARGS) --file=$(CONTROLPLANE_CONTAINERFILE) -t $(BLIXT_CONTROLPLANE_IMAGE):$(TAG) ./
94-
104+
mkdir -p target/
105+
$(CONTAINER_RUNTIME) build $(CONTAINER_BUILD_ARGS) --file=$(CONTROLPLANE_CONTAINERFILE) --tag $(BLIXT_CONTROLPLANE_IMAGE):$(TAG)
95106
.PHONY: build.image.udp-test-server
96107
build.image.udp-test-server:
97-
$(CONTAINER_RUNTIME) build $(BUILD_ARGS) --file=$(UDP_SERVER_CONTAINERFILE) -t $(BLIXT_UDP_SERVER_IMAGE):$(TAG) ./
108+
mkdir -p target/
109+
$(CONTAINER_RUNTIME) build $(CONTAINER_BUILD_ARGS) --file=$(UDP_SERVER_CONTAINERFILE) --tag $(BLIXT_UDP_SERVER_IMAGE):$(TAG)
98110

99111
.PHONY: build.image.dataplane
100112
build.image.dataplane:
101-
$(CONTAINER_RUNTIME) build $(BUILD_ARGS) --file=$(DATAPLANE_CONTAINERFILE) -t $(BLIXT_DATAPLANE_IMAGE):$(TAG) ./
113+
mkdir -p target/
114+
$(CONTAINER_RUNTIME) build $(CONTAINER_BUILD_ARGS) --file=$(DATAPLANE_CONTAINERFILE) --tag $(BLIXT_DATAPLANE_IMAGE):$(TAG)
102115

103116
.PHONY: build.all.images
104117
build.all.images:
@@ -131,7 +144,10 @@ test:
131144
cargo test -vv --workspace --exclude tests-integration
132145

133146
test.integration:
134-
cargo test --package tests-integration
147+
cargo test --package tests-integration
148+
149+
test.integration.reuse:
150+
CARGO_HOME="$(WORK_DIR)/target" RUSTUP_HOME="$(WORK_DIR)/target" cargo test --package tests-integration
135151

136152
.PHONY: test.gencert
137153
test.gencert: cfssl cfssljson

‎build/Containerfile.controlplane‎

Lines changed: 44 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2,42 +2,61 @@
22
# Builder
33
# ------------------------------------------------------------------------------
44

5-
FROM rust:alpine AS builder
6-
7-
RUN apk add --no-cache clang lld
8-
9-
WORKDIR /workspace
10-
11-
ARG PROJECT_DIR=/workspace
12-
13-
ARG BUILD_DIR=$PROJECT_DIR/build
14-
15-
COPY Cargo.toml Cargo.lock ./
16-
17-
COPY controlplane/ controlplane/
18-
19-
COPY dataplane/ dataplane/
20-
21-
COPY tests-integration/ tests-integration/
22-
23-
COPY tools/ tools/
24-
25-
COPY xtask/ xtask/
26-
27-
RUN cargo build -p controlplane --target x86_64-unknown-linux-musl
5+
FROM debian:trixie-slim AS builder
6+
7+
RUN apt update \
8+
&& apt install -y build-essential rustup
9+
10+
ARG UID
11+
ARG GID
12+
ARG WORK_DIR
13+
RUN chown "${UID}:${GID}" ${WORK_DIR}
14+
USER ${UID}:${GID}
15+
16+
ENV RUSTUP_HOME=${WORK_DIR}/target
17+
# allow re-using of cargo downloads trough saving in target/
18+
ENV CARGO_HOME=${WORK_DIR}/target
19+
20+
RUN rustup install stable \
21+
&& rustup default stable
22+
23+
ARG BUILD_TIMESTAMP
24+
WORKDIR ${WORK_DIR}
25+
26+
# /workspace needs to be volume mounted
27+
# ${WORK_DIR}/target can be optionally volume mounted
28+
RUN echo "${BUILD_TIMESTAMP}" \
29+
&& date +%s \
30+
&& cp -a /workspace/Cargo.toml \
31+
/workspace/Cargo.lock \
32+
/workspace/.cargo/ \
33+
/workspace/controlplane/ \
34+
/workspace/dataplane/ \
35+
/workspace/tests-integration/ \
36+
/workspace/tools/ \
37+
/workspace/xtask/ \
38+
${WORK_DIR}
39+
40+
RUN date +%s \
41+
&& cargo build --package controlplane \
42+
&& date +%s \
43+
&& mkdir -p results/ \
44+
&& cp target/debug/controller results/
2845

2946
# ------------------------------------------------------------------------------
3047
# Image
3148
# ------------------------------------------------------------------------------
3249

33-
FROM alpine:latest
50+
FROM debian:trixie-slim
3451

3552
LABEL org.opencontainers.image.source=https://github.com/kubernetes-sigs/blixt
3653

3754
WORKDIR /
3855

3956
USER 1000:1000
4057

41-
COPY --from=builder /workspace/target/x86_64-unknown-linux-musl/debug/controller /controller
58+
ARG WORK_DIR
59+
60+
COPY --from=builder ${WORK_DIR}/results/controller /controller
4261

4362
ENTRYPOINT [ "/controller" ]

‎build/Containerfile.dataplane‎

Lines changed: 55 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -2,84 +2,73 @@
22
# Builder
33
# ------------------------------------------------------------------------------
44

5-
FROM rust:slim-bookworm AS builder
6-
7-
RUN apt-get update
8-
9-
RUN apt-get install --yes \
10-
build-essential \
11-
llvm-19 \
12-
protobuf-compiler \
13-
pkg-config \
14-
musl-tools \
15-
clang \
16-
wget \
17-
lsb-release \
18-
software-properties-common \
19-
gnupg
20-
21-
RUN rustup default stable
22-
23-
RUN rustup install nightly
24-
25-
RUN rustup component add rust-src --toolchain nightly
26-
27-
RUN cargo install bpf-linker
28-
29-
WORKDIR /workspace
30-
31-
RUN rustup target add x86_64-unknown-linux-musl
32-
33-
ARG PROJECT_DIR=/workspace
34-
35-
ARG BUILD_DIR=$PROJECT_DIR/build
36-
37-
COPY Cargo.toml Cargo.lock ./
38-
39-
COPY controlplane/ controlplane/
40-
41-
COPY dataplane/ dataplane/
42-
43-
COPY tests-integration/ tests-integration/
44-
45-
COPY tools/ tools/
46-
47-
COPY xtask/ xtask/
48-
49-
COPY .cargo/config.toml .cargo/config.toml
50-
51-
# We need to tell bpf-linker where it can find LLVM's shared library file.
52-
# Ref: https://github.com/aya-rs/rustc-llvm-proxy/blob/cbcb3c6/src/lib.rs#L48
53-
ENV LD_LIBRARY_PATH="/usr/lib/llvm-19/lib"
54-
55-
ENV CC_x86_64_unknown_linux_musl="/usr/bin/clang"
56-
57-
ENV AR_x86_64_unknown_linux_musl="/usr/lib/llvm-19/bin/llvm-ar"
58-
59-
RUN cargo xtask build-ebpf
60-
61-
RUN RUSTFLAGS=-Ctarget-feature=+crt-static cargo build \
62-
--workspace \
63-
--exclude ebpf \
64-
--package loader \
65-
--target=x86_64-unknown-linux-musl
5+
FROM debian:trixie-slim AS builder
6+
7+
RUN apt update \
8+
&& apt install -y build-essential rustup
9+
10+
ARG UID
11+
ARG GID
12+
ARG WORK_DIR
13+
RUN chown "${UID}:${GID}" ${WORK_DIR}
14+
USER ${UID}:${GID}
15+
16+
ENV RUSTUP_HOME=${WORK_DIR}/target
17+
# allow re-using of cargo downloads trough saving in target/
18+
ENV CARGO_HOME=${WORK_DIR}/target
19+
20+
RUN rustup install stable \
21+
&& rustup default stable
22+
23+
RUN rustup install nightly \
24+
&& rustup component add rust-src --toolchain nightly \
25+
&& cargo install bpf-linker
26+
27+
ARG BUILD_TIMESTAMP
28+
WORKDIR ${WORK_DIR}
29+
30+
# /workspace needs to be volume mounted
31+
# ${WORK_DIR}/target can be optionally volume mounted
32+
RUN echo "${BUILD_TIMESTAMP}" \
33+
&& date +%s \
34+
&& cp -a /workspace/Cargo.toml \
35+
/workspace/Cargo.lock \
36+
/workspace/.cargo/ \
37+
/workspace/controlplane/ \
38+
/workspace/dataplane/ \
39+
/workspace/tests-integration/ \
40+
/workspace/tools/ \
41+
/workspace/xtask/ \
42+
${WORK_DIR}
43+
44+
RUN date +%s \
45+
&& cargo xtask build-ebpf \
46+
&& date +%s \
47+
&& cargo build --package loader \
48+
&& date +%s \
49+
&& mkdir -p results/ \
50+
&& cp target/debug/loader results/ \
51+
&& cp dataplane/LICENSE.BSD-2-Clause results/ \
52+
&& cp dataplane/LICENSE.GPL-2.0 results/
6653

6754
# ------------------------------------------------------------------------------
6855
# Image
6956
# ------------------------------------------------------------------------------
7057

71-
FROM alpine
58+
FROM debian:trixie-slim
7259

7360
LABEL org.opencontainers.image.source=https://github.com/kubernetes-sigs/blixt
7461

7562
LABEL org.opencontainers.image.licenses=GPL-2.0-only,BSD-2-Clause
7663

7764
WORKDIR /
7865

79-
COPY --from=builder /workspace/target/x86_64-unknown-linux-musl/debug/loader /dataplane
66+
ARG WORK_DIR
67+
68+
COPY --from=builder ${WORK_DIR}/results/LICENSE.BSD-2-Clause /LICENSE.BSD-2-Clause
8069

81-
COPY dataplane/LICENSE.GPL-2.0 /LICENSE.GPL-2.0
70+
COPY --from=builder ${WORK_DIR}/results/LICENSE.GPL-2.0 /LICENSE.GPL-2.0
8271

83-
COPY dataplane/LICENSE.BSD-2-Clause /LICENSE.BSD-2-Clause
72+
COPY --from=builder ${WORK_DIR}/results/loader /dataplane
8473

85-
ENTRYPOINT ["/dataplane"]
74+
ENTRYPOINT ["/dataplane"]

0 commit comments

Comments
 (0)