Skip to content

Commit 5fdcefe

Browse files
velovelo
committed
Simplied uber-jar creation process
Signed-off-by: Marvin Froeder <marvin@datasqrl.com>
1 parent 14eb41b commit 5fdcefe

File tree

13 files changed

+151
-289
lines changed

13 files changed

+151
-289
lines changed

‎.github/workflows/uber-jar.yml

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,12 @@ jobs:
1515
runs-on: ubuntu-latest
1616
strategy:
1717
matrix:
18-
SQRL_VERSION: [0.5.6, 0.5.7, 0.5.8, 0.5.9, 0.5.10, dev]
18+
SQRL_VERSION: [0.5.6, 0.5.7, 0.5.8, 0.5.9, 0.5.10, 0.6.0-SNAPSHOT]
19+
FLINK_PROFILE: [flink18, flink19]
1920

2021
env:
2122
DOCKER_REPO: datasqrl/flink-jar-runner
22-
VERSION: ${{ github.event_name == 'release' && github.event.action == 'created' && github.ref_name || '1.0.0-SNAPSHOT' }}
23+
VERSION: ${{ github.event_name == 'release' && github.event.action == 'created' && github.ref_name || 'snapshot' }}
2324

2425
steps:
2526
- name: Checkout code
@@ -37,24 +38,16 @@ jobs:
3738

3839
- name: Pull base images
3940
run: |
40-
docker pull maven:3.9.9-eclipse-temurin-11
4141
docker pull alpine:3
4242
4343
- name: Run Maven Build
4444
if: github.event_name == 'release' && github.event.action == 'created'
4545
run: |
4646
mvn -B versions:set -DnewVersion=${{ env.VERSION }}
47-
mvn -B clean install -DskipTests=true
47+
mvn -B clean install -P${{ matrix.FLINK_PROFILE }} -Dsqrl.version=${{ matrix.SQRL_VERSION }}
4848
4949
- name: Build Docker image
5050
run: |
51-
cp uber-jar/Dockerfile .
52-
DOCKER_TAG=$DOCKER_REPO:jar-only-${{ env.VERSION }}-sqrlv${{ matrix.SQRL_VERSION }}
53-
docker build --build-arg SQRL_VERSION=${{ matrix.SQRL_VERSION }} --build-arg GITHUB_ACTOR=${{ github.actor }} --build-arg GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} --build-arg RUNNER_VERSION=${{ env.VERSION }} -t $DOCKER_TAG .
54-
55-
- name: Push Docker image
56-
if: github.event_name == 'release' && github.event.action == 'created'
57-
run: |
58-
DOCKER_TAG=$DOCKER_REPO:${{ env.VERSION }}-sqrlv${{ matrix.SQRL_VERSION }}
51+
DOCKER_TAG=$DOCKER_REPO:${{ env.VERSION }}-sqrlv${{ matrix.SQRL_VERSION }}-flink${{ matrix.FLINK_PROFILE }}
52+
docker build --build-arg SQRL_VERSION=${{ matrix.SQRL_VERSION }} --build-arg RUNNER_VERSION=${{ env.VERSION }} -t $DOCKER_TAG .
5953
docker push $DOCKER_TAG
60-

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ tmp/
6161
sqrl-core/src/test/resources/local.properties
6262
.loadpath
6363
.factorypath
64+
m2e-target/
6465

6566
##############################
6667
## NetBeans

‎uber-jar/Dockerfile renamed to ‎Dockerfile

Lines changed: 1 addition & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -14,33 +14,9 @@
1414
# limitations under the License.
1515
#
1616

17-
FROM maven:3.9.9-eclipse-temurin-11 AS build
18-
19-
ARG RUNNER_VERSION="1.0.0-SNAPSHOT"
20-
ARG SQRL_VERSION="0.5.7"
21-
ARG GITHUB_ACTOR
22-
ARG GITHUB_TOKEN
23-
24-
ENV GITHUB_ACTOR=${GITHUB_ACTOR}
25-
ENV GITHUB_TOKEN=${GITHUB_TOKEN}
26-
27-
ENV SQRL_VERSION=${SQRL_VERSION}
28-
29-
WORKDIR /src
30-
COPY * /src/
31-
RUN mvn clean install -B -Pquickbuild
32-
33-
WORKDIR /uber-jar
34-
COPY uber-jar/uber-pom.xml /uber-jar/pom.xml
35-
RUN mvn clean install -B
36-
3717
FROM alpine:3
3818

39-
ARG RUNNER_VERSION="1.0.0-SNAPSHOT"
40-
ENV RUNNER_VERSION=${RUNNER_VERSION}
41-
4219
USER 9999
4320

4421
WORKDIR /runner
45-
46-
COPY --from=build /uber-jar/target/flink-jar-runner-uber-jar-${RUNNER_VERSION}.jar /runner/
22+
COPY target/flink-jar-runner.uber.jar /runner/

‎flink-dockerfile-example/README.md

Lines changed: 0 additions & 11 deletions
This file was deleted.

‎flink-dockerfile-example/build.gradle

Lines changed: 0 additions & 72 deletions
This file was deleted.

‎pom.xml

Lines changed: 136 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,15 @@
3737
<maven-jar-plugin.version>3.3.0</maven-jar-plugin.version>
3838
<surefire-plugin.version>3.5.0</surefire-plugin.version>
3939

40-
<flink.version>1.19.1</flink.version>
4140
<slf4j.version>1.7.36</slf4j.version>
4241
<log4j.version>2.17.1</log4j.version>
4342
<feign.version>13.5</feign.version>
44-
4543
<picocli.version>4.7.6</picocli.version>
44+
45+
<sqrl.version>0.5.10</sqrl.version>
46+
<flink.version>1.19.1</flink.version>
47+
<jdbc.version>3.2.0-1.19</jdbc.version>
48+
<kafka.version>3.2.0-1.19</kafka.version>
4649
</properties>
4750

4851
<dependencies>
@@ -112,13 +115,13 @@
112115
<dependency>
113116
<groupId>org.apache.flink</groupId>
114117
<artifactId>flink-test-utils</artifactId>
115-
<version>1.19.1</version>
118+
<version>${flink.version}</version>
116119
<scope>test</scope>
117120
</dependency>
118121
<dependency>
119122
<groupId>org.apache.flink</groupId>
120123
<artifactId>flink-table-api-java-bridge</artifactId>
121-
<version>1.19.1</version>
124+
<version>${flink.version}</version>
122125
<scope>provided</scope>
123126
</dependency>
124127
<dependency>
@@ -141,9 +144,89 @@
141144
<version>4.2.2</version>
142145
<scope>test</scope>
143146
</dependency>
147+
148+
<dependency>
149+
<groupId>org.apache.flink</groupId>
150+
<artifactId>flink-connector-kafka</artifactId>
151+
<version>${kafka.version}</version>
152+
</dependency>
153+
<dependency>
154+
<groupId>org.apache.flink</groupId>
155+
<artifactId>flink-connector-jdbc</artifactId>
156+
<version>${jdbc.version}</version>
157+
</dependency>
158+
<dependency>
159+
<groupId>org.apache.flink</groupId>
160+
<artifactId>flink-avro-confluent-registry</artifactId>
161+
<version>${flink.version}</version>
162+
</dependency>
163+
<dependency>
164+
<groupId>com.datasqrl</groupId>
165+
<artifactId>sqrl-lib-common</artifactId>
166+
<version>${sqrl.version}</version>
167+
</dependency>
168+
<dependency>
169+
<groupId>com.datasqrl</groupId>
170+
<artifactId>sqrl-json</artifactId>
171+
<version>${sqrl.version}</version>
172+
</dependency>
173+
<dependency>
174+
<groupId>com.datasqrl</groupId>
175+
<artifactId>sqrl-secure</artifactId>
176+
<version>${sqrl.version}</version>
177+
</dependency>
178+
<dependency>
179+
<groupId>com.datasqrl</groupId>
180+
<artifactId>sqrl-time</artifactId>
181+
<version>${sqrl.version}</version>
182+
</dependency>
183+
<dependency>
184+
<groupId>com.datasqrl</groupId>
185+
<artifactId>sqrl-text</artifactId>
186+
<version>${sqrl.version}</version>
187+
</dependency>
188+
<dependency>
189+
<groupId>com.datasqrl</groupId>
190+
<artifactId>sqrl-flexible-json</artifactId>
191+
<version>${sqrl.version}</version>
192+
</dependency>
193+
<dependency>
194+
<groupId>com.datasqrl</groupId>
195+
<artifactId>sqrl-flexible-csv</artifactId>
196+
<version>${sqrl.version}</version>
197+
</dependency>
144198
</dependencies>
199+
<repositories>
200+
<repository>
201+
<id>central</id>
202+
<url>https://repo.maven.apache.org/maven2</url>
203+
</repository>
204+
<repository>
205+
<id>confluent</id>
206+
<url>https://packages.confluent.io/maven/</url>
207+
</repository>
208+
<repository>
209+
<releases>
210+
<enabled>true</enabled>
211+
</releases>
212+
<snapshots>
213+
<enabled>true</enabled>
214+
</snapshots>
215+
<id>datasqrl</id>
216+
<url>https://maven.pkg.github.com/DataSQRL/sqrl</url>
217+
</repository>
218+
</repositories>
145219

146220
<build>
221+
<testResources>
222+
<testResource>
223+
<filtering>true</filtering>
224+
<directory>src/test/docker</directory>
225+
</testResource>
226+
<testResource>
227+
<directory>src/test/resources</directory>
228+
</testResource>
229+
</testResources>
147230
<plugins>
148231
<plugin>
149232
<groupId>org.apache.maven.plugins</groupId>
@@ -157,6 +240,7 @@
157240
</goals>
158241
<phase>package</phase>
159242
<configuration>
243+
<finalName>flink-jar-runner.uber</finalName>
160244
<artifactSet>
161245
<excludes>
162246
<exclude>org.apache.flink:flink-shaded-force-shading</exclude>
@@ -279,7 +363,7 @@
279363
<version>5.0.0</version>
280364
<configuration>
281365
<skip>${skipTests}</skip>
282-
<composeFile>${project.basedir}/src/test/resources/docker-compose.yml</composeFile>
366+
<composeFile>${project.basedir}/target/test-classes/docker-compose.yml</composeFile>
283367
</configuration>
284368
<executions>
285369
<execution>
@@ -332,6 +416,40 @@
332416
</build>
333417

334418
<profiles>
419+
<profile>
420+
<id>flink19</id>
421+
<properties>
422+
<flink.version>1.19.1</flink.version>
423+
<jdbc.version>3.2.0-1.19</jdbc.version>
424+
<kafka.version>3.2.0-1.19</kafka.version>
425+
<test-image-tag>1.19.1-scala_2.12-java11</test-image-tag>
426+
</properties>
427+
428+
<dependencies>
429+
<dependency>
430+
<groupId>com.datasqrl</groupId>
431+
<artifactId>sqrl-jdbc-1.19</artifactId>
432+
<version>${sqrl.version}</version>
433+
</dependency>
434+
</dependencies>
435+
</profile>
436+
<profile>
437+
<id>flink18</id>
438+
<properties>
439+
<flink.version>1.18.1</flink.version>
440+
<jdbc.version>3.2.0-1.18</jdbc.version>
441+
<kafka.version>3.2.0-1.18</kafka.version>
442+
<test-image-tag>1.18.1-scala_2.12-java11</test-image-tag>
443+
</properties>
444+
445+
<dependencies>
446+
<dependency>
447+
<groupId>com.datasqrl</groupId>
448+
<artifactId>sqrl-jdbc-1.19</artifactId>
449+
<version>${sqrl.version}</version>
450+
</dependency>
451+
</dependencies>
452+
</profile>
335453
<profile>
336454
<!-- useful profile for local development -->
337455
<id>dev</id>
@@ -433,5 +551,18 @@
433551
<license.skip>true</license.skip>
434552
</properties>
435553
</profile>
554+
555+
<profile>
556+
<!-- helper profile to proper configure eclipse -->
557+
<id>m2e</id>
558+
<activation>
559+
<property>
560+
<name>m2e.version</name>
561+
</property>
562+
</activation>
563+
<build>
564+
<directory>${project.basedir}/m2e-target</directory>
565+
</build>
566+
</profile>
436567
</profiles>
437568
</project>

‎src/test/resources/docker-compose.yml renamed to ‎src/test/docker/docker-compose.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ services:
3838
- datasqrl_network
3939

4040
jobmanager:
41-
image: datasqrl/flink-jar-runner:0.0.1-RC1-sqrl0.5.7
41+
image: flink:${test-image-tag}
4242
environment:
4343
- JDBC_URL=jdbc:postgresql://postgres:5432/datasqrl
4444
- JDBC_USERNAME=postgres
@@ -70,7 +70,7 @@ services:
7070
- datasqrl_network
7171

7272
taskmanager:
73-
image: datasqrl/flink-jar-runner:0.0.1-RC1-sqrl0.5.7
73+
image: flink:${test-image-tag}
7474
environment:
7575
- JDBC_URL=jdbc:postgresql://postgres:5432/datasqrl
7676
- JDBC_USERNAME=postgres

0 commit comments

Comments
 (0)