Run integration tests on docker image produced by the project

Signed-off-by: Marvin Froeder <marvin@datasqrl.com>

Author: velo

.github/workflows/deploy.yml

@@ -59,7 +59,8 @@ jobs:
       - name: Download dependencies
         run: |
           mvn -B org.apache.maven.plugins:maven-dependency-plugin:3.8.1:go-offline de.qaware.maven:go-offline-maven-plugin:1.2.8:resolve-dependencies -Pci,flink-1.19
-          mvn -B -f flink-sql-runner/pom.xml org.apache.maven.plugins:maven-resources-plugin:3.3.1:resources com.marvinformatics:docker-compose-maven-plugin:5.0.0:pull -Pci,flink-1.19
+          mvn -B -f flink-sql-runner/pom.xml org.apache.maven.plugins:maven-resources-plugin:3.3.1:resources -Pci,flink-1.19
+          grep '^FROM' flink-sql-runner/target/Dockerfile | awk '{print $2}' | xargs -n1 docker pull
 
       - name: Update version
         if: github.event_name == 'release' && github.event.action == 'created'

.github/workflows/uber-jar.yml

@@ -62,7 +62,8 @@ jobs:
       - name: Download dependencies
         run: |
           mvn -B org.apache.maven.plugins:maven-dependency-plugin:3.8.1:go-offline de.qaware.maven:go-offline-maven-plugin:1.2.8:resolve-dependencies -P${{ matrix.FLINK_PROFILE }}
-          mvn -B -f flink-sql-runner/pom.xml org.apache.maven.plugins:maven-resources-plugin:3.3.1:resources com.marvinformatics:docker-compose-maven-plugin:5.0.0:pull -P${{ matrix.FLINK_PROFILE }}
+          mvn -B -f flink-sql-runner/pom.xml org.apache.maven.plugins:maven-resources-plugin:3.3.1:resources -P${{ matrix.FLINK_PROFILE }}
+          grep '^FROM' flink-sql-runner/target/Dockerfile | awk '{print $2}' | xargs -n1 docker pull
 
       - name: Update version
         if: github.event_name == 'release' && github.event.action == 'created'

flink-sql-runner/pom.xml

@@ -258,6 +258,31 @@
         </executions>
       </plugin>
 
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <version>${exec.plugin.version}</version>
+        <executions>
+          <execution>
+            <id>build-docker-image</id>
+            <goals>
+              <goal>exec</goal>
+            </goals>
+            <phase>package</phase>
+            <configuration>
+              <workingDirectory>${project.basedir}/target/</workingDirectory>
+              <executable>docker</executable>
+              <skip>${skipTests}</skip>
+              <arguments>
+                <argument>build</argument>
+                <argument>--tag</argument>
+                <argument>flink-sql-runner</argument>
+                <argument>.</argument>
+              </arguments>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>com.marvinformatics</groupId>
         <artifactId>docker-compose-maven-plugin</artifactId>

flink-sql-runner/src/main/docker/Dockerfile

@@ -18,3 +18,4 @@ FROM flink:${flink-base-image}
 
 RUN mkdir -p /opt/flink/plugins/flink-jar-runner
 COPY flink-jar-runner.uber.jar /opt/flink/plugins/flink-jar-runner
+COPY system-functions-discovery-${project.version}.jar /opt/flink/lib/system-functions-discovery.jar

flink-sql-runner/src/main/docker/docker-compose.yml

@@ -40,7 +40,7 @@ services:
       - datasqrl_network
 
   jobmanager:
-    image: flink:${flink-base-image}
+    image: flink-sql-runner
     environment:
       - JDBC_URL=jdbc:postgresql://postgres:5432/datasqrl
       - JDBC_USERNAME=postgres
@@ -64,7 +64,6 @@ services:
       - ./test-classes/:/opt/flink/usrlib/
       - ./test-classes/datasources/:/datasources/
       - /tmp:/tmp
-      - ./system-functions-discovery-${project.version}.jar:/opt/flink/lib/system-functions-discovery.jar
       - ./test-classes/systemfunction/system-functions-sample-${project.version}.jar:/opt/flink/lib/systemfunction.jar
     healthcheck:
       test: ["CMD-SHELL", "curl -f http://localhost:8081/ || exit 1"]
@@ -80,7 +79,7 @@ services:
       - datasqrl_network
 
   taskmanager:
-    image: flink:${flink-base-image}
+    image: flink-sql-runner
     environment:
       - JDBC_URL=jdbc:postgresql://postgres:5432/datasqrl
       - JDBC_USERNAME=postgres
@@ -100,7 +99,6 @@ services:
       - ./test-classes/:/opt/flink/usrlib/
       - ./test-classes/datasources/:/datasources/
       - /tmp:/tmp
-      - ./system-functions-discovery-${project.version}.jar:/opt/flink/lib/system-functions-discovery.jar
       - ./test-classes/systemfunction/system-functions-sample-${project.version}.jar:/opt/flink/lib/systemfunction.jar
     restart: always
     deploy:

pom.xml

@@ -420,7 +420,7 @@
                 <excludeArtifact>system-functions-sample</excludeArtifact>
                 <excludeArtifact>udf-sample</excludeArtifact>
               </excludeArtifacts>
-              
+
               <autoPublish>true</autoPublish>
             </configuration>
           </plugin>