Merge pull request #207 from vinodbhorge/gcp-checkpoint-support

Gcp checkpoint support

Author: pallakartheekreddy

jobs-distribution/Dockerfile

@@ -10,4 +10,15 @@ RUN tar -xvf /tmp/jobs-distribution-1.0.tar.gz -C $FLINK_HOME/lib/
 RUN cp $FLINK_HOME/opt/flink-s3-fs-presto-1.13.5.jar $FLINK_HOME/lib/flink-aaa-s3-fs-presto-1.13.5.jar
 USER root
 RUN rm -f /tmp/jobs-distribution-1.0.tar.gz
+
+COPY lib/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar $FLINK_HOME/lib/
+COPY lib/gcs-connector-hadoop2-2.2.0.jar $FLINK_HOME/lib/
+COPY lib/hadoop-azure-3.3.1.jar $FLINK_HOME/lib/
+
+RUN mkdir -p $FLINK_HOME/plugins/gs-fs-hadoop && mkdir -p $FLINK_HOME/plugins/azure-fs-hadoop
+
+COPY lib/flink-gs-fs-hadoop-1.16.2.jar $FLINK_HOME/plugins/gs-fs-hadoop
+COPY lib/flink-azure-fs-hadoop-1.16.2.jar $FLINK_HOME/plugins/azure-fs-hadoop
+
+COPY lib/core-site.xml $FLINK_HOME/conf/
 USER flink

jobs-distribution/lib/core-site.xml

@@ -0,0 +1,29 @@
+<configuration>
+  <!-- GCP Configuration -->
+  <property>
+    <name>fs.gs.impl</name>
+    <value>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem</value>
+  </property>
+  <property>
+    <name>fs.AbstractFileSystem.gs.impl</name>
+    <value>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS</value>
+  </property>
+  <property>
+    <name>fs.gs.auth.type</name>
+    <value>ADC</value>
+  </property>
+
+  <!-- Azure Configuration -->
+  <property>
+    <name>fs.azure</name>
+    <value>org.apache.hadoop.fs.azure.NativeAzureFileSystem</value>
+  </property>
+  <property>
+    <name>fs.azure.account.auth.type</name>
+    <value>OAuth</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth.provider.type</name>
+    <value>org.apache.hadoop.fs.azurebfs.oauth2.ManagedIdentityTokenProvider</value>
+  </property>
+</configuration>