Parent images CI/CD

.github/workflows/pi_build.yml

@@ -0,0 +1,130 @@
+name: 'Parent Images: Build'
+
+on:
+  workflow_call:
+
+# Special permissions required for OIDC authentication
+permissions:
+  id-token: write
+  contents: read
+  actions: read
+
+env:
+  GH_TOKEN: ${{ secrets.GH_TOKEN }}
+
+jobs:
+  pi-matrix:
+    uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/pi_matrix.yml@parent-images
+    secrets: inherit
+    with:
+      repository: "${{ github.repository_owner }}/refinery-submodule-parent-images"
+
+  pi-build:
+    name: 'Parent Images: Docker Build'
+    runs-on: ubuntu-latest
+    needs: [pi-matrix]
+    environment: dev
+    env:
+      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+      DEV_CONTAINER_REGISTRY: ${{ vars.DEV_CONTAINER_REGISTRY }}
+      DEV_LOGIN_USERNAME: ${{ secrets.DEV_LOGIN_USERNAME }}
+      DEV_LOGIN_PASSWORD: ${{ secrets.DEV_LOGIN_PASSWORD }}
+      IMAGE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.pull_request.head.ref }}
+    strategy:
+      matrix:
+        parent_image_type: ${{ fromJson(needs.pi-matrix.outputs.parent_image_type) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GH_TOKEN }}
+          repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
+          fetch-depth: 0
+          submodules: 'true'
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install Dependencies
+        run: python -m pip install pip-tools
+
+      - name: Compile Requirements
+        run: |
+          cd ${{ github.workspace }}/submodules/parent-images
+          git checkout ${{ github.event.pull_request.head.ref }}
+          pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64,arm
+
+      - name: Log into DEV registry
+        uses: docker/login-action@v3
+        with:
+          registry: "${{ env.DEV_CONTAINER_REGISTRY }}"
+          username: "${{ env.DEV_LOGIN_USERNAME }}"
+          password: "${{ env.DEV_LOGIN_PASSWORD }}"
+
+      - name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
+          platforms: linux/amd64
+          file: Dockerfile
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}
+          push: true
+          build-args: |
+            platform=linux/amd64
+            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+
+      - name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
+          platforms: linux/arm64
+          file: Dockerfile
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64
+          push: true
+          build-args: |
+            platform=linux/arm64
+            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+
+      - name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
+          platforms: linux/amd64
+          file: Dockerfile
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}
+          push: true
+          build-args: |
+            platform=linux/amd64
+            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+
+      - name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}-arm64
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
+          platforms: linux/arm64
+          file: Dockerfile
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64
+          push: true
+          build-args: |
+            platform=linux/arm64
+            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile

.github/workflows/pi_matrix.yml

@@ -0,0 +1,50 @@
+name: 'Parent Images: Matrix'
+
+on:
+  workflow_call:
+    inputs:
+      repository:
+        description: 'Repository for actions/checkout'
+        required: false
+        type: string
+        default: ${{ github.repository }}
+      parent_image_type:
+        description: 'If specified, "include" only outputs apps associated to this parent image type'
+        required: false
+        type: string
+        default: ''
+    outputs:
+      parent_image_type:
+        description: 'List[str] of parent image types'
+        value: ${{ jobs.pi-matrix.outputs.parent_image_type }}
+      include:
+        description: 'List[Dict] of apps associated to parent image types'
+        value: ${{ jobs.pi-matrix.outputs.include }}
+
+jobs:
+  pi-matrix:
+    name: 'Parent Images: Generate Matrix'
+    runs-on: ubuntu-latest
+    environment: dev
+    env:
+      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+    outputs:
+      parent_image_type: ${{ steps.generate-matrix.outputs.parent_image_type }}
+      include: ${{ steps.generate-matrix.outputs.include }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.repository }}/refinery-submodule-parent-images
+          ref: ${{ github.event.pull_request.head.ref || github.event.repository.default_branch }}
+
+      - name: Clone cicd-deployment-scripts
+        run: git clone --branch parent-images https://oauth2:${{ secrets.GH_TOKEN }}@github.com/code-kern-ai/cicd-deployment-scripts.git
+
+      - name: Generate Matrix
+        id: generate-matrix
+        run: |
+          bash cicd-deployment-scripts/pi/matrix.sh \
+            -p "${{ github.event.pull_request.number || '' }}" \
+            -s cicd-deployment-scripts/pi/settings.sh \
+            -t "${{ inputs.parent_image_type }}"

.github/workflows/pi_merge.yml

@@ -0,0 +1,133 @@
+name: 'Parent Images: Submodules Merge'
+
+on:
+  workflow_call:
+
+# Special permissions required for OIDC authentication
+permissions:
+  id-token: write
+  contents: read
+  actions: read
+
+env:
+  GH_TOKEN: ${{ secrets.GH_TOKEN }}
+
+jobs:
+  pi-matrix:
+    uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/pi_matrix.yml@parent-images
+    secrets: inherit
+    with:
+      repository: "${{ github.repository_owner }}/refinery-submodule-parent-images"
+
+  pi-update-submodule:
+    name: 'Parent Images: Submodules'
+    runs-on: ubuntu-latest
+    needs: [pi-matrix]
+    environment: dev
+    env:
+      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+    strategy:
+      matrix:
+        parent_image_type: ${{ fromJson(needs.pi-matrix.outputs.parent_image_type) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GH_TOKEN }}
+          repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
+          fetch-depth: 0
+          submodules: 'true'
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install Dependencies
+        run: python -m pip install pip-tools
+
+      - name: Perform Edit/Git Operations
+        run: |
+          cd ${{ github.workspace }}/submodules/parent-images
+          git checkout ${{ github.event.pull_request.base.ref }}
+
+          cd ${{ github.workspace }}
+          git checkout ${{ github.event.pull_request.head.ref }} || git checkout -b ${{ github.event.pull_request.head.ref }}
+          git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
+
+          git config user.email "devtools@kern.ai"
+          git config user.name "GitHub Actions"
+
+          git add submodules
+          git commit -m "ci: update submodules to origin/${{ github.event.pull_request.head.ref }}" || true
+          git push origin ${{ github.event.pull_request.head.ref }}
+          echo "::notice::${{ github.event.repository.name }} updated to origin/${{ github.event.pull_request.head.ref }}"
+
+          gh pr create --draft \
+            --title "${{ github.event.pull_request.title }}" \
+            --body "${{ github.event.pull_request.body }}" \
+            --base dev \
+            --head ${{ github.event.pull_request.head.ref }} \
+            --repo ${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image
+
+  pi-update-app:
+    name: 'Parent Images: ' # suffix populated by matrix
+    runs-on: ubuntu-latest
+    needs: [pi-matrix]
+    environment: dev
+    continue-on-error: true
+    env:
+      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+    strategy:
+      matrix: 
+        include: ${{ fromJson(needs.pi-matrix.outputs.include) }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
+          submodules: 'true'
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install Dependencies
+        run: python -m pip install pip-tools
+
+      - name: Clone ${{ matrix.app }}
+        run: git clone https://oauth2:${{ secrets.GH_TOKEN }}@github.com/${{ github.repository_owner }}/${{ matrix.app }}.git
+
+      - name: Compile Requirements
+        run: |
+          pip-compile --quiet \
+            --output-file ${{ matrix.app }}/requirements/${{ matrix.parent_image_type }}-requirements.txt \
+            submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in
+
+          pip-compile --quiet \
+            --output-file ${{ matrix.app }}/requirements.txt \
+            ${{ matrix.app }}/requirements/requirements.in
+
+      - name: Perform Edit/Git Operations
+        run: |
+          cd ${{ matrix.app }}
+
+          git config user.email "devtools@kern.ai"
+          git config user.name "GitHub Actions"
+
+          git checkout -b ${{ github.event.pull_request.head.ref }} || git checkout ${{ github.event.pull_request.head.ref }}
+          git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
+
+          git add requirements*
+          git commit -m "ci: update ${{ matrix.parent_image_type }}-requirements.txt"
+          git push origin ${{ github.event.pull_request.head.ref }}
+          echo "::notice::${{ matrix.app }} updated to origin/${{ github.event.pull_request.head.ref }}"
+
+          gh pr create --draft \
+            --title "${{ github.event.pull_request.title }}" \
+            --body "${{ github.event.pull_request.body }}" \
+            --base dev \
+            --head ${{ github.event.pull_request.head.ref }} \
+            --repo ${{ github.repository_owner }}/${{ matrix.app }}
+

.github/workflows/pi_release.yml

@@ -0,0 +1,81 @@
+name: 'Parent Images: Release'
+
+on:
+  workflow_call:
+
+# Special permissions required for OIDC authentication
+permissions:
+  id-token: write
+  contents: read
+  actions: read
+
+env:
+  GH_TOKEN: ${{ secrets.GH_TOKEN }}
+
+jobs:
+  pi-matrix:
+    uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/pi_matrix.yml@parent-images
+    secrets: inherit
+    with:
+      repository: ${{ github.repository }}
+      parent_image_type: ${{ vars.PARENT_IMAGE_TYPE }}
+
+  pi-edit:
+    name: 'Parent Images: Dockerfile ' # suffix populated by matrix
+    runs-on: ubuntu-latest
+    needs: [pi-matrix]
+    environment: dev
+    env:
+      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+      DOCKERHUB_CONTAINER_REGISTRY: ${{ vars.DOCKERHUB_CONTAINER_REGISTRY }}
+      DOCKERHUB_LOGIN_USERNAME: ${{ secrets.DOCKERHUB_LOGIN_USERNAME }}
+      DOCKERHUB_LOGIN_PASSWORD: ${{ secrets.DOCKERHUB_LOGIN_PASSWORD }}
+      PARENT_IMAGE_NAME: ${{ vars.PARENT_IMAGE_NAME }}
+      PARENT_IMAGE_TYPE: ${{ vars.PARENT_IMAGE_TYPE }}
+      DOCKERFILE: ${{ vars.DOCKERFILE }}
+    strategy:
+      matrix:
+        include: ${{ fromJson(needs.pi-matrix.outputs.include) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GH_TOKEN }}
+          repository: ${{ github.repository_owner }}/${{ matrix.app }}
+
+      - name: Clone cicd-deployment-scripts
+        run: git clone --branch parent-images https://oauth2:${{ secrets.GH_TOKEN }}@github.com/code-kern-ai/cicd-deployment-scripts.git
+
+      - name: Perform Edit/Git Operations
+        run: |
+          bash cicd-deployment-scripts/pi/edit_dockerfile.sh \
+            -i ${{ env.PARENT_IMAGE_NAME }} \
+            -t ${{ matrix.parent_image_type }} \
+            -l ${{ github.event.release.tag_name }} \
+            -r ${{ env.DOCKERHUB_CONTAINER_REGISTRY }} \
+            -d ${{ env.DOCKERFILE }}
+
+          LATEST_IMAGE_TAG="${{ env.DOCKERHUB_CONTAINER_REGISTRY }}/${{ matrix.parent_image_type }}:${{ github.event.release.tag_name }}"
+          BASE_REF="dev"
+          HEAD_REF="parent-images"
+          PR_TITLE="ci(pi): update to $LATEST_IMAGE_TAG"
+
+          git config user.email "devtools@kern.ai"
+          git config user.name "GitHub Actions"
+
+          git checkout -b parent-images || git checkout parent-images
+          git push origin parent-images && git pull origin parent-images
+
+          git add ${{ env.DOCKERFILE }}
+          git commit -m "$PR_TITLE" || true
+          git push origin parent-images
+          echo "::notice::${{ matrix.app }} updated to $LATEST_IMAGE_TAG"
+
+          bash cicd-deployment-scripts/pi/pr_create.sh \
+            -b "$BASE_REF" \
+            -h "$HEAD_REF" \
+            -t "ci(pi): update to $LATEST_IMAGE_TAG" \
+            -o ${{ github.repository_owner }} \}}
+            -r ${{ github.event.repository.name }} \}}
+            -n ${{ github.event.release.tag_name }} \}}
+            -a ${{ matrix.app }}