diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml new file mode 100644 index 0000000..2874579 --- /dev/null +++ b/.github/workflows/pi_build.yml @@ -0,0 +1,130 @@ +name: 'Parent Images: Build' + +on: + workflow_call: + +# Special permissions required for OIDC authentication +permissions: + id-token: write + contents: read + actions: read + +env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + +jobs: + pi-matrix: + uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/pi_matrix.yml@parent-images + secrets: inherit + with: + repository: "${{ github.repository_owner }}/refinery-submodule-parent-images" + + pi-build: + name: 'Parent Images: Docker Build' + runs-on: ubuntu-latest + needs: [pi-matrix] + environment: dev + env: + PYTHON_VERSION: ${{ vars.PYTHON_VERSION }} + DEV_CONTAINER_REGISTRY: ${{ vars.DEV_CONTAINER_REGISTRY }} + DEV_LOGIN_USERNAME: ${{ secrets.DEV_LOGIN_USERNAME }} + DEV_LOGIN_PASSWORD: ${{ secrets.DEV_LOGIN_PASSWORD }} + IMAGE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.pull_request.head.ref }} + strategy: + matrix: + parent_image_type: ${{ fromJson(needs.pi-matrix.outputs.parent_image_type) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GH_TOKEN }} + repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image' + fetch-depth: 0 + submodules: 'true' + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Dependencies + run: python -m pip install pip-tools + + - name: Compile Requirements + run: | + cd ${{ github.workspace }}/submodules/parent-images + git checkout ${{ github.event.pull_request.head.ref }} + pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + platforms: linux/amd64,linux/arm64 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64,arm + + - name: Log into DEV registry + uses: docker/login-action@v3 + with: + registry: "${{ env.DEV_CONTAINER_REGISTRY }}" + username: "${{ env.DEV_LOGIN_USERNAME }}" + password: "${{ env.DEV_LOGIN_PASSWORD }}" + + - name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }} + uses: docker/build-push-action@v5 + with: + context: . + cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache + cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true + platforms: linux/amd64 + file: Dockerfile + tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }} + push: true + build-args: | + platform=linux/amd64 + label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile + + - name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64 + uses: docker/build-push-action@v5 + with: + context: . + cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache + cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true + platforms: linux/arm64 + file: Dockerfile + tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64 + push: true + build-args: | + platform=linux/arm64 + label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile + + - name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }} + uses: docker/build-push-action@v5 + with: + context: . + cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache + cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true + platforms: linux/amd64 + file: Dockerfile + tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }} + push: true + build-args: | + platform=linux/amd64 + label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile + + - name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}-arm64 + uses: docker/build-push-action@v5 + with: + context: . + cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache + cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true + platforms: linux/arm64 + file: Dockerfile + tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64 + push: true + build-args: | + platform=linux/arm64 + label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile diff --git a/.github/workflows/pi_matrix.yml b/.github/workflows/pi_matrix.yml new file mode 100644 index 0000000..3d5f767 --- /dev/null +++ b/.github/workflows/pi_matrix.yml @@ -0,0 +1,50 @@ +name: 'Parent Images: Matrix' + +on: + workflow_call: + inputs: + repository: + description: 'Repository for actions/checkout' + required: false + type: string + default: ${{ github.repository }} + parent_image_type: + description: 'If specified, "include" only outputs apps associated to this parent image type' + required: false + type: string + default: '' + outputs: + parent_image_type: + description: 'List[str] of parent image types' + value: ${{ jobs.pi-matrix.outputs.parent_image_type }} + include: + description: 'List[Dict] of apps associated to parent image types' + value: ${{ jobs.pi-matrix.outputs.include }} + +jobs: + pi-matrix: + name: 'Parent Images: Generate Matrix' + runs-on: ubuntu-latest + environment: dev + env: + PYTHON_VERSION: ${{ vars.PYTHON_VERSION }} + outputs: + parent_image_type: ${{ steps.generate-matrix.outputs.parent_image_type }} + include: ${{ steps.generate-matrix.outputs.include }} + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + repository: ${{ inputs.repository }}/refinery-submodule-parent-images + ref: ${{ github.event.pull_request.head.ref || github.event.repository.default_branch }} + + - name: Clone cicd-deployment-scripts + run: git clone --branch parent-images https://oauth2:${{ secrets.GH_TOKEN }}@github.com/code-kern-ai/cicd-deployment-scripts.git + + - name: Generate Matrix + id: generate-matrix + run: | + bash cicd-deployment-scripts/pi/matrix.sh \ + -p "${{ github.event.pull_request.number || '' }}" \ + -s cicd-deployment-scripts/pi/settings.sh \ + -t "${{ inputs.parent_image_type }}" diff --git a/.github/workflows/pi_merge.yml b/.github/workflows/pi_merge.yml new file mode 100644 index 0000000..347854f --- /dev/null +++ b/.github/workflows/pi_merge.yml @@ -0,0 +1,133 @@ +name: 'Parent Images: Submodules Merge' + +on: + workflow_call: + +# Special permissions required for OIDC authentication +permissions: + id-token: write + contents: read + actions: read + +env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + +jobs: + pi-matrix: + uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/pi_matrix.yml@parent-images + secrets: inherit + with: + repository: "${{ github.repository_owner }}/refinery-submodule-parent-images" + + pi-update-submodule: + name: 'Parent Images: Submodules' + runs-on: ubuntu-latest + needs: [pi-matrix] + environment: dev + env: + PYTHON_VERSION: ${{ vars.PYTHON_VERSION }} + strategy: + matrix: + parent_image_type: ${{ fromJson(needs.pi-matrix.outputs.parent_image_type) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GH_TOKEN }} + repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image' + fetch-depth: 0 + submodules: 'true' + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Dependencies + run: python -m pip install pip-tools + + - name: Perform Edit/Git Operations + run: | + cd ${{ github.workspace }}/submodules/parent-images + git checkout ${{ github.event.pull_request.base.ref }} + + cd ${{ github.workspace }} + git checkout ${{ github.event.pull_request.head.ref }} || git checkout -b ${{ github.event.pull_request.head.ref }} + git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }} + + git config user.email "devtools@kern.ai" + git config user.name "GitHub Actions" + + git add submodules + git commit -m "ci: update submodules to origin/${{ github.event.pull_request.head.ref }}" || true + git push origin ${{ github.event.pull_request.head.ref }} + echo "::notice::${{ github.event.repository.name }} updated to origin/${{ github.event.pull_request.head.ref }}" + + gh pr create --draft \ + --title "${{ github.event.pull_request.title }}" \ + --body "${{ github.event.pull_request.body }}" \ + --base dev \ + --head ${{ github.event.pull_request.head.ref }} \ + --repo ${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image + + pi-update-app: + name: 'Parent Images: ' # suffix populated by matrix + runs-on: ubuntu-latest + needs: [pi-matrix] + environment: dev + continue-on-error: true + env: + PYTHON_VERSION: ${{ vars.PYTHON_VERSION }} + strategy: + matrix: + include: ${{ fromJson(needs.pi-matrix.outputs.include) }} + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image' + submodules: 'true' + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Dependencies + run: python -m pip install pip-tools + + - name: Clone ${{ matrix.app }} + run: git clone https://oauth2:${{ secrets.GH_TOKEN }}@github.com/${{ github.repository_owner }}/${{ matrix.app }}.git + + - name: Compile Requirements + run: | + pip-compile --quiet \ + --output-file ${{ matrix.app }}/requirements/${{ matrix.parent_image_type }}-requirements.txt \ + submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in + + pip-compile --quiet \ + --output-file ${{ matrix.app }}/requirements.txt \ + ${{ matrix.app }}/requirements/requirements.in + + - name: Perform Edit/Git Operations + run: | + cd ${{ matrix.app }} + + git config user.email "devtools@kern.ai" + git config user.name "GitHub Actions" + + git checkout -b ${{ github.event.pull_request.head.ref }} || git checkout ${{ github.event.pull_request.head.ref }} + git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }} + + git add requirements* + git commit -m "ci: update ${{ matrix.parent_image_type }}-requirements.txt" + git push origin ${{ github.event.pull_request.head.ref }} + echo "::notice::${{ matrix.app }} updated to origin/${{ github.event.pull_request.head.ref }}" + + gh pr create --draft \ + --title "${{ github.event.pull_request.title }}" \ + --body "${{ github.event.pull_request.body }}" \ + --base dev \ + --head ${{ github.event.pull_request.head.ref }} \ + --repo ${{ github.repository_owner }}/${{ matrix.app }} + \ No newline at end of file diff --git a/.github/workflows/pi_release.yml b/.github/workflows/pi_release.yml new file mode 100644 index 0000000..5409a2a --- /dev/null +++ b/.github/workflows/pi_release.yml @@ -0,0 +1,81 @@ +name: 'Parent Images: Release' + +on: + workflow_call: + +# Special permissions required for OIDC authentication +permissions: + id-token: write + contents: read + actions: read + +env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + +jobs: + pi-matrix: + uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/pi_matrix.yml@parent-images + secrets: inherit + with: + repository: ${{ github.repository }} + parent_image_type: ${{ vars.PARENT_IMAGE_TYPE }} + + pi-edit: + name: 'Parent Images: Dockerfile ' # suffix populated by matrix + runs-on: ubuntu-latest + needs: [pi-matrix] + environment: dev + env: + PYTHON_VERSION: ${{ vars.PYTHON_VERSION }} + DOCKERHUB_CONTAINER_REGISTRY: ${{ vars.DOCKERHUB_CONTAINER_REGISTRY }} + DOCKERHUB_LOGIN_USERNAME: ${{ secrets.DOCKERHUB_LOGIN_USERNAME }} + DOCKERHUB_LOGIN_PASSWORD: ${{ secrets.DOCKERHUB_LOGIN_PASSWORD }} + PARENT_IMAGE_NAME: ${{ vars.PARENT_IMAGE_NAME }} + PARENT_IMAGE_TYPE: ${{ vars.PARENT_IMAGE_TYPE }} + DOCKERFILE: ${{ vars.DOCKERFILE }} + strategy: + matrix: + include: ${{ fromJson(needs.pi-matrix.outputs.include) }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GH_TOKEN }} + repository: ${{ github.repository_owner }}/${{ matrix.app }} + + - name: Clone cicd-deployment-scripts + run: git clone --branch parent-images https://oauth2:${{ secrets.GH_TOKEN }}@github.com/code-kern-ai/cicd-deployment-scripts.git + + - name: Perform Edit/Git Operations + run: | + bash cicd-deployment-scripts/pi/edit_dockerfile.sh \ + -i ${{ env.PARENT_IMAGE_NAME }} \ + -t ${{ matrix.parent_image_type }} \ + -l ${{ github.event.release.tag_name }} \ + -r ${{ env.DOCKERHUB_CONTAINER_REGISTRY }} \ + -d ${{ env.DOCKERFILE }} + + LATEST_IMAGE_TAG="${{ env.DOCKERHUB_CONTAINER_REGISTRY }}/${{ matrix.parent_image_type }}:${{ github.event.release.tag_name }}" + BASE_REF="dev" + HEAD_REF="parent-images" + PR_TITLE="ci(pi): update to $LATEST_IMAGE_TAG" + + git config user.email "devtools@kern.ai" + git config user.name "GitHub Actions" + + git checkout -b parent-images || git checkout parent-images + git push origin parent-images && git pull origin parent-images + + git add ${{ env.DOCKERFILE }} + git commit -m "$PR_TITLE" || true + git push origin parent-images + echo "::notice::${{ matrix.app }} updated to $LATEST_IMAGE_TAG" + + bash cicd-deployment-scripts/pi/pr_create.sh \ + -b "$BASE_REF" \ + -h "$HEAD_REF" \ + -t "ci(pi): update to $LATEST_IMAGE_TAG" \ + -o ${{ github.repository_owner }} \}} + -r ${{ github.event.repository.name }} \}} + -n ${{ github.event.release.tag_name }} \}} + -a ${{ matrix.app }} \ No newline at end of file diff --git a/pi/edit_dockerfile.sh b/pi/edit_dockerfile.sh new file mode 100644 index 0000000..3a19319 --- /dev/null +++ b/pi/edit_dockerfile.sh @@ -0,0 +1,27 @@ +# !/bin/bash + +set -e + +PARENT_IMAGE_NAME="refinery-parent-images" +PARENT_IMAGE_TYPE="" +RELEASE_TAG="" +DOCKER_REGISTRY="kernai" +DOCKERFILE_PATH="Dockerfile" + +while getopts i:t:l:r:d: flag +do + case "${flag}" in + i) PARENT_IMAGE_NAME=${OPTARG};; + t) PARENT_IMAGE_TYPE=$(echo ${OPTARG} | sed 's|_|-|g');; + l) RELEASE_TAG=${OPTARG};; + r) DOCKER_REGISTRY=${OPTARG};; + d) DOCKERFILE_PATH=${OPTARG};; + esac +done + +PI_EXISTING_TAG=$(grep "${DOCKER_REGISTRY}/${PARENT_IMAGE_NAME}:v.*-${PARENT_IMAGE_TYPE}" $DOCKERFILE_PATH | sed 's|FROM ||g' | cut -d ':' -f 2) +PI_EXISTING_IMAGE="${DOCKER_REGISTRY}/${PARENT_IMAGE_NAME}:${PI_EXISTING_TAG}" +PI_NEW_IMAGE="${DOCKER_REGISTRY}/${PARENT_IMAGE_NAME}:${RELEASE_TAG}" + +echo "$(sed "s|${PI_EXISTING_IMAGE}|${PI_NEW_IMAGE}|g" ${DOCKERFILE_PATH})" > $DOCKERFILE_PATH +echo "::notice::Dockerfile updated with new image: ${PI_NEW_IMAGE}" \ No newline at end of file diff --git a/pi/matrix.sh b/pi/matrix.sh new file mode 100644 index 0000000..189862f --- /dev/null +++ b/pi/matrix.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -e + +PARENT_IMAGE_TYPE="" +PR_NUMBER="" +SOURCE_SCRIPT="pi/settings.sh" + +while getopts t:p:s: flag +do + case "${flag}" in + t) PARENT_IMAGE_TYPE=${OPTARG};; + p) PR_NUMBER=${OPTARG};; + s) SOURCE_SCRIPT=${OPTARG};; + esac +done + +source $SOURCE_SCRIPT + +UPDATED_PARENT_TYPES=() + +if [ -n $PR_NUMBER ] && [ -z $PARENT_IMAGE_TYPE ]; then + UPDATED_FILES=$(gh pr diff $PR_NUMBER --name-only) + while IFS= read -r file; do + if [[ $file != requirements/* ]] || [[ $file != *.in ]]; then + continue + fi + + parent_image_type=$(basename $file | sed 's|-requirements.in||g') + UPDATED_PARENT_TYPES+=($parent_image_type) + + done <<< "$UPDATED_FILES" + echo "::notice::Exporting matrix for parent image types: $UPDATED_PARENT_TYPES" +elif [ -z $PR_NUMBER ] && [ -n $PARENT_IMAGE_TYPE ]; then + echo "::notice::Exporting matrix for parent image type: $PARENT_IMAGE_TYPE" + UPDATED_PARENT_TYPES=( $PARENT_IMAGE_TYPE ) +fi + +PARENT_IMAGE_TYPES="" +INCLUDES="" +for parent_image_type in "${UPDATED_PARENT_TYPES[@]}"; do + PARENT_IMAGE_TYPES+="\"$parent_image_type\"," + eval 'APP_REPOS=( "${'$(echo ${parent_image_type} | sed "s|-|_|g")'[@]}" )' + for app in "${APP_REPOS[@]}"; do + INCLUDES+='{ "parent_image_type": "'${parent_image_type}'", "app": "'${app}'" },' + done +done + +MATRIX='{"include": ['${INCLUDES::-1}']}' +echo $MATRIX | jq -C --indent 2 '.' +echo "include=[${INCLUDES::-1}]" >> $GITHUB_OUTPUT +echo "parent_image_type=[${PARENT_IMAGE_TYPES::-1}]" >> $GITHUB_OUTPUT \ No newline at end of file diff --git a/pi/pr_create.sh b/pi/pr_create.sh new file mode 100644 index 0000000..d49ec70 --- /dev/null +++ b/pi/pr_create.sh @@ -0,0 +1,58 @@ +# !/bin/bash +set -e + +BASE_REF="dev" +HEAD_REF="parent-images" +PR_TITLE="ci(pi): update parent image" +REPOSITORY_OWNER="code-kern-ai" +REPOSITORY_NAME="" +RELEASE_TAG="" +APP="" + +while getopts b:h:t:o:r:n:a: flag +do + case "${flag}" in + b) BASE_REF=${OPTARG};; + h) HEAD_REF=${OPTARG};; + t) PR_TITLE=${OPTARG};; + o) REPOSITORY_OWNER=${OPTARG};; + r) REPOSITORY_NAME=${OPTARG};; + n) RELEASE_TAG=${OPTARG};; + a) APP=${OPTARG};; + esac +done + +EXISTING_PR_NUMBER="" +EXISTING_PR_BODY=$(gh pr list --base $BASE_REF --head $HEAD_REF --json body --jq '.[].body') + +if [ -z "$EXISTING_PR_BODY" ]; then + PR_BODY=$(cat <