improve layer build for 7.0.0

Author: xuan-cao-swi

.github/workflows/build_publish_lambda_layer.yml

@@ -24,6 +24,10 @@ on:
           - staging
           - production
 
+      ruby-version:
+        description: 'Supported ruby version for lambda layer'
+        default: '3.2 3.3 3.4'
+
 permissions:
   id-token: write   # This is required for requesting the JWT
   contents: read    # This is required for actions/checkout
@@ -32,12 +36,19 @@ jobs:
   # build layer on arm64 and amd64, then upload to artifacts
   # act -j build_layer --container-architecture linux/arm64
   build_layer:
-    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          - x86_64
+          - arm64
+
+    runs-on: ${{ matrix.arch == 'arm64' && fromJSON('{"group":"apm-arm-runner"}') || 'ubuntu-latest' }}
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: Build ruby lambda layer
+      - name: Build ruby lambda layer on ${{ matrix.arch }}
         run: |
           uname -a
           ./build.sh
@@ -46,6 +57,9 @@ jobs:
         env:
           GITHUB_RUBY_TOKEN: ${{ secrets.PACKAGE_GITHUB_TOKEN }}
           SOLARWINDS_SOURCE: ${{ github.event.inputs.solarwinds-source }}
+          PUBLISH_DEST: ${{ github.event.inputs.publish-dest }}
+          ALLOWED_RUBY_VERSION: ${{ github.event.inputs.ruby-version }}
+          ARCHITECTURE: ${{ matrix.arch }}
 
       - name: Show directory contents
         run: |
@@ -55,30 +69,47 @@ jobs:
       - name: Upload to artifact
         uses: actions/upload-artifact@v4
         with:
-          name: ruby-layer.zip
-          path: lambda/build/ruby-layer.zip
+          name: ruby-layer-${{ matrix.arch }}.zip
+          path: lambda/build/ruby-layer-${{ matrix.arch }}.zip
 
   # extract the built layer from artifacts, then scan it with reverselab
   reverselab_scan_layer:
     needs:
       - build_layer
-    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
+      matrix:
+        arch:
+          - x86_64
+          - arm64
+
+    runs-on: ubuntu-latest
+
+    outputs:
+      apm_ruby_version: ${{ steps.version.outputs.SOLARWINDS_APM_VERSION }}
 
     steps:
       - uses: actions/checkout@v4
 
       - name: extract layer zip from artifacts
         uses: actions/download-artifact@v4
         with:
-          name: ruby-layer.zip
+          name: ruby-layer-${{ matrix.arch }}.zip
           path: lambda
 
       - name: extract current solarwinds_apm version
+        id: version
         run: |
-          APM_VERSION=$(grep "gem 'solarwinds_apm'" lambda/otel/layer/Gemfile | awk -F"'" '{print $4}')
+          if [ $SOLARWINDS_SOURCE = 'Local' ]; then \
+            APM_VERSION=$(ruby -r './lib/solarwinds_apm/version.rb' -e 'puts SolarWindsAPM::Version::STRING'); \
+          else \
+            apt-get update && apt-get install -y jq curl; \
+            APM_VERSION=$(curl -s https://rubygems.org/api/v1/gems/solarwinds_apm.json | jq -r .version); \
+          fi
           echo "SOLARWINDS_APM_VERSION=$APM_VERSION" >> $GITHUB_ENV
+          echo "SOLARWINDS_APM_VERSION=$APM_VERSION" >> $GITHUB_OUTPUT
+        env:
+          SOLARWINDS_SOURCE: ${{ github.event.inputs.solarwinds-source }}
 
       - name: Scan build artifact on the Portal
         id: rl-scan
@@ -88,12 +119,12 @@ jobs:
 
         uses: reversinglabs/gh-action-rl-scanner-cloud-only@v1
         with:
-          artifact-to-scan: ./lambda/ruby-layer.zip
+          artifact-to-scan: ./lambda/ruby-layer-${{ matrix.arch }}.zip
           rl-verbose: true
           rl-portal-server: solarwinds
           rl-portal-org: SolarWinds
           rl-portal-group: SaaS-Agents-SWO
-          rl-package-url: solarwinds-apm-ruby/apm-ruby-lambda-layer@${{ env.SOLARWINDS_APM_VERSION }}
+          rl-package-url: solarwinds-apm-ruby/apm-ruby-lambda-layer-${{ matrix.arch }}@${{ env.SOLARWINDS_APM_VERSION }}
 
       - name: report the scan status
         if: success() || failure()
@@ -104,9 +135,9 @@ jobs:
   # extract the built layer from artifacts, then publish it based on region
   publish_layer:
     needs:
-      - build_layer
-    runs-on: ubuntu-latest
+      - reverselab_scan_layer
     strategy:
+      fail-fast: false
       matrix:
         aws_region: 
           - ap-northeast-1
@@ -125,6 +156,11 @@ jobs:
           - us-east-2
           - us-west-1
           - us-west-2
+        arch:
+          - x86_64
+          - arm64
+
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v4
@@ -138,30 +174,33 @@ jobs:
       - name: extract layer zip from artifacts
         uses: actions/download-artifact@v4
         with:
-          name: ruby-layer.zip
+          name: ruby-layer-${{ matrix.arch }}.zip
           path: lambda
 
-      - name: extract current solarwinds_apm version
+      - name: format version for aws lambda and define lambda ruby version
         run: |
-          APM_VERSION=$(grep "gem 'solarwinds_apm'" lambda/otel/layer/Gemfile | awk -F"'" '{print $4}')
-          APM_VERSION="${APM_VERSION//./_}"
+          APM_VERSION=$(echo "$APM_VERSION" | tr '.' '_')
           echo "SOLARWINDS_APM_VERSION=$APM_VERSION" >> $GITHUB_ENV
+          echo "LAMBDA_RUBY_VERSION=$(echo $ALLOWED_RUBY_VERSION | sed 's/\([0-9]\+\.[0-9]\+\)/ruby\1/g')" >> $GITHUB_ENV
+        env:
+          APM_VERSION: ${{needs.reverselab_scan_layer.outputs.apm_ruby_version}}
+          ALLOWED_RUBY_VERSION: ${{ github.event.inputs.ruby-version }}
 
       - name: publish lambda layer
         run: |
           cd lambda/
           aws lambda publish-layer-version \
-            --layer-name solarwinds-apm-ruby-${{ env.SOLARWINDS_APM_VERSION }} \
+            --layer-name solarwinds-apm-ruby-${{ matrix.arch }}-${{ env.SOLARWINDS_APM_VERSION }} \
             --license-info "Apache 2.0" \
-            --compatible-architectures x86_64 arm64 \
-            --compatible-runtimes ruby3.2 ruby3.3 ruby3.4 \
-            --zip-file fileb://ruby-layer.zip \
+            --compatible-architectures ${{ matrix.arch }} \
+            --compatible-runtimes ${{ env.LAMBDA_RUBY_VERSION }} \
+            --zip-file fileb://ruby-layer-${{ matrix.arch }}.zip \
             --query 'LayerVersionArn' \
             --output text
 
       - name: grant permissions to public for the published layer
         run: |
-          layer_name=solarwinds-apm-ruby-${{ env.SOLARWINDS_APM_VERSION }}
+          layer_name=solarwinds-apm-ruby-${{ matrix.arch }}-${{ env.SOLARWINDS_APM_VERSION }}
           latest_version=$(aws lambda list-layer-versions --layer-name $layer_name | jq -r '.LayerVersions | max_by(.Version) | .Version')
           aws lambda add-layer-version-permission \
             --layer-name $layer_name \

.github/workflows/run_unit_tests.yml

@@ -10,6 +10,7 @@ on:
     paths-ignore:
       - 'test/Dockerfile'
       - '.github/*'
+      - 'lambda/*'
   workflow_dispatch:
 
 jobs:

lambda/build.sh

@@ -1,21 +1,22 @@
 #!/bin/sh
-set -e
+set -xe
 
+echo "Build from source: $SOLARWINDS_SOURCE. Publish to $PUBLISH_DEST."
 if [ $SOLARWINDS_SOURCE = 'Local' ]; then
-    cd ../
-    sudo apt-get update && sudo apt-get install -y --no-install-recommends ruby ruby-dev g++ make
-    sudo gem install bundler
-    sudo echo 'gem: --no-document' >> ~/.gemrc
-    sudo bundle config set --local without 'test development'
-    sudo gem build solarwinds_apm.gemspec
-    CURRENT_GEM=$(ls | grep solarwinds_apm-*.gem)
-    mv $CURRENT_GEM lambda/otel/layer/
-    cd -
+  cd ../
+  sudo apt-get update && sudo apt-get install -y --no-install-recommends ruby ruby-dev g++ make
+  sudo gem install bundler
+  sudo echo 'gem: --no-document' >> ~/.gemrc
+  sudo bundle config set --local without 'test development'
+  sudo gem build solarwinds_apm.gemspec
+  CURRENT_GEM=$(ls | grep solarwinds_apm-*.gem)
+  mv $CURRENT_GEM lambda/otel/layer/
+  cd -
 fi
 
 mkdir -p build
 
-for ruby_version in 3.2 3.3 3.4; do
+for ruby_version in $ALLOWED_RUBY_VERSION; do
   docker build --no-cache \
                --build-arg RUBY_VERSION=${ruby_version} \
                --progress plain \
@@ -26,7 +27,12 @@ for ruby_version in 3.2 3.3 3.4; do
 done
 
 cd build/
-mkdir solarwinds-apm && mkdir ruby && mkdir ruby/gems
-unzip -q gems-3.2.0.zip -d ruby/gems/ && unzip -q gems-3.3.0.zip -d ruby/gems/ && unzip -q gems-3.4.0.zip -d ruby/gems/
+mkdir -p ruby/gems
+
+for ruby_version in $ALLOWED_RUBY_VERSION; do
+  unzip -q gems-$ruby_version.0.zip -d ruby/gems/
+done
+
+mkdir solarwinds-apm
 cp ../otel/layer/otel_wrapper.rb . && cp ../otel/layer/wrapper solarwinds-apm/
-zip -qr ruby-layer.zip ruby/ solarwinds-apm/ otel_wrapper.rb
+zip -qr ruby-layer-${ARCHITECTURE}.zip ruby/ solarwinds-apm/ otel_wrapper.rb

lambda/otel/Dockerfile

@@ -4,35 +4,15 @@ FROM public.ecr.aws/sam/build-ruby${RUBY_VERSION}:latest
 ARG RUBY_VERSION
 ENV RUBY_VERSION=${RUBY_VERSION}
 
-RUN mkdir /build
-COPY . /build
-
 WORKDIR /build/layer
-RUN bundle config set --local path 'ruby'
+COPY . /build
+RUN mkdir -p ruby/ruby/${RUBY_VERSION}.0
 
 # Save gem filename if present
 RUN ls | grep solarwinds_apm-*.gem > gem_file.txt 2>/dev/null || true
 
-# Install dependencies
-RUN bundle update
-
-# Install wget
-RUN sh -c '(command -v dnf && dnf install -y wget) || (command -v yum && yum install -y wget)'
-
-# Install aarch64 version of protobuf gem
-ENV ARCH=aarch64
-RUN PROTOBUF_VERSION=$(bundle exec ruby -e 'puts Gem.loaded_specs["google-protobuf"].version.to_s') && \
-    wget "https://rubygems.org/downloads/google-protobuf-${PROTOBUF_VERSION}-${ARCH}-linux-gnu.gem" && \
-    bundle exec gem install "google-protobuf-${PROTOBUF_VERSION}-${ARCH}-linux-gnu.gem"
-
-# Install solarwinds_apm gem (from file, if exists; for local build only)
-RUN gem_file=$(cat gem_file.txt) && \
-    if [ -f "$gem_file" ]; then \
-        gem install $gem_file; \
-        bundle exec gem uninstall solarwinds_apm; \
-        mv /var/lang/lib/ruby/gems/${RUBY_VERSION}.0/gems/solarwinds_apm-*/ /build/layer/ruby/ruby/${RUBY_VERSION}.0/gems/; \
-        mv /var/lang/lib/ruby/gems/${RUBY_VERSION}.0/specifications/solarwinds_apm-* /build/layer/ruby/ruby/${RUBY_VERSION}.0/specifications/; \
-    fi
+RUN gem_file=$(ls | grep solarwinds_apm-*.gem 2>/dev/null || true) && \
+    gem install ${gem_file:-solarwinds_apm} --install-dir ruby/ruby/${RUBY_VERSION}.0 --no-document
 
 # Clean up cache and docs
 RUN rm -rf /build/layer/ruby/ruby/${RUBY_VERSION}.0/cache/* && \