azcopy-ci #1629
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: azcopy-ci | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - '**/README.md' | |
| - '**/LICENSE' | |
| - 'visuals/**' | |
| pull_request: | |
| paths-ignore: | |
| - '**/README.md' | |
| - '**/LICENSE' | |
| - 'visuals/**' | |
| schedule: | |
| - cron: '0 3 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| destroy: | |
| description: 'Destroy Infrastructure' | |
| required: false | |
| type: boolean | |
| default: true | |
| useSAS: | |
| description: 'Use SAS' | |
| required: false | |
| type: boolean | |
| default: false | |
| runnerImage: | |
| type: choice | |
| required: true | |
| description: 'Runner image' | |
| default: ubuntu-latest | |
| options: | |
| - macos-latest | |
| - ubuntu-latest | |
| - windows-latest | |
| env: | |
| ACTIONS_STEP_DEBUG: ${{ vars.ACTIONS_STEP_DEBUG }} | |
| AZCOPY_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_EXTENSION_USE_DYNAMIC_INSTALL: yes_without_prompt | |
| AZURE_DEFAULTS_LOCATION: westeurope | |
| FILES_SYNC_STORAGE_CONTAINER: artifacts | |
| FILES_SYNC_USE_SAS: $${{ inputs.useSAS || 'false' }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| syncTest: | |
| name: Create storage, sync & destroy | |
| defaults: | |
| run: | |
| working-directory: scripts | |
| runs-on: ${{ inputs.runnerImage || 'windows-latest' }} | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v2 | |
| - name: Install AzCopy | |
| run: | | |
| if ($env:ACTIONS_STEP_DEBUG -ieq "true") { | |
| $InformationPreference = "Continue" | |
| $VerbosePreference = "Continue" | |
| $DebugPreference = "Continue" | |
| Set-PSDebug -Trace 1 | |
| } | |
| # FIX: https://github.com/Azure/azure-storage-azcopy/issues/2714 | |
| ./install_azcopy.ps1 -ExcludeVersion "10.25","10.26.0" | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| shell: pwsh | |
| - name: Use Azure CLI | |
| uses: azure/login@v1 | |
| with: | |
| # Using OpenID Connect / Workload Identity | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| - name: Prepare environment variables | |
| run: | | |
| Write-Output "ACTIONS_STEP_DEBUG=${{ secrets.ACTIONS_RUNNER_DEBUG }}" >> $env:GITHUB_ENV | |
| Write-Output "FILES_SYNC_STORAGE_ACCOUNT=filesyncci${env:GITHUB_RUN_ID}" >> $env:GITHUB_ENV | |
| Write-Output "FILES_SYNC_STORAGE_ACCOUNT_CLONE=filesync2ci${env:GITHUB_RUN_ID}" >> $env:GITHUB_ENV | |
| Write-Output "FILES_SYNC_RESOURCE_GROUP=file-sync-ci-rg${env:GITHUB_RUN_ID}" >> $env:GITHUB_ENV | |
| if ($IsMacOS) { | |
| Write-Output "FILES_SYNC_AZCOPY_PUBLIC_ACCESS=true" >> $env:GITHUB_ENV | |
| } | |
| shell: pwsh | |
| - name: List environment variables | |
| run: | | |
| # List environment variables | |
| Get-ChildItem -Path Env: -Recurse -Include ACTIONS_*,AZCOPY_*,AZURE_*,FILES_SYNC_*,GH_*,GITHUB_*,*TEMP*,*TMP* | Sort-Object -Property Name | |
| Get-ChildItem -Path Env: -Recurse -Include * | Sort-Object -Property Name | |
| shell: pwsh | |
| - name: Show runner Azure region | |
| run: | | |
| try { | |
| $vmMetadata = (Invoke-RestMethod -Headers @{"Metadata"="true"} -Method GET -NoProxy -Uri "http://169.254.169.254/metadata/instance/compute?api-version=2021-02-01" -TimeoutSec 1) | |
| } catch { | |
| $vmMetadata = $null | |
| } | |
| if ($vmMetadata) { | |
| Write-Host "Running in '$($vmMetadata.location)'" | |
| if ($vmMetadata.location -eq $env:AZURE_DEFAULTS_LOCATION) { | |
| Write-Waning "Deployment region is also '${env:AZURE_DEFAULTS_LOCATION}'. Deployment may fail due to this Azure Storage limitation: https://docs.microsoft.com/en-us/azure/storage/common/storage-network-security?tabs=azure-portal#grant-access-from-an-internet-ip-range" | |
| } | |
| } else { | |
| Write-Host "Could not determine region" | |
| } | |
| shell: pwsh | |
| - name: Create Storage Account | |
| run: | | |
| if ($env:ACTIONS_STEP_DEBUG -ieq "true") { | |
| $InformationPreference = "Continue" | |
| $VerbosePreference = "Continue" | |
| $DebugPreference = "Continue" | |
| Set-PSDebug -Trace 1 | |
| } | |
| ./create_storage_account.ps1 -Container $env:FILES_SYNC_STORAGE_CONTAINER ` | |
| -Name $env:FILES_SYNC_STORAGE_ACCOUNT ` | |
| -ResourceGroup $env:FILES_SYNC_RESOURCE_GROUP ` | |
| -SkipResourceLock ` | |
| -SubscriptionId ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| Write-Host "Waiting for resource graph to populate..." | |
| Start-Sleep -Seconds 30 | |
| shell: pwsh | |
| - name: Prepare upload | |
| run: | | |
| if ($env:ACTIONS_STEP_DEBUG -ieq "true") { | |
| $InformationPreference = "Continue" | |
| $VerbosePreference = "Continue" | |
| $DebugPreference = "Continue" | |
| Set-PSDebug -Trace 1 | |
| } | |
| # Create upload directory | |
| $uploadDirectory = (Join-Path $env:RUNNER_WORKSPACE "upload") | |
| New-Item $uploadDirectory -ItemType "directory" -Force | Write-Debug | |
| Get-ChildItem $env:GITHUB_WORKSPACE -Recurse ` | |
| | Where-Object { $_.FullName -notlike "*bin*" } ` | |
| | Copy-Item -Destination {Join-Path $uploadDirectory $_.FullName.Substring(($env:GITHUB_WORKSPACE).Length)} | |
| if ($env:ACTIONS_STEP_DEBUG -ieq "true") { | |
| Write-Host "Files to be uploaded:" | |
| Get-ChildItem $uploadDirectory -Recurse -Force | % { | |
| Write-Host $_.FullName | |
| } | |
| } | |
| Write-Output "FILES_SYNC_UPLOAD_DIRECTORY=${uploadDirectory}" >> $env:GITHUB_ENV | |
| shell: pwsh | |
| - name: Sync with AzCopy | |
| run: | | |
| if ($env:ACTIONS_STEP_DEBUG -ieq "true") { | |
| $InformationPreference = "Continue" | |
| $VerbosePreference = "Continue" | |
| $DebugPreference = "Continue" | |
| Set-PSDebug -Trace 1 | |
| } | |
| $ErrorActionPreference = "Stop" | |
| # Wait for resource graph to populate | |
| $waitSeconds = 45 | |
| Write-Host "Waiting $waitSeconds seconds for Azure resource graph to reflect..." | |
| Start-Sleep -Seconds $waitSeconds | |
| Get-Content $env:GITHUB_WORKSPACE/scripts/azcopy-settings.jsonc | ConvertFrom-Json | Set-Variable settings | |
| $settings.tenantId = $env:AZCOPY_TENANT_ID | |
| $settings.syncPairs[0].source = $env:FILES_SYNC_UPLOAD_DIRECTORY | |
| $settings.syncPairs[0].target = "https://${env:FILES_SYNC_STORAGE_ACCOUNT}.blob.core.windows.net/${env:FILES_SYNC_STORAGE_CONTAINER}" | |
| $jobSettingsFile = (New-TemporaryFile).FullName | |
| $settings | ConvertTo-Json | Out-File $jobSettingsFile | |
| Get-Content $jobSettingsFile | Write-Debug | |
| ./sync_with_azcopy.ps1 -SettingsFile $jobSettingsFile -UseSasToken:([bool]${env:FILES_SYNC_USE_SAS}) | |
| shell: pwsh | |
| - name: Clone Storage Account | |
| run: | | |
| if ($env:ACTIONS_STEP_DEBUG -ieq "true") { | |
| $InformationPreference = "Continue" | |
| $VerbosePreference = "Continue" | |
| $DebugPreference = "Continue" | |
| Set-PSDebug -Trace 1 | |
| } | |
| $ErrorActionPreference = "Stop" | |
| ./clone_storage_account.ps1 -SourceName $env:FILES_SYNC_STORAGE_ACCOUNT ` | |
| -TargetName $env:FILES_SYNC_STORAGE_ACCOUNT_CLONE ` | |
| -SkipResourceLock ` | |
| -UseSasToken:([bool]${env:FILES_SYNC_USE_SAS}) | |
| shell: pwsh | |
| - name: Download with AzCopy | |
| run: | | |
| if ($env:ACTIONS_STEP_DEBUG -ieq "true") { | |
| $InformationPreference = "Continue" | |
| $VerbosePreference = "Continue" | |
| $DebugPreference = "Continue" | |
| Set-PSDebug -Trace 1 | |
| } | |
| $ErrorActionPreference = "Stop" | |
| # Wait for resource graph to populate | |
| $waitSeconds = 45 | |
| Write-Host "Waiting $waitSeconds seconds for Azure resource graph to reflect..." | |
| Start-Sleep -Seconds $waitSeconds | |
| $downloadDirectory = (Join-Path $env:RUNNER_WORKSPACE "download") | |
| $downloadDirectoryStorageContainer = (Join-Path $downloadDirectory $env:FILES_SYNC_STORAGE_CONTAINER) | |
| New-Item $downloadDirectory -ItemType "directory" -Force | Write-Debug | |
| ./download_from_storage.ps1 -Source "https://${env:FILES_SYNC_STORAGE_ACCOUNT_CLONE}.blob.core.windows.net/${env:FILES_SYNC_STORAGE_CONTAINER}" ` | |
| -Destination $downloadDirectory ` | |
| -UseSasToken:([bool]${env:FILES_SYNC_USE_SAS}) | |
| Write-Output "FILES_SYNC_DOWNLOAD_DIRECTORY=${downloadDirectory}" >> $env:GITHUB_ENV | |
| Write-Output "FILES_SYNC_DOWNLOAD_DIRECTORY_STORAGE_CONTAINER=${downloadDirectoryStorageContainer}" >> $env:GITHUB_ENV | |
| shell: pwsh | |
| - name: Compare download with upload | |
| run: | | |
| if ($env:ACTIONS_STEP_DEBUG -ieq "true") { | |
| $InformationPreference = "Continue" | |
| $VerbosePreference = "Continue" | |
| $DebugPreference = "Continue" | |
| Set-PSDebug -Trace 1 | |
| } | |
| $uploadDirectory = $env:FILES_SYNC_UPLOAD_DIRECTORY | |
| $upload = (Get-ChildItem $uploadDirectory -Recurse -Force | Sort-Object -Property FullName) | |
| $downloadDirectory = $env:FILES_SYNC_DOWNLOAD_DIRECTORY_STORAGE_CONTAINER | |
| $download = (Get-ChildItem $downloadDirectory -Recurse -Force | Sort-Object -Property FullName) | |
| if (!$download -or ($download.Length -eq 0)) { | |
| throw "Download was empty" | |
| } | |
| if ($env:ACTIONS_STEP_DEBUG -ieq "true") { | |
| Write-Host "Files downloaded:" | |
| $download | % { | |
| Write-Host $_.FullName | |
| } | |
| Write-Host " " | |
| } | |
| # Compare file names | |
| $diff = (Compare-Object -ReferenceObject $upload.Name -DifferenceObject $download.Name -SyncWindow ($upload.length / 2)) | |
| if ($diff -and $diff.Count -gt 0) { | |
| Write-Warning "Differences found between upload & download:" | |
| $diff | Format-Table | |
| throw "Test failed, download is different from upload" | |
| } else { | |
| Write-Host "Names of files uploaded & downloaded match" | |
| } | |
| # Compare file sizes | |
| $diff = (Compare-Object -ReferenceObject $upload -DifferenceObject $download -Property Name, Length -SyncWindow ($upload.length / 2)) | |
| if ($diff -and $diff.Count -gt 0) { | |
| Write-Warning "Differences found between upload & download file sizes:" | |
| $diff | Format-Table | |
| throw "Test failed, download file size is different from upload" | |
| } else { | |
| Write-Host "Sizes of files uploaded & downloaded match" | |
| } | |
| # Calculate file hashes | |
| foreach ($file in ($upload + $download)) { | |
| $hash = $null | |
| if ($file.Attributes -inotcontains "Directory") { | |
| $hash = (Get-FileHash –Path $file.FullName -ErrorAction SilentlyContinue) | |
| } | |
| $file | Add-Member -NotePropertyName Hash -NotePropertyValue $hash | |
| } | |
| # Compare file hashes | |
| $diff = (Compare-Object -ReferenceObject $upload -DifferenceObject $download -Property Name, Hash -SyncWindow ($upload.length / 2)) | |
| if ($diff -and $diff.Count -gt 0) { | |
| Write-Warning "Differences found between upload & download file hashes:" | |
| $diff | Format-Table | |
| throw "Test failed, download file hash is different from upload" | |
| } else { | |
| Write-Host "Hashes of files uploaded & downloaded match" | |
| } | |
| shell: pwsh | |
| - name: Clean Up | |
| if: ${{ github.event_name != 'workflow_dispatch' || inputs.destroy }} | |
| run: | | |
| if ($env:ACTIONS_STEP_DEBUG -ieq "true") { | |
| $InformationPreference = "Continue" | |
| $VerbosePreference = "Continue" | |
| $DebugPreference = "Continue" | |
| Set-PSDebug -Trace 1 | |
| } | |
| $ErrorActionPreference = "Stop" | |
| if (!$env:GITHUB_RUN_ID) { | |
| throw "GITHUB_RUN_ID not set, we can't identify resources to be cleaned up" | |
| } | |
| if (!$env:FILES_SYNC_RESOURCE_GROUP) { | |
| throw "FILES_SYNC_RESOURCE_GROUP not set, we can't identify resources to be cleaned up" | |
| } | |
| $ErrorActionPreference = "Continue" | |
| # Remove resource locks first | |
| az lock list -g $env:FILES_SYNC_RESOURCE_GROUP --query [].id -o tsv | Set-Variable resourceLockIds | |
| if ($resourceLockIds) { | |
| Write-Host "Removing resource locks $resourceLockIds..." | |
| az lock delete --ids $resourceLockIds | |
| } | |
| # Build JMESPath expression | |
| $tagQuery = "[?tags.application == 'files-sync' && tags.runid == '${env:GITHUB_RUN_ID}' && properties.provisioningState != 'Deleting'].id" | |
| Write-Host "Removing resource group identified by `"$tagQuery`"..." | |
| $resourceGroupIDs = $(az group list --query "$tagQuery" -o tsv) | |
| if ($resourceGroupIDs) { | |
| Write-Host "az resource delete --ids ${resourceGroupIDs}..." | |
| az resource delete --ids $resourceGroupIDs --verbose | |
| } else { | |
| Write-Host "Nothing to remove" | |
| } | |
| shell: pwsh |