Data Processing Pipeline #11
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Data Processing Pipeline | |
| on: | |
| schedule: | |
| # Run every 6 hours (at 00:00, 06:00, 12:00, 18:00 UTC) | |
| - cron: '0 */6 * * *' | |
| # Allow manual triggering | |
| workflow_dispatch: | |
| inputs: | |
| run_bridge_data: | |
| description: 'Run bridge data processing' | |
| required: false | |
| default: 'true' | |
| type: boolean | |
| run_musd_data: | |
| description: 'Run MUSD data processing' | |
| required: false | |
| default: 'true' | |
| type: boolean | |
| env: | |
| PYTHON_VERSION: '3.11' | |
| jobs: | |
| process-bridge-data: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_bridge_data == 'true') | |
| steps: | |
| - name: 🔄 Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: 🐍 Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: 📦 Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: 🌉 Process Bridge Data | |
| env: | |
| SUPABASE_URL_PROD: ${{ secrets.SUPABASE_URL_PROD }} | |
| SUPABASE_KEY_PROD: ${{ secrets.SUPABASE_KEY_PROD }} | |
| SUPABASE_DATA_URL: ${{ secrets.SUPABASE_DATA_URL }} | |
| SUPABASE_DATA_KEY: ${{ secrets.SUPABASE_DATA_KEY }} | |
| COINGECKO_KEY: ${{ secrets.COINGECKO_KEY }} | |
| run: | | |
| echo "🚀 Starting bridge data processing..." | |
| python scripts/process_bridge_data.py | |
| echo "✅ Bridge data processing completed" | |
| - name: 📊 Upload Bridge Processing Artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: bridge-processing-logs-${{ github.run_number }} | |
| path: | | |
| data/*.csv | |
| retention-days: 7 | |
| process-musd-data: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.run_musd_data == 'true') | |
| steps: | |
| - name: 🔄 Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: 🐍 Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: 'pip' | |
| - name: 📦 Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: 💰 Process MUSD Data | |
| env: | |
| SUPABASE_URL_PROD: ${{ secrets.SUPABASE_URL_PROD }} | |
| SUPABASE_KEY_PROD: ${{ secrets.SUPABASE_KEY_PROD }} | |
| SUPABASE_DATA_URL: ${{ secrets.SUPABASE_DATA_URL }} | |
| SUPABASE_DATA_KEY: ${{ secrets.SUPABASE_DATA_KEY }} | |
| COINGECKO_KEY: ${{ secrets.COINGECKO_KEY }} | |
| run: | | |
| echo "🚀 Starting MUSD data processing..." | |
| python scripts/process_musd_data.py | |
| echo "✅ MUSD data processing completed" | |
| - name: 📊 Upload MUSD Processing Artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: musd-processing-logs-${{ github.run_number }} | |
| path: | | |
| data/*.csv | |
| retention-days: 7 | |
| notify-completion: | |
| runs-on: ubuntu-latest | |
| needs: [process-bridge-data, process-musd-data] | |
| if: always() | |
| steps: | |
| - name: 📝 Create processing summary | |
| run: | | |
| echo "## 📊 Data Processing Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Job | Status | Duration |" >> $GITHUB_STEP_SUMMARY | |
| echo "|-----|--------|----------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| 🌉 Bridge Data | ${{ needs.process-bridge-data.result }} | - |" >> $GITHUB_STEP_SUMMARY | |
| echo "| 💰 MUSD Data | ${{ needs.process-musd-data.result }} | - |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**⏰ Timestamp:** $(date -u)" >> $GITHUB_STEP_SUMMARY | |
| echo "**🔢 Run Number:** ${{ github.run_number }}" >> $GITHUB_STEP_SUMMARY | |
| echo "**🔗 Workflow:** [View Details](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> $GITHUB_STEP_SUMMARY | |
| - name: 🔔 Check for failures | |
| if: needs.process-bridge-data.result == 'failure' || needs.process-musd-data.result == 'failure' | |
| run: | | |
| echo "❌ One or more data processing jobs failed!" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Failed Jobs:**" >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ needs.process-bridge-data.result }}" == "failure" ]; then | |
| echo "- 🌉 Bridge Data Processing" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| if [ "${{ needs.process-musd-data.result }}" == "failure" ]; then | |
| echo "- 💰 MUSD Data Processing" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Check the logs for details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_STEP_SUMMARY | |
| exit 1 | |
| - name: ✅ Success notification | |
| if: needs.process-bridge-data.result == 'success' && needs.process-musd-data.result == 'success' | |
| run: | | |
| echo "🎉 All data processing jobs completed successfully!" >> $GITHUB_STEP_SUMMARY |