Create credential_phishing_nifty.com_domain_abuse.yml #15260
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rules PR CI | |
| on: | |
| push: | |
| branches: [ "main", "test-rules" ] | |
| pull_request_target: | |
| branches: [ "**" ] | |
| workflow_dispatch: {} | |
| issue_comment: | |
| types: [ created ] | |
| merge_group: {} | |
| concurrency: | |
| # For pull_request_target workflows we want to use head_ref -- the branch triggering the workflow. Otherwise, | |
| # use ref, which is the branch for a push event or workflow trigger. And for an issue comment just give up grouping. | |
| group: ${{ github.event_name == 'pull_request_target' && github.head_ref || (github.event_name == 'issue_comment' && github.run_id || github.ref) }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request_target' }} | |
| jobs: | |
| tests: | |
| name: Run Rule Validation | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| issues: read | |
| pull-requests: read | |
| checks: write | |
| if: github.event_name != 'issue_comment' || github.event.issue.pull_request && contains(github.event.comment.body, '/mql-mimic-exempt') | |
| steps: | |
| - name: Set up yq | |
| uses: mikefarah/yq@8bf425b4d1344db7cd469a8d10a390876e0c77fd # v4.45.1 | |
| - name: Get PR branch | |
| if: github.event_name == 'issue_comment' | |
| uses: alessbell/pull-request-comment-branch@ef3408c9757d05f89cb525036383033a313758a0 # v2.1.0 | |
| id: comment_branch | |
| - name: Get Refs | |
| id: get_head_ref | |
| run: | | |
| # Accurate for push events, merge queues, and workflow dispatch. | |
| head_ref="${{ github.ref }}" | |
| repo="${{ github.repository }}" | |
| if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then | |
| head_ref="${{ github.head_ref }}" | |
| repo="${{ github.event.pull_request.head.repo.full_name }}" | |
| elif [[ "${{ github.event_name }}" == 'issue_comment' ]]; then | |
| # Rely on comment_branch to figure out the head and base | |
| head_ref="${{ steps.comment_branch.outputs.head_ref }}" | |
| repo="${{ steps.comment_branch.outputs.head_owner }}/${{ steps.comment_branch.outputs.head_repo }}" | |
| fi | |
| echo "##[set-output name=head_ref;]$head_ref" | |
| echo "##[set-output name=repo;]$repo" | |
| - name: Checkout | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: ${{ steps.get_head_ref.outputs.repo }} | |
| ref: ${{ steps.get_head_ref.outputs.head_ref }} | |
| fetch-depth: 0 | |
| - name: Validate Branch vs. Trigerring SHA | |
| run: | | |
| # If this is from a pull request validate that what we checked out is the same as the PR head. | |
| # If not we'll just fail -- the workflow will be cancelled momentarily. | |
| if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then | |
| if [[ "${{ github.event.pull_request.head.sha }}" != "$(git rev-parse HEAD)" ]]; then | |
| echo "Workflow is out of date with branch, cancelling" | |
| exit 1 | |
| fi | |
| fi | |
| - name: Get Refs | |
| id: get_base_ref | |
| run: | | |
| run_all="" | |
| base_ref="" | |
| if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then | |
| # Ensure we have the latest base branch ref for accurate merge-base calculation | |
| git fetch origin ${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }} | |
| # Use the merge base to avoid including changes from target branch | |
| # that happened after this PR branch was created. | |
| base_ref="$(git merge-base HEAD origin/${{ github.base_ref }})" | |
| elif [[ "${{ github.event_name }}" == 'push' || "${{ github.event_name }}" == 'merge_group' ]]; then | |
| # Detect changes based on the previous commit | |
| base_ref="$(git rev-parse HEAD^)" | |
| elif [[ "${{ github.event_name }}" == 'workflow_dispatch' ]]; then | |
| # Run on a target, so run for all rules. | |
| run_all="true" | |
| elif [[ "${{ github.event_name }}" == 'issue_comment' ]]; then | |
| # Rely on comment_branch to figure out base | |
| base_ref="${{ steps.comment_branch.outputs.base_ref }}" | |
| fi | |
| echo "##[set-output name=run_all;]$run_all" | |
| echo "##[set-output name=base_ref;]$base_ref" | |
| - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 | |
| with: | |
| python-version: '3.10' | |
| - name: Add Rule IDs as Needed & Check for Duplicates | |
| if: github.event_name != 'issue_comment' | |
| # Run before testing, just in case this could invalidate the rule itself | |
| run: | | |
| pip install -r scripts/generate-rule-ids/requirements.txt | |
| python scripts/generate-rule-ids/main.py | |
| - name: Validate Rules | |
| if: github.event_name != 'issue_comment' | |
| run: | | |
| BATCH_SIZE=100 | |
| counter=0 | |
| batch_num=0 | |
| # Collect all files | |
| all_files=($(ls -1 *-rules/*.yml insights/**/*.yml)) | |
| total_files=${#all_files[@]} | |
| echo "Total files to validate: $total_files" | |
| # Start first batch | |
| echo '{"rules_or_queries": [' > bulk_validate_request.json | |
| for i in "${!all_files[@]}"; do | |
| f="${all_files[$i]}" | |
| # Determine if this is a rule or insight | |
| if [[ "$f" == insights/* ]]; then | |
| yq -o=json eval 'del(.type) | .source = "length([\n\n" + .source + "\n]) >= 0"' "$f" >> bulk_validate_request.json | |
| else | |
| yq -o=json eval 'del(.type)' "$f" >> bulk_validate_request.json | |
| fi | |
| counter=$((counter + 1)) | |
| # Check if we need to submit this batch | |
| should_submit=false | |
| if [[ $counter -eq $BATCH_SIZE ]]; then | |
| should_submit=true | |
| elif [[ $((i + 1)) -eq $total_files ]]; then | |
| # Last file | |
| should_submit=true | |
| else | |
| # Not submitting yet, add comma | |
| echo "," >> bulk_validate_request.json | |
| fi | |
| if [[ "$should_submit" == "true" ]]; then | |
| # Close JSON and submit | |
| echo "]}" >> bulk_validate_request.json | |
| batch_num=$((batch_num + 1)) | |
| echo "Submitting batch $batch_num with $counter files..." | |
| http_code=$(curl -H "Content-Type: application/json" -X POST -d @bulk_validate_request.json -o response.txt -w "%{http_code}" --silent https://play.sublime.security/v1/rules/bulk_validate) | |
| echo '' >> response.txt | |
| cat response.txt | |
| if [[ "$http_code" != "200" ]]; then | |
| echo "Unexpected response $http_code for batch $batch_num" | |
| exit 1 | |
| fi | |
| # Reset for next batch if there are more files | |
| if [[ $((i + 1)) -lt $total_files ]]; then | |
| counter=0 | |
| echo '{"rules_or_queries": [' > bulk_validate_request.json | |
| fi | |
| fi | |
| done | |
| echo "All batches submitted successfully!" | |
| - name: Verify no .yaml files exist | |
| if: github.event_name != 'issue_comment' | |
| run: | | |
| ! /bin/sh -c 'ls **/*.yaml' | |
| - name: Verify no .yml files exist in the top directory | |
| if: github.event_name != 'issue_comment' | |
| run: | | |
| ! /bin/sh -c 'ls *.yml' | |
| - name: Commit & Push Results, if needed | |
| if: github.event_name != 'issue_comment' | |
| id: final_basic_validation | |
| run: | | |
| rm response.txt | |
| rm bulk_validate_request.json | |
| if [ -z "$(git status --porcelain)" ]; then | |
| echo "No files changed, nothing to do" | |
| exit 0 | |
| fi | |
| git config user.name 'ID Generator' | |
| git config user.email 'hello@sublimesecurity.com' | |
| git add **/*.yml | |
| git commit -m "Auto add rule ID" | |
| # This will only work when running for a pull_request_target, but rather than filter we'll let this expose | |
| # any issues. | |
| git push origin ${{ steps.get_head_ref.outputs.head_ref }} | |
| - name: Get the head SHA | |
| id: get_head | |
| if: ${{ always() }} | |
| run: echo "##[set-output name=HEAD;]$(git rev-parse HEAD)" | |
| # When we add a commit, GitHub won't trigger actions on the auto commit, so we're missing a required check on the | |
| # HEAD commit. | |
| # Various alternatives were explored, but all run into issues when dealing with forks. This sets a "Check" for | |
| # the latest commit, and we can depend on that as a required check. | |
| - name: "Create a check run" | |
| # this needs to be upgraded to v7 but need to get this working now | |
| # actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| uses: actions/github-script@v6 | |
| if: (github.event_name == 'pull_request_target' || github.event_name == 'merge_group' ) && always() | |
| env: | |
| run_url: "${{ format('https://github.com/{0}/actions/runs/{1}', steps.get_head_ref.outputs.repo, github.run_id) }}" | |
| conclusion: "${{ steps.final_basic_validation.outcome == 'success' && 'success' || 'failure' }}" | |
| with: | |
| debug: ${{ secrets.ACTIONS_STEP_DEBUG || false }} | |
| retries: 3 | |
| # Default includes 422 which GitHub returns when it doesn't know about the head_sha we set the status for. | |
| # This occurs when the previous push succeeds, but the checks/pull request component of GitHub isn't yet aware | |
| # of the new commit. This isn't the common case, but it comes up enough to be annoying. | |
| retry-exempt-status-codes: 400, 401, 403, 404 | |
| script: | | |
| // any JavaScript code can go here, you can use Node JS APIs too. | |
| // Docs: https://docs.github.com/en/rest/checks/runs#create-a-check-run | |
| // Rest: https://octokit.github.io/rest.js/v18#checks-create | |
| await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| head_sha: "${{ steps.get_head.outputs.HEAD }}", | |
| name: "Rule Tests and ID Updated", | |
| status: "completed", | |
| conclusion: process.env.conclusion, | |
| details_url: process.env.run_url, | |
| output: { | |
| title: "Rule Tests and ID Updated", | |
| summary: "Rule Tests and ID Updated", | |
| text: "Rule Tests and ID Updated", | |
| }, | |
| }); | |
| - name: Get changed detection-rules | |
| id: changed-files | |
| uses: tj-actions/changed-files@2f7c5bfce28377bc069a65ba478de0a74aa0ca32 # v46.0.1 | |
| with: | |
| files: "detection-rules/**" | |
| recover_deleted_files: true | |
| - name: Checkout base | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| if: ${{ steps.get_base_ref.outputs.run_all != 'true' }} | |
| with: | |
| ref: ${{ steps.get_base_ref.outputs.base_ref }} | |
| repository: sublime-security/sublime-rules | |
| depth: 0 | |
| path: sr-main | |
| - name: Rename files in sr-main based on rule id | |
| if: ${{ steps.get_base_ref.outputs.run_all != 'true' }} | |
| run: | | |
| cd sr-main/detection-rules | |
| for file in *.yml | |
| do | |
| id=$(yq '.id' "$file") | |
| mv "$file" "${id}.yml" | |
| done | |
| - name: "Find updated rule IDs" | |
| id: find_ids | |
| run: | | |
| for file in detection-rules/*.yml; do | |
| rule_id=$(yq '.id' $file) | |
| if [[ "${{ steps.get_base_ref.outputs.run_all }}" == "true" ]]; then | |
| altered_rule_ids=$(echo "$rule_id"" ""$altered_rule_ids") | |
| continue | |
| fi | |
| new_source=$(yq '.source' "$file") | |
| old_source=$(yq '.source' "sr-main/detection-rules/$rule_id.yml" || echo '') | |
| # We only need to care when rule source is changed. This will handle renames, tag changes, etc. | |
| if [[ "$new_source" != "$old_source" ]]; then | |
| echo "$file ($rule_id) has altered source" | |
| altered_rule_ids=$(echo "$rule_id"" ""$altered_rule_ids") | |
| fi | |
| done | |
| for file in ${{ steps.changed-files.outputs.deleted_files }}; do | |
| rule_id=$(yq '.id' $file) | |
| echo "$file ($rule_id) was deleted" | |
| altered_rule_ids=$(echo "$rule_id"" ""$altered_rule_ids") | |
| done | |
| echo "Altered Ruled IDs: [$altered_rule_ids]" | |
| echo "##[set-output name=rule_ids;]$(echo $altered_rule_ids)" | |
| # TODO: This doesn't solve for a modified rule_id. We could merge with any files known on 'main', but changing | |
| # a rule ID is a separate problem. | |
| - name: Get PR Number | |
| if: github.event_name == 'pull_request_target' || github.event_name == 'issue_comment' | |
| id: find_pr_number | |
| run: | | |
| if [[ "${{ github.event_name }}" == 'pull_request_target' || "${{ github.event_name }}" == 'merge_group' ]]; then | |
| result="${{ github.event.number }}" | |
| elif [[ "${{ github.event_name }}" == 'issue_comment' ]]; then | |
| result="${{ github.event.issue.number }}" | |
| fi | |
| echo "PR $result" | |
| echo "##[set-output name=result;]$result" | |
| - name: "Find mql-mimic-exempt Comments" | |
| # this needs to be upgraded to v7 but need to get this working now | |
| # actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| uses: actions/github-script@v6 | |
| id: find_emls_to_skip | |
| if: steps.find_pr_number.outputs.result != '' | |
| with: | |
| debug: ${{ secrets.ACTIONS_STEP_DEBUG || false }} | |
| result-encoding: string | |
| script: | | |
| const opts = github.rest.issues.listComments.endpoint.merge({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: "${{ steps.find_pr_number.outputs.result }}", | |
| }) | |
| const comments = await github.paginate(opts) | |
| const seperatorRegex = /[\s:,;\/]+/ | |
| const exemptRegex = /\/mql-mimic-exempt((?:[\s:,;\/]+#*\d+)+)/gis | |
| let allEMLsToSkip = [] | |
| for (const comment of comments) { | |
| if (comment.author_association !== "MEMBER") { | |
| console.log("Ignoring comment from non-member" + comment.user.login) | |
| } | |
| while ((m = exemptRegex.exec(comment.body)) !== null) { | |
| if (m.index === exemptRegex.lastIndex) { | |
| break | |
| } | |
| // The result can be accessed through the `m`-variable. | |
| m.forEach((match, groupIndex) => { | |
| if (groupIndex != 1) { | |
| return | |
| } | |
| console.log("Found MQL Mimic Exemption EMLs: " + match) | |
| // First cut out all (optional) # | |
| match = match.replaceAll("#", "") | |
| let emls = match.split(seperatorRegex) | |
| console.log("Split EMLs: " + JSON.stringify(emls)) | |
| allEMLsToSkip = allEMLsToSkip.concat(emls.filter((s) => s !== "")) | |
| }); | |
| } | |
| } | |
| console.log("All EMLs: " + JSON.stringify(allEMLsToSkip)) | |
| // MQL Mimic will handle duplicates gracefully, no need to handle here. | |
| return allEMLsToSkip.join(" ") | |
| - name: "Find Existing MQL Mimic Test Results" | |
| # this needs to be upgraded to v7 but need to get this working now | |
| # actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| uses: actions/github-script@v6 | |
| id: find_mql_mimic_results | |
| if: ${{ github.event_name != 'merge_group' }} | |
| env: | |
| sha: '${{ steps.get_head.outputs.HEAD }}' | |
| with: | |
| debug: ${{ secrets.ACTIONS_STEP_DEBUG || false }} | |
| script: | | |
| const result = await github.rest.checks.listForRef({ | |
| check_name: "MQL Mimic Tests", | |
| owner: "sublime-security", | |
| repo: "sublime-rules", | |
| ref: process.env.sha | |
| }) | |
| let existingRuns = result.data.check_runs.map((r) => r.id) | |
| console.log(existingRuns) | |
| return existingRuns | |
| - name: "Trigger MQL Mimic Tests" | |
| if: ${{ github.event_name != 'merge_group' }} | |
| env: | |
| trigger_url: '${{ secrets.MQL_MOCK_TRIGGER }}' | |
| branch: '${{ steps.get_head_ref.outputs.head_ref }}' | |
| repo: '${{ steps.get_head_ref.outputs.repo }}' | |
| token: '${{ secrets.GITHUB_TOKEN }}' | |
| sha: '${{ steps.get_head.outputs.HEAD }}' | |
| only_rule_ids: '${{ steps.find_ids.outputs.rule_ids }}' | |
| skip_eml_ids: '${{ steps.find_emls_to_skip.outputs.result }}' | |
| run: | | |
| body='{"branch":"'$branch'","repo":"'$repo'","token":"'$token'","sha":"'$sha'","only_rule_ids":"'$only_rule_ids'","skip_eml_ids":"'$skip_eml_ids'"}' | |
| echo $body | |
| curl -X POST $trigger_url \ | |
| -H 'Content-Type: application/json' \ | |
| -d "$body" | |
| - name: Wait for MQL Mimic check to be completed | |
| if: ${{ github.event_name != 'merge_group' }} | |
| uses: sublime-security/action-wait-for-check@master | |
| # Wait for results so that the token remains valid while the test suite is executing and posting a check here. | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| checkName: "MQL Mimic Tests" | |
| ref: ${{ steps.get_head.outputs.HEAD }} | |
| timeoutSeconds: 3600 | |
| ignoreIDs: ${{ steps.find_mql_mimic_results.outputs.result }} | |
| - name: "Create MQL Mimic Check" | |
| # this needs to be upgraded to v7 but need to get this working now | |
| # actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 | |
| uses: actions/github-script@v6 | |
| if: ${{ github.event_name == 'merge_group' }} | |
| id: create_check | |
| env: | |
| run_url: "${{ format('https://github.com/{0}/actions/runs/{1}', steps.get_head_ref.outputs.repo, github.run_id) }}" | |
| with: | |
| debug: ${{ secrets.ACTIONS_STEP_DEBUG || false }} | |
| retries: 3 | |
| # Default includes 422 which GitHub returns when it doesn't know about the head_sha we set the status for. | |
| # This occurs when the previous push succeeds, but the checks/pull request component of GitHub isn't yet aware | |
| # of the new commit. This isn't the common case, but it comes up enough to be annoying. | |
| retry-exempt-status-codes: 400, 401, 403, 404 | |
| script: | | |
| // any JavaScript code can go here, you can use Node JS APIs too. | |
| // Docs: https://docs.github.com/en/rest/checks/runs#create-a-check-run | |
| // Rest: https://octokit.github.io/rest.js/v18#checks-create | |
| const response = await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| head_sha: "${{ steps.get_head.outputs.HEAD }}", | |
| name: "MQL Mimic Tests", | |
| status: "completed", | |
| conclusion: "success", | |
| details_url: process.env.run_url, | |
| output: { | |
| title: "MQL Mimic Tests", | |
| summary: "MQL Mimic tests are not run on merge queues", | |
| text: "MQL Mimic auto pass", | |
| }, | |
| }); | |
| return response["data"]["id"] |