From 6316de484b1156f9a2a11077660f46c288e01805 Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Sun, 16 Oct 2022 14:08:15 -0400 Subject: [PATCH 01/20] Initialize the SAM app --- sam-notifier/.gitignore | 249 ++++++++++++++++++++ sam-notifier/README.md | 55 +++++ sam-notifier/__init__.py | 0 sam-notifier/functions/__init__.py | 0 sam-notifier/statemachine/notifier.asl.json | 44 ++++ sam-notifier/template.yaml | 85 +++++++ 6 files changed, 433 insertions(+) create mode 100644 sam-notifier/.gitignore create mode 100644 sam-notifier/README.md create mode 100644 sam-notifier/__init__.py create mode 100644 sam-notifier/functions/__init__.py create mode 100644 sam-notifier/statemachine/notifier.asl.json create mode 100644 sam-notifier/template.yaml diff --git a/sam-notifier/.gitignore b/sam-notifier/.gitignore new file mode 100644 index 0000000..30ee366 --- /dev/null +++ b/sam-notifier/.gitignore @@ -0,0 +1,249 @@ + +# Created by https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### OSX ### +*.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### PyCharm ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Ruby plugin and RubyMine +/.rakeTasks + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +### PyCharm Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +.idea/sonarlint + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +.pytest_cache/ +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule.* + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +.history + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +ehthumbs.db +ehthumbs_vista.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# Build folder + +*/build/* + +# End of https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode + +# AWS Serverless Application Model +.aws-sam +.aws-sam/* +samconfig.toml diff --git a/sam-notifier/README.md b/sam-notifier/README.md new file mode 100644 index 0000000..712d018 --- /dev/null +++ b/sam-notifier/README.md @@ -0,0 +1,55 @@ +# SAM Notifier (sam-notifier) + +This project contains source code and supporting files for a serverless application that you can deploy with the SAM CLI. It includes the following files and folders: + +- functions - Code for the application's Lambda functions to respond to CIS AWS Foundations Benchmark CloudWatch Alarms. +- statemachines - Definition for the state machine that orchestrates the notification workflow. +- template.yaml - A template that defines the application's AWS resources. + +This application creates a AWS Step Functions workflow coupled with event-driven approach using Amazon EventBridge to respond to CIS AWS Foundations Benchmark CloudWatch Alarms provisioned via the [CIS-alarms-cfn.yml](https://github.com/rewindio/aws-security-hub-CIS-metrics/blob/main/CIS-alarms-cfn.yml) CloudFormation template. + +The application uses several AWS resources, including Step Functions state machines, Lambda functions and an EventBridge rule. These resources are defined in the `template.yaml` file in this project. You can update the template to add AWS resources through the same deployment process that updates your application code. + +## Deploy the sample application + +The Serverless Application Model Command Line Interface (SAM CLI) is an extension of the AWS CLI that adds functionality for building and testing Lambda applications. It uses Docker to run your functions in an Amazon Linux environment that matches Lambda. + +To use the SAM CLI, you need the following tools: + +* SAM CLI - [Install the SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html) +* [Python 3 installed](https://www.python.org/downloads/) +* Docker - [Install Docker community edition](https://hub.docker.com/search/?type=edition&offering=community) + +To build and deploy your application for the first time, run the following in your shell: + +```bash +sam build --use-container +sam deploy --guided +``` + +The first command will build the source of your application. The second command will package and deploy your application to AWS, with a series of prompts: + +* **Stack Name**: The name of the stack to deploy to CloudFormation. This should be unique to your account and region, and a good starting point would be something matching your project name. +* **AWS Region**: The AWS region you want to deploy your app to. +* **Parameter LogGroupName**: Name of the CloudWatch Logs log group used CloudTrail +* **Confirm changes before deploy**: If set to yes, any change sets will be shown to you before execution for manual review. If set to no, the AWS SAM CLI will automatically deploy application changes. +* **Allow SAM CLI IAM role creation**: Many AWS SAM templates, including this example, create AWS IAM roles required for the AWS Lambda function(s) included to access AWS services. By default, these are scoped down to minimum required permissions. To deploy an AWS CloudFormation stack which creates or modifies IAM roles, the `CAPABILITY_IAM` value for `capabilities` must be provided. If permission isn't provided through this prompt, to deploy this example you must explicitly pass `--capabilities CAPABILITY_IAM` to the `sam deploy` command. +* **Save arguments to samconfig.toml**: If set to yes, your choices will be saved to a configuration file inside the project, so that in the future you can just re-run `sam deploy` without parameters to deploy changes to your application. + +## Use the SAM CLI to build locally + +Build the Lambda functions in your application with the `sam build --use-container` command. + +```bash +sam-notifier$ sam build --use-container +``` + +The SAM CLI installs dependencies defined in `functions/*/requirements.txt`, creates a deployment package, and saves it in the `.aws-sam/build` folder. + +## Cleanup + +To delete the sample application that you created, use the AWS CLI. Assuming you used your project name for the stack name, you can run the following: + +```bash +aws cloudformation delete-stack --stack-name sam-notifier +``` diff --git a/sam-notifier/__init__.py b/sam-notifier/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sam-notifier/functions/__init__.py b/sam-notifier/functions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sam-notifier/statemachine/notifier.asl.json b/sam-notifier/statemachine/notifier.asl.json new file mode 100644 index 0000000..6b715f5 --- /dev/null +++ b/sam-notifier/statemachine/notifier.asl.json @@ -0,0 +1,44 @@ +{ + "Comment": "A state machine for CIS Alarm notifer.", + "StartAt": "Validate Choice", + "States": { + "Validate Choice": { + "Type": "Choice", + "Choices": [ + { + "And": [ + { + "Variable": "$.detail.alarmName", + "IsPresent": true + } + ], + "Next": "Epoch Converter Function", + "Comment": "Alarm Name Found" + } + ], + "Default": "Improper Input Fail" + }, + "Epoch Converter Function": { + "Type": "Task", + "Resource": "${EpochFunctionArn}", + "InputPath": "$", + "ResultPath": "$.epoch", + "Retry": [ + { + "ErrorEquals": [ + "States.TaskFailed" + ], + "IntervalSeconds": 10, + "MaxAttempts": 1, + "BackoffRate": 1 + } + ], + "End": true + }, + "Improper Input Fail": { + "Type": "Fail", + "Error": "Improper Input Fail", + "Cause": "Missing Alert Name" + } + } +} diff --git a/sam-notifier/template.yaml b/sam-notifier/template.yaml new file mode 100644 index 0000000..116dfa5 --- /dev/null +++ b/sam-notifier/template.yaml @@ -0,0 +1,85 @@ +AWSTemplateFormatVersion: "2010-09-09" +Transform: AWS::Serverless-2016-10-31 +Description: > + SAM Template to create state machine for CIS Alarm notifer + +Parameters: + LogGroupName: + Type: String + Description: Name of the CloudWatch Logs log group used CloudTrail + +Resources: + StateMachine: + Type: AWS::Serverless::StateMachine + Properties: + DefinitionUri: statemachine/notifier.asl.json + DefinitionSubstitutions: + EpochFunctionArn: !GetAtt EpochFunction.Arn + LogGroupName: !Ref LogGroupName + Events: + EBRule: + Type: EventBridgeRule + Properties: + Pattern: + source: + - aws.cloudwatch + detail: + state: + value: + - ALARM + alarmName: + - CIS-Unauthorized Activity Attempt + - CIS-Console Signin Without MFA + - CIS-Root Activity + - CIS-IAM Policy Changes + - CIS-Cloudtrail Config Changes + - CIS-Console Login Failures + - CIS-KMS Key Disabled or Scheduled for Deletion + - CIS-S3 Bucket Policy Changed + - CIS-AWS Config Configuration has changed + - CIS-Security Groups Have Changed + - CIS-NACLs Have Changed + - CIS-Network Gateways Have Changed + - CIS-Route Tables Have Changed + - CIS-VPC Has Changed + Policies: + - LambdaInvokePolicy: + FunctionName: !Ref EpochFunction + + EpochFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: functions/epoch/ + Handler: app.lambda_handler + Runtime: python3.9 + Timeout: 5 + Architectures: + - x86_64 + + CloudWatchLogsPolicy: + Type: 'AWS::IAM::Policy' + Properties: + PolicyName: CloudWatchLogsPolicyForSamNotifier + PolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: CloudWatchLogsPolicy + Effect: Allow + Action: + - 'logs:Describe*' + - 'logs:Get*' + - 'logs:List*' + - 'logs:StartQuery' + - 'logs:StopQuery' + - 'logs:TestMetricFilter' + - 'logs:FilterLogEvents' + Resource: + - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:${LogGroupName}:*' + - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group::log-stream:*' + Roles: + - !Ref StateMachineRole + +Outputs: + StateMachineArn: + Description: "State machine ARN" + Value: !Ref StateMachine From 3b7f9e6483152b45d9c909f20584ec0fa51be016 Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Sun, 16 Oct 2022 14:09:19 -0400 Subject: [PATCH 02/20] Add EpochFunction Lambda --- sam-notifier/functions/epoch/__init__.py | 0 sam-notifier/functions/epoch/app.py | 26 +++++++++++++++++++ sam-notifier/functions/epoch/requirements.txt | 0 3 files changed, 26 insertions(+) create mode 100644 sam-notifier/functions/epoch/__init__.py create mode 100644 sam-notifier/functions/epoch/app.py create mode 100644 sam-notifier/functions/epoch/requirements.txt diff --git a/sam-notifier/functions/epoch/__init__.py b/sam-notifier/functions/epoch/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sam-notifier/functions/epoch/app.py b/sam-notifier/functions/epoch/app.py new file mode 100644 index 0000000..f9eeebe --- /dev/null +++ b/sam-notifier/functions/epoch/app.py @@ -0,0 +1,26 @@ +import logging + +import datetime +from datetime import timedelta + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +def check_time(event): + if event.get("time"): + return datetime.datetime.strptime(event.get("time"), "%Y-%m-%dT%H:%M:%SZ") + return datetime.datetime.now() + + +def lambda_handler(event, context): + alarm_time = check_time(event) + + logger.info(f"Event time: {alarm_time}") + + start = round((alarm_time - timedelta(minutes=30)).timestamp()) + end = round(alarm_time.timestamp()) + + logger.info(f"Start timestamp ({start}), end timetamps ({end}).") + + return {"start": start, "end": end} diff --git a/sam-notifier/functions/epoch/requirements.txt b/sam-notifier/functions/epoch/requirements.txt new file mode 100644 index 0000000..e69de29 From e06cc9a4dd5c67208ba535403902c86f79261f83 Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Sun, 16 Oct 2022 14:56:40 -0400 Subject: [PATCH 03/20] Add Choice logic per each AlarmName type --- sam-notifier/statemachine/notifier.asl.json | 219 +++++++++++++++++++- 1 file changed, 217 insertions(+), 2 deletions(-) diff --git a/sam-notifier/statemachine/notifier.asl.json b/sam-notifier/statemachine/notifier.asl.json index 6b715f5..0db2b21 100644 --- a/sam-notifier/statemachine/notifier.asl.json +++ b/sam-notifier/statemachine/notifier.asl.json @@ -33,7 +33,222 @@ "BackoffRate": 1 } ], - "End": true + "Next": "Event Handler Choice" + }, + "Event Handler Choice": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.detail.alarmName", + "StringMatches": "CIS-Unauthorized Activity Attempt", + "Next": "CIS-Unauthorized Activity Attempt", + "Comment": "CIS-Unauthorized Activity Attempt Alarm" + }, + { + "Variable": "$.detail.alarmName", + "StringMatches": "CIS-Console Signin Without MFA", + "Next": "CIS-Console Signin Without MFA", + "Comment": "CIS-Console Signin Without MFA Alarm" + }, + { + "Variable": "$.detail.alarmName", + "StringMatches": "CIS-Root Activity", + "Next": "CIS-Root Activity", + "Comment": "CIS-Root Activity Alarm" + }, + { + "Variable": "$.detail.alarmName", + "StringMatches": "CIS-IAM Policy Changes", + "Next": "CIS-IAM Policy Changes", + "Comment": "CIS-IAM Policy Changes Alarm" + }, + { + "Variable": "$.detail.alarmName", + "StringMatches": "CIS-Cloudtrail Config Changes", + "Next": "CIS-Cloudtrail Config Changes", + "Comment": "CIS-Cloudtrail Config Changes Alarm" + }, + { + "Variable": "$.detail.alarmName", + "StringMatches": "CIS-Console Login Failures", + "Next": "CIS-Console Login Failures", + "Comment": "CIS-Console Login Failures Alarm" + }, + { + "Variable": "$.detail.alarmName", + "StringMatches": "CIS-KMS Key Disabled or Scheduled for Deletion", + "Next": "CIS-KMS Key Disabled or Scheduled for Deletion", + "Comment": "CIS-KMS Key Disabled or Scheduled for Deletion Alarm" + }, + { + "Variable": "$.detail.alarmName", + "StringMatches": "CIS-S3 Bucket Policy Changed", + "Next": "CIS-S3 Bucket Policy Changed", + "Comment": "CIS-S3 Bucket Policy Changed Alarm" + }, + { + "Variable": "$.detail.alarmName", + "StringMatches": "CIS-AWS Config Configuration has changed", + "Next": "CIS-AWS Config Configuration has changed", + "Comment": "CIS-AWS Config Configuration has changed Alarm" + }, + { + "Variable": "$.detail.alarmName", + "StringMatches": "CIS-Security Groups Have Changed", + "Next": "CIS-Security Groups Have Changed", + "Comment": "CIS-Security Groups Have Changed Alarm" + }, + { + "Variable": "$.detail.alarmName", + "StringMatches": "CIS-NACLs Have Changed", + "Next": "CIS-NACLs Have Changed", + "Comment": "CIS-NACLs Have Changed Alarm" + }, + { + "Variable": "$.detail.alarmName", + "StringMatches": "CIS-Network Gateways Have Changed", + "Next": "CIS-Network Gateways Have Changed", + "Comment": "CIS-Network Gateways Have Changed Alarm" + }, + { + "Variable": "$.detail.alarmName", + "StringMatches": "CIS-Route Tables Have Changed", + "Next": "CIS-Route Tables Have Changed", + "Comment": "CIS-Route Tables Have Changed Alarm" + }, + { + "Variable": "$.detail.alarmName", + "StringMatches": "CIS-VPC Has Changed", + "Next": "CIS-VPC Has Changed", + "Comment": "CIS-VPC Has Changed Alarm" + } + ] + }, + "CIS-Unauthorized Activity Attempt": { + "Type": "Pass", + "End": true, + "ResultPath": "$.query", + "Result": { + "string": "fields @timestamp, @message | sort @timestamp desc | filter errorCode == '*UnauthorizedOperation' or errorCode == 'AccessDenied*'", + "severity": "HIGH" + } + }, + "CIS-Console Signin Without MFA": { + "Type": "Pass", + "End": true, + "ResultPath": "$.query", + "Result": { + "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName == 'ConsoleLogin' and responseElements.ConsoleLogin == 'Success' and additionalEventData.MFAUsed != 'Yes'", + "severity": "MEDIUM" + } + }, + "CIS-Root Activity": { + "Type": "Pass", + "End": true, + "ResultPath": "$.query", + "Result": { + "string": "fields @timestamp, @message | sort @timestamp desc | filter userIdentity.type == 'Root' and eventType != 'AwsServiceEvent'", + "severity": "HIGH" + } + }, + "CIS-IAM Policy Changes": { + "Type": "Pass", + "End": true, + "ResultPath": "$.query", + "Result": { + "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['AttachGroupPolicy', 'AttachRolePolicy', 'AttachUserPolicy', 'CreatePolicy', 'CreatePolicyVersion', 'DeleteGroupPolicy', 'DeletePolicy', 'DeletePolicyVersion', 'DeleteRolePolicy', 'DeleteUserPolicy', 'DetachGroupPolicy', 'DetachRolePolicy', 'DetachUserPolicy', 'PutGroupPolicy', 'PutRolePolicy', 'PutUserPolicy']", + "severity": "LOW" + } + }, + "CIS-Cloudtrail Config Changes": { + "Type": "Pass", + "End": true, + "ResultPath": "$.query", + "Result": { + "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['CreateTrail', 'DeleteTrail', 'StartLogging', 'StopLogging', 'UpdateTrail']", + "severity": "HIGH" + } + }, + "CIS-Console Login Failures": { + "Type": "Pass", + "End": true, + "ResultPath": "$.query", + "Result": { + "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName == 'ConsoleLogin' and errorMessage == 'Failed authentication'", + "severity": "HIGH" + } + }, + "CIS-KMS Key Disabled or Scheduled for Deletion": { + "Type": "Pass", + "End": true, + "ResultPath": "$.query", + "Result": { + "string": "fields @timestamp, @message | sort @timestamp desc | filter eventSource == 'kms.amazonaws.com' and eventName in ['DisableKey', 'ScheduleKeyDeletion']", + "severity": "MEDIUM" + } + }, + "CIS-S3 Bucket Policy Changed": { + "Type": "Pass", + "End": true, + "ResultPath": "$.query", + "Result": { + "string": "fields @timestamp, @message | sort @timestamp desc | filter eventSource = 's3.amazonaws.com' and eventName in ['DeleteBucketCors', 'DeleteBucketLifecycle', 'DeleteBucketPolicy', 'DeleteBucketReplication', 'PutBucketAcl', 'PutBucketCors', 'PutBucketLifecycle', 'PutBucketPolicy', 'PutBucketReplication']", + "severity": "MEDIUM" + } + }, + "CIS-AWS Config Configuration has changed": { + "Type": "Pass", + "End": true, + "ResultPath": "$.query", + "Result": { + "string": "fields @timestamp, @message | sort @timestamp desc | filter eventSource = 'config.amazonaws.com' and eventName in ['DeleteDeliveryChannel', 'StopConfigurationRecorder', 'PutConfigurationRecorder', 'PutDeliveryChannel']", + "severity": "HIGH" + } + }, + "CIS-Security Groups Have Changed": { + "Type": "Pass", + "End": true, + "ResultPath": "$.query", + "Result": { + "string": "fields @timestamp, @message | sort @timestamp desc | filter eventSource = 'config.amazonaws.com' and eventName in ['DeleteDeliveryChannel', 'StopConfigurationRecorder', 'PutConfigurationRecorder', 'PutDeliveryChannel']", + "severity": "LOW" + } + }, + "CIS-NACLs Have Changed": { + "Type": "Pass", + "End": true, + "ResultPath": "$.query", + "Result": { + "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['CreateNetworkAcl', 'CreateNetworkAclEntry', 'DeleteNetworkAcl', 'DeleteNetworkAclEntry', 'ReplaceNetworkAclEntry', 'ReplaceNetworkAclAssociation']", + "severity": "LOW" + } + }, + "CIS-Network Gateways Have Changed": { + "Type": "Pass", + "End": true, + "ResultPath": "$.query", + "Result": { + "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['AttachInternetGateway', 'CreateCustomerGateway', 'CreateInternetGateway', 'DeleteCustomerGateway', 'DeleteInternetGateway', 'DetachInternetGateway']", + "severity": "LOW" + } + }, + "CIS-Route Tables Have Changed": { + "Type": "Pass", + "End": true, + "ResultPath": "$.query", + "Result": { + "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['CreateRoute', 'CreateRouteTable', 'DeleteRoute', 'DeleteRouteTable', 'DisassociateRouteTable', 'ReplaceRoute', 'ReplaceRouteTableAssociation']", + "severity": "LOW" + } + }, + "CIS-VPC Has Changed": { + "Type": "Pass", + "End": true, + "ResultPath": "$.query", + "Result": { + "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['AcceptVpcPeeringConnection', 'AttachClassicLinkVpc', 'CreateVpc', 'CreateVpcPeeringConnection', 'DeleteVpc', 'DeleteVpcPeeringConnection', 'DetachClassicLinkVpc', 'DisableVpcClassicLink', 'EnableVpcClassicLink', 'ModifyVpcAttribute', 'RejectVpcPeeringConnection']", + "severity": "LOW" + } }, "Improper Input Fail": { "Type": "Fail", @@ -41,4 +256,4 @@ "Cause": "Missing Alert Name" } } -} +} \ No newline at end of file From 21497c6af40346d5aee38480db211fa1a73fae88 Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Sun, 16 Oct 2022 15:44:46 -0400 Subject: [PATCH 04/20] Add LogQueryFunction Lambda --- sam-notifier/functions/log-query/__init__.py | 0 sam-notifier/functions/log-query/app.py | 67 +++++++++++++++++++ .../functions/log-query/requirements.txt | 0 sam-notifier/statemachine/notifier.asl.json | 51 ++++++++++---- sam-notifier/template.yaml | 14 ++++ 5 files changed, 118 insertions(+), 14 deletions(-) create mode 100644 sam-notifier/functions/log-query/__init__.py create mode 100644 sam-notifier/functions/log-query/app.py create mode 100644 sam-notifier/functions/log-query/requirements.txt diff --git a/sam-notifier/functions/log-query/__init__.py b/sam-notifier/functions/log-query/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sam-notifier/functions/log-query/app.py b/sam-notifier/functions/log-query/app.py new file mode 100644 index 0000000..e34060b --- /dev/null +++ b/sam-notifier/functions/log-query/app.py @@ -0,0 +1,67 @@ +import time +import logging +import boto3 + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +client = boto3.client('logs') + + +def get_query_results(query_id): + response = client.get_query_results( + queryId=query_id + ) + + logger.info(f"Get Query Results: {response}") + + # Wait for the Query to complete + while response.get("status") in ("Running", "Scheduled"): + time.sleep(5) + response = client.get_query_results( + queryId=query_id + ) + if response.get("status") == "Complete": + logger.info(f"Query results: {response}") + logger.info(f"Query statistics: {response.get('statistics')}") + else: + logger.error(f"CloudWatch Query failed. {response.get('status')}") + return response + + +def process_query_results(results): + log_entries = list() + logger.info(f"Process Query Results: {results}") + for result in results.get("results"): + for entry in result: + if entry.get("field") == "@message": + logger.info(entry.get("value")) + log_entries.append(entry.get("value")) + return log_entries + + +def lambda_handler(event, context): + log_group_name = event.get("CloudWatchLogsLogGroupName") + + logger.info(f"StartTime: {event.get('epoch').get('start')}") + logger.info(f"EndTime: {event.get('epoch').get('end')}") + logger.info(f"QueryString: {event.get('query').get('string')}") + logger.info(f"LogGroupName: {log_group_name}") + + response = client.start_query( + logGroupName=log_group_name, + startTime=event.get("epoch").get("start"), + endTime=event.get("epoch").get("end"), + queryString=event.get("query").get("string"), + limit=10 + ) + + logger.info(f"Response: {response}") + logger.info(f"CloudWatch Logs QueryId: {response.get('queryId')}") + + results = get_query_results(response.get("queryId")) + logs = process_query_results(results) + + return { + "logs": logs + } diff --git a/sam-notifier/functions/log-query/requirements.txt b/sam-notifier/functions/log-query/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/sam-notifier/statemachine/notifier.asl.json b/sam-notifier/statemachine/notifier.asl.json index 0db2b21..e598f80 100644 --- a/sam-notifier/statemachine/notifier.asl.json +++ b/sam-notifier/statemachine/notifier.asl.json @@ -126,7 +126,7 @@ }, "CIS-Unauthorized Activity Attempt": { "Type": "Pass", - "End": true, + "Next": "GetLogQuery Function", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter errorCode == '*UnauthorizedOperation' or errorCode == 'AccessDenied*'", @@ -135,7 +135,7 @@ }, "CIS-Console Signin Without MFA": { "Type": "Pass", - "End": true, + "Next": "GetLogQuery Function", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName == 'ConsoleLogin' and responseElements.ConsoleLogin == 'Success' and additionalEventData.MFAUsed != 'Yes'", @@ -144,7 +144,7 @@ }, "CIS-Root Activity": { "Type": "Pass", - "End": true, + "Next": "GetLogQuery Function", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter userIdentity.type == 'Root' and eventType != 'AwsServiceEvent'", @@ -153,7 +153,7 @@ }, "CIS-IAM Policy Changes": { "Type": "Pass", - "End": true, + "Next": "GetLogQuery Function", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['AttachGroupPolicy', 'AttachRolePolicy', 'AttachUserPolicy', 'CreatePolicy', 'CreatePolicyVersion', 'DeleteGroupPolicy', 'DeletePolicy', 'DeletePolicyVersion', 'DeleteRolePolicy', 'DeleteUserPolicy', 'DetachGroupPolicy', 'DetachRolePolicy', 'DetachUserPolicy', 'PutGroupPolicy', 'PutRolePolicy', 'PutUserPolicy']", @@ -162,7 +162,7 @@ }, "CIS-Cloudtrail Config Changes": { "Type": "Pass", - "End": true, + "Next": "GetLogQuery Function", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['CreateTrail', 'DeleteTrail', 'StartLogging', 'StopLogging', 'UpdateTrail']", @@ -171,7 +171,7 @@ }, "CIS-Console Login Failures": { "Type": "Pass", - "End": true, + "Next": "GetLogQuery Function", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName == 'ConsoleLogin' and errorMessage == 'Failed authentication'", @@ -180,7 +180,7 @@ }, "CIS-KMS Key Disabled or Scheduled for Deletion": { "Type": "Pass", - "End": true, + "Next": "GetLogQuery Function", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventSource == 'kms.amazonaws.com' and eventName in ['DisableKey', 'ScheduleKeyDeletion']", @@ -189,7 +189,7 @@ }, "CIS-S3 Bucket Policy Changed": { "Type": "Pass", - "End": true, + "Next": "GetLogQuery Function", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventSource = 's3.amazonaws.com' and eventName in ['DeleteBucketCors', 'DeleteBucketLifecycle', 'DeleteBucketPolicy', 'DeleteBucketReplication', 'PutBucketAcl', 'PutBucketCors', 'PutBucketLifecycle', 'PutBucketPolicy', 'PutBucketReplication']", @@ -198,7 +198,7 @@ }, "CIS-AWS Config Configuration has changed": { "Type": "Pass", - "End": true, + "Next": "GetLogQuery Function", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventSource = 'config.amazonaws.com' and eventName in ['DeleteDeliveryChannel', 'StopConfigurationRecorder', 'PutConfigurationRecorder', 'PutDeliveryChannel']", @@ -207,7 +207,7 @@ }, "CIS-Security Groups Have Changed": { "Type": "Pass", - "End": true, + "Next": "GetLogQuery Function", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventSource = 'config.amazonaws.com' and eventName in ['DeleteDeliveryChannel', 'StopConfigurationRecorder', 'PutConfigurationRecorder', 'PutDeliveryChannel']", @@ -216,7 +216,7 @@ }, "CIS-NACLs Have Changed": { "Type": "Pass", - "End": true, + "Next": "GetLogQuery Function", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['CreateNetworkAcl', 'CreateNetworkAclEntry', 'DeleteNetworkAcl', 'DeleteNetworkAclEntry', 'ReplaceNetworkAclEntry', 'ReplaceNetworkAclAssociation']", @@ -225,7 +225,7 @@ }, "CIS-Network Gateways Have Changed": { "Type": "Pass", - "End": true, + "Next": "GetLogQuery Function", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['AttachInternetGateway', 'CreateCustomerGateway', 'CreateInternetGateway', 'DeleteCustomerGateway', 'DeleteInternetGateway', 'DetachInternetGateway']", @@ -234,7 +234,7 @@ }, "CIS-Route Tables Have Changed": { "Type": "Pass", - "End": true, + "Next": "GetLogQuery Function", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['CreateRoute', 'CreateRouteTable', 'DeleteRoute', 'DeleteRouteTable', 'DisassociateRouteTable', 'ReplaceRoute', 'ReplaceRouteTableAssociation']", @@ -243,13 +243,36 @@ }, "CIS-VPC Has Changed": { "Type": "Pass", - "End": true, + "Next": "GetLogQuery Function", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['AcceptVpcPeeringConnection', 'AttachClassicLinkVpc', 'CreateVpc', 'CreateVpcPeeringConnection', 'DeleteVpc', 'DeleteVpcPeeringConnection', 'DetachClassicLinkVpc', 'DisableVpcClassicLink', 'EnableVpcClassicLink', 'ModifyVpcAttribute', 'RejectVpcPeeringConnection']", "severity": "LOW" } }, + "GetLogQuery Function": { + "Type": "Task", + "Resource": "${LogQueryFunctionArn}", + "InputPath": "$", + "ResultPath": "$.logs", + "Parameters": { + "CloudWatchLogsLogGroupName": "${LogGroupName}", + "alarmName.$": "$.detail.alarmName", + "epoch.$": "$.epoch", + "query.$": "$.query" + }, + "Retry": [ + { + "ErrorEquals": [ + "States.TaskFailed" + ], + "IntervalSeconds": 300, + "MaxAttempts": 1, + "BackoffRate": 1 + } + ], + "End": true + }, "Improper Input Fail": { "Type": "Fail", "Error": "Improper Input Fail", diff --git a/sam-notifier/template.yaml b/sam-notifier/template.yaml index 116dfa5..a8bea42 100644 --- a/sam-notifier/template.yaml +++ b/sam-notifier/template.yaml @@ -15,6 +15,7 @@ Resources: DefinitionUri: statemachine/notifier.asl.json DefinitionSubstitutions: EpochFunctionArn: !GetAtt EpochFunction.Arn + LogQueryFunctionArn: !GetAtt LogQueryFunction.Arn LogGroupName: !Ref LogGroupName Events: EBRule: @@ -45,6 +46,8 @@ Resources: Policies: - LambdaInvokePolicy: FunctionName: !Ref EpochFunction + - LambdaInvokePolicy: + FunctionName: !Ref LogQueryFunction EpochFunction: Type: AWS::Serverless::Function @@ -56,6 +59,16 @@ Resources: Architectures: - x86_64 + LogQueryFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: functions/log-query/ + Handler: app.lambda_handler + Runtime: python3.9 + Timeout: 300 + Architectures: + - x86_64 + CloudWatchLogsPolicy: Type: 'AWS::IAM::Policy' Properties: @@ -78,6 +91,7 @@ Resources: - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group::log-stream:*' Roles: - !Ref StateMachineRole + - !Ref LogQueryFunctionRole Outputs: StateMachineArn: From 81a90266a209c1b7d92438471823c8e6ac54677c Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Sun, 16 Oct 2022 16:11:21 -0400 Subject: [PATCH 05/20] Add LogProcessorFunction Lambda --- .../functions/log-processor/__init__.py | 0 sam-notifier/functions/log-processor/app.py | 47 +++++++++++++++++ .../functions/log-processor/requirements.txt | 0 sam-notifier/statemachine/notifier.asl.json | 50 ++++++++++++++----- sam-notifier/template.yaml | 29 +++++++++++ 5 files changed, 113 insertions(+), 13 deletions(-) create mode 100644 sam-notifier/functions/log-processor/__init__.py create mode 100644 sam-notifier/functions/log-processor/app.py create mode 100644 sam-notifier/functions/log-processor/requirements.txt diff --git a/sam-notifier/functions/log-processor/__init__.py b/sam-notifier/functions/log-processor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sam-notifier/functions/log-processor/app.py b/sam-notifier/functions/log-processor/app.py new file mode 100644 index 0000000..7b05a02 --- /dev/null +++ b/sam-notifier/functions/log-processor/app.py @@ -0,0 +1,47 @@ +import logging +import json +import boto3 + + +dynamodb = boto3.resource('dynamodb') + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +def event_exists(event_id, table): + response = table.get_item(Key={'EventId': event_id}) + logger.info(f"AuditTable query ({event_id}): {response}") + return True if response.get("Item") else False + + +def process_log_entry(event, table, alarm_type): + logger.info(f"Log entry ({type(event)}): {event}") + if not event_exists(event.get("eventID"), table): + logger.info(f"Storing EventID ({event.get('eventID')}) in AuditTable.") + table.put_item( + Item={ + 'EventId': event.get("eventID"), + 'TicketId': None, + 'LogRecord': event, + 'AlarmType': alarm_type + } + ) + return event.get("eventID") + logger.info(f"Event ({event.get('eventID')}) has already been processed.") + + +def lambda_handler(event, context): + table = dynamodb.Table(event.get("DDBAuditTableName")) + new_events = list() + + for log_event in event.get("logs").get("logs"): + log_event = json.loads(log_event) + logger.info(f"Processing log event: {log_event.get('eventID')}") + result = process_log_entry(log_event, table, event.get("alarmName")) + if result: + new_events.append(result) + + return { + "events": new_events + } diff --git a/sam-notifier/functions/log-processor/requirements.txt b/sam-notifier/functions/log-processor/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/sam-notifier/statemachine/notifier.asl.json b/sam-notifier/statemachine/notifier.asl.json index e598f80..a97fba2 100644 --- a/sam-notifier/statemachine/notifier.asl.json +++ b/sam-notifier/statemachine/notifier.asl.json @@ -256,23 +256,47 @@ "InputPath": "$", "ResultPath": "$.logs", "Parameters": { - "CloudWatchLogsLogGroupName": "${LogGroupName}", - "alarmName.$": "$.detail.alarmName", - "epoch.$": "$.epoch", - "query.$": "$.query" + "CloudWatchLogsLogGroupName": "${LogGroupName}", + "alarmName.$": "$.detail.alarmName", + "epoch.$": "$.epoch", + "query.$": "$.query" }, "Retry": [ - { - "ErrorEquals": [ - "States.TaskFailed" - ], - "IntervalSeconds": 300, - "MaxAttempts": 1, - "BackoffRate": 1 - } + { + "ErrorEquals": [ + "States.TaskFailed" + ], + "IntervalSeconds": 300, + "MaxAttempts": 1, + "BackoffRate": 1 + } + ], + "Next": "LogProcessor Function" + }, + "LogProcessor Function": { + "Type": "Task", + "Resource": "${LogProcessorFunctionArn}", + "InputPath": "$", + "ResultPath": "$", + "Parameters": { + "DDBAuditTableName": "${DDBAuditTableName}", + "alarmName.$": "$.detail.alarmName", + "epoch.$": "$.epoch", + "query.$": "$.query", + "logs.$": "$.logs" + }, + "Retry": [ + { + "ErrorEquals": [ + "States.TaskFailed" + ], + "IntervalSeconds": 300, + "MaxAttempts": 1, + "BackoffRate": 1 + } ], "End": true - }, + }, "Improper Input Fail": { "Type": "Fail", "Error": "Improper Input Fail", diff --git a/sam-notifier/template.yaml b/sam-notifier/template.yaml index a8bea42..48806c3 100644 --- a/sam-notifier/template.yaml +++ b/sam-notifier/template.yaml @@ -16,7 +16,9 @@ Resources: DefinitionSubstitutions: EpochFunctionArn: !GetAtt EpochFunction.Arn LogQueryFunctionArn: !GetAtt LogQueryFunction.Arn + LogProcessorFunctionArn: !GetAtt LogProcessorFunction.Arn LogGroupName: !Ref LogGroupName + DDBAuditTableName: !Ref AuditTable Events: EBRule: Type: EventBridgeRule @@ -48,6 +50,8 @@ Resources: FunctionName: !Ref EpochFunction - LambdaInvokePolicy: FunctionName: !Ref LogQueryFunction + - LambdaInvokePolicy: + FunctionName: !Ref LogProcessorFunction EpochFunction: Type: AWS::Serverless::Function @@ -69,6 +73,31 @@ Resources: Architectures: - x86_64 + LogProcessorFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: functions/log-processor/ + Handler: app.lambda_handler + Runtime: python3.9 + Timeout: 300 + Architectures: + - x86_64 + Policies: + - DynamoDBWritePolicy: + TableName: !Ref AuditTable + - DynamoDBReadPolicy: + TableName: !Ref AuditTable + + AuditTable: + Type: AWS::Serverless::SimpleTable + Properties: + PrimaryKey: + Name: EventId + Type: String + ProvisionedThroughput: + ReadCapacityUnits: 1 + WriteCapacityUnits: 1 + CloudWatchLogsPolicy: Type: 'AWS::IAM::Policy' Properties: From 2917f61d0f7947852508d13c034756ceaa783f44 Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Sun, 16 Oct 2022 16:23:32 -0400 Subject: [PATCH 06/20] Refactor StateMachine task names --- sam-notifier/statemachine/notifier.asl.json | 38 ++++++++++----------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/sam-notifier/statemachine/notifier.asl.json b/sam-notifier/statemachine/notifier.asl.json index a97fba2..735febb 100644 --- a/sam-notifier/statemachine/notifier.asl.json +++ b/sam-notifier/statemachine/notifier.asl.json @@ -12,13 +12,13 @@ "IsPresent": true } ], - "Next": "Epoch Converter Function", + "Next": "EpochFunction", "Comment": "Alarm Name Found" } ], "Default": "Improper Input Fail" }, - "Epoch Converter Function": { + "EpochFunction": { "Type": "Task", "Resource": "${EpochFunctionArn}", "InputPath": "$", @@ -126,7 +126,7 @@ }, "CIS-Unauthorized Activity Attempt": { "Type": "Pass", - "Next": "GetLogQuery Function", + "Next": "LogQueryFunction", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter errorCode == '*UnauthorizedOperation' or errorCode == 'AccessDenied*'", @@ -135,7 +135,7 @@ }, "CIS-Console Signin Without MFA": { "Type": "Pass", - "Next": "GetLogQuery Function", + "Next": "LogQueryFunction", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName == 'ConsoleLogin' and responseElements.ConsoleLogin == 'Success' and additionalEventData.MFAUsed != 'Yes'", @@ -144,7 +144,7 @@ }, "CIS-Root Activity": { "Type": "Pass", - "Next": "GetLogQuery Function", + "Next": "LogQueryFunction", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter userIdentity.type == 'Root' and eventType != 'AwsServiceEvent'", @@ -153,7 +153,7 @@ }, "CIS-IAM Policy Changes": { "Type": "Pass", - "Next": "GetLogQuery Function", + "Next": "LogQueryFunction", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['AttachGroupPolicy', 'AttachRolePolicy', 'AttachUserPolicy', 'CreatePolicy', 'CreatePolicyVersion', 'DeleteGroupPolicy', 'DeletePolicy', 'DeletePolicyVersion', 'DeleteRolePolicy', 'DeleteUserPolicy', 'DetachGroupPolicy', 'DetachRolePolicy', 'DetachUserPolicy', 'PutGroupPolicy', 'PutRolePolicy', 'PutUserPolicy']", @@ -162,7 +162,7 @@ }, "CIS-Cloudtrail Config Changes": { "Type": "Pass", - "Next": "GetLogQuery Function", + "Next": "LogQueryFunction", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['CreateTrail', 'DeleteTrail', 'StartLogging', 'StopLogging', 'UpdateTrail']", @@ -171,7 +171,7 @@ }, "CIS-Console Login Failures": { "Type": "Pass", - "Next": "GetLogQuery Function", + "Next": "LogQueryFunction", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName == 'ConsoleLogin' and errorMessage == 'Failed authentication'", @@ -180,7 +180,7 @@ }, "CIS-KMS Key Disabled or Scheduled for Deletion": { "Type": "Pass", - "Next": "GetLogQuery Function", + "Next": "LogQueryFunction", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventSource == 'kms.amazonaws.com' and eventName in ['DisableKey', 'ScheduleKeyDeletion']", @@ -189,7 +189,7 @@ }, "CIS-S3 Bucket Policy Changed": { "Type": "Pass", - "Next": "GetLogQuery Function", + "Next": "LogQueryFunction", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventSource = 's3.amazonaws.com' and eventName in ['DeleteBucketCors', 'DeleteBucketLifecycle', 'DeleteBucketPolicy', 'DeleteBucketReplication', 'PutBucketAcl', 'PutBucketCors', 'PutBucketLifecycle', 'PutBucketPolicy', 'PutBucketReplication']", @@ -198,7 +198,7 @@ }, "CIS-AWS Config Configuration has changed": { "Type": "Pass", - "Next": "GetLogQuery Function", + "Next": "LogQueryFunction", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventSource = 'config.amazonaws.com' and eventName in ['DeleteDeliveryChannel', 'StopConfigurationRecorder', 'PutConfigurationRecorder', 'PutDeliveryChannel']", @@ -207,7 +207,7 @@ }, "CIS-Security Groups Have Changed": { "Type": "Pass", - "Next": "GetLogQuery Function", + "Next": "LogQueryFunction", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventSource = 'config.amazonaws.com' and eventName in ['DeleteDeliveryChannel', 'StopConfigurationRecorder', 'PutConfigurationRecorder', 'PutDeliveryChannel']", @@ -216,7 +216,7 @@ }, "CIS-NACLs Have Changed": { "Type": "Pass", - "Next": "GetLogQuery Function", + "Next": "LogQueryFunction", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['CreateNetworkAcl', 'CreateNetworkAclEntry', 'DeleteNetworkAcl', 'DeleteNetworkAclEntry', 'ReplaceNetworkAclEntry', 'ReplaceNetworkAclAssociation']", @@ -225,7 +225,7 @@ }, "CIS-Network Gateways Have Changed": { "Type": "Pass", - "Next": "GetLogQuery Function", + "Next": "LogQueryFunction", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['AttachInternetGateway', 'CreateCustomerGateway', 'CreateInternetGateway', 'DeleteCustomerGateway', 'DeleteInternetGateway', 'DetachInternetGateway']", @@ -234,7 +234,7 @@ }, "CIS-Route Tables Have Changed": { "Type": "Pass", - "Next": "GetLogQuery Function", + "Next": "LogQueryFunction", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['CreateRoute', 'CreateRouteTable', 'DeleteRoute', 'DeleteRouteTable', 'DisassociateRouteTable', 'ReplaceRoute', 'ReplaceRouteTableAssociation']", @@ -243,14 +243,14 @@ }, "CIS-VPC Has Changed": { "Type": "Pass", - "Next": "GetLogQuery Function", + "Next": "LogQueryFunction", "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName in ['AcceptVpcPeeringConnection', 'AttachClassicLinkVpc', 'CreateVpc', 'CreateVpcPeeringConnection', 'DeleteVpc', 'DeleteVpcPeeringConnection', 'DetachClassicLinkVpc', 'DisableVpcClassicLink', 'EnableVpcClassicLink', 'ModifyVpcAttribute', 'RejectVpcPeeringConnection']", "severity": "LOW" } }, - "GetLogQuery Function": { + "LogQueryFunction": { "Type": "Task", "Resource": "${LogQueryFunctionArn}", "InputPath": "$", @@ -271,9 +271,9 @@ "BackoffRate": 1 } ], - "Next": "LogProcessor Function" + "Next": "LogProcessorFunction" }, - "LogProcessor Function": { + "LogProcessorFunction": { "Type": "Task", "Resource": "${LogProcessorFunctionArn}", "InputPath": "$", From 3a8781b2077cb85752fc36c0f74ffa1fc7e2d0e2 Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Sun, 23 Oct 2022 19:21:06 -0400 Subject: [PATCH 07/20] Add CreateTicketFunction Lambda --- .../functions/create-ticket/__init__.py | 0 sam-notifier/functions/create-ticket/app.py | 140 ++++++++++++++++++ .../functions/create-ticket/requirements.txt | 1 + sam-notifier/statemachine/notifier.asl.json | 26 +++- sam-notifier/template.yaml | 42 ++++++ 5 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 sam-notifier/functions/create-ticket/__init__.py create mode 100644 sam-notifier/functions/create-ticket/app.py create mode 100644 sam-notifier/functions/create-ticket/requirements.txt diff --git a/sam-notifier/functions/create-ticket/__init__.py b/sam-notifier/functions/create-ticket/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sam-notifier/functions/create-ticket/app.py b/sam-notifier/functions/create-ticket/app.py new file mode 100644 index 0000000..9b8e770 --- /dev/null +++ b/sam-notifier/functions/create-ticket/app.py @@ -0,0 +1,140 @@ +import logging +import json +import requests + +import boto3 +from botocore.exceptions import ClientError + + +dynamodb = boto3.resource('dynamodb') + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +def get_secret(jira_auth_token_secret_arn): + session = boto3.session.Session() + client = session.client( + service_name='secretsmanager' + ) + + try: + get_secret_value_response = client.get_secret_value( + SecretId=jira_auth_token_secret_arn + ) + except ClientError as e: + logger.error( + f"Error, unable to access Secret '{jira_auth_token_secret_arn}'.") + logger.error(e) + + if 'SecretString' in get_secret_value_response: + return get_secret_value_response['SecretString'] + + +def get_jira_auth_header(jira_auth_token_secret_arn): + headers = { + "Authorization": f"Basic {get_secret(jira_auth_token_secret_arn)}", + "Content-Type": "application/json" + } + return headers + + +def create_jira_issue(jira_url, headers, issue_data): + r = requests.post(f"{jira_url}/issue/", headers=headers, + data=json.dumps(issue_data)) + logger.info(f"Jira Response: {r} ({r.url})") + + if r.status_code == 201 or r.status_code == requests.codes.ok: + ticket = r.json() + logger.info(f"Successfully created ticket: {ticket.get('key')}") + return ticket.get('key') + + +def get_ddb_event(event_id, table): + logger.info(f"Get EventID: {event_id}, DynamoDB table: {table}") + try: + response = table.get_item(Key={'EventId': event_id}) + except ClientError as err: + logger.error( + "Couldn't get event %s from table %s. Here's why: %s: %s", + event_id, table.name, + err.response['Error']['Code'], err.response['Error']['Message']) + raise + else: + return response['Item'] + + +def update_ddb_event(event_id, ticket_id, table): + logger.info( + f"Update EventID: {event_id}, Ticket: {ticket_id}, DynamoDB table: {table}") + try: + response = table.update_item( + Key={'EventId': event_id}, + UpdateExpression='SET TicketId=:newTicketId', + ExpressionAttributeValues={ + ':newTicketId': ticket_id + }, + ReturnValues="UPDATED_NEW" + ) + except ClientError as err: + logger.error( + "Couldn't update event %s from table %s. Here's why: %s: %s", + ticket_id, table.name, + err.response['Error']['Code'], err.response['Error']['Message']) + raise + else: + return response + + +def process_log_record(log_record, jira_project_key, alarm_name): + logger.info(f"LogRecord ({type(log_record)}): {log_record}") + issue_data = { + "fields": { + "project": + { + "key": jira_project_key + }, + "summary": f"{alarm_name} ({log_record.get('eventID')})", + "description": None, + "issuetype": { + "name": "Task" + } + } + } + + formatted = f"{alarm_name} ({log_record.get('eventID')}) \n\n" + + for k, v in log_record.items(): + formatted += f"{k}: {v if v else 'NO_VALUE_SPECIFIED'} \n" + + issue_data["fields"]["description"] = formatted + return issue_data + + +def lambda_handler(event, context): + table = dynamodb.Table(event.get("DDBAuditTableName")) + + jira_url = event.get("JiraUrl") + jira_project_key = event.get("JiraProjectKey") + jira_auth_token_secret_arn = event.get("JiraAuthTokenSecretArn") + + logger.info(f"Event: {event}") + + alarm_name = event.get("alarmName", "AWS CIS Benchmark Alarm") + ticket_id = None + + if event.get("EventId"): + event_record = get_ddb_event(event.get("EventId"), table) + logger.info(f"Event record: {event_record}") + + if event_record.get("LogRecord"): + jira_issue_data = process_log_record( + event_record.get("LogRecord"), jira_project_key, alarm_name) + logger.info(f"Jira Issue Data payload: {jira_issue_data}") + ticket_id = create_jira_issue(jira_url, get_jira_auth_header( + jira_auth_token_secret_arn), jira_issue_data) + update_ddb_event(event.get("EventId"), ticket_id, table) + + return { + "TicketID": ticket_id + } diff --git a/sam-notifier/functions/create-ticket/requirements.txt b/sam-notifier/functions/create-ticket/requirements.txt new file mode 100644 index 0000000..f229360 --- /dev/null +++ b/sam-notifier/functions/create-ticket/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/sam-notifier/statemachine/notifier.asl.json b/sam-notifier/statemachine/notifier.asl.json index 735febb..04882f4 100644 --- a/sam-notifier/statemachine/notifier.asl.json +++ b/sam-notifier/statemachine/notifier.asl.json @@ -277,7 +277,7 @@ "Type": "Task", "Resource": "${LogProcessorFunctionArn}", "InputPath": "$", - "ResultPath": "$", + "ResultPath": "$.events", "Parameters": { "DDBAuditTableName": "${DDBAuditTableName}", "alarmName.$": "$.detail.alarmName", @@ -295,6 +295,30 @@ "BackoffRate": 1 } ], + "Next": "EventsMap" + }, + "EventsMap": { + "Type": "Map", + "InputPath": "$", + "ItemsPath": "$.events.events", + "Parameters": { + "DDBAuditTableName": "${DDBAuditTableName}", + "JiraUrl": "${JiraUrl}", + "JiraProjectKey": "${JiraProjectKey}", + "JiraAuthTokenSecretArn": "${JiraAuthTokenSecretArn}", + "alarmName.$": "$.detail.alarmName", + "EventId.$": "$$.Map.Item.Value" + }, + "Iterator": { + "StartAt": "CreateTicketFunction", + "States": { + "CreateTicketFunction": { + "Type": "Task", + "Resource": "${CreateTicketFunctionArn}", + "End": true + } + } + }, "End": true }, "Improper Input Fail": { diff --git a/sam-notifier/template.yaml b/sam-notifier/template.yaml index 48806c3..75a9ddc 100644 --- a/sam-notifier/template.yaml +++ b/sam-notifier/template.yaml @@ -7,6 +7,16 @@ Parameters: LogGroupName: Type: String Description: Name of the CloudWatch Logs log group used CloudTrail + JiraUrl: + Type: String + Description: Jira REST API URL (ex. https://.atlassian.net/rest/api/2) + JiraProjectKey: + Type: String + Description: Jira project key + JiraAuthToken: + Type: String + Description: Jira Basic Auth token (https://developer.atlassian.com/cloud/jira/platform/basic-auth-for-rest-apis/) + NoEcho: true Resources: StateMachine: @@ -17,8 +27,12 @@ Resources: EpochFunctionArn: !GetAtt EpochFunction.Arn LogQueryFunctionArn: !GetAtt LogQueryFunction.Arn LogProcessorFunctionArn: !GetAtt LogProcessorFunction.Arn + CreateTicketFunctionArn: !GetAtt CreateTicketFunction.Arn LogGroupName: !Ref LogGroupName DDBAuditTableName: !Ref AuditTable + JiraUrl: !Ref JiraUrl + JiraProjectKey: !Ref JiraProjectKey + JiraAuthTokenSecretArn: !Ref JiraAuthTokenSecret Events: EBRule: Type: EventBridgeRule @@ -52,6 +66,8 @@ Resources: FunctionName: !Ref LogQueryFunction - LambdaInvokePolicy: FunctionName: !Ref LogProcessorFunction + - LambdaInvokePolicy: + FunctionName: !Ref CreateTicketFunction EpochFunction: Type: AWS::Serverless::Function @@ -88,6 +104,23 @@ Resources: - DynamoDBReadPolicy: TableName: !Ref AuditTable + CreateTicketFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: functions/create-ticket/ + Handler: app.lambda_handler + Runtime: python3.9 + Timeout: 300 + Architectures: + - x86_64 + Policies: + - DynamoDBWritePolicy: + TableName: !Ref AuditTable + - DynamoDBReadPolicy: + TableName: !Ref AuditTable + - AWSSecretsManagerGetSecretValuePolicy: + SecretArn: !Ref JiraAuthTokenSecret + AuditTable: Type: AWS::Serverless::SimpleTable Properties: @@ -98,6 +131,12 @@ Resources: ReadCapacityUnits: 1 WriteCapacityUnits: 1 + JiraAuthTokenSecret: + Type: AWS::SecretsManager::Secret + Properties: + Description: String + SecretString: !Ref JiraAuthToken + CloudWatchLogsPolicy: Type: 'AWS::IAM::Policy' Properties: @@ -126,3 +165,6 @@ Outputs: StateMachineArn: Description: "State machine ARN" Value: !Ref StateMachine + JiraApiTokenSecret: + Description: "Secrets Manager Secret with value of JiraAuthToken" + Value: !Ref JiraAuthTokenSecret From 33fab47053cc05f205ae567cb5ae25d2ead59db9 Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Mon, 24 Oct 2022 20:34:38 -0400 Subject: [PATCH 08/20] Retry query function if CloudWatch Logs query returned zero logs --- sam-notifier/functions/log-query/app.py | 3 +++ sam-notifier/statemachine/notifier.asl.json | 12 +++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/sam-notifier/functions/log-query/app.py b/sam-notifier/functions/log-query/app.py index e34060b..a32013b 100644 --- a/sam-notifier/functions/log-query/app.py +++ b/sam-notifier/functions/log-query/app.py @@ -62,6 +62,9 @@ def lambda_handler(event, context): results = get_query_results(response.get("queryId")) logs = process_query_results(results) + # Raise AssertionError if CloudWatch Logs query returned zero logs + assert len(logs) >= 1 + return { "logs": logs } diff --git a/sam-notifier/statemachine/notifier.asl.json b/sam-notifier/statemachine/notifier.asl.json index 04882f4..7ac4741 100644 --- a/sam-notifier/statemachine/notifier.asl.json +++ b/sam-notifier/statemachine/notifier.asl.json @@ -263,12 +263,14 @@ }, "Retry": [ { - "ErrorEquals": [ - "States.TaskFailed" - ], + "ErrorEquals": ["AssertionError"], "IntervalSeconds": 300, - "MaxAttempts": 1, - "BackoffRate": 1 + "MaxAttempts": 2 + }, + { + "ErrorEquals": ["States.TaskFailed"], + "IntervalSeconds": 300, + "MaxAttempts": 1 } ], "Next": "LogProcessorFunction" From 04ed2c66c017f42c92e5d4313fefea726d0832ef Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Mon, 24 Oct 2022 20:36:20 -0400 Subject: [PATCH 09/20] Update epoch lambda to add additional logging --- sam-notifier/functions/epoch/app.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sam-notifier/functions/epoch/app.py b/sam-notifier/functions/epoch/app.py index f9eeebe..d539be1 100644 --- a/sam-notifier/functions/epoch/app.py +++ b/sam-notifier/functions/epoch/app.py @@ -18,9 +18,12 @@ def lambda_handler(event, context): logger.info(f"Event time: {alarm_time}") - start = round((alarm_time - timedelta(minutes=30)).timestamp()) - end = round(alarm_time.timestamp()) + start_time = alarm_time - timedelta(minutes=30) + start_timestamp = round(start_time.timestamp()) + end_time = datetime.datetime.now() + end_timestamp = round(end_time.timestamp()) - logger.info(f"Start timestamp ({start}), end timetamps ({end}).") + logger.info(f"Start time ({start_time}), end time ({end_time}).") + logger.info(f"Start timestamp ({start_timestamp}), end timetamps ({end_timestamp}).") - return {"start": start, "end": end} + return {"start": start_timestamp, "end": end_timestamp} From 00ac2ffc22a6124c3f332e8fcc1e2217c6a440fd Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Mon, 24 Oct 2022 21:43:30 -0400 Subject: [PATCH 10/20] Update Jira ticket description fields formatting --- sam-notifier/functions/create-ticket/app.py | 30 +++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/sam-notifier/functions/create-ticket/app.py b/sam-notifier/functions/create-ticket/app.py index 9b8e770..db526b8 100644 --- a/sam-notifier/functions/create-ticket/app.py +++ b/sam-notifier/functions/create-ticket/app.py @@ -102,10 +102,36 @@ def process_log_record(log_record, jira_project_key, alarm_name): } } - formatted = f"{alarm_name} ({log_record.get('eventID')}) \n\n" + formatted = f"Alarm Name: {alarm_name} \n" + formatted = f"Event ID: {log_record.get('eventID')}) \n\n" + + event_details = "Event Details \n" + user_details = "User Details \n" + additional_details = "Additional Details \n" for k, v in log_record.items(): - formatted += f"{k}: {v if v else 'NO_VALUE_SPECIFIED'} \n" + if k in ("eventID", "eventVersion"): + # Skipping these Event entries + continue + + if k in ("eventName", "errorMessage", "eventSource", "eventTime", "eventType", "eventCategory"): + event_details += f"{k}: {v if v else 'NO_VALUE_SPECIFIED'} \n" + continue + + if k == "userIdentity": + for k, v in log_record.get("userIdentity").items(): + user_details += f"{k}: {v if v else 'NO_VALUE_SPECIFIED'} \n" + continue + + if k in ("sourceIPAddress", "userAgent", "recipientAccountId", "awsRegion"): + user_details += f"{k}: {v if v else 'NO_VALUE_SPECIFIED'} \n" + continue + + additional_details += f"{k}: {v if v else 'NO_VALUE_SPECIFIED'} \n" + + formatted += f"{event_details} \n" + formatted += f"{user_details} \n" + formatted += f"{additional_details} \n" issue_data["fields"]["description"] = formatted return issue_data From ef472a6070f30ea41bd520e68da256f08126baa9 Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Thu, 3 Nov 2022 21:39:10 -0400 Subject: [PATCH 11/20] Add choice to skip Jira issue creation based on Severity --- sam-notifier/statemachine/notifier.asl.json | 39 +++++++++++++++------ 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/sam-notifier/statemachine/notifier.asl.json b/sam-notifier/statemachine/notifier.asl.json index 7ac4741..229b392 100644 --- a/sam-notifier/statemachine/notifier.asl.json +++ b/sam-notifier/statemachine/notifier.asl.json @@ -129,8 +129,8 @@ "Next": "LogQueryFunction", "ResultPath": "$.query", "Result": { - "string": "fields @timestamp, @message | sort @timestamp desc | filter errorCode == '*UnauthorizedOperation' or errorCode == 'AccessDenied*'", - "severity": "HIGH" + "string": "fields @timestamp, @message | sort @timestamp desc | filter errorCode LIKE 'UnauthorizedOperation' or errorCode LIKE 'AccessDenied'", + "severity": "LOW" } }, "CIS-Console Signin Without MFA": { @@ -139,7 +139,7 @@ "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventName == 'ConsoleLogin' and responseElements.ConsoleLogin == 'Success' and additionalEventData.MFAUsed != 'Yes'", - "severity": "MEDIUM" + "severity": "HIGH" } }, "CIS-Root Activity": { @@ -184,7 +184,7 @@ "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventSource == 'kms.amazonaws.com' and eventName in ['DisableKey', 'ScheduleKeyDeletion']", - "severity": "MEDIUM" + "severity": "LOW" } }, "CIS-S3 Bucket Policy Changed": { @@ -193,7 +193,7 @@ "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventSource = 's3.amazonaws.com' and eventName in ['DeleteBucketCors', 'DeleteBucketLifecycle', 'DeleteBucketPolicy', 'DeleteBucketReplication', 'PutBucketAcl', 'PutBucketCors', 'PutBucketLifecycle', 'PutBucketPolicy', 'PutBucketReplication']", - "severity": "MEDIUM" + "severity": "LOW" } }, "CIS-AWS Config Configuration has changed": { @@ -202,7 +202,7 @@ "ResultPath": "$.query", "Result": { "string": "fields @timestamp, @message | sort @timestamp desc | filter eventSource = 'config.amazonaws.com' and eventName in ['DeleteDeliveryChannel', 'StopConfigurationRecorder', 'PutConfigurationRecorder', 'PutDeliveryChannel']", - "severity": "HIGH" + "severity": "LOW" } }, "CIS-Security Groups Have Changed": { @@ -210,7 +210,7 @@ "Next": "LogQueryFunction", "ResultPath": "$.query", "Result": { - "string": "fields @timestamp, @message | sort @timestamp desc | filter eventSource = 'config.amazonaws.com' and eventName in ['DeleteDeliveryChannel', 'StopConfigurationRecorder', 'PutConfigurationRecorder', 'PutDeliveryChannel']", + "string": "filter eventName in ['AuthorizeSecurityGroupIngress', 'AuthorizeSecurityGroupEgress', 'CreateSecurityGroup', 'DeleteSecurityGroup', 'RevokeSecurityGroupIngress', 'RevokeSecurityGroupEgress']", "severity": "LOW" } }, @@ -263,12 +263,16 @@ }, "Retry": [ { - "ErrorEquals": ["AssertionError"], + "ErrorEquals": [ + "AssertionError" + ], "IntervalSeconds": 300, "MaxAttempts": 2 }, { - "ErrorEquals": ["States.TaskFailed"], + "ErrorEquals": [ + "States.TaskFailed" + ], "IntervalSeconds": 300, "MaxAttempts": 1 } @@ -297,7 +301,18 @@ "BackoffRate": 1 } ], - "Next": "EventsMap" + "Next": "SeverityChoice" + }, + "SeverityChoice": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.query.severity", + "StringMatches": "LOW", + "Next": "Severity is LOW" + } + ], + "Default": "EventsMap" }, "EventsMap": { "Type": "Map", @@ -327,6 +342,10 @@ "Type": "Fail", "Error": "Improper Input Fail", "Cause": "Missing Alert Name" + }, + "Severity is LOW": { + "Type": "Succeed", + "Comment": "Skipping Jira issue creation, as the severity is below threshold." } } } \ No newline at end of file From 4f447678bbecdc94881fc444a4eda9ade21d18ce Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Thu, 3 Nov 2022 21:55:30 -0400 Subject: [PATCH 12/20] Update assert error message --- sam-notifier/functions/log-query/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sam-notifier/functions/log-query/app.py b/sam-notifier/functions/log-query/app.py index a32013b..b38c43f 100644 --- a/sam-notifier/functions/log-query/app.py +++ b/sam-notifier/functions/log-query/app.py @@ -63,7 +63,7 @@ def lambda_handler(event, context): logs = process_query_results(results) # Raise AssertionError if CloudWatch Logs query returned zero logs - assert len(logs) >= 1 + assert len(logs) >= 1, "CloudWatch Logs query did not return any entries." return { "logs": logs From 96be551bf7a5d8193c992ba5d874987c93edc836 Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Sun, 6 Nov 2022 16:28:09 -0500 Subject: [PATCH 13/20] Update README.md --- sam-notifier/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sam-notifier/README.md b/sam-notifier/README.md index 712d018..582411e 100644 --- a/sam-notifier/README.md +++ b/sam-notifier/README.md @@ -6,7 +6,7 @@ This project contains source code and supporting files for a serverless applicat - statemachines - Definition for the state machine that orchestrates the notification workflow. - template.yaml - A template that defines the application's AWS resources. -This application creates a AWS Step Functions workflow coupled with event-driven approach using Amazon EventBridge to respond to CIS AWS Foundations Benchmark CloudWatch Alarms provisioned via the [CIS-alarms-cfn.yml](https://github.com/rewindio/aws-security-hub-CIS-metrics/blob/main/CIS-alarms-cfn.yml) CloudFormation template. +This application creates a AWS Step Functions workflow coupled with event-driven approach using Amazon EventBridge to respond to CIS AWS Foundations Benchmark CloudWatch Alarms provisioned via the [CIS-alarms-cfn.yml](https://github.com/rewindio/aws-security-hub-CIS-metrics/blob/main/CIS-alarms-cfn.yml) CloudFormation template. The event detail information is saved in a Amazon DynamoDB table, as well as supplied to a Jira Cloud issue. The application uses several AWS resources, including Step Functions state machines, Lambda functions and an EventBridge rule. These resources are defined in the `template.yaml` file in this project. You can update the template to add AWS resources through the same deployment process that updates your application code. @@ -32,6 +32,9 @@ The first command will build the source of your application. The second command * **Stack Name**: The name of the stack to deploy to CloudFormation. This should be unique to your account and region, and a good starting point would be something matching your project name. * **AWS Region**: The AWS region you want to deploy your app to. * **Parameter LogGroupName**: Name of the CloudWatch Logs log group used CloudTrail +* **Jira Url**: Jira REST API URL (ex. https://.atlassian.net/rest/api/2) +* **Jira Project Key**: Jira project key +* **Jira Auth Token**: Jira Basic Auth token [Atlassian Developer - Basic auth for REST APIs](https://developer.atlassian.com/cloud/jira/platform/basic-auth-for-rest-apis/) * **Confirm changes before deploy**: If set to yes, any change sets will be shown to you before execution for manual review. If set to no, the AWS SAM CLI will automatically deploy application changes. * **Allow SAM CLI IAM role creation**: Many AWS SAM templates, including this example, create AWS IAM roles required for the AWS Lambda function(s) included to access AWS services. By default, these are scoped down to minimum required permissions. To deploy an AWS CloudFormation stack which creates or modifies IAM roles, the `CAPABILITY_IAM` value for `capabilities` must be provided. If permission isn't provided through this prompt, to deploy this example you must explicitly pass `--capabilities CAPABILITY_IAM` to the `sam deploy` command. * **Save arguments to samconfig.toml**: If set to yes, your choices will be saved to a configuration file inside the project, so that in the future you can just re-run `sam deploy` without parameters to deploy changes to your application. From d3673427852306604a6bec1daf4d04115271475d Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Tue, 8 Nov 2022 14:10:01 -0500 Subject: [PATCH 14/20] Address Pylint - logging-fstring-interpolation / W1203 - invalid-name / C0103 - unused-argument / W0613 - missing-timeout / W3101 --- sam-notifier/functions/create-ticket/app.py | 60 ++++++++++++--------- sam-notifier/functions/epoch/app.py | 14 +++-- sam-notifier/functions/log-processor/app.py | 17 ++++-- sam-notifier/functions/log-query/app.py | 29 ++++++---- 4 files changed, 76 insertions(+), 44 deletions(-) diff --git a/sam-notifier/functions/create-ticket/app.py b/sam-notifier/functions/create-ticket/app.py index db526b8..1250fbb 100644 --- a/sam-notifier/functions/create-ticket/app.py +++ b/sam-notifier/functions/create-ticket/app.py @@ -1,3 +1,8 @@ +r""" +The ``create-ticket`` Lambda retrieves the Event details from the +DynamoDB table and creates a Jira issue. +""" + import logging import json import requests @@ -22,10 +27,10 @@ def get_secret(jira_auth_token_secret_arn): get_secret_value_response = client.get_secret_value( SecretId=jira_auth_token_secret_arn ) - except ClientError as e: + except ClientError as err: logger.error( - f"Error, unable to access Secret '{jira_auth_token_secret_arn}'.") - logger.error(e) + "Error, unable to access Secret '%s'.", jira_auth_token_secret_arn) + logger.error(err) if 'SecretString' in get_secret_value_response: return get_secret_value_response['SecretString'] @@ -40,18 +45,18 @@ def get_jira_auth_header(jira_auth_token_secret_arn): def create_jira_issue(jira_url, headers, issue_data): - r = requests.post(f"{jira_url}/issue/", headers=headers, - data=json.dumps(issue_data)) - logger.info(f"Jira Response: {r} ({r.url})") + resp = requests.post(f"{jira_url}/issue/", headers=headers, + data=json.dumps(issue_data), timeout=60) + logger.info("Jira Response: %s (%s)", resp, resp.url) - if r.status_code == 201 or r.status_code == requests.codes.ok: - ticket = r.json() - logger.info(f"Successfully created ticket: {ticket.get('key')}") + if resp.status_code == 201 or resp.status_code == requests.codes.ok: #pylint: disable=E1101 + ticket = resp.json() + logger.info("Successfully created ticket: %s", ticket.get('key')) return ticket.get('key') def get_ddb_event(event_id, table): - logger.info(f"Get EventID: {event_id}, DynamoDB table: {table}") + logger.info("Get EventID: %s, DynamoDB table: %s", event_id, table) try: response = table.get_item(Key={'EventId': event_id}) except ClientError as err: @@ -66,7 +71,7 @@ def get_ddb_event(event_id, table): def update_ddb_event(event_id, ticket_id, table): logger.info( - f"Update EventID: {event_id}, Ticket: {ticket_id}, DynamoDB table: {table}") + "Update EventID: %s, Ticket: %s, DynamoDB table: %s", event_id, ticket_id, table) try: response = table.update_item( Key={'EventId': event_id}, @@ -87,7 +92,7 @@ def update_ddb_event(event_id, ticket_id, table): def process_log_record(log_record, jira_project_key, alarm_name): - logger.info(f"LogRecord ({type(log_record)}): {log_record}") + logger.info("LogRecord (%s): %s", type(log_record), log_record) issue_data = { "fields": { "project": @@ -109,25 +114,28 @@ def process_log_record(log_record, jira_project_key, alarm_name): user_details = "User Details \n" additional_details = "Additional Details \n" - for k, v in log_record.items(): - if k in ("eventID", "eventVersion"): + for key, val in log_record.items(): + if key in ("eventID", "eventVersion"): # Skipping these Event entries continue - if k in ("eventName", "errorMessage", "eventSource", "eventTime", "eventType", "eventCategory"): - event_details += f"{k}: {v if v else 'NO_VALUE_SPECIFIED'} \n" + if key in ( + "eventName", "errorMessage", "eventSource", + "eventTime", "eventType", "eventCategory" + ): + event_details += f"{key}: {val if val else 'NO_VALUE_SPECIFIED'} \n" continue - if k == "userIdentity": - for k, v in log_record.get("userIdentity").items(): - user_details += f"{k}: {v if v else 'NO_VALUE_SPECIFIED'} \n" + if key == "userIdentity": + for key, val in log_record.get("userIdentity").items(): + user_details += f"{key}: {val if val else 'NO_VALUE_SPECIFIED'} \n" continue - if k in ("sourceIPAddress", "userAgent", "recipientAccountId", "awsRegion"): - user_details += f"{k}: {v if v else 'NO_VALUE_SPECIFIED'} \n" + if key in ("sourceIPAddress", "userAgent", "recipientAccountId", "awsRegion"): + user_details += f"{key}: {val if val else 'NO_VALUE_SPECIFIED'} \n" continue - additional_details += f"{k}: {v if v else 'NO_VALUE_SPECIFIED'} \n" + additional_details += f"{key}: {val if val else 'NO_VALUE_SPECIFIED'} \n" formatted += f"{event_details} \n" formatted += f"{user_details} \n" @@ -138,25 +146,27 @@ def process_log_record(log_record, jira_project_key, alarm_name): def lambda_handler(event, context): + logger.debug("Context: %s", context) + table = dynamodb.Table(event.get("DDBAuditTableName")) jira_url = event.get("JiraUrl") jira_project_key = event.get("JiraProjectKey") jira_auth_token_secret_arn = event.get("JiraAuthTokenSecretArn") - logger.info(f"Event: {event}") + logger.info("Event: %s", event) alarm_name = event.get("alarmName", "AWS CIS Benchmark Alarm") ticket_id = None if event.get("EventId"): event_record = get_ddb_event(event.get("EventId"), table) - logger.info(f"Event record: {event_record}") + logger.info("Event record: %s", event_record) if event_record.get("LogRecord"): jira_issue_data = process_log_record( event_record.get("LogRecord"), jira_project_key, alarm_name) - logger.info(f"Jira Issue Data payload: {jira_issue_data}") + logger.info("Jira Issue Data payload: %s", jira_issue_data) ticket_id = create_jira_issue(jira_url, get_jira_auth_header( jira_auth_token_secret_arn), jira_issue_data) update_ddb_event(event.get("EventId"), ticket_id, table) diff --git a/sam-notifier/functions/epoch/app.py b/sam-notifier/functions/epoch/app.py index d539be1..01161f7 100644 --- a/sam-notifier/functions/epoch/app.py +++ b/sam-notifier/functions/epoch/app.py @@ -1,3 +1,8 @@ +r""" +The ``epoch`` Lambda converts the Event date and time to epoch time +as per the Request Parameter for the CloudWatch Log StartQuery call. +""" + import logging import datetime @@ -14,16 +19,19 @@ def check_time(event): def lambda_handler(event, context): + logger.debug("Context: %s", context) + alarm_time = check_time(event) - logger.info(f"Event time: {alarm_time}") + logger.info("Event time: %s", alarm_time) start_time = alarm_time - timedelta(minutes=30) start_timestamp = round(start_time.timestamp()) end_time = datetime.datetime.now() end_timestamp = round(end_time.timestamp()) - logger.info(f"Start time ({start_time}), end time ({end_time}).") - logger.info(f"Start timestamp ({start_timestamp}), end timetamps ({end_timestamp}).") + logger.info("Start time (%s), end time (%s).", start_time, end_time) + logger.info("Start timestamp (%s), end timetamps (%s).", + start_timestamp, end_timestamp) return {"start": start_timestamp, "end": end_timestamp} diff --git a/sam-notifier/functions/log-processor/app.py b/sam-notifier/functions/log-processor/app.py index 7b05a02..19273f6 100644 --- a/sam-notifier/functions/log-processor/app.py +++ b/sam-notifier/functions/log-processor/app.py @@ -1,3 +1,8 @@ +r""" +The ``log-processor`` Lambda processes the Events returned by ``log-query`` +Lambda. This is required as multiple Log entries can be returned. +""" + import logging import json import boto3 @@ -11,14 +16,14 @@ def event_exists(event_id, table): response = table.get_item(Key={'EventId': event_id}) - logger.info(f"AuditTable query ({event_id}): {response}") + logger.info("AuditTable query (%s): %s", event_id, response) return True if response.get("Item") else False def process_log_entry(event, table, alarm_type): - logger.info(f"Log entry ({type(event)}): {event}") + logger.info("Log entry (%s): %s", type(event), event) if not event_exists(event.get("eventID"), table): - logger.info(f"Storing EventID ({event.get('eventID')}) in AuditTable.") + logger.info("Storing EventID (%s) in AuditTable.", event.get('eventID')) table.put_item( Item={ 'EventId': event.get("eventID"), @@ -28,16 +33,18 @@ def process_log_entry(event, table, alarm_type): } ) return event.get("eventID") - logger.info(f"Event ({event.get('eventID')}) has already been processed.") + logger.info("Event (%s) has already been processed.", event.get('eventID')) def lambda_handler(event, context): + logger.debug("Context: %s", context) + table = dynamodb.Table(event.get("DDBAuditTableName")) new_events = list() for log_event in event.get("logs").get("logs"): log_event = json.loads(log_event) - logger.info(f"Processing log event: {log_event.get('eventID')}") + logger.info("Processing log event: %s", log_event.get('eventID')) result = process_log_entry(log_event, table, event.get("alarmName")) if result: new_events.append(result) diff --git a/sam-notifier/functions/log-query/app.py b/sam-notifier/functions/log-query/app.py index b38c43f..8930195 100644 --- a/sam-notifier/functions/log-query/app.py +++ b/sam-notifier/functions/log-query/app.py @@ -1,3 +1,8 @@ +r""" +The ``log-query`` Lambda executes CloudWatch Log query and returns +log Events assocaited with the CloudWatch Alarm. +""" + import time import logging import boto3 @@ -13,7 +18,7 @@ def get_query_results(query_id): queryId=query_id ) - logger.info(f"Get Query Results: {response}") + logger.info("Get Query Results: %s", response) # Wait for the Query to complete while response.get("status") in ("Running", "Scheduled"): @@ -22,16 +27,16 @@ def get_query_results(query_id): queryId=query_id ) if response.get("status") == "Complete": - logger.info(f"Query results: {response}") - logger.info(f"Query statistics: {response.get('statistics')}") + logger.info("Query results: %s", response) + logger.info("Query statistics: %s", response.get('statistics')) else: - logger.error(f"CloudWatch Query failed. {response.get('status')}") + logger.error("CloudWatch Query failed. %s", response.get('status')) return response def process_query_results(results): log_entries = list() - logger.info(f"Process Query Results: {results}") + logger.info("Process Query Results: %s", results) for result in results.get("results"): for entry in result: if entry.get("field") == "@message": @@ -41,12 +46,14 @@ def process_query_results(results): def lambda_handler(event, context): + logger.debug("Context: %s", context) + log_group_name = event.get("CloudWatchLogsLogGroupName") - logger.info(f"StartTime: {event.get('epoch').get('start')}") - logger.info(f"EndTime: {event.get('epoch').get('end')}") - logger.info(f"QueryString: {event.get('query').get('string')}") - logger.info(f"LogGroupName: {log_group_name}") + logger.info("StartTime: %s", event.get('epoch').get('start')) + logger.info("EndTime: %s", event.get('epoch').get('end')) + logger.info("QueryString: %s", event.get('query').get('string')) + logger.info("LogGroupName: %s", log_group_name) response = client.start_query( logGroupName=log_group_name, @@ -56,8 +63,8 @@ def lambda_handler(event, context): limit=10 ) - logger.info(f"Response: {response}") - logger.info(f"CloudWatch Logs QueryId: {response.get('queryId')}") + logger.info("Response: %s", response) + logger.info("CloudWatch Logs QueryId: %s", response.get('queryId')) results = get_query_results(response.get("queryId")) logs = process_query_results(results) From f3a110da579834423c342e2b0c58c19200e0e3db Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Sat, 19 Nov 2022 15:26:09 -0500 Subject: [PATCH 15/20] Set Name property of the AWS::Serverless::StateMachine resource --- sam-notifier/template.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sam-notifier/template.yaml b/sam-notifier/template.yaml index 75a9ddc..dae0794 100644 --- a/sam-notifier/template.yaml +++ b/sam-notifier/template.yaml @@ -4,6 +4,9 @@ Description: > SAM Template to create state machine for CIS Alarm notifer Parameters: + StateMachineName: + Type: String + Description: The name of the state machine LogGroupName: Type: String Description: Name of the CloudWatch Logs log group used CloudTrail @@ -22,6 +25,7 @@ Resources: StateMachine: Type: AWS::Serverless::StateMachine Properties: + Name: !Ref StateMachineName DefinitionUri: statemachine/notifier.asl.json DefinitionSubstitutions: EpochFunctionArn: !GetAtt EpochFunction.Arn From 5049b5da69a983447edf4e33f206cfee601bfb7b Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Sat, 19 Nov 2022 15:50:46 -0500 Subject: [PATCH 16/20] Add "EventTime" key to the AuditTable object --- sam-notifier/functions/log-processor/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sam-notifier/functions/log-processor/app.py b/sam-notifier/functions/log-processor/app.py index 19273f6..761d5b8 100644 --- a/sam-notifier/functions/log-processor/app.py +++ b/sam-notifier/functions/log-processor/app.py @@ -27,6 +27,7 @@ def process_log_entry(event, table, alarm_type): table.put_item( Item={ 'EventId': event.get("eventID"), + 'EventTime': event.get("eventTime"), 'TicketId': None, 'LogRecord': event, 'AlarmType': alarm_type From 4c69dcf7d550e7fc0859055f9c5134c770590946 Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Sat, 19 Nov 2022 16:15:41 -0500 Subject: [PATCH 17/20] Run "black", the Uncompromising Code Formatter --- sam-notifier/functions/create-ticket/app.py | 90 +++++++++++---------- sam-notifier/functions/epoch/app.py | 5 +- sam-notifier/functions/log-processor/app.py | 24 +++--- sam-notifier/functions/log-query/app.py | 28 +++---- 4 files changed, 73 insertions(+), 74 deletions(-) diff --git a/sam-notifier/functions/create-ticket/app.py b/sam-notifier/functions/create-ticket/app.py index 1250fbb..caea849 100644 --- a/sam-notifier/functions/create-ticket/app.py +++ b/sam-notifier/functions/create-ticket/app.py @@ -11,7 +11,7 @@ from botocore.exceptions import ClientError -dynamodb = boto3.resource('dynamodb') +dynamodb = boto3.resource("dynamodb") logger = logging.getLogger() logger.setLevel(logging.INFO) @@ -19,73 +19,78 @@ def get_secret(jira_auth_token_secret_arn): session = boto3.session.Session() - client = session.client( - service_name='secretsmanager' - ) + client = session.client(service_name="secretsmanager") try: get_secret_value_response = client.get_secret_value( SecretId=jira_auth_token_secret_arn ) except ClientError as err: - logger.error( - "Error, unable to access Secret '%s'.", jira_auth_token_secret_arn) + logger.error("Error, unable to access Secret '%s'.", jira_auth_token_secret_arn) logger.error(err) - if 'SecretString' in get_secret_value_response: - return get_secret_value_response['SecretString'] + if "SecretString" in get_secret_value_response: + return get_secret_value_response["SecretString"] def get_jira_auth_header(jira_auth_token_secret_arn): headers = { "Authorization": f"Basic {get_secret(jira_auth_token_secret_arn)}", - "Content-Type": "application/json" + "Content-Type": "application/json", } return headers def create_jira_issue(jira_url, headers, issue_data): - resp = requests.post(f"{jira_url}/issue/", headers=headers, - data=json.dumps(issue_data), timeout=60) + resp = requests.post( + f"{jira_url}/issue/", headers=headers, data=json.dumps(issue_data), timeout=60 + ) logger.info("Jira Response: %s (%s)", resp, resp.url) - if resp.status_code == 201 or resp.status_code == requests.codes.ok: #pylint: disable=E1101 + if ( + resp.status_code == 201 or resp.status_code == requests.codes.ok # pylint: disable=E1101 + ): ticket = resp.json() - logger.info("Successfully created ticket: %s", ticket.get('key')) - return ticket.get('key') + logger.info("Successfully created ticket: %s", ticket.get("key")) + return ticket.get("key") def get_ddb_event(event_id, table): logger.info("Get EventID: %s, DynamoDB table: %s", event_id, table) try: - response = table.get_item(Key={'EventId': event_id}) + response = table.get_item(Key={"EventId": event_id}) except ClientError as err: logger.error( "Couldn't get event %s from table %s. Here's why: %s: %s", - event_id, table.name, - err.response['Error']['Code'], err.response['Error']['Message']) + event_id, + table.name, + err.response["Error"]["Code"], + err.response["Error"]["Message"], + ) raise else: - return response['Item'] + return response["Item"] def update_ddb_event(event_id, ticket_id, table): logger.info( - "Update EventID: %s, Ticket: %s, DynamoDB table: %s", event_id, ticket_id, table) + "Update EventID: %s, Ticket: %s, DynamoDB table: %s", event_id, ticket_id, table + ) try: response = table.update_item( - Key={'EventId': event_id}, - UpdateExpression='SET TicketId=:newTicketId', - ExpressionAttributeValues={ - ':newTicketId': ticket_id - }, - ReturnValues="UPDATED_NEW" + Key={"EventId": event_id}, + UpdateExpression="SET TicketId=:newTicketId", + ExpressionAttributeValues={":newTicketId": ticket_id}, + ReturnValues="UPDATED_NEW", ) except ClientError as err: logger.error( "Couldn't update event %s from table %s. Here's why: %s: %s", - ticket_id, table.name, - err.response['Error']['Code'], err.response['Error']['Message']) + ticket_id, + table.name, + err.response["Error"]["Code"], + err.response["Error"]["Message"], + ) raise else: return response @@ -95,15 +100,10 @@ def process_log_record(log_record, jira_project_key, alarm_name): logger.info("LogRecord (%s): %s", type(log_record), log_record) issue_data = { "fields": { - "project": - { - "key": jira_project_key - }, + "project": {"key": jira_project_key}, "summary": f"{alarm_name} ({log_record.get('eventID')})", "description": None, - "issuetype": { - "name": "Task" - } + "issuetype": {"name": "Task"}, } } @@ -120,8 +120,12 @@ def process_log_record(log_record, jira_project_key, alarm_name): continue if key in ( - "eventName", "errorMessage", "eventSource", - "eventTime", "eventType", "eventCategory" + "eventName", + "errorMessage", + "eventSource", + "eventTime", + "eventType", + "eventCategory", ): event_details += f"{key}: {val if val else 'NO_VALUE_SPECIFIED'} \n" continue @@ -165,12 +169,14 @@ def lambda_handler(event, context): if event_record.get("LogRecord"): jira_issue_data = process_log_record( - event_record.get("LogRecord"), jira_project_key, alarm_name) + event_record.get("LogRecord"), jira_project_key, alarm_name + ) logger.info("Jira Issue Data payload: %s", jira_issue_data) - ticket_id = create_jira_issue(jira_url, get_jira_auth_header( - jira_auth_token_secret_arn), jira_issue_data) + ticket_id = create_jira_issue( + jira_url, + get_jira_auth_header(jira_auth_token_secret_arn), + jira_issue_data, + ) update_ddb_event(event.get("EventId"), ticket_id, table) - return { - "TicketID": ticket_id - } + return {"TicketID": ticket_id} diff --git a/sam-notifier/functions/epoch/app.py b/sam-notifier/functions/epoch/app.py index 01161f7..1faa54f 100644 --- a/sam-notifier/functions/epoch/app.py +++ b/sam-notifier/functions/epoch/app.py @@ -31,7 +31,8 @@ def lambda_handler(event, context): end_timestamp = round(end_time.timestamp()) logger.info("Start time (%s), end time (%s).", start_time, end_time) - logger.info("Start timestamp (%s), end timetamps (%s).", - start_timestamp, end_timestamp) + logger.info( + "Start timestamp (%s), end timetamps (%s).", start_timestamp, end_timestamp + ) return {"start": start_timestamp, "end": end_timestamp} diff --git a/sam-notifier/functions/log-processor/app.py b/sam-notifier/functions/log-processor/app.py index 761d5b8..6f36e0f 100644 --- a/sam-notifier/functions/log-processor/app.py +++ b/sam-notifier/functions/log-processor/app.py @@ -8,14 +8,14 @@ import boto3 -dynamodb = boto3.resource('dynamodb') +dynamodb = boto3.resource("dynamodb") logger = logging.getLogger() logger.setLevel(logging.INFO) def event_exists(event_id, table): - response = table.get_item(Key={'EventId': event_id}) + response = table.get_item(Key={"EventId": event_id}) logger.info("AuditTable query (%s): %s", event_id, response) return True if response.get("Item") else False @@ -23,18 +23,18 @@ def event_exists(event_id, table): def process_log_entry(event, table, alarm_type): logger.info("Log entry (%s): %s", type(event), event) if not event_exists(event.get("eventID"), table): - logger.info("Storing EventID (%s) in AuditTable.", event.get('eventID')) + logger.info("Storing EventID (%s) in AuditTable.", event.get("eventID")) table.put_item( Item={ - 'EventId': event.get("eventID"), - 'EventTime': event.get("eventTime"), - 'TicketId': None, - 'LogRecord': event, - 'AlarmType': alarm_type + "EventId": event.get("eventID"), + "EventTime": event.get("eventTime"), + "TicketId": None, + "LogRecord": event, + "AlarmType": alarm_type, } ) return event.get("eventID") - logger.info("Event (%s) has already been processed.", event.get('eventID')) + logger.info("Event (%s) has already been processed.", event.get("eventID")) def lambda_handler(event, context): @@ -45,11 +45,9 @@ def lambda_handler(event, context): for log_event in event.get("logs").get("logs"): log_event = json.loads(log_event) - logger.info("Processing log event: %s", log_event.get('eventID')) + logger.info("Processing log event: %s", log_event.get("eventID")) result = process_log_entry(log_event, table, event.get("alarmName")) if result: new_events.append(result) - return { - "events": new_events - } + return {"events": new_events} diff --git a/sam-notifier/functions/log-query/app.py b/sam-notifier/functions/log-query/app.py index 8930195..0bdb90c 100644 --- a/sam-notifier/functions/log-query/app.py +++ b/sam-notifier/functions/log-query/app.py @@ -10,27 +10,23 @@ logger = logging.getLogger() logger.setLevel(logging.INFO) -client = boto3.client('logs') +client = boto3.client("logs") def get_query_results(query_id): - response = client.get_query_results( - queryId=query_id - ) + response = client.get_query_results(queryId=query_id) logger.info("Get Query Results: %s", response) # Wait for the Query to complete while response.get("status") in ("Running", "Scheduled"): time.sleep(5) - response = client.get_query_results( - queryId=query_id - ) + response = client.get_query_results(queryId=query_id) if response.get("status") == "Complete": logger.info("Query results: %s", response) - logger.info("Query statistics: %s", response.get('statistics')) + logger.info("Query statistics: %s", response.get("statistics")) else: - logger.error("CloudWatch Query failed. %s", response.get('status')) + logger.error("CloudWatch Query failed. %s", response.get("status")) return response @@ -50,9 +46,9 @@ def lambda_handler(event, context): log_group_name = event.get("CloudWatchLogsLogGroupName") - logger.info("StartTime: %s", event.get('epoch').get('start')) - logger.info("EndTime: %s", event.get('epoch').get('end')) - logger.info("QueryString: %s", event.get('query').get('string')) + logger.info("StartTime: %s", event.get("epoch").get("start")) + logger.info("EndTime: %s", event.get("epoch").get("end")) + logger.info("QueryString: %s", event.get("query").get("string")) logger.info("LogGroupName: %s", log_group_name) response = client.start_query( @@ -60,11 +56,11 @@ def lambda_handler(event, context): startTime=event.get("epoch").get("start"), endTime=event.get("epoch").get("end"), queryString=event.get("query").get("string"), - limit=10 + limit=10, ) logger.info("Response: %s", response) - logger.info("CloudWatch Logs QueryId: %s", response.get('queryId')) + logger.info("CloudWatch Logs QueryId: %s", response.get("queryId")) results = get_query_results(response.get("queryId")) logs = process_query_results(results) @@ -72,6 +68,4 @@ def lambda_handler(event, context): # Raise AssertionError if CloudWatch Logs query returned zero logs assert len(logs) >= 1, "CloudWatch Logs query did not return any entries." - return { - "logs": logs - } + return {"logs": logs} From 89d1a9af175ca38e2c958fbe6527175b96fd296f Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Sat, 19 Nov 2022 17:10:36 -0500 Subject: [PATCH 18/20] Handle '429 Too Many Requests' response from Atlassian API --- sam-notifier/functions/create-ticket/app.py | 16 ++++++++++++++-- sam-notifier/statemachine/notifier.asl.json | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/sam-notifier/functions/create-ticket/app.py b/sam-notifier/functions/create-ticket/app.py index caea849..88b28b0 100644 --- a/sam-notifier/functions/create-ticket/app.py +++ b/sam-notifier/functions/create-ticket/app.py @@ -47,12 +47,24 @@ def create_jira_issue(jira_url, headers, issue_data): ) logger.info("Jira Response: %s (%s)", resp, resp.url) + # Raise AssertionError if Atlassian API returns status code 429 + assert resp.status_code != 429, "Atlassian API returned '429 Too Many Requests'." + if ( - resp.status_code == 201 or resp.status_code == requests.codes.ok # pylint: disable=E1101 + resp.status_code == 201 + or resp.status_code == requests.codes.ok # pylint: disable=E1101 ): ticket = resp.json() logger.info("Successfully created ticket: %s", ticket.get("key")) return ticket.get("key") + else: + error_msg = ( + "Atlassian API responded with (%s): %s", + resp.status_code, + resp.json(), + ) + logger.error(error_msg) + raise Exception(error_msg) def get_ddb_event(event_id, table): @@ -108,7 +120,7 @@ def process_log_record(log_record, jira_project_key, alarm_name): } formatted = f"Alarm Name: {alarm_name} \n" - formatted = f"Event ID: {log_record.get('eventID')}) \n\n" + formatted = f"Event ID: {log_record.get('eventID')} \n\n" event_details = "Event Details \n" user_details = "User Details \n" diff --git a/sam-notifier/statemachine/notifier.asl.json b/sam-notifier/statemachine/notifier.asl.json index 229b392..6865612 100644 --- a/sam-notifier/statemachine/notifier.asl.json +++ b/sam-notifier/statemachine/notifier.asl.json @@ -332,6 +332,22 @@ "CreateTicketFunction": { "Type": "Task", "Resource": "${CreateTicketFunctionArn}", + "Retry": [ + { + "ErrorEquals": [ + "AssertionError" + ], + "IntervalSeconds": 300, + "MaxAttempts": 2 + }, + { + "ErrorEquals": [ + "States.TaskFailed" + ], + "IntervalSeconds": 300, + "MaxAttempts": 1 + } + ], "End": true } } From 5aa988a9a8e7104e72775686ad237ccaa6f4e67a Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Sat, 19 Nov 2022 17:22:52 -0500 Subject: [PATCH 19/20] Format "responseElements" values in Create-Ticket function payload --- sam-notifier/functions/create-ticket/app.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sam-notifier/functions/create-ticket/app.py b/sam-notifier/functions/create-ticket/app.py index 88b28b0..a649360 100644 --- a/sam-notifier/functions/create-ticket/app.py +++ b/sam-notifier/functions/create-ticket/app.py @@ -125,6 +125,7 @@ def process_log_record(log_record, jira_project_key, alarm_name): event_details = "Event Details \n" user_details = "User Details \n" additional_details = "Additional Details \n" + response_elements = "Response Elements \n" for key, val in log_record.items(): if key in ("eventID", "eventVersion"): @@ -151,11 +152,17 @@ def process_log_record(log_record, jira_project_key, alarm_name): user_details += f"{key}: {val if val else 'NO_VALUE_SPECIFIED'} \n" continue + if key == "responseElements": + for key, val in log_record.get("responseElements").items(): + response_elements += f"{key}: {val if val else 'NO_VALUE_SPECIFIED'} \n" + continue + additional_details += f"{key}: {val if val else 'NO_VALUE_SPECIFIED'} \n" formatted += f"{event_details} \n" formatted += f"{user_details} \n" formatted += f"{additional_details} \n" + formatted += f"{response_elements} \n" issue_data["fields"]["description"] = formatted return issue_data From 5ed341f8c67576236d723d0843efb8bf858933d0 Mon Sep 17 00:00:00 2001 From: Greg Sienkiewicz Date: Sat, 19 Nov 2022 17:23:46 -0500 Subject: [PATCH 20/20] Reduce the epoch timedelta to 10 minutes to decrease CloudWatch Logs batch --- sam-notifier/functions/epoch/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sam-notifier/functions/epoch/app.py b/sam-notifier/functions/epoch/app.py index 1faa54f..23ec1df 100644 --- a/sam-notifier/functions/epoch/app.py +++ b/sam-notifier/functions/epoch/app.py @@ -25,7 +25,7 @@ def lambda_handler(event, context): logger.info("Event time: %s", alarm_time) - start_time = alarm_time - timedelta(minutes=30) + start_time = alarm_time - timedelta(minutes=10) start_timestamp = round(start_time.timestamp()) end_time = datetime.datetime.now() end_timestamp = round(end_time.timestamp())