From 04f2aa1a42f019230339a1ff778f1a24553f925f Mon Sep 17 00:00:00 2001 From: Md Azhar Date: Mon, 8 Jun 2026 13:11:54 +0530 Subject: [PATCH 1/4] making relevant changes against solution template mp changes --- .../DLQLambdaCloudFormation.json | 365 ---------------- ...mbdaCloudFormationWithSecuredEndpoint.json | 391 ------------------ .../dlq_lambda_cloudformation.template.yaml | 353 ++++++++++++++++ upload_artifacts.py | 14 +- 4 files changed, 362 insertions(+), 761 deletions(-) delete mode 100644 cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json delete mode 100644 cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json create mode 100644 cloudwatchlogs-with-dlq/dlq_lambda_cloudformation.template.yaml diff --git a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json deleted file mode 100644 index a935406..0000000 --- a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json +++ /dev/null @@ -1,365 +0,0 @@ -{ - "AWSTemplateFormatVersion": "2010-09-09", - "Description" : "Sumo Logic CloudWatch log collector", - "Parameters" : { - "SumoEndPointURL" : { - "Type" : "String", - "Default" : "", - "Description" : "Enter SUMO_ENDPOINT created while configuring HTTP Source" - }, - "EmailID": { - "Type": "String", - "Default": "test@gmail.com", - "Description": "Enter your email for receiving alerts.You will receive confirmation email after the deployment is complete, confirm it to subscribe for alerts." - }, - "NumOfWorkers": { - "Type": "Number", - "Default": 4, - "Description": "Enter the number of lambda function invocations for faster Dead Letter Queue processing." - }, - "LogFormat": { - "Type": "String", - "Default": "Others", - "AllowedValues" : ["VPC-RAW" ,"VPC-JSON", "Others"], - "Description": "Choose the Service" - }, - "IncludeLogGroupInfo": { - "Type": "String", - "Default": "false", - "AllowedValues" : ["true" ,"false"], - "Description": "Select true to get loggroup/logstream values in logs" - }, - "LogStreamPrefix": { - "Type": "String", - "Description": "(Optional) Enter comma separated list of logStream name prefixes to filter by logStream. Please note this is seperate from a logGroup. This is used to only send certain logStreams within a cloudwatch logGroup(s). LogGroups still need to be subscribed to the created Lambda funciton, regardless of what is input for this value.", - "Default": "" - } - }, - "Mappings" : { - "RegionMap" : { - "us-east-1": {"bucketname": "appdevzipfiles-us-east-1"}, - "us-east-2": {"bucketname": "appdevzipfiles-us-east-2"}, - "us-west-1": {"bucketname": "appdevzipfiles-us-west-1"}, - "us-west-2": {"bucketname": "appdevzipfiles-us-west-2"}, - "ap-south-1": {"bucketname": "appdevzipfiles-ap-south-1"}, - "ap-northeast-2": {"bucketname": "appdevzipfiles-ap-northeast-2"}, - "ap-southeast-1": {"bucketname": "appdevzipfiles-ap-southeast-1"}, - "ap-southeast-2": {"bucketname": "appdevzipfiles-ap-southeast-2"}, - "ap-northeast-1": {"bucketname": "appdevzipfiles-ap-northeast-1"}, - "ap-east-1": {"bucketname": "appdevzipfiles-ap-east-1s"}, - "af-south-1": {"bucketname": "appdevzipfiles-af-south-1s"}, - "ca-central-1": {"bucketname": "appdevzipfiles-ca-central-1"}, - "eu-central-1": {"bucketname": "appdevzipfiles-eu-central-1"}, - "eu-west-1": {"bucketname": "appdevzipfiles-eu-west-1"}, - "eu-west-2": {"bucketname": "appdevzipfiles-eu-west-2"}, - "eu-west-3": {"bucketname": "appdevzipfiles-eu-west-3"}, - "eu-north-1": {"bucketname": "appdevzipfiles-eu-north-1s"}, - "eu-south-1": {"bucketname": "appdevzipfiles-eu-south-1"}, - "me-south-1": {"bucketname": "appdevzipfiles-me-south-1s"}, - "sa-east-1": {"bucketname": "appdevzipfiles-sa-east-1"}, - "me-central-1": {"bucketname": "appdevzipfiles-me-central-1"}, - "eu-central-2": {"bucketname": "appdevzipfiles-eu-central-2ss"}, - "ap-northeast-3": {"bucketname": "appdevzipfiles-ap-northeast-3s"}, - "ap-southeast-3": {"bucketname": "appdevzipfiles-ap-southeast-3"} - } - }, - "Resources": { - "SumoCWLogGroup": { - "Type": "AWS::Logs::LogGroup", - "Properties": { - "LogGroupName": { "Fn::Join": [ "-", [ "SumoCWLogGroup", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, - "RetentionInDays": 7 - } - }, - "SumoCWLogSubsriptionFilter": { - "Type": "AWS::Logs::SubscriptionFilter", - "Properties": { - "LogGroupName": { - "Ref": "SumoCWLogGroup" - }, - "DestinationArn": { - "Fn::GetAtt": [ - "SumoCWLogsLambda", - "Arn" - ] - }, - "FilterPattern": "" - }, - "DependsOn": [ - "SumoCWLogGroup", - "SumoCWLambdaPermission", - "SumoCWLogsLambda" - ] - }, - "SumoCWLambdaPermission": { - - "Type": "AWS::Lambda::Permission", - - "Properties": { - - "FunctionName": { - "Fn::GetAtt": [ - "SumoCWLogsLambda", - "Arn" - ] - }, - "Action": "lambda:InvokeFunction", - - "Principal": { "Fn::Join": [ ".", [ "logs", { "Ref": "AWS::Region" }, "amazonaws.com" ] ] }, - - "SourceAccount": { "Ref": "AWS::AccountId" } - } - }, - "SumoCWDeadLetterQueue": { - "Type": "AWS::SQS::Queue", - "Properties": { - "QueueName": { "Fn::Join": [ "-", [ "SumoCWDeadLetterQueue", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] } - } - }, - "SumoCWLambdaExecutionRole": { - "Type": "AWS::IAM::Role", - "Properties": { - "AssumeRolePolicyDocument": { - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Principal": {"Service": ["lambda.amazonaws.com"] }, - "Action": ["sts:AssumeRole"] - } ] - }, - "Path": "/", - "Policies": [ - { - "PolicyName": { "Fn::Join": [ "-", [ "SQSCreateLogsRolePolicy", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, - "PolicyDocument": { - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Action": [ - "sqs:DeleteMessage", - "sqs:GetQueueUrl", - "sqs:ListQueues", - "sqs:ChangeMessageVisibility", - "sqs:SendMessageBatch", - "sqs:ReceiveMessage", - "sqs:SendMessage", - "sqs:GetQueueAttributes", - "sqs:ListQueueTags", - "sqs:ListDeadLetterSourceQueues", - "sqs:DeleteMessageBatch", - "sqs:PurgeQueue", - "sqs:DeleteQueue", - "sqs:CreateQueue", - "sqs:ChangeMessageVisibilityBatch", - "sqs:SetQueueAttributes" - ], - "Resource": [ - { - "Fn::GetAtt": [ - "SumoCWDeadLetterQueue", - "Arn" - ] - } - ] - }] - } - }, - { - "PolicyName": { "Fn::Join": [ "-", [ "CloudWatchCreateLogsRolePolicy", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, - "PolicyDocument": { - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Action": [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents", - "logs:DescribeLogStreams" - ], - "Resource": [ - { "Fn::Join": [ ":", [ "arn", { "Ref" : "AWS::Partition" }, "logs", { "Ref" : "AWS::Region" }, { "Ref" : "AWS::AccountId" },"log-group","*" ] ] } - ] - }] - } - }, - { - "PolicyName": "InvokeLambdaRolePolicy", - "PolicyDocument": { - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Action": [ - "lambda:InvokeFunction" - ], - "Resource": [ - { "Fn::Join": [ ":", [ "arn", { "Ref" : "AWS::Partition" }, "lambda", { "Ref" : "AWS::Region" }, { "Ref" : "AWS::AccountId" }, "function", { "Fn::Join": [ "-", [ "SumoCWProcessDLQLambda", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] } ] ] } - ] - }] - } - } - ] - } - }, - "SumoCWLogsLambda": { - "Type": "AWS::Lambda::Function", - "DependsOn": [ - "SumoCWLambdaExecutionRole", - "SumoCWDeadLetterQueue" - ], - "Properties": { - "FunctionName": { "Fn::Join": [ "-", [ "SumoCWLogsLambda", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, - "Code": { - "S3Bucket": { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "bucketname"]}, - "S3Key": "cloudwatchLogsDLQ/v1.3.0/cloudwatchlogs-with-dlq.zip" - }, - "Role": { - "Fn::GetAtt": [ - "SumoCWLambdaExecutionRole", - "Arn" - ] - }, - "Timeout": 300, - "DeadLetterConfig": { - "TargetArn" : { - "Fn::GetAtt": [ - "SumoCWDeadLetterQueue", - "Arn" - ] - } - }, - "Handler": "cloudwatchlogs_lambda.handler", - "Runtime": "nodejs22.x", - "MemorySize": 128, - "Environment": { - "Variables": { - "SUMO_ENDPOINT": {"Ref": "SumoEndPointURL"}, - "LOG_FORMAT": {"Ref": "LogFormat"}, - "INCLUDE_LOG_INFO": {"Ref": "IncludeLogGroupInfo"}, - "LOG_STREAM_PREFIX": {"Ref": "LogStreamPrefix"} - - } - } - } - }, - "SumoCWEventsInvokeLambdaPermission": { - "Type": "AWS::Lambda::Permission", - "Properties": { - "FunctionName": { "Ref": "SumoCWProcessDLQLambda" }, - "Action": "lambda:InvokeFunction", - "Principal": "events.amazonaws.com", - "SourceArn": { "Fn::GetAtt": ["SumoCWProcessDLQScheduleRule", "Arn"] } - } - }, - "SumoCWProcessDLQScheduleRule": { - "Type": "AWS::Events::Rule", - "Properties": { - "Description": "Events rule for Cron", - "ScheduleExpression": "rate(5 minutes)", - "State": "ENABLED", - "Targets": [{ - "Arn": { "Fn::GetAtt": ["SumoCWProcessDLQLambda", "Arn"] }, - "Id": "TargetFunctionV1" - }] - } - }, - "SumoCWProcessDLQLambda": { - "Type": "AWS::Lambda::Function", - "DependsOn": [ - "SumoCWLambdaExecutionRole", - "SumoCWDeadLetterQueue" - ], - "Properties": { - "FunctionName": { "Fn::Join": [ "-", [ "SumoCWProcessDLQLambda", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, - "Code": { - "S3Bucket": { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "bucketname"]}, - "S3Key": "cloudwatchLogsDLQ/v1.3.0/cloudwatchlogs-with-dlq.zip" - }, - "Role": { - "Fn::GetAtt": [ - "SumoCWLambdaExecutionRole", - "Arn" - ] - }, - "Timeout": 300, - "Handler": "DLQProcessor.handler", - "DeadLetterConfig": { - "TargetArn" : { - "Fn::GetAtt": [ - "SumoCWDeadLetterQueue", - "Arn" - ] - } - }, - "Runtime": "nodejs22.x", - "MemorySize": 128, - "Environment": { - "Variables": { - "SUMO_ENDPOINT": {"Ref": "SumoEndPointURL"}, - "TASK_QUEUE_URL": { - "Fn::Join": [ - "", - [ - "https://sqs.", - { "Ref" : "AWS::Region" }, - ".amazonaws.com/", - { "Ref" : "AWS::AccountId" }, - "/", - {"Fn::GetAtt": ["SumoCWDeadLetterQueue", "QueueName"]} - ] - ] - }, - "NUM_OF_WORKERS": {"Ref": "NumOfWorkers"}, - "LOG_FORMAT": {"Ref": "LogFormat"}, - "INCLUDE_LOG_INFO": {"Ref": "IncludeLogGroupInfo"}, - "LOG_STREAM_PREFIX": {"Ref": "LogStreamPrefix"} - } - } - } - }, - "SumoCWEmailSNSTopic": { - "Type":"AWS::SNS::Topic", - "Properties":{ - "Subscription":[ { - "Endpoint" : {"Ref": "EmailID"}, - "Protocol" : "email" - }] - } - }, - "SumoCWSpilloverAlarm":{ - "Type":"AWS::CloudWatch::Alarm", - "Properties":{ - "AlarmActions":[ - { - "Ref":"SumoCWEmailSNSTopic" - } - ], - "AlarmDescription":"Notify via email if number of messages in DeadLetterQueue exceeds threshold", - "ComparisonOperator":"GreaterThanThreshold", - "Dimensions":[ - { - "Name": "QueueName", - "Value": {"Fn::GetAtt": ["SumoCWDeadLetterQueue", "QueueName"]} - } - ], - "EvaluationPeriods":"1", - "MetricName":"ApproximateNumberOfMessagesVisible", - "Namespace":"AWS/SQS", - "Period":"3600", - "Statistic":"Sum", - "Threshold":"100000" - }, - "DependsOn": ["SumoCWEmailSNSTopic"] - } - }, - "Outputs": { - "SumoCWLogsLambdaArn" : { - "Description": "The ARN of the sumologic cloudwatch logs lambda", - "Value" : { "Fn::GetAtt" : ["SumoCWLogsLambda", "Arn"] }, - "Export" : { - "Name" : { "Fn::Join": [ "-", [ "SumoCWLogsLambdaArn", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] } - - } - } - } -} - diff --git a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json deleted file mode 100644 index 4200d9b..0000000 --- a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json +++ /dev/null @@ -1,391 +0,0 @@ -{ - "AWSTemplateFormatVersion": "2010-09-09", - "Description" : "Sumo Logic CloudWatch log collector", - "Parameters" : { - "SumoURLDecryptKeyID" : { - "Type" : "String", - "Default" : "", - "Description" : "Enter the KMS key ID used for encrypting SUMO_ENDPOINT" - }, - "EmailID": { - "Type": "String", - "Default": "test@gmail.com", - "Description": "Enter your email for receiving alerts.You will receive confirmation email after the deployment is complete, confirm it to subscribe for alerts." - }, - "NumOfWorkers": { - "Type": "Number", - "Default": 4, - "Description": "Enter the number of lambda function invocations for faster Dead Letter Queue processing." - }, - "LogFormat": { - "Type": "String", - "Default": "Others", - "AllowedValues" : ["VPC-RAW" ,"VPC-JSON", "Others"], - "Description": "Choose the Service" - }, - "IncludeLogGroupInfo": { - "Type": "String", - "Default": "false", - "AllowedValues" : ["true" ,"false"], - "Description": "Select true to get loggroup/logstream values in logs" - }, - "LogStreamPrefix": { - "Type": "String", - "Description": "(Optional) Enter comma separated list of logStream name prefixes to filter by logStream. Please note this is seperate from a logGroup. This is used to only send certain logStreams within a cloudwatch logGroup(s). LogGroups still need to be subscribed to the created Lambda funciton, regardless of what is input for this value.", - "Default": "" - } - }, - "Mappings" : { - "RegionMap" : { - "us-east-1": {"bucketname": "appdevzipfiles-us-east-1"}, - "us-east-2": {"bucketname": "appdevzipfiles-us-east-2"}, - "us-west-1": {"bucketname": "appdevzipfiles-us-west-1"}, - "us-west-2": {"bucketname": "appdevzipfiles-us-west-2"}, - "ap-south-1": {"bucketname": "appdevzipfiles-ap-south-1"}, - "ap-northeast-2": {"bucketname": "appdevzipfiles-ap-northeast-2"}, - "ap-southeast-1": {"bucketname": "appdevzipfiles-ap-southeast-1"}, - "ap-southeast-2": {"bucketname": "appdevzipfiles-ap-southeast-2"}, - "ap-northeast-1": {"bucketname": "appdevzipfiles-ap-northeast-1"}, - "ap-east-1": {"bucketname": "appdevzipfiles-ap-east-1s"}, - "af-south-1": {"bucketname": "appdevzipfiles-af-south-1s"}, - "ca-central-1": {"bucketname": "appdevzipfiles-ca-central-1"}, - "eu-central-1": {"bucketname": "appdevzipfiles-eu-central-1"}, - "eu-west-1": {"bucketname": "appdevzipfiles-eu-west-1"}, - "eu-west-2": {"bucketname": "appdevzipfiles-eu-west-2"}, - "eu-west-3": {"bucketname": "appdevzipfiles-eu-west-3"}, - "eu-north-1": {"bucketname": "appdevzipfiles-eu-north-1s"}, - "eu-south-1": {"bucketname": "appdevzipfiles-eu-south-1"}, - "me-south-1": {"bucketname": "appdevzipfiles-me-south-1s"}, - "sa-east-1": {"bucketname": "appdevzipfiles-sa-east-1"}, - "me-central-1": {"bucketname": "appdevzipfiles-me-central-1"}, - "eu-central-2": {"bucketname": "appdevzipfiles-eu-central-2ss"}, - "ap-northeast-3": {"bucketname": "appdevzipfiles-ap-northeast-3s"}, - "ap-southeast-3": {"bucketname": "appdevzipfiles-ap-southeast-3"} - } - }, - "Resources": { - "SumoCWLogGroup": { - "Type": "AWS::Logs::LogGroup", - "Properties": { - "LogGroupName": { "Fn::Join": [ "-", [ "SumoCWLogGroup", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, - "RetentionInDays": 7 - } - }, - "SumoCWLogSubsriptionFilter": { - "Type": "AWS::Logs::SubscriptionFilter", - "Properties": { - "LogGroupName": { - "Ref": "SumoCWLogGroup" - }, - "DestinationArn": { - "Fn::GetAtt": [ - "SumoCWLogsLambda", - "Arn" - ] - }, - "FilterPattern": "" - }, - "DependsOn": [ - "SumoCWLogGroup", - "SumoCWLambdaPermission", - "SumoCWLogsLambda" - ] - }, - "SumoCWLambdaPermission": { - - "Type": "AWS::Lambda::Permission", - - "Properties": { - - "FunctionName": { - "Fn::GetAtt": [ - "SumoCWLogsLambda", - "Arn" - ] - }, - "Action": "lambda:InvokeFunction", - - "Principal": { "Fn::Join": [ ".", [ "logs", { "Ref": "AWS::Region" }, "amazonaws.com" ] ] }, - - "SourceAccount": { "Ref": "AWS::AccountId" } - } - }, - "SumoCWDeadLetterQueue": { - "Type": "AWS::SQS::Queue", - "Properties": { - "QueueName": { "Fn::Join": [ "-", [ "SumoCWDeadLetterQueue", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] } - } - }, - "SumoCWLambdaExecutionRole": { - "Type": "AWS::IAM::Role", - "Properties": { - "AssumeRolePolicyDocument": { - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Principal": {"Service": ["lambda.amazonaws.com"] }, - "Action": ["sts:AssumeRole"] - } ] - }, - "Path": "/", - "Policies": [ - { - "PolicyName": { "Fn::Join": [ "-", [ "SQSCreateLogsRolePolicy", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, - "PolicyDocument": { - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Action": [ - "sqs:DeleteMessage", - "sqs:GetQueueUrl", - "sqs:ListQueues", - "sqs:ChangeMessageVisibility", - "sqs:SendMessageBatch", - "sqs:ReceiveMessage", - "sqs:SendMessage", - "sqs:GetQueueAttributes", - "sqs:ListQueueTags", - "sqs:ListDeadLetterSourceQueues", - "sqs:DeleteMessageBatch", - "sqs:PurgeQueue", - "sqs:DeleteQueue", - "sqs:CreateQueue", - "sqs:ChangeMessageVisibilityBatch", - "sqs:SetQueueAttributes" - ], - "Resource": [ - { - "Fn::GetAtt": [ - "SumoCWDeadLetterQueue", - "Arn" - ] - } - ] - }] - } - }, - { - "PolicyName": { "Fn::Join": [ "-", [ "CloudWatchCreateLogsRolePolicy", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, - "PolicyDocument": { - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Action": [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:PutLogEvents", - "logs:DescribeLogStreams" - ], - "Resource": [ - { "Fn::Join": [ ":", ["arn", { "Ref" : "AWS::Partition" }, "logs", { "Ref" : "AWS::Region" }, { "Ref" : "AWS::AccountId" },"log-group","*" ] ] } - ] - }] - } - }, - { - "PolicyName": "InvokeLambdaRolePolicy", - "PolicyDocument": { - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Action": [ - "lambda:InvokeFunction" - ], - "Resource": [ - { "Fn::Join": [ ":", [ "arn", { "Ref" : "AWS::Partition" }, "lambda", { "Ref" : "AWS::Region" }, { "Ref" : "AWS::AccountId" }, "function", { "Fn::Join": [ "-", [ "SumoCWProcessDLQLambda", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] } ] ] } - ] - }] - } - }, - { - "PolicyName": { "Fn::Join": [ "-", [ "SSMSumoLambdaPolicy", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, - "PolicyDocument": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "ssm:GetParameter" - ], - "Resource": { - "Fn::Join": [ ":", [ "arn", { "Ref" : "AWS::Partition" }, "ssm", { "Ref" : "AWS::Region" }, { "Ref" : "AWS::AccountId" }, "parameter/SUMO_ENDPOINT"] - ] - } - }, - { - "Effect": "Allow", - "Action": [ - "kms:Decrypt" - ], - "Resource": { - "Fn::Join": [ ":", [ "arn", { "Ref" : "AWS::Partition" }, "kms", { "Ref" : "AWS::Region" }, { "Ref" : "AWS::AccountId" }, { "Fn::Sub": [ "key/${KeyID}", { "KeyID": {"Ref" : "SumoURLDecryptKeyID" }} ]}] - ] - } - } - ] - } - } - ] - } - }, - "SumoCWLogsLambda": { - "Type": "AWS::Lambda::Function", - "DependsOn": [ - "SumoCWLambdaExecutionRole", - "SumoCWDeadLetterQueue" - ], - "Properties": { - "FunctionName": { "Fn::Join": [ "-", [ "SumoCWLogsLambda", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, - "Code": { - "S3Bucket": { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "bucketname"]}, - "S3Key": "cloudwatchLogsDLQ/v1.3.0/cloudwatchlogs-with-dlq.zip" - }, - "Role": { - "Fn::GetAtt": [ - "SumoCWLambdaExecutionRole", - "Arn" - ] - }, - "Timeout": 300, - "DeadLetterConfig": { - "TargetArn" : { - "Fn::GetAtt": [ - "SumoCWDeadLetterQueue", - "Arn" - ] - } - }, - "Handler": "cloudwatchlogs_lambda.handler", - "Runtime": "nodejs22.x", - "MemorySize": 128, - "Environment": { - "Variables": { - "LOG_FORMAT": {"Ref": "LogFormat"}, - "INCLUDE_LOG_INFO": {"Ref": "IncludeLogGroupInfo"}, - "LOG_STREAM_PREFIX": {"Ref": "LogStreamPrefix"} - - } - } - } - }, - "SumoCWEventsInvokeLambdaPermission": { - "Type": "AWS::Lambda::Permission", - "Properties": { - "FunctionName": { "Ref": "SumoCWProcessDLQLambda" }, - "Action": "lambda:InvokeFunction", - "Principal": "events.amazonaws.com", - "SourceArn": { "Fn::GetAtt": ["SumoCWProcessDLQScheduleRule", "Arn"] } - } - }, - "SumoCWProcessDLQScheduleRule": { - "Type": "AWS::Events::Rule", - "Properties": { - "Description": "Events rule for Cron", - "ScheduleExpression": "rate(5 minutes)", - "State": "ENABLED", - "Targets": [{ - "Arn": { "Fn::GetAtt": ["SumoCWProcessDLQLambda", "Arn"] }, - "Id": "TargetFunctionV1" - }] - } - }, - "SumoCWProcessDLQLambda": { - "Type": "AWS::Lambda::Function", - "DependsOn": [ - "SumoCWLambdaExecutionRole", - "SumoCWDeadLetterQueue" - ], - "Properties": { - "FunctionName": { "Fn::Join": [ "-", [ "SumoCWProcessDLQLambda", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, - "Code": { - "S3Bucket": { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "bucketname"]}, - "S3Key": "cloudwatchLogsDLQ/v1.3.0/cloudwatchlogs-with-dlq.zip" - }, - "Role": { - "Fn::GetAtt": [ - "SumoCWLambdaExecutionRole", - "Arn" - ] - }, - "Timeout": 300, - "Handler": "DLQProcessor.handler", - "DeadLetterConfig": { - "TargetArn" : { - "Fn::GetAtt": [ - "SumoCWDeadLetterQueue", - "Arn" - ] - } - }, - "Runtime": "nodejs22.x", - "MemorySize": 128, - "Environment": { - "Variables": { - "TASK_QUEUE_URL": { - "Fn::Join": [ - "", - [ - "https://sqs.", - { "Ref" : "AWS::Region" }, - ".amazonaws.com/", - { "Ref" : "AWS::AccountId" }, - "/", - {"Fn::GetAtt": ["SumoCWDeadLetterQueue", "QueueName"]} - ] - ] - }, - "NUM_OF_WORKERS": {"Ref": "NumOfWorkers"}, - "LOG_FORMAT": {"Ref": "LogFormat"}, - "INCLUDE_LOG_INFO": {"Ref": "IncludeLogGroupInfo"}, - "LOG_STREAM_PREFIX": {"Ref": "LogStreamPrefix"} - } - } - } - }, - "SumoCWEmailSNSTopic": { - "Type":"AWS::SNS::Topic", - "Properties":{ - "Subscription":[ { - "Endpoint" : {"Ref": "EmailID"}, - "Protocol" : "email" - }] - } - }, - "SumoCWSpilloverAlarm":{ - "Type":"AWS::CloudWatch::Alarm", - "Properties":{ - "AlarmActions":[ - { - "Ref":"SumoCWEmailSNSTopic" - } - ], - "AlarmDescription":"Notify via email if number of messages in DeadLetterQueue exceeds threshold", - "ComparisonOperator":"GreaterThanThreshold", - "Dimensions":[ - { - "Name": "QueueName", - "Value": {"Fn::GetAtt": ["SumoCWDeadLetterQueue", "QueueName"]} - } - ], - "EvaluationPeriods":"1", - "MetricName":"ApproximateNumberOfMessagesVisible", - "Namespace":"AWS/SQS", - "Period":"3600", - "Statistic":"Sum", - "Threshold":"100000" - }, - "DependsOn": ["SumoCWEmailSNSTopic"] - } - }, - "Outputs": { - "SumoCWLogsLambdaArn" : { - "Description": "The ARN of the sumologic cloudwatch logs lambda", - "Value" : { "Fn::GetAtt" : ["SumoCWLogsLambda", "Arn"] }, - "Export" : { - "Name" : { "Fn::Join": [ "-", [ "SumoCWLogsLambdaArn", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] } - - } - } - } -} - diff --git a/cloudwatchlogs-with-dlq/dlq_lambda_cloudformation.template.yaml b/cloudwatchlogs-with-dlq/dlq_lambda_cloudformation.template.yaml new file mode 100644 index 0000000..0253ba5 --- /dev/null +++ b/cloudwatchlogs-with-dlq/dlq_lambda_cloudformation.template.yaml @@ -0,0 +1,353 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: Sumo Logic CloudWatch log collector +Parameters: + SumoEndPointURL: + Type: String + Default: + Description: Enter SUMO_ENDPOINT created while configuring HTTP Source + EmailID: + Type: String + Default: test@gmail.com + Description: Enter your email for receiving alerts.You will receive confirmation email after the deployment is complete, confirm it to subscribe for alerts. + NumOfWorkers: + Type: Number + Default: 4 + Description: Enter the number of lambda function invocations for faster Dead Letter Queue processing. + LogFormat: + Type: String + Default: Others + AllowedValues: + - VPC-RAW + - VPC-JSON + - Others + Description: Choose the Service + IncludeLogGroupInfo: + Type: String + Default: 'false' + AllowedValues: + - 'true' + - 'false' + Description: Select true to get loggroup/logstream values in logs + LogStreamPrefix: + Type: String + Description: (Optional) Enter comma separated list of logStream name prefixes to filter by logStream. Please note this is seperate from a logGroup. This is used to only send certain logStreams within a cloudwatch logGroup(s). LogGroups still need to be subscribed to the created Lambda funciton, regardless of what is input for this value. + Default: '' +Mappings: + RegionMap: + us-east-1: + bucketname: appdevzipfiles-us-east-1 + us-east-2: + bucketname: appdevzipfiles-us-east-2 + us-west-1: + bucketname: appdevzipfiles-us-west-1 + us-west-2: + bucketname: appdevzipfiles-us-west-2 + ap-south-1: + bucketname: appdevzipfiles-ap-south-1 + ap-northeast-2: + bucketname: appdevzipfiles-ap-northeast-2 + ap-southeast-1: + bucketname: appdevzipfiles-ap-southeast-1 + ap-southeast-2: + bucketname: appdevzipfiles-ap-southeast-2 + ap-northeast-1: + bucketname: appdevzipfiles-ap-northeast-1 + ap-east-1: + bucketname: appdevzipfiles-ap-east-1s + af-south-1: + bucketname: appdevzipfiles-af-south-1s + ca-central-1: + bucketname: appdevzipfiles-ca-central-1 + eu-central-1: + bucketname: appdevzipfiles-eu-central-1 + eu-west-1: + bucketname: appdevzipfiles-eu-west-1 + eu-west-2: + bucketname: appdevzipfiles-eu-west-2 + eu-west-3: + bucketname: appdevzipfiles-eu-west-3 + eu-north-1: + bucketname: appdevzipfiles-eu-north-1s + eu-south-1: + bucketname: appdevzipfiles-eu-south-1 + me-south-1: + bucketname: appdevzipfiles-me-south-1s + sa-east-1: + bucketname: appdevzipfiles-sa-east-1 + me-central-1: + bucketname: appdevzipfiles-me-central-1 + eu-central-2: + bucketname: appdevzipfiles-eu-central-2ss + ap-northeast-3: + bucketname: appdevzipfiles-ap-northeast-3s + ap-southeast-3: + bucketname: appdevzipfiles-ap-southeast-3 +Resources: + SumoCWLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Join + - '-' + - - SumoCWLogGroup + - !Select + - '2' + - !Split + - / + - !Ref AWS::StackId + RetentionInDays: 7 + SumoCWLogSubsriptionFilter: + Type: AWS::Logs::SubscriptionFilter + Properties: + LogGroupName: !Ref SumoCWLogGroup + DestinationArn: !GetAtt SumoCWLogsLambda.Arn + FilterPattern: '' + DependsOn: + - SumoCWLogGroup + - SumoCWLambdaPermission + - SumoCWLogsLambda + SumoCWLambdaPermission: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !GetAtt SumoCWLogsLambda.Arn + Action: lambda:InvokeFunction + Principal: !Join + - . + - - logs + - !Ref AWS::Region + - amazonaws.com + SourceAccount: !Ref AWS::AccountId + SumoCWDeadLetterQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: !Join + - '-' + - - SumoCWDeadLetterQueue + - !Select + - '2' + - !Split + - / + - !Ref AWS::StackId + SumoCWLambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + Path: / + Policies: + - PolicyName: !Join + - '-' + - - SQSCreateLogsRolePolicy + - !Select + - '2' + - !Split + - / + - !Ref AWS::StackId + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - sqs:DeleteMessage + - sqs:GetQueueUrl + - sqs:ListQueues + - sqs:ChangeMessageVisibility + - sqs:SendMessageBatch + - sqs:ReceiveMessage + - sqs:SendMessage + - sqs:GetQueueAttributes + - sqs:ListQueueTags + - sqs:ListDeadLetterSourceQueues + - sqs:DeleteMessageBatch + - sqs:PurgeQueue + - sqs:DeleteQueue + - sqs:CreateQueue + - sqs:ChangeMessageVisibilityBatch + - sqs:SetQueueAttributes + Resource: + - !GetAtt SumoCWDeadLetterQueue.Arn + - PolicyName: !Join + - '-' + - - CloudWatchCreateLogsRolePolicy + - !Select + - '2' + - !Split + - / + - !Ref AWS::StackId + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - logs:DescribeLogStreams + Resource: + - !Join + - ':' + - - arn + - !Ref AWS::Partition + - logs + - !Ref AWS::Region + - !Ref AWS::AccountId + - log-group + - '*' + - PolicyName: InvokeLambdaRolePolicy + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - lambda:InvokeFunction + Resource: + - !Join + - ':' + - - arn + - !Ref AWS::Partition + - lambda + - !Ref AWS::Region + - !Ref AWS::AccountId + - function + - !Join + - '-' + - - SumoCWProcessDLQLambda + - !Select + - '2' + - !Split + - / + - !Ref AWS::StackId + SumoCWLogsLambda: + Type: AWS::Lambda::Function + DependsOn: + - SumoCWLambdaExecutionRole + - SumoCWDeadLetterQueue + Properties: + FunctionName: !Join + - '-' + - - SumoCWLogsLambda + - !Select + - '2' + - !Split + - / + - !Ref AWS::StackId + Code: + S3Bucket: !FindInMap + - RegionMap + - !Ref AWS::Region + - bucketname + S3Key: sumologic-aws-observability/functions/cloudwatch-logs-dlq/v1.4.0/cloudwatchlogs-with-dlq.zip + Role: !GetAtt SumoCWLambdaExecutionRole.Arn + Timeout: 300 + DeadLetterConfig: + TargetArn: !GetAtt SumoCWDeadLetterQueue.Arn + Handler: cloudwatchlogs_lambda.handler + Runtime: nodejs22.x + MemorySize: 128 + Environment: + Variables: + SUMO_ENDPOINT: !Ref SumoEndPointURL + LOG_FORMAT: !Ref LogFormat + INCLUDE_LOG_INFO: !Ref IncludeLogGroupInfo + LOG_STREAM_PREFIX: !Ref LogStreamPrefix + SumoCWEventsInvokeLambdaPermission: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref SumoCWProcessDLQLambda + Action: lambda:InvokeFunction + Principal: events.amazonaws.com + SourceArn: !GetAtt SumoCWProcessDLQScheduleRule.Arn + SumoCWProcessDLQScheduleRule: + Type: AWS::Events::Rule + Properties: + Description: Events rule for Cron + ScheduleExpression: rate(5 minutes) + State: ENABLED + Targets: + - Arn: !GetAtt SumoCWProcessDLQLambda.Arn + Id: TargetFunctionV1 + SumoCWProcessDLQLambda: + Type: AWS::Lambda::Function + DependsOn: + - SumoCWLambdaExecutionRole + - SumoCWDeadLetterQueue + Properties: + FunctionName: !Join + - '-' + - - SumoCWProcessDLQLambda + - !Select + - '2' + - !Split + - / + - !Ref AWS::StackId + Code: + S3Bucket: !FindInMap + - RegionMap + - !Ref AWS::Region + - bucketname + S3Key: sumologic-aws-observability/functions/cloudwatch-logs-dlq/v1.4.0/cloudwatchlogs-with-dlq.zip + Role: !GetAtt SumoCWLambdaExecutionRole.Arn + Timeout: 300 + Handler: DLQProcessor.handler + DeadLetterConfig: + TargetArn: !GetAtt SumoCWDeadLetterQueue.Arn + Runtime: nodejs22.x + MemorySize: 128 + Environment: + Variables: + SUMO_ENDPOINT: !Ref SumoEndPointURL + TASK_QUEUE_URL: !Join + - '' + - - https://sqs. + - !Ref AWS::Region + - .amazonaws.com/ + - !Ref AWS::AccountId + - / + - !GetAtt SumoCWDeadLetterQueue.QueueName + NUM_OF_WORKERS: !Ref NumOfWorkers + LOG_FORMAT: !Ref LogFormat + INCLUDE_LOG_INFO: !Ref IncludeLogGroupInfo + LOG_STREAM_PREFIX: !Ref LogStreamPrefix + SumoCWEmailSNSTopic: + Type: AWS::SNS::Topic + Properties: + Subscription: + - Endpoint: !Ref EmailID + Protocol: email + SumoCWSpilloverAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmActions: + - !Ref SumoCWEmailSNSTopic + AlarmDescription: Notify via email if number of messages in DeadLetterQueue exceeds threshold + ComparisonOperator: GreaterThanThreshold + Dimensions: + - Name: QueueName + Value: !GetAtt SumoCWDeadLetterQueue.QueueName + EvaluationPeriods: '1' + MetricName: ApproximateNumberOfMessagesVisible + Namespace: AWS/SQS + Period: '3600' + Statistic: Sum + Threshold: '100000' + DependsOn: + - SumoCWEmailSNSTopic +Outputs: + SumoCWLogsLambdaArn: + Description: The ARN of the sumologic cloudwatch logs lambda + Value: !GetAtt SumoCWLogsLambda.Arn + Export: + Name: !Join + - '-' + - - SumoCWLogsLambdaArn + - !Select + - '2' + - !Split + - / + - !Ref AWS::StackId diff --git a/upload_artifacts.py b/upload_artifacts.py index 449b458..0638276 100644 --- a/upload_artifacts.py +++ b/upload_artifacts.py @@ -62,10 +62,10 @@ def get_bucket_name(region): return region_map[region] -def upload_code_in_multiple_regions(filepath, bucket_prefix): +def upload_code_in_multiple_regions(filepath, bucket_prefix, s3_key_prefix=""): for region in regions: - upload_code_in_S3(filepath, get_bucket_name(region), region) + upload_code_in_S3(filepath, get_bucket_name(region), region, s3_key_prefix) def create_buckets(bucket_prefix): @@ -87,11 +87,12 @@ def create_buckets(bucket_prefix): -def upload_code_in_S3(filepath, bucket_name, region): +def upload_code_in_S3(filepath, bucket_name, region, s3_key_prefix=""): print("Uploading zip file in S3", region) s3 = boto3.client('s3', region) filename = os.path.basename(filepath) - s3.upload_file(filepath, bucket_name, filename, + s3_key = s3_key_prefix + filename if s3_key_prefix else filename + s3.upload_file(filepath, bucket_name, s3_key, ExtraArgs={'ACL': 'public-read'}) @@ -115,6 +116,9 @@ def upload_cftemplate(templatepath, bucket_name, region='us-east-1'): parser.add_argument("-d", "--deployment", dest="deployment", default="dev", help="aws account type") + parser.add_argument("-p", "--s3prefix", dest="s3prefix", default="", + help="S3 key prefix path for the zip file (e.g. sumologic-aws-observability/functions/cloudwatch-logs-dlq/v1.4.0/)") + args = parser.parse_args() if args.deployment == "prod": zip_bucket_prefix = "appdevzipfiles" @@ -135,6 +139,6 @@ def upload_cftemplate(templatepath, bucket_name, region='us-east-1'): if not os.path.isfile(args.zipfile): raise Exception("zipfile does not exists") else: - upload_code_in_multiple_regions(args.zipfile, zip_bucket_prefix) + upload_code_in_multiple_regions(args.zipfile, zip_bucket_prefix, args.s3prefix) print("Deployment Successfull: ALL files copied to %s" % args.deployment) From 7bdb59fd857bdf769c7d546d44caf33bf9b402f2 Mon Sep 17 00:00:00 2001 From: Md Azhar Date: Mon, 8 Jun 2026 17:44:33 +0530 Subject: [PATCH 2/4] keeping json files for backward compatibility --- .../DLQLambdaCloudFormation.json | 365 ++++++++++++++++ ...mbdaCloudFormationWithSecuredEndpoint.json | 391 ++++++++++++++++++ 2 files changed, 756 insertions(+) create mode 100644 cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json create mode 100644 cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json diff --git a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json new file mode 100644 index 0000000..d0cc50e --- /dev/null +++ b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json @@ -0,0 +1,365 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Description" : "Sumo Logic CloudWatch log collector", + "Parameters" : { + "SumoEndPointURL" : { + "Type" : "String", + "Default" : "", + "Description" : "Enter SUMO_ENDPOINT created while configuring HTTP Source" + }, + "EmailID": { + "Type": "String", + "Default": "test@gmail.com", + "Description": "Enter your email for receiving alerts.You will receive confirmation email after the deployment is complete, confirm it to subscribe for alerts." + }, + "NumOfWorkers": { + "Type": "Number", + "Default": 4, + "Description": "Enter the number of lambda function invocations for faster Dead Letter Queue processing." + }, + "LogFormat": { + "Type": "String", + "Default": "Others", + "AllowedValues" : ["VPC-RAW" ,"VPC-JSON", "Others"], + "Description": "Choose the Service" + }, + "IncludeLogGroupInfo": { + "Type": "String", + "Default": "false", + "AllowedValues" : ["true" ,"false"], + "Description": "Select true to get loggroup/logstream values in logs" + }, + "LogStreamPrefix": { + "Type": "String", + "Description": "(Optional) Enter comma separated list of logStream name prefixes to filter by logStream. Please note this is seperate from a logGroup. This is used to only send certain logStreams within a cloudwatch logGroup(s). LogGroups still need to be subscribed to the created Lambda funciton, regardless of what is input for this value.", + "Default": "" + } + }, + "Mappings" : { + "RegionMap" : { + "us-east-1": {"bucketname": "appdevzipfiles-us-east-1"}, + "us-east-2": {"bucketname": "appdevzipfiles-us-east-2"}, + "us-west-1": {"bucketname": "appdevzipfiles-us-west-1"}, + "us-west-2": {"bucketname": "appdevzipfiles-us-west-2"}, + "ap-south-1": {"bucketname": "appdevzipfiles-ap-south-1"}, + "ap-northeast-2": {"bucketname": "appdevzipfiles-ap-northeast-2"}, + "ap-southeast-1": {"bucketname": "appdevzipfiles-ap-southeast-1"}, + "ap-southeast-2": {"bucketname": "appdevzipfiles-ap-southeast-2"}, + "ap-northeast-1": {"bucketname": "appdevzipfiles-ap-northeast-1"}, + "ap-east-1": {"bucketname": "appdevzipfiles-ap-east-1s"}, + "af-south-1": {"bucketname": "appdevzipfiles-af-south-1s"}, + "ca-central-1": {"bucketname": "appdevzipfiles-ca-central-1"}, + "eu-central-1": {"bucketname": "appdevzipfiles-eu-central-1"}, + "eu-west-1": {"bucketname": "appdevzipfiles-eu-west-1"}, + "eu-west-2": {"bucketname": "appdevzipfiles-eu-west-2"}, + "eu-west-3": {"bucketname": "appdevzipfiles-eu-west-3"}, + "eu-north-1": {"bucketname": "appdevzipfiles-eu-north-1s"}, + "eu-south-1": {"bucketname": "appdevzipfiles-eu-south-1"}, + "me-south-1": {"bucketname": "appdevzipfiles-me-south-1s"}, + "sa-east-1": {"bucketname": "appdevzipfiles-sa-east-1"}, + "me-central-1": {"bucketname": "appdevzipfiles-me-central-1"}, + "eu-central-2": {"bucketname": "appdevzipfiles-eu-central-2ss"}, + "ap-northeast-3": {"bucketname": "appdevzipfiles-ap-northeast-3s"}, + "ap-southeast-3": {"bucketname": "appdevzipfiles-ap-southeast-3"} + } + }, + "Resources": { + "SumoCWLogGroup": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "LogGroupName": { "Fn::Join": [ "-", [ "SumoCWLogGroup", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, + "RetentionInDays": 7 + } + }, + "SumoCWLogSubsriptionFilter": { + "Type": "AWS::Logs::SubscriptionFilter", + "Properties": { + "LogGroupName": { + "Ref": "SumoCWLogGroup" + }, + "DestinationArn": { + "Fn::GetAtt": [ + "SumoCWLogsLambda", + "Arn" + ] + }, + "FilterPattern": "" + }, + "DependsOn": [ + "SumoCWLogGroup", + "SumoCWLambdaPermission", + "SumoCWLogsLambda" + ] + }, + "SumoCWLambdaPermission": { + + "Type": "AWS::Lambda::Permission", + + "Properties": { + + "FunctionName": { + "Fn::GetAtt": [ + "SumoCWLogsLambda", + "Arn" + ] + }, + "Action": "lambda:InvokeFunction", + + "Principal": { "Fn::Join": [ ".", [ "logs", { "Ref": "AWS::Region" }, "amazonaws.com" ] ] }, + + "SourceAccount": { "Ref": "AWS::AccountId" } + } + }, + "SumoCWDeadLetterQueue": { + "Type": "AWS::SQS::Queue", + "Properties": { + "QueueName": { "Fn::Join": [ "-", [ "SumoCWDeadLetterQueue", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] } + } + }, + "SumoCWLambdaExecutionRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": ["lambda.amazonaws.com"] }, + "Action": ["sts:AssumeRole"] + } ] + }, + "Path": "/", + "Policies": [ + { + "PolicyName": { "Fn::Join": [ "-", [ "SQSCreateLogsRolePolicy", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:ListQueues", + "sqs:ChangeMessageVisibility", + "sqs:SendMessageBatch", + "sqs:ReceiveMessage", + "sqs:SendMessage", + "sqs:GetQueueAttributes", + "sqs:ListQueueTags", + "sqs:ListDeadLetterSourceQueues", + "sqs:DeleteMessageBatch", + "sqs:PurgeQueue", + "sqs:DeleteQueue", + "sqs:CreateQueue", + "sqs:ChangeMessageVisibilityBatch", + "sqs:SetQueueAttributes" + ], + "Resource": [ + { + "Fn::GetAtt": [ + "SumoCWDeadLetterQueue", + "Arn" + ] + } + ] + }] + } + }, + { + "PolicyName": { "Fn::Join": [ "-", [ "CloudWatchCreateLogsRolePolicy", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:DescribeLogStreams" + ], + "Resource": [ + { "Fn::Join": [ ":", [ "arn", { "Ref" : "AWS::Partition" }, "logs", { "Ref" : "AWS::Region" }, { "Ref" : "AWS::AccountId" },"log-group","*" ] ] } + ] + }] + } + }, + { + "PolicyName": "InvokeLambdaRolePolicy", + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "lambda:InvokeFunction" + ], + "Resource": [ + { "Fn::Join": [ ":", [ "arn", { "Ref" : "AWS::Partition" }, "lambda", { "Ref" : "AWS::Region" }, { "Ref" : "AWS::AccountId" }, "function", { "Fn::Join": [ "-", [ "SumoCWProcessDLQLambda", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] } ] ] } + ] + }] + } + } + ] + } + }, + "SumoCWLogsLambda": { + "Type": "AWS::Lambda::Function", + "DependsOn": [ + "SumoCWLambdaExecutionRole", + "SumoCWDeadLetterQueue" + ], + "Properties": { + "FunctionName": { "Fn::Join": [ "-", [ "SumoCWLogsLambda", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, + "Code": { + "S3Bucket": { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "bucketname"]}, + "S3Key": "sumologic-aws-observability/functions/cloudwatch-logs-dlq/v1.4.0/cloudwatchlogs-with-dlq.zip" + }, + "Role": { + "Fn::GetAtt": [ + "SumoCWLambdaExecutionRole", + "Arn" + ] + }, + "Timeout": 300, + "DeadLetterConfig": { + "TargetArn" : { + "Fn::GetAtt": [ + "SumoCWDeadLetterQueue", + "Arn" + ] + } + }, + "Handler": "cloudwatchlogs_lambda.handler", + "Runtime": "nodejs24.x", + "MemorySize": 128, + "Environment": { + "Variables": { + "SUMO_ENDPOINT": {"Ref": "SumoEndPointURL"}, + "LOG_FORMAT": {"Ref": "LogFormat"}, + "INCLUDE_LOG_INFO": {"Ref": "IncludeLogGroupInfo"}, + "LOG_STREAM_PREFIX": {"Ref": "LogStreamPrefix"} + + } + } + } + }, + "SumoCWEventsInvokeLambdaPermission": { + "Type": "AWS::Lambda::Permission", + "Properties": { + "FunctionName": { "Ref": "SumoCWProcessDLQLambda" }, + "Action": "lambda:InvokeFunction", + "Principal": "events.amazonaws.com", + "SourceArn": { "Fn::GetAtt": ["SumoCWProcessDLQScheduleRule", "Arn"] } + } + }, + "SumoCWProcessDLQScheduleRule": { + "Type": "AWS::Events::Rule", + "Properties": { + "Description": "Events rule for Cron", + "ScheduleExpression": "rate(5 minutes)", + "State": "ENABLED", + "Targets": [{ + "Arn": { "Fn::GetAtt": ["SumoCWProcessDLQLambda", "Arn"] }, + "Id": "TargetFunctionV1" + }] + } + }, + "SumoCWProcessDLQLambda": { + "Type": "AWS::Lambda::Function", + "DependsOn": [ + "SumoCWLambdaExecutionRole", + "SumoCWDeadLetterQueue" + ], + "Properties": { + "FunctionName": { "Fn::Join": [ "-", [ "SumoCWProcessDLQLambda", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, + "Code": { + "S3Bucket": { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "bucketname"]}, + "S3Key": "sumologic-aws-observability/functions/cloudwatch-logs-dlq/v1.4.0/cloudwatchlogs-with-dlq.zip" + }, + "Role": { + "Fn::GetAtt": [ + "SumoCWLambdaExecutionRole", + "Arn" + ] + }, + "Timeout": 300, + "Handler": "DLQProcessor.handler", + "DeadLetterConfig": { + "TargetArn" : { + "Fn::GetAtt": [ + "SumoCWDeadLetterQueue", + "Arn" + ] + } + }, + "Runtime": "nodejs24.x", + "MemorySize": 128, + "Environment": { + "Variables": { + "SUMO_ENDPOINT": {"Ref": "SumoEndPointURL"}, + "TASK_QUEUE_URL": { + "Fn::Join": [ + "", + [ + "https://sqs.", + { "Ref" : "AWS::Region" }, + ".amazonaws.com/", + { "Ref" : "AWS::AccountId" }, + "/", + {"Fn::GetAtt": ["SumoCWDeadLetterQueue", "QueueName"]} + ] + ] + }, + "NUM_OF_WORKERS": {"Ref": "NumOfWorkers"}, + "LOG_FORMAT": {"Ref": "LogFormat"}, + "INCLUDE_LOG_INFO": {"Ref": "IncludeLogGroupInfo"}, + "LOG_STREAM_PREFIX": {"Ref": "LogStreamPrefix"} + } + } + } + }, + "SumoCWEmailSNSTopic": { + "Type":"AWS::SNS::Topic", + "Properties":{ + "Subscription":[ { + "Endpoint" : {"Ref": "EmailID"}, + "Protocol" : "email" + }] + } + }, + "SumoCWSpilloverAlarm":{ + "Type":"AWS::CloudWatch::Alarm", + "Properties":{ + "AlarmActions":[ + { + "Ref":"SumoCWEmailSNSTopic" + } + ], + "AlarmDescription":"Notify via email if number of messages in DeadLetterQueue exceeds threshold", + "ComparisonOperator":"GreaterThanThreshold", + "Dimensions":[ + { + "Name": "QueueName", + "Value": {"Fn::GetAtt": ["SumoCWDeadLetterQueue", "QueueName"]} + } + ], + "EvaluationPeriods":"1", + "MetricName":"ApproximateNumberOfMessagesVisible", + "Namespace":"AWS/SQS", + "Period":"3600", + "Statistic":"Sum", + "Threshold":"100000" + }, + "DependsOn": ["SumoCWEmailSNSTopic"] + } + }, + "Outputs": { + "SumoCWLogsLambdaArn" : { + "Description": "The ARN of the sumologic cloudwatch logs lambda", + "Value" : { "Fn::GetAtt" : ["SumoCWLogsLambda", "Arn"] }, + "Export" : { + "Name" : { "Fn::Join": [ "-", [ "SumoCWLogsLambdaArn", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] } + + } + } + } +} + diff --git a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json new file mode 100644 index 0000000..505a428 --- /dev/null +++ b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json @@ -0,0 +1,391 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Description" : "Sumo Logic CloudWatch log collector", + "Parameters" : { + "SumoURLDecryptKeyID" : { + "Type" : "String", + "Default" : "", + "Description" : "Enter the KMS key ID used for encrypting SUMO_ENDPOINT" + }, + "EmailID": { + "Type": "String", + "Default": "test@gmail.com", + "Description": "Enter your email for receiving alerts.You will receive confirmation email after the deployment is complete, confirm it to subscribe for alerts." + }, + "NumOfWorkers": { + "Type": "Number", + "Default": 4, + "Description": "Enter the number of lambda function invocations for faster Dead Letter Queue processing." + }, + "LogFormat": { + "Type": "String", + "Default": "Others", + "AllowedValues" : ["VPC-RAW" ,"VPC-JSON", "Others"], + "Description": "Choose the Service" + }, + "IncludeLogGroupInfo": { + "Type": "String", + "Default": "false", + "AllowedValues" : ["true" ,"false"], + "Description": "Select true to get loggroup/logstream values in logs" + }, + "LogStreamPrefix": { + "Type": "String", + "Description": "(Optional) Enter comma separated list of logStream name prefixes to filter by logStream. Please note this is seperate from a logGroup. This is used to only send certain logStreams within a cloudwatch logGroup(s). LogGroups still need to be subscribed to the created Lambda funciton, regardless of what is input for this value.", + "Default": "" + } + }, + "Mappings" : { + "RegionMap" : { + "us-east-1": {"bucketname": "appdevzipfiles-us-east-1"}, + "us-east-2": {"bucketname": "appdevzipfiles-us-east-2"}, + "us-west-1": {"bucketname": "appdevzipfiles-us-west-1"}, + "us-west-2": {"bucketname": "appdevzipfiles-us-west-2"}, + "ap-south-1": {"bucketname": "appdevzipfiles-ap-south-1"}, + "ap-northeast-2": {"bucketname": "appdevzipfiles-ap-northeast-2"}, + "ap-southeast-1": {"bucketname": "appdevzipfiles-ap-southeast-1"}, + "ap-southeast-2": {"bucketname": "appdevzipfiles-ap-southeast-2"}, + "ap-northeast-1": {"bucketname": "appdevzipfiles-ap-northeast-1"}, + "ap-east-1": {"bucketname": "appdevzipfiles-ap-east-1s"}, + "af-south-1": {"bucketname": "appdevzipfiles-af-south-1s"}, + "ca-central-1": {"bucketname": "appdevzipfiles-ca-central-1"}, + "eu-central-1": {"bucketname": "appdevzipfiles-eu-central-1"}, + "eu-west-1": {"bucketname": "appdevzipfiles-eu-west-1"}, + "eu-west-2": {"bucketname": "appdevzipfiles-eu-west-2"}, + "eu-west-3": {"bucketname": "appdevzipfiles-eu-west-3"}, + "eu-north-1": {"bucketname": "appdevzipfiles-eu-north-1s"}, + "eu-south-1": {"bucketname": "appdevzipfiles-eu-south-1"}, + "me-south-1": {"bucketname": "appdevzipfiles-me-south-1s"}, + "sa-east-1": {"bucketname": "appdevzipfiles-sa-east-1"}, + "me-central-1": {"bucketname": "appdevzipfiles-me-central-1"}, + "eu-central-2": {"bucketname": "appdevzipfiles-eu-central-2ss"}, + "ap-northeast-3": {"bucketname": "appdevzipfiles-ap-northeast-3s"}, + "ap-southeast-3": {"bucketname": "appdevzipfiles-ap-southeast-3"} + } + }, + "Resources": { + "SumoCWLogGroup": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "LogGroupName": { "Fn::Join": [ "-", [ "SumoCWLogGroup", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, + "RetentionInDays": 7 + } + }, + "SumoCWLogSubsriptionFilter": { + "Type": "AWS::Logs::SubscriptionFilter", + "Properties": { + "LogGroupName": { + "Ref": "SumoCWLogGroup" + }, + "DestinationArn": { + "Fn::GetAtt": [ + "SumoCWLogsLambda", + "Arn" + ] + }, + "FilterPattern": "" + }, + "DependsOn": [ + "SumoCWLogGroup", + "SumoCWLambdaPermission", + "SumoCWLogsLambda" + ] + }, + "SumoCWLambdaPermission": { + + "Type": "AWS::Lambda::Permission", + + "Properties": { + + "FunctionName": { + "Fn::GetAtt": [ + "SumoCWLogsLambda", + "Arn" + ] + }, + "Action": "lambda:InvokeFunction", + + "Principal": { "Fn::Join": [ ".", [ "logs", { "Ref": "AWS::Region" }, "amazonaws.com" ] ] }, + + "SourceAccount": { "Ref": "AWS::AccountId" } + } + }, + "SumoCWDeadLetterQueue": { + "Type": "AWS::SQS::Queue", + "Properties": { + "QueueName": { "Fn::Join": [ "-", [ "SumoCWDeadLetterQueue", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] } + } + }, + "SumoCWLambdaExecutionRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": ["lambda.amazonaws.com"] }, + "Action": ["sts:AssumeRole"] + } ] + }, + "Path": "/", + "Policies": [ + { + "PolicyName": { "Fn::Join": [ "-", [ "SQSCreateLogsRolePolicy", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:ListQueues", + "sqs:ChangeMessageVisibility", + "sqs:SendMessageBatch", + "sqs:ReceiveMessage", + "sqs:SendMessage", + "sqs:GetQueueAttributes", + "sqs:ListQueueTags", + "sqs:ListDeadLetterSourceQueues", + "sqs:DeleteMessageBatch", + "sqs:PurgeQueue", + "sqs:DeleteQueue", + "sqs:CreateQueue", + "sqs:ChangeMessageVisibilityBatch", + "sqs:SetQueueAttributes" + ], + "Resource": [ + { + "Fn::GetAtt": [ + "SumoCWDeadLetterQueue", + "Arn" + ] + } + ] + }] + } + }, + { + "PolicyName": { "Fn::Join": [ "-", [ "CloudWatchCreateLogsRolePolicy", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:DescribeLogStreams" + ], + "Resource": [ + { "Fn::Join": [ ":", ["arn", { "Ref" : "AWS::Partition" }, "logs", { "Ref" : "AWS::Region" }, { "Ref" : "AWS::AccountId" },"log-group","*" ] ] } + ] + }] + } + }, + { + "PolicyName": "InvokeLambdaRolePolicy", + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "lambda:InvokeFunction" + ], + "Resource": [ + { "Fn::Join": [ ":", [ "arn", { "Ref" : "AWS::Partition" }, "lambda", { "Ref" : "AWS::Region" }, { "Ref" : "AWS::AccountId" }, "function", { "Fn::Join": [ "-", [ "SumoCWProcessDLQLambda", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] } ] ] } + ] + }] + } + }, + { + "PolicyName": { "Fn::Join": [ "-", [ "SSMSumoLambdaPolicy", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ssm:GetParameter" + ], + "Resource": { + "Fn::Join": [ ":", [ "arn", { "Ref" : "AWS::Partition" }, "ssm", { "Ref" : "AWS::Region" }, { "Ref" : "AWS::AccountId" }, "parameter/SUMO_ENDPOINT"] + ] + } + }, + { + "Effect": "Allow", + "Action": [ + "kms:Decrypt" + ], + "Resource": { + "Fn::Join": [ ":", [ "arn", { "Ref" : "AWS::Partition" }, "kms", { "Ref" : "AWS::Region" }, { "Ref" : "AWS::AccountId" }, { "Fn::Sub": [ "key/${KeyID}", { "KeyID": {"Ref" : "SumoURLDecryptKeyID" }} ]}] + ] + } + } + ] + } + } + ] + } + }, + "SumoCWLogsLambda": { + "Type": "AWS::Lambda::Function", + "DependsOn": [ + "SumoCWLambdaExecutionRole", + "SumoCWDeadLetterQueue" + ], + "Properties": { + "FunctionName": { "Fn::Join": [ "-", [ "SumoCWLogsLambda", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, + "Code": { + "S3Bucket": { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "bucketname"]}, + "S3Key": "sumologic-aws-observability/functions/cloudwatch-logs-dlq/v1.4.0/cloudwatchlogs-with-dlq.zip" + }, + "Role": { + "Fn::GetAtt": [ + "SumoCWLambdaExecutionRole", + "Arn" + ] + }, + "Timeout": 300, + "DeadLetterConfig": { + "TargetArn" : { + "Fn::GetAtt": [ + "SumoCWDeadLetterQueue", + "Arn" + ] + } + }, + "Handler": "cloudwatchlogs_lambda.handler", + "Runtime": "nodejs24.x", + "MemorySize": 128, + "Environment": { + "Variables": { + "LOG_FORMAT": {"Ref": "LogFormat"}, + "INCLUDE_LOG_INFO": {"Ref": "IncludeLogGroupInfo"}, + "LOG_STREAM_PREFIX": {"Ref": "LogStreamPrefix"} + + } + } + } + }, + "SumoCWEventsInvokeLambdaPermission": { + "Type": "AWS::Lambda::Permission", + "Properties": { + "FunctionName": { "Ref": "SumoCWProcessDLQLambda" }, + "Action": "lambda:InvokeFunction", + "Principal": "events.amazonaws.com", + "SourceArn": { "Fn::GetAtt": ["SumoCWProcessDLQScheduleRule", "Arn"] } + } + }, + "SumoCWProcessDLQScheduleRule": { + "Type": "AWS::Events::Rule", + "Properties": { + "Description": "Events rule for Cron", + "ScheduleExpression": "rate(5 minutes)", + "State": "ENABLED", + "Targets": [{ + "Arn": { "Fn::GetAtt": ["SumoCWProcessDLQLambda", "Arn"] }, + "Id": "TargetFunctionV1" + }] + } + }, + "SumoCWProcessDLQLambda": { + "Type": "AWS::Lambda::Function", + "DependsOn": [ + "SumoCWLambdaExecutionRole", + "SumoCWDeadLetterQueue" + ], + "Properties": { + "FunctionName": { "Fn::Join": [ "-", [ "SumoCWProcessDLQLambda", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, + "Code": { + "S3Bucket": { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "bucketname"]}, + "S3Key": "sumologic-aws-observability/functions/cloudwatch-logs-dlq/v1.4.0/cloudwatchlogs-with-dlq.zip" + }, + "Role": { + "Fn::GetAtt": [ + "SumoCWLambdaExecutionRole", + "Arn" + ] + }, + "Timeout": 300, + "Handler": "DLQProcessor.handler", + "DeadLetterConfig": { + "TargetArn" : { + "Fn::GetAtt": [ + "SumoCWDeadLetterQueue", + "Arn" + ] + } + }, + "Runtime": "nodejs24.x", + "MemorySize": 128, + "Environment": { + "Variables": { + "TASK_QUEUE_URL": { + "Fn::Join": [ + "", + [ + "https://sqs.", + { "Ref" : "AWS::Region" }, + ".amazonaws.com/", + { "Ref" : "AWS::AccountId" }, + "/", + {"Fn::GetAtt": ["SumoCWDeadLetterQueue", "QueueName"]} + ] + ] + }, + "NUM_OF_WORKERS": {"Ref": "NumOfWorkers"}, + "LOG_FORMAT": {"Ref": "LogFormat"}, + "INCLUDE_LOG_INFO": {"Ref": "IncludeLogGroupInfo"}, + "LOG_STREAM_PREFIX": {"Ref": "LogStreamPrefix"} + } + } + } + }, + "SumoCWEmailSNSTopic": { + "Type":"AWS::SNS::Topic", + "Properties":{ + "Subscription":[ { + "Endpoint" : {"Ref": "EmailID"}, + "Protocol" : "email" + }] + } + }, + "SumoCWSpilloverAlarm":{ + "Type":"AWS::CloudWatch::Alarm", + "Properties":{ + "AlarmActions":[ + { + "Ref":"SumoCWEmailSNSTopic" + } + ], + "AlarmDescription":"Notify via email if number of messages in DeadLetterQueue exceeds threshold", + "ComparisonOperator":"GreaterThanThreshold", + "Dimensions":[ + { + "Name": "QueueName", + "Value": {"Fn::GetAtt": ["SumoCWDeadLetterQueue", "QueueName"]} + } + ], + "EvaluationPeriods":"1", + "MetricName":"ApproximateNumberOfMessagesVisible", + "Namespace":"AWS/SQS", + "Period":"3600", + "Statistic":"Sum", + "Threshold":"100000" + }, + "DependsOn": ["SumoCWEmailSNSTopic"] + } + }, + "Outputs": { + "SumoCWLogsLambdaArn" : { + "Description": "The ARN of the sumologic cloudwatch logs lambda", + "Value" : { "Fn::GetAtt" : ["SumoCWLogsLambda", "Arn"] }, + "Export" : { + "Name" : { "Fn::Join": [ "-", [ "SumoCWLogsLambdaArn", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] } + + } + } + } +} + From 1081fcdc03919bd625d0079c5f00f36b3c831694 Mon Sep 17 00:00:00 2001 From: Md Azhar Date: Mon, 8 Jun 2026 17:51:41 +0530 Subject: [PATCH 3/4] backward compatibility --- cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json | 4 ++-- .../DLQLambdaCloudFormationWithSecuredEndpoint.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json index d0cc50e..a6aeae4 100644 --- a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json +++ b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json @@ -210,7 +210,7 @@ "FunctionName": { "Fn::Join": [ "-", [ "SumoCWLogsLambda", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, "Code": { "S3Bucket": { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "bucketname"]}, - "S3Key": "sumologic-aws-observability/functions/cloudwatch-logs-dlq/v1.4.0/cloudwatchlogs-with-dlq.zip" + "S3Key": "cloudwatchLogsDLQ/v1.3.0/cloudwatchlogs-with-dlq.zip" }, "Role": { "Fn::GetAtt": [ @@ -272,7 +272,7 @@ "FunctionName": { "Fn::Join": [ "-", [ "SumoCWProcessDLQLambda", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, "Code": { "S3Bucket": { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "bucketname"]}, - "S3Key": "sumologic-aws-observability/functions/cloudwatch-logs-dlq/v1.4.0/cloudwatchlogs-with-dlq.zip" + "S3Key": "cloudwatchLogsDLQ/v1.3.0/cloudwatchlogs-with-dlq.zip" }, "Role": { "Fn::GetAtt": [ diff --git a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json index 505a428..6bdaadf 100644 --- a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json +++ b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json @@ -238,7 +238,7 @@ "FunctionName": { "Fn::Join": [ "-", [ "SumoCWLogsLambda", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, "Code": { "S3Bucket": { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "bucketname"]}, - "S3Key": "sumologic-aws-observability/functions/cloudwatch-logs-dlq/v1.4.0/cloudwatchlogs-with-dlq.zip" + "S3Key": "cloudwatchLogsDLQ/v1.3.0/cloudwatchlogs-with-dlq.zip" }, "Role": { "Fn::GetAtt": [ @@ -299,7 +299,7 @@ "FunctionName": { "Fn::Join": [ "-", [ "SumoCWProcessDLQLambda", { "Fn::Select" : [ "2", {"Fn::Split" : [ "/" , { "Ref": "AWS::StackId" } ]}] } ] ] }, "Code": { "S3Bucket": { "Fn::FindInMap" : [ "RegionMap", { "Ref" : "AWS::Region" }, "bucketname"]}, - "S3Key": "sumologic-aws-observability/functions/cloudwatch-logs-dlq/v1.4.0/cloudwatchlogs-with-dlq.zip" + "S3Key": "cloudwatchLogsDLQ/v1.3.0/cloudwatchlogs-with-dlq.zip" }, "Role": { "Fn::GetAtt": [ From 57975d8b87afaadb441e6beaa1ee1c058f9108ff Mon Sep 17 00:00:00 2001 From: Md Azhar Date: Tue, 9 Jun 2026 22:40:10 +0530 Subject: [PATCH 4/4] refractored with new regions --- .../guardduty/cloudwatchevents.json | 13 ++- .../DLQLambdaCloudFormation.json | 11 +- ...mbdaCloudFormationWithSecuredEndpoint.json | 11 +- cloudwatchlogs-with-dlq/deploy_cwl_lambda.py | 5 +- .../dlq_lambda_cloudformation.template.yaml | 104 +++++++++--------- upload_artifacts.py | 59 +++++----- 6 files changed, 117 insertions(+), 86 deletions(-) diff --git a/cloudwatchevents/guardduty/cloudwatchevents.json b/cloudwatchevents/guardduty/cloudwatchevents.json index d8d78b3..3fb8b5f 100644 --- a/cloudwatchevents/guardduty/cloudwatchevents.json +++ b/cloudwatchevents/guardduty/cloudwatchevents.json @@ -25,6 +25,8 @@ "ap-northeast-2": {"bucketname": "appdevzipfiles-ap-northeast-2"}, "ap-southeast-1": {"bucketname": "appdevzipfiles-ap-southeast-1"}, "ap-southeast-2": {"bucketname": "appdevzipfiles-ap-southeast-2"}, + "ap-southeast-4": {"bucketname": "appdevzipfiles-ap-southeast-4s"}, + "ap-southeast-6": {"bucketname": "appdevzipfiles-ap-southeast-6ss"}, "ap-northeast-1": {"bucketname": "appdevzipfiles-ap-northeast-1"}, "ca-central-1": {"bucketname": "appdevzipfiles-ca-central-1"}, "eu-central-1": {"bucketname": "appdevzipfiles-eu-central-1"}, @@ -32,7 +34,16 @@ "eu-west-2": {"bucketname": "appdevzipfiles-eu-west-2"}, "eu-west-3": {"bucketname": "appdevzipfiles-eu-west-3"}, "eu-north-1": {"bucketname": "appdevzipfiles-eu-north-1s"}, - "sa-east-1": {"bucketname": "appdevzipfiles-sa-east-1"} + "sa-east-1": {"bucketname": "appdevzipfiles-sa-east-1"}, + "ap-east-1": {"bucketname": "appdevzipfiles-ap-east-1s"}, + "af-south-1": {"bucketname": "appdevzipfiles-af-south-1s"}, + "eu-south-1": {"bucketname": "appdevzipfiles-eu-south-1"}, + "me-south-1": {"bucketname": "appdevzipfiles-me-south-1s"}, + "me-central-1": {"bucketname": "appdevzipfiles-me-central-1"}, + "eu-central-2": {"bucketname": "appdevzipfiles-eu-central-2ss"}, + "ap-northeast-3": {"bucketname": "appdevzipfiles-ap-northeast-3s"}, + "ap-southeast-3": {"bucketname": "appdevzipfiles-ap-southeast-3"}, + "il-central-1": {"bucketname": "appdevzipfiles-il-central-1"} } }, "Resources": { diff --git a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json index a6aeae4..15ce53e 100644 --- a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json +++ b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormation.json @@ -45,22 +45,25 @@ "ap-northeast-2": {"bucketname": "appdevzipfiles-ap-northeast-2"}, "ap-southeast-1": {"bucketname": "appdevzipfiles-ap-southeast-1"}, "ap-southeast-2": {"bucketname": "appdevzipfiles-ap-southeast-2"}, + "ap-southeast-4": {"bucketname": "appdevzipfiles-ap-southeast-4s"}, + "ap-southeast-6": {"bucketname": "appdevzipfiles-ap-southeast-6ss"}, "ap-northeast-1": {"bucketname": "appdevzipfiles-ap-northeast-1"}, - "ap-east-1": {"bucketname": "appdevzipfiles-ap-east-1s"}, - "af-south-1": {"bucketname": "appdevzipfiles-af-south-1s"}, "ca-central-1": {"bucketname": "appdevzipfiles-ca-central-1"}, "eu-central-1": {"bucketname": "appdevzipfiles-eu-central-1"}, "eu-west-1": {"bucketname": "appdevzipfiles-eu-west-1"}, "eu-west-2": {"bucketname": "appdevzipfiles-eu-west-2"}, "eu-west-3": {"bucketname": "appdevzipfiles-eu-west-3"}, "eu-north-1": {"bucketname": "appdevzipfiles-eu-north-1s"}, + "sa-east-1": {"bucketname": "appdevzipfiles-sa-east-1"}, + "ap-east-1": {"bucketname": "appdevzipfiles-ap-east-1s"}, + "af-south-1": {"bucketname": "appdevzipfiles-af-south-1s"}, "eu-south-1": {"bucketname": "appdevzipfiles-eu-south-1"}, "me-south-1": {"bucketname": "appdevzipfiles-me-south-1s"}, - "sa-east-1": {"bucketname": "appdevzipfiles-sa-east-1"}, "me-central-1": {"bucketname": "appdevzipfiles-me-central-1"}, "eu-central-2": {"bucketname": "appdevzipfiles-eu-central-2ss"}, "ap-northeast-3": {"bucketname": "appdevzipfiles-ap-northeast-3s"}, - "ap-southeast-3": {"bucketname": "appdevzipfiles-ap-southeast-3"} + "ap-southeast-3": {"bucketname": "appdevzipfiles-ap-southeast-3"}, + "il-central-1": {"bucketname": "appdevzipfiles-il-central-1"} } }, "Resources": { diff --git a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json index 6bdaadf..83ff96c 100644 --- a/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json +++ b/cloudwatchlogs-with-dlq/DLQLambdaCloudFormationWithSecuredEndpoint.json @@ -45,22 +45,25 @@ "ap-northeast-2": {"bucketname": "appdevzipfiles-ap-northeast-2"}, "ap-southeast-1": {"bucketname": "appdevzipfiles-ap-southeast-1"}, "ap-southeast-2": {"bucketname": "appdevzipfiles-ap-southeast-2"}, + "ap-southeast-4": {"bucketname": "appdevzipfiles-ap-southeast-4s"}, + "ap-southeast-6": {"bucketname": "appdevzipfiles-ap-southeast-6ss"}, "ap-northeast-1": {"bucketname": "appdevzipfiles-ap-northeast-1"}, - "ap-east-1": {"bucketname": "appdevzipfiles-ap-east-1s"}, - "af-south-1": {"bucketname": "appdevzipfiles-af-south-1s"}, "ca-central-1": {"bucketname": "appdevzipfiles-ca-central-1"}, "eu-central-1": {"bucketname": "appdevzipfiles-eu-central-1"}, "eu-west-1": {"bucketname": "appdevzipfiles-eu-west-1"}, "eu-west-2": {"bucketname": "appdevzipfiles-eu-west-2"}, "eu-west-3": {"bucketname": "appdevzipfiles-eu-west-3"}, "eu-north-1": {"bucketname": "appdevzipfiles-eu-north-1s"}, + "sa-east-1": {"bucketname": "appdevzipfiles-sa-east-1"}, + "ap-east-1": {"bucketname": "appdevzipfiles-ap-east-1s"}, + "af-south-1": {"bucketname": "appdevzipfiles-af-south-1s"}, "eu-south-1": {"bucketname": "appdevzipfiles-eu-south-1"}, "me-south-1": {"bucketname": "appdevzipfiles-me-south-1s"}, - "sa-east-1": {"bucketname": "appdevzipfiles-sa-east-1"}, "me-central-1": {"bucketname": "appdevzipfiles-me-central-1"}, "eu-central-2": {"bucketname": "appdevzipfiles-eu-central-2ss"}, "ap-northeast-3": {"bucketname": "appdevzipfiles-ap-northeast-3s"}, - "ap-southeast-3": {"bucketname": "appdevzipfiles-ap-southeast-3"} + "ap-southeast-3": {"bucketname": "appdevzipfiles-ap-southeast-3"}, + "il-central-1": {"bucketname": "appdevzipfiles-il-central-1"} } }, "Resources": { diff --git a/cloudwatchlogs-with-dlq/deploy_cwl_lambda.py b/cloudwatchlogs-with-dlq/deploy_cwl_lambda.py index be0744d..5ebd11f 100644 --- a/cloudwatchlogs-with-dlq/deploy_cwl_lambda.py +++ b/cloudwatchlogs-with-dlq/deploy_cwl_lambda.py @@ -14,6 +14,8 @@ "ap-northeast-2": "appdevzipfiles-ap-northeast-2", "ap-southeast-1": "appdevzipfiles-ap-southeast-1", "ap-southeast-2": "appdevzipfiles-ap-southeast-2", + "ap-southeast-4": "appdevzipfiles-ap-southeast-4s", + "ap-southeast-6": "appdevzipfiles-ap-southeast-6ss", "ap-northeast-1": "appdevzipfiles-ap-northeast-1", "ca-central-1": "appdevzipfiles-ca-central-1", "eu-central-1": "appdevzipfiles-eu-central-1", @@ -29,7 +31,8 @@ "me-central-1": "appdevzipfiles-me-central-1", "eu-central-2": "appdevzipfiles-eu-central-2ss", "ap-northeast-3": "appdevzipfiles-ap-northeast-3s", - "ap-southeast-3": "appdevzipfiles-ap-southeast-3" + "ap-southeast-3": "appdevzipfiles-ap-southeast-3", + "il-central-1": "appdevzipfiles-il-central-1" } def get_bucket_name(region): diff --git a/cloudwatchlogs-with-dlq/dlq_lambda_cloudformation.template.yaml b/cloudwatchlogs-with-dlq/dlq_lambda_cloudformation.template.yaml index 0253ba5..80c9692 100644 --- a/cloudwatchlogs-with-dlq/dlq_lambda_cloudformation.template.yaml +++ b/cloudwatchlogs-with-dlq/dlq_lambda_cloudformation.template.yaml @@ -33,55 +33,61 @@ Parameters: Description: (Optional) Enter comma separated list of logStream name prefixes to filter by logStream. Please note this is seperate from a logGroup. This is used to only send certain logStreams within a cloudwatch logGroup(s). LogGroups still need to be subscribed to the created Lambda funciton, regardless of what is input for this value. Default: '' Mappings: - RegionMap: - us-east-1: - bucketname: appdevzipfiles-us-east-1 - us-east-2: - bucketname: appdevzipfiles-us-east-2 - us-west-1: - bucketname: appdevzipfiles-us-west-1 - us-west-2: - bucketname: appdevzipfiles-us-west-2 - ap-south-1: - bucketname: appdevzipfiles-ap-south-1 - ap-northeast-2: - bucketname: appdevzipfiles-ap-northeast-2 - ap-southeast-1: - bucketname: appdevzipfiles-ap-southeast-1 - ap-southeast-2: - bucketname: appdevzipfiles-ap-southeast-2 - ap-northeast-1: - bucketname: appdevzipfiles-ap-northeast-1 - ap-east-1: - bucketname: appdevzipfiles-ap-east-1s - af-south-1: - bucketname: appdevzipfiles-af-south-1s - ca-central-1: - bucketname: appdevzipfiles-ca-central-1 - eu-central-1: - bucketname: appdevzipfiles-eu-central-1 - eu-west-1: - bucketname: appdevzipfiles-eu-west-1 - eu-west-2: - bucketname: appdevzipfiles-eu-west-2 - eu-west-3: - bucketname: appdevzipfiles-eu-west-3 - eu-north-1: - bucketname: appdevzipfiles-eu-north-1s - eu-south-1: - bucketname: appdevzipfiles-eu-south-1 - me-south-1: - bucketname: appdevzipfiles-me-south-1s - sa-east-1: - bucketname: appdevzipfiles-sa-east-1 - me-central-1: - bucketname: appdevzipfiles-me-central-1 - eu-central-2: - bucketname: appdevzipfiles-eu-central-2ss - ap-northeast-3: - bucketname: appdevzipfiles-ap-northeast-3s - ap-southeast-3: - bucketname: appdevzipfiles-ap-southeast-3 + RegionMap: + us-east-1: + bucketname: appdevzipfiles-us-east-1 + us-east-2: + bucketname: appdevzipfiles-us-east-2 + us-west-1: + bucketname: appdevzipfiles-us-west-1 + us-west-2: + bucketname: appdevzipfiles-us-west-2 + ap-south-1: + bucketname: appdevzipfiles-ap-south-1 + ap-northeast-2: + bucketname: appdevzipfiles-ap-northeast-2 + ap-southeast-1: + bucketname: appdevzipfiles-ap-southeast-1 + ap-southeast-2: + bucketname: appdevzipfiles-ap-southeast-2 + ap-southeast-4: + bucketname: appdevzipfiles-ap-southeast-4s + ap-southeast-6: + bucketname: appdevzipfiles-ap-southeast-6ss + ap-northeast-1: + bucketname: appdevzipfiles-ap-northeast-1 + ca-central-1: + bucketname: appdevzipfiles-ca-central-1 + eu-central-1: + bucketname: appdevzipfiles-eu-central-1 + eu-west-1: + bucketname: appdevzipfiles-eu-west-1 + eu-west-2: + bucketname: appdevzipfiles-eu-west-2 + eu-west-3: + bucketname: appdevzipfiles-eu-west-3 + eu-north-1: + bucketname: appdevzipfiles-eu-north-1s + sa-east-1: + bucketname: appdevzipfiles-sa-east-1 + ap-east-1: + bucketname: appdevzipfiles-ap-east-1s + af-south-1: + bucketname: appdevzipfiles-af-south-1s + eu-south-1: + bucketname: appdevzipfiles-eu-south-1 + me-south-1: + bucketname: appdevzipfiles-me-south-1s + me-central-1: + bucketname: appdevzipfiles-me-central-1 + eu-central-2: + bucketname: appdevzipfiles-eu-central-2ss + ap-northeast-3: + bucketname: appdevzipfiles-ap-northeast-3s + ap-southeast-3: + bucketname: appdevzipfiles-ap-southeast-3 + il-central-1: + bucketname: appdevzipfiles-il-central-1 Resources: SumoCWLogGroup: Type: AWS::Logs::LogGroup diff --git a/upload_artifacts.py b/upload_artifacts.py index 0638276..cde84e2 100644 --- a/upload_artifacts.py +++ b/upload_artifacts.py @@ -3,18 +3,18 @@ from argparse import ArgumentParser regions = [ - "us-east-2", "us-east-1", + "us-east-2", "us-west-1", "us-west-2", "ap-south-1", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-4", + "ap-southeast-6", "ap-northeast-1", "ca-central-1", - # "cn-north-1", - "ap-northeast-3", "eu-central-1", "eu-west-1", "eu-west-2", @@ -22,39 +22,44 @@ "eu-north-1", "sa-east-1", "ap-east-1", - "me-south-1", - "eu-south-1", "af-south-1", + "eu-south-1", + "me-south-1", "me-central-1", "eu-central-2", - "ap-southeast-3" + "ap-northeast-3", + "ap-southeast-3", + "il-central-1" ] region_map = { - "us-east-1" : "appdevzipfiles-us-east-1", - "us-east-2" : "appdevzipfiles-us-east-2", - "us-west-1" : "appdevzipfiles-us-west-1", - "us-west-2" : "appdevzipfiles-us-west-2", + "us-east-1": "appdevzipfiles-us-east-1", + "us-east-2": "appdevzipfiles-us-east-2", + "us-west-1": "appdevzipfiles-us-west-1", + "us-west-2": "appdevzipfiles-us-west-2", "ap-south-1": "appdevzipfiles-ap-south-1", - "ap-northeast-2":"appdevzipfiles-ap-northeast-2", - "ap-southeast-1":"appdevzipfiles-ap-southeast-1", - "ap-southeast-2":"appdevzipfiles-ap-southeast-2", - "ap-northeast-1":"appdevzipfiles-ap-northeast-1", + "ap-northeast-2": "appdevzipfiles-ap-northeast-2", + "ap-southeast-1": "appdevzipfiles-ap-southeast-1", + "ap-southeast-2": "appdevzipfiles-ap-southeast-2", + "ap-southeast-4": "appdevzipfiles-ap-southeast-4s", + "ap-southeast-6": "appdevzipfiles-ap-southeast-6ss", + "ap-northeast-1": "appdevzipfiles-ap-northeast-1", "ca-central-1": "appdevzipfiles-ca-central-1", - "eu-central-1":"appdevzipfiles-eu-central-1", - "eu-west-1":"appdevzipfiles-eu-west-1", - "eu-west-2":"appdevzipfiles-eu-west-2", - "eu-west-3":"appdevzipfiles-eu-west-3", - "eu-north-1":"appdevzipfiles-eu-north-1s", - "sa-east-1":"appdevzipfiles-sa-east-1", - "ap-east-1":"appdevzipfiles-ap-east-1s", - "af-south-1":"appdevzipfiles-af-south-1s", - "eu-south-1":"appdevzipfiles-eu-south-1", - "me-south-1":"appdevzipfiles-me-south-1s", + "eu-central-1": "appdevzipfiles-eu-central-1", + "eu-west-1": "appdevzipfiles-eu-west-1", + "eu-west-2": "appdevzipfiles-eu-west-2", + "eu-west-3": "appdevzipfiles-eu-west-3", + "eu-north-1": "appdevzipfiles-eu-north-1s", + "sa-east-1": "appdevzipfiles-sa-east-1", + "ap-east-1": "appdevzipfiles-ap-east-1s", + "af-south-1": "appdevzipfiles-af-south-1s", + "eu-south-1": "appdevzipfiles-eu-south-1", + "me-south-1": "appdevzipfiles-me-south-1s", "me-central-1": "appdevzipfiles-me-central-1", - "eu-central-2":"appdevzipfiles-eu-central-2ss", - "ap-northeast-3" :"appdevzipfiles-ap-northeast-3s", - "ap-southeast-3": "appdevzipfiles-ap-southeast-3" + "eu-central-2": "appdevzipfiles-eu-central-2ss", + "ap-northeast-3": "appdevzipfiles-ap-northeast-3s", + "ap-southeast-3": "appdevzipfiles-ap-southeast-3", + "il-central-1": "appdevzipfiles-il-central-1" }