Add pattern for VTL message filtering

apigw-sqs-msg-filtering-cdk/README.md

@@ -0,0 +1,86 @@
+# Amazon API Gateway to Amazon SQS with message filtering
+
+This project contains sample AWS CDK code to create an API Gateway Rest API, an SQS Queue and the correct VTL integration mapping template to filter out parts of the message body.
+
+Learn more about this pattern at Serverless Land Patterns: https://serverlessland.com/patterns/apigw-sqs-msg-filtering-cdk
+
+Important: this application uses various AWS services and there are costs associated with these services after the Free Tier usage - please see the AWS Pricing page for details. You are responsible for any AWS costs incurred. No warranty is implied in this example.
+
+## Requirements
+
+* [Create an AWS account](https://portal.aws.amazon.com/gp/aws/developer/registration/index.html) if you do not already have one and log in. The IAM user that you use must have sufficient permissions to make necessary AWS service calls and manage AWS resources.
+* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) installed and configured
+* [Git Installed](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
+* [Node and NPM](https://nodejs.org/en/download/) installed
+* [AWS Cloud Development Kit](https://docs.aws.amazon.com/cdk/latest/guide/cli.html) (AWS CDK) installed
+
+## Deployment Instructions
+
+1. Create a new directory, navigate to that directory in a terminal and clone the GitHub repository:
+
+```
+git clone https://github.com/aws-samples/serverless-patterns
+```
+
+2. Change directory to the pattern directory:
+
+```
+cd apigw-sqs-msg-filtering-cdk
+```
+
+3. Install dependencies
+
+```
+npm install
+```
+
+4. This project uses typescript as client language for AWS CDK. Run the given command to compile TypeScript to JavaScript
+
+```
+npm run build
+```
+
+5. Synthesize CloudFormation template from the AWS CDK app
+
+```
+cdk synth
+```
+
+6. Deploy the stack to your default AWS account and region.
+
+```
+cdk deploy
+```
+
+## Testing
+
+Run the following command to send a POST request to the REST API endpoint that will then send the filtered message to the SQS queue. Run the commands from the `apigw-sqs-msg-filtering-cdk` folder.
+
+```bash
+export API_GATEWAY_SQS_RESOURCE_ENDPOINT=$(aws cloudformation describe-stacks --stack-name ApigwSqsMsgFilteringCdkStack --query 'Stacks[0].Outputs[?OutputKey==`ApiGatewaySqsResourceEndpoint`].OutputValue' --output text)
+
+curl --location --request POST $API_GATEWAY_SQS_RESOURCE_ENDPOINT \  
+--header 'Content-Type: application/json' \
+-d @test/test-payload.json
+```
+
+To check and receive messages in the queue, it can be done in the AWS console or by running the following command. Note, replace {MyQueueUrl} placeholder in the command with the endpoint that has been deployed. The endpoint can be found in the CloudFormation stack output.
+
+```bash
+export SQS_QUEUE_URL=$(aws cloudformation describe-stacks --stack-name ApigwSqsMsgFilteringCdkStack --query 'Stacks[0].Outputs[?OutputKey==`SqsQueueUrl`].OutputValue' --output text)
+
+aws sqs receive-message --queue-url $SQS_QUEUE_URL
+```
+
+## Cleanup
+
+1. Delete the stack
+   ```bash
+   cdk destroy --all
+   ```
+
+---
+
+Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+SPDX-License-Identifier: MIT-0

apigw-sqs-msg-filtering-cdk/cdk/.gitignore

@@ -0,0 +1,8 @@
+*.js
+!jest.config.js
+*.d.ts
+node_modules
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

apigw-sqs-msg-filtering-cdk/cdk/.npmignore

@@ -0,0 +1,6 @@
+*.ts
+!*.d.ts
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

apigw-sqs-msg-filtering-cdk/cdk/bin/apigw-sqs-msg-filtering-cdk.ts

@@ -0,0 +1,20 @@
+#!/usr/bin/env node
+import * as cdk from 'aws-cdk-lib';
+import { ApigwSqsMsgFilteringCdkStack } from '../lib/apigw-sqs-msg-filtering-cdk-stack';
+
+const app = new cdk.App();
+new ApigwSqsMsgFilteringCdkStack(app, 'ApigwSqsMsgFilteringCdkStack', {
+  /* If you don't specify 'env', this stack will be environment-agnostic.
+   * Account/Region-dependent features and context lookups will not work,
+   * but a single synthesized template can be deployed anywhere. */
+
+  /* Uncomment the next line to specialize this stack for the AWS Account
+   * and Region that are implied by the current CLI configuration. */
+  // env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION },
+
+  /* Uncomment the next line if you know exactly what Account and Region you
+   * want to deploy the stack to. */
+  // env: { account: '123456789012', region: 'us-east-1' },
+
+  /* For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html */
+});

apigw-sqs-msg-filtering-cdk/cdk/cdk.json

@@ -0,0 +1,94 @@
+{
+  "app": "npx ts-node --prefer-ts-exts bin/apigw-sqs-msg-filtering-cdk.ts",
+  "watch": {
+    "include": ["**"],
+    "exclude": [
+      "README.md",
+      "cdk*.json",
+      "**/*.d.ts",
+      "**/*.js",
+      "tsconfig.json",
+      "package*.json",
+      "yarn.lock",
+      "node_modules",
+      "test"
+    ]
+  },
+  "context": {
+    "@aws-cdk/aws-lambda:recognizeLayerVersion": true,
+    "@aws-cdk/core:checkSecretUsage": true,
+    "@aws-cdk/core:target-partitions": ["aws", "aws-cn"],
+    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
+    "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
+    "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true,
+    "@aws-cdk/aws-iam:minimizePolicies": true,
+    "@aws-cdk/core:validateSnapshotRemovalPolicy": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
+    "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true,
+    "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true,
+    "@aws-cdk/aws-apigateway:disableCloudWatchRole": true,
+    "@aws-cdk/core:enablePartitionLiterals": true,
+    "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true,
+    "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true,
+    "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true,
+    "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true,
+    "@aws-cdk/aws-route53-patters:useCertificate": true,
+    "@aws-cdk/customresources:installLatestAwsSdkDefault": false,
+    "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true,
+    "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true,
+    "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true,
+    "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true,
+    "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true,
+    "@aws-cdk/aws-redshift:columnId": true,
+    "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true,
+    "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true,
+    "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true,
+    "@aws-cdk/aws-kms:aliasNameRef": true,
+    "@aws-cdk/aws-kms:applyImportedAliasPermissionsToPrincipal": true,
+    "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true,
+    "@aws-cdk/core:includePrefixInUniqueNameGeneration": true,
+    "@aws-cdk/aws-efs:denyAnonymousAccess": true,
+    "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true,
+    "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true,
+    "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true,
+    "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true,
+    "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true,
+    "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true,
+    "@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true,
+    "@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeysDefaultValueToFalse": true,
+    "@aws-cdk/aws-codepipeline:defaultPipelineTypeToV2": true,
+    "@aws-cdk/aws-kms:reduceCrossAccountRegionPolicyScope": true,
+    "@aws-cdk/aws-eks:nodegroupNameAttribute": true,
+    "@aws-cdk/aws-ec2:ebsDefaultGp3Volume": true,
+    "@aws-cdk/aws-ecs:removeDefaultDeploymentAlarm": true,
+    "@aws-cdk/custom-resources:logApiResponseDataPropertyTrueDefault": false,
+    "@aws-cdk/aws-s3:keepNotificationInImportedBucket": false,
+    "@aws-cdk/core:explicitStackTags": true,
+    "@aws-cdk/aws-ecs:enableImdsBlockingDeprecatedFeature": false,
+    "@aws-cdk/aws-ecs:disableEcsImdsBlocking": true,
+    "@aws-cdk/aws-ecs:reduceEc2FargateCloudWatchPermissions": true,
+    "@aws-cdk/aws-dynamodb:resourcePolicyPerReplica": true,
+    "@aws-cdk/aws-ec2:ec2SumTImeoutEnabled": true,
+    "@aws-cdk/aws-appsync:appSyncGraphQLAPIScopeLambdaPermission": true,
+    "@aws-cdk/aws-rds:setCorrectValueForDatabaseInstanceReadReplicaInstanceResourceId": true,
+    "@aws-cdk/core:cfnIncludeRejectComplexResourceUpdateCreatePolicyIntrinsics": true,
+    "@aws-cdk/aws-lambda-nodejs:sdkV3ExcludeSmithyPackages": true,
+    "@aws-cdk/aws-stepfunctions-tasks:fixRunEcsTaskPolicy": true,
+    "@aws-cdk/aws-ec2:bastionHostUseAmazonLinux2023ByDefault": true,
+    "@aws-cdk/aws-route53-targets:userPoolDomainNameMethodWithoutCustomResource": true,
+    "@aws-cdk/aws-elasticloadbalancingV2:albDualstackWithoutPublicIpv4SecurityGroupRulesDefault": true,
+    "@aws-cdk/aws-iam:oidcRejectUnauthorizedConnections": true,
+    "@aws-cdk/core:enableAdditionalMetadataCollection": true,
+    "@aws-cdk/aws-lambda:createNewPoliciesWithAddToRolePolicy": false,
+    "@aws-cdk/aws-s3:setUniqueReplicationRoleName": true,
+    "@aws-cdk/aws-events:requireEventBusPolicySid": true,
+    "@aws-cdk/core:aspectPrioritiesMutating": true,
+    "@aws-cdk/aws-dynamodb:retainTableReplica": true,
+    "@aws-cdk/aws-stepfunctions:useDistributedMapResultWriterV2": true,
+    "@aws-cdk/s3-notifications:addS3TrustKeyPolicyForSnsSubscriptions": true,
+    "@aws-cdk/aws-ec2:requirePrivateSubnetsForEgressOnlyInternetGateway": true,
+    "@aws-cdk/aws-s3:publicAccessBlockedByDefault": true,
+    "@aws-cdk/aws-lambda:useCdkManagedLogGroup": true
+  }
+}

apigw-sqs-msg-filtering-cdk/cdk/jest.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  testEnvironment: 'node',
+  roots: ['<rootDir>/test'],
+  testMatch: ['**/*.test.ts'],
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest'
+  }
+};

apigw-sqs-msg-filtering-cdk/cdk/lib/apigw-sqs-msg-filtering-cdk-stack.ts

@@ -0,0 +1,80 @@
+import * as cdk from 'aws-cdk-lib';
+import { Construct } from 'constructs';
+import { ApiGwSqsConstruct } from './apigw-sqs-msg-filtering';
+
+export class ApigwSqsMsgFilteringCdkStack extends cdk.Stack {
+  constructor(scope: Construct, id: string, props?: cdk.StackProps) {
+    super(scope, id, props);
+
+    // Create custom sqs queue
+    const sqsQueue = new cdk.aws_sqs.Queue(this, 'apigwSqs-queue', {
+      queueName: 'MyQueueName',
+    });
+
+    // Create custom api gateway
+    const apiGateway = new cdk.aws_apigateway.RestApi(this, 'apigwSqs-restApi', {
+      description: 'APIGW-SQS REST API Gateway',
+      restApiName: 'apiGatewayToSqs',
+      deployOptions: {
+        stageName: 'dev',
+      },
+      // Enable CORS
+      defaultCorsPreflightOptions: {
+        allowHeaders: [
+          'Content-Type',
+          'X-Amz-Date',
+          'Authorization',
+          'X-Api-Key',
+        ],
+        allowMethods: ['OPTIONS', 'GET', 'POST', 'PUT', 'PATCH', 'DELETE'],
+        allowOrigins: ['*'],
+      },
+    });
+
+    // https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html
+    const vtlMappingTemplate = `
+{
+  #set($payload = $input.path('$'))
+  #set($messageBody = $input.path('$.messageBody'))
+  #if($messageBody.notNeededElement)
+  #set($partToRemove = $messageBody.remove("notNeededElement"))
+  #end
+  #set($payload.filteredMessageBody = $messageBody)
+  "MessageBody": "$util.escapeJavaScript($input.json('$.filteredMessageBody'))",
+  #if($payload.delaySeconds > 0)
+  "DelaySeconds": $payload.delaySeconds,
+  #end
+  #if($payload.messageAttributes && $payload.messageAttributes.size() > 0)
+  "MessageAttributes": {
+    #foreach($attrName in $payload.messageAttributes.keySet())
+      #set($attr = $payload.messageAttributes.get($attrName))
+      "$attrName": {
+        "DataType": "$attr.dataType",
+        #if($attr.dataType == "String" || $attr.dataType.startsWith("String."))
+        "StringValue": "$attr.stringValue"
+        #elseif($attr.dataType == "Binary" || $attr.dataType.startsWith("Binary."))
+        "BinaryValue": "$attr.binaryValue"
+        #else
+        "StringValue": "$attr.stringValue"
+        #end
+      }#if($foreach.hasNext),#end
+    #end
+  },
+  #end
+  "QueueUrl": "${sqsQueue.queueUrl}"
+}
+`;
+
+    new ApiGwSqsConstruct(this, 'apiGwSqs', {
+      apiGateway: apiGateway,
+      sqsQueue: sqsQueue,
+      vtlMappingTemplate: vtlMappingTemplate,
+    });
+
+    new cdk.CfnOutput(this, 'ApiGatewayName', { value: apiGateway.restApiName });
+    new cdk.CfnOutput(this, 'ApiGatewayUrl', { value: apiGateway.url });
+    new cdk.CfnOutput(this, 'ApiGatewaySqsResourceEndpoint', { value: `${apiGateway.url}sqs` });
+    new cdk.CfnOutput(this, 'SqsQueueName', { value: sqsQueue.queueName });
+    new cdk.CfnOutput(this, 'SqsQueueUrl', { value: sqsQueue.queueUrl });
+  }
+}