From af94a5f6cc134162ec0f276d81a7cbbc9dc19800 Mon Sep 17 00:00:00 2001 From: Tony Sherman Date: Sat, 22 Oct 2022 13:52:52 -0400 Subject: [PATCH 1/8] fix(assetlibraryhistory): utilize kinesis stream to batch assetlibrary history updates ISSUES CLOSED: #87 --- .../cfn-assetLibraryHistory.yml | 24 ++++++++++++++++--- .../src/lambda_iot_rule.ts | 15 ++++++------ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml b/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml index 36ad0666e..ea6c586c3 100644 --- a/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml +++ b/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml @@ -135,11 +135,13 @@ Resources: Properties: TopicRulePayload: Actions: - - Lambda: - FunctionArn: !Sub 'arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${EventsLambdaFunction}' + - Kinesis: + PartitionKey: ${topic(4)} + RoleArn: !GetAtt LambdaExecutionRole.Arn + StreamName: !Ref EventsKinesisStream Description: 'Saves AssetLibrary configuration changes' AwsIotSqlVersion: '2016-03-23' - RuleDisabled: 'false' + RuleDisabled: false Sql: !Sub "SELECT * FROM '${AssetLibraryEventsTopic}'" LambdaInvocationPermission: @@ -285,6 +287,13 @@ Resources: ReadCapacityUnits: '5' WriteCapacityUnits: '5' + EventsKinesisStream: + Type: AWS::Kinesis::Stream + Properties: + Name: !Sub 'cdf-assetlibrary-history-stream-${Environment}' + StreamModeDetails: + StreamMode: 'on-demand' + EventsLambdaFunction: Type: AWS::Serverless::Function Properties: @@ -301,6 +310,15 @@ Resources: APP_CONFIG: !Ref ApplicationConfigurationOverride AWS_DYNAMODB_TABLE_EVENTS: !Ref HistoryTable Tracing: Active + Events: + Stream: + Type: Kinesis + Properties: + Stream: !GetAtt EventsKinesisStream.Arn + BatchSize: 100 # What should this be? + MaximumBatchingWindowInSeconds: 30 + StartingPosition: LATEST + MaximumRetryAttempts: 1 RESTLambdaFunction: Type: AWS::Serverless::Function diff --git a/source/packages/services/assetlibraryhistory/src/lambda_iot_rule.ts b/source/packages/services/assetlibraryhistory/src/lambda_iot_rule.ts index 268d8d9b6..6b3aa037b 100644 --- a/source/packages/services/assetlibraryhistory/src/lambda_iot_rule.ts +++ b/source/packages/services/assetlibraryhistory/src/lambda_iot_rule.ts @@ -15,17 +15,18 @@ import 'reflect-metadata'; import { container } from './di/inversify.config'; import { logger } from '@awssolutions/simple-cdf-logger'; import { TYPES } from './di/types'; -import { EventModel } from './events/events.models'; import { EventsService } from './events/events.service'; const eventsService: EventsService = container.get(TYPES.EventsService); -exports.iot_rule_handler = async (event: EventModel, _context: unknown) => { +exports.iot_rule_handler = async (event: any, _context: unknown) => { logger.debug(`events.service create: in: event: ${JSON.stringify(event)}`); - // TODO validation - - await eventsService.create(event); - - logger.debug('events.service create: exit:'); + for (const record of event.Records) { + // Kinesis record data is base64 encoded + const payload = JSON.parse(Buffer.from(record.kinesis.data, 'base64').toString('ascii')); + // TODO validation + await eventsService.create(payload); + logger.debug('events.service create: exit:'); + } }; From a2948d5523fb40afc714f658131598b2a62d3d5d Mon Sep 17 00:00:00 2001 From: Tony Sherman Date: Sat, 22 Oct 2022 13:59:57 -0400 Subject: [PATCH 2/8] docs(assetlibrary): correct typo in device monitoring docs --- source/packages/services/device-monitoring/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/packages/services/device-monitoring/README.md b/source/packages/services/device-monitoring/README.md index 3ada3ed25..1e9f358cf 100644 --- a/source/packages/services/device-monitoring/README.md +++ b/source/packages/services/device-monitoring/README.md @@ -4,7 +4,7 @@ A device monitoring module provides near real-time device status - connected and ## Introduction -The device monitoring module utilizes [AWS IoT Lifecycle Events](https://docs.aws.amazon.com/iot/latest/developerguide/life-cycle-events.html) feature. Whenever the device connects or disconnects, AWS IoT Core emits a smaple event that is then picked up by AWS Lambda. It parses the eventType and then updating the device attributes in Asset Library to connected as `true` or `false` +The device monitoring module utilizes [AWS IoT Lifecycle Events](https://docs.aws.amazon.com/iot/latest/developerguide/life-cycle-events.html) feature. Whenever the device connects or disconnects, AWS IoT Core emits a sample event that is then picked up by AWS Lambda. It parses the eventType and then updating the device attributes in Asset Library to connected as `true` or `false` The following sample represents the schema that AWS IoT core emits when a device connects or disconnects From 774026d464d7822791a63275623c8296d6301584 Mon Sep 17 00:00:00 2001 From: Tony Sherman Date: Sat, 22 Oct 2022 14:12:46 -0400 Subject: [PATCH 3/8] fix(assetlibraryhistory): add kinesis permissions to lambda execution role --- .../infrastructure/cfn-assetLibraryHistory.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml b/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml index ea6c586c3..aaf7de996 100644 --- a/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml +++ b/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml @@ -184,10 +184,18 @@ Resources: - 'dynamodb:PutItem' - 'dynamodb:UpdateItem' - 'dynamodb:DeleteItem' + - 'kinesis:DescribeStream' + - 'kinesis:DescribeStreamSummary' + - 'kinesis:GetRecords' + - 'kinesis:GetShardIterator' + - 'kinesis:ListShards' + - 'kinesis:ListStreams' + - 'kinesis:SubscribeToShard' Effect: Allow Resource: - !GetAtt HistoryTable.Arn - !Sub '${HistoryTable.Arn}/index/type-time-index' + - !GetAtt EventsKinesisStream.Arn RESTLambdaExecutionRole: Type: AWS::IAM::Role From 29ca433bdec0138fdb2c0acc80e6a0827fdf101c Mon Sep 17 00:00:00 2001 From: Aaron Pittenger Date: Sun, 17 Sep 2023 10:03:07 -0400 Subject: [PATCH 4/8] style(assetlibraryhistory): removed unnecessary comment --- .../infrastructure/cfn-assetLibraryHistory.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml b/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml index aaf7de996..671fd5e9e 100644 --- a/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml +++ b/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml @@ -323,7 +323,7 @@ Resources: Type: Kinesis Properties: Stream: !GetAtt EventsKinesisStream.Arn - BatchSize: 100 # What should this be? + BatchSize: 100 MaximumBatchingWindowInSeconds: 30 StartingPosition: LATEST MaximumRetryAttempts: 1 From 0a74dc1fe5fe8fed1a12085639ec8cca56e1bf3d Mon Sep 17 00:00:00 2001 From: Aaron Pittenger Date: Sun, 17 Sep 2023 10:04:14 -0400 Subject: [PATCH 5/8] chore(assetlibraryhistory): changelog --- ...assetlibrary_history_batching_2023-09-17-14-03.json | 10 ++++++++++ ...assetlibrary_history_batching_2023-09-17-14-03.json | 10 ++++++++++ 2 files changed, 20 insertions(+) create mode 100644 source/common/changes/@awssolutions/cdf-assetlibrary-history/fix_assetlibrary_history_batching_2023-09-17-14-03.json create mode 100644 source/common/changes/@awssolutions/cdf-device-monitoring/fix_assetlibrary_history_batching_2023-09-17-14-03.json diff --git a/source/common/changes/@awssolutions/cdf-assetlibrary-history/fix_assetlibrary_history_batching_2023-09-17-14-03.json b/source/common/changes/@awssolutions/cdf-assetlibrary-history/fix_assetlibrary_history_batching_2023-09-17-14-03.json new file mode 100644 index 000000000..82cb2c584 --- /dev/null +++ b/source/common/changes/@awssolutions/cdf-assetlibrary-history/fix_assetlibrary_history_batching_2023-09-17-14-03.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@awssolutions/cdf-assetlibrary-history", + "comment": "added Kinesis batching for assetlibraryhistory", + "type": "none" + } + ], + "packageName": "@awssolutions/cdf-assetlibrary-history" +} \ No newline at end of file diff --git a/source/common/changes/@awssolutions/cdf-device-monitoring/fix_assetlibrary_history_batching_2023-09-17-14-03.json b/source/common/changes/@awssolutions/cdf-device-monitoring/fix_assetlibrary_history_batching_2023-09-17-14-03.json new file mode 100644 index 000000000..374b6f04b --- /dev/null +++ b/source/common/changes/@awssolutions/cdf-device-monitoring/fix_assetlibrary_history_batching_2023-09-17-14-03.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@awssolutions/cdf-device-monitoring", + "comment": "", + "type": "none" + } + ], + "packageName": "@awssolutions/cdf-device-monitoring" +} \ No newline at end of file From c5eb972ef3481c47b7e14422f66ec7e73209fd7e Mon Sep 17 00:00:00 2001 From: Aaron Pittenger Date: Wed, 28 Feb 2024 14:52:59 -0500 Subject: [PATCH 6/8] fix(assetlibraryhistory): better infrastructure setup --- .../cfn-assetLibraryHistory.yml | 54 ++++++++++++++----- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml b/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml index 671fd5e9e..6fc0f9cd8 100644 --- a/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml +++ b/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml @@ -137,21 +137,41 @@ Resources: Actions: - Kinesis: PartitionKey: ${topic(4)} - RoleArn: !GetAtt LambdaExecutionRole.Arn + RoleArn: !GetAtt KinesisExecutionRole.Arn StreamName: !Ref EventsKinesisStream Description: 'Saves AssetLibrary configuration changes' AwsIotSqlVersion: '2016-03-23' RuleDisabled: false Sql: !Sub "SELECT * FROM '${AssetLibraryEventsTopic}'" - LambdaInvocationPermission: - Type: AWS::Lambda::Permission + KinesisExecutionRole: + Type: AWS::IAM::Role Properties: - SourceArn: !Sub 'arn:aws:iot:${AWS::Region}:${AWS::AccountId}:rule/${AssetLibraryEventsRule}' - Action: lambda:InvokeFunction - Principal: iot.amazonaws.com - FunctionName: !GetAtt EventsLambdaFunction.Arn - SourceAccount: !Ref AWS::AccountId + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - iot.amazonaws.com + Action: sts:AssumeRole + Path: '/cdf/assetlibraryhistory/' + ManagedPolicyArns: + - !Ref KinesisPolicy + + KinesisPolicy: + Type: 'AWS::IAM::ManagedPolicy' + Properties: + Description: 'Role policy for IoT rule engine' + Path: '/cdf/assetlibraryhistory/' + PolicyDocument: + Version: '2012-10-17' + Statement: + - Action: + - 'kinesis:PutRecord' + Effect: Allow + Resource: + - !GetAtt EventsKinesisStream.Arn LambdaExecutionRole: Type: AWS::IAM::Role @@ -184,6 +204,16 @@ Resources: - 'dynamodb:PutItem' - 'dynamodb:UpdateItem' - 'dynamodb:DeleteItem' + Effect: Allow + Resource: + - !GetAtt HistoryTable.Arn + - !Sub '${HistoryTable.Arn}/index/type-time-index' + - !GetAtt EventsKinesisStream.Arn + - PolicyName: 'Kinesis' + PolicyDocument: + Version: '2012-10-17' + Statement: + - Action: - 'kinesis:DescribeStream' - 'kinesis:DescribeStreamSummary' - 'kinesis:GetRecords' @@ -193,8 +223,6 @@ Resources: - 'kinesis:SubscribeToShard' Effect: Allow Resource: - - !GetAtt HistoryTable.Arn - - !Sub '${HistoryTable.Arn}/index/type-time-index' - !GetAtt EventsKinesisStream.Arn RESTLambdaExecutionRole: @@ -300,7 +328,7 @@ Resources: Properties: Name: !Sub 'cdf-assetlibrary-history-stream-${Environment}' StreamModeDetails: - StreamMode: 'on-demand' + StreamMode: 'ON_DEMAND' EventsLambdaFunction: Type: AWS::Serverless::Function @@ -323,8 +351,8 @@ Resources: Type: Kinesis Properties: Stream: !GetAtt EventsKinesisStream.Arn - BatchSize: 100 - MaximumBatchingWindowInSeconds: 30 + BatchSize: 1000 + MaximumBatchingWindowInSeconds: 60 StartingPosition: LATEST MaximumRetryAttempts: 1 From 53b56a6cd0e7989a3684512cb5de99c98cb74329 Mon Sep 17 00:00:00 2001 From: Aaron Pittenger Date: Wed, 28 Feb 2024 14:58:37 -0500 Subject: [PATCH 7/8] fix(assetlibraryhistory): added some type definition to the lambda handler --- .../src/events/events.models.ts | 21 +++++++++++++++++++ .../src/lambda_iot_rule.ts | 9 +++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/source/packages/services/assetlibraryhistory/src/events/events.models.ts b/source/packages/services/assetlibraryhistory/src/events/events.models.ts index e6f961ad2..7b3e0f7fc 100644 --- a/source/packages/services/assetlibraryhistory/src/events/events.models.ts +++ b/source/packages/services/assetlibraryhistory/src/events/events.models.ts @@ -21,6 +21,27 @@ export interface EventModel { attributes: { [key: string]: string }; } +export interface KinesisRecord { + kinesis: { + kinesisSchemaVersion: string; + partitionKey: string; + sequenceNumber: string; + data: string; + approimateArrivalTimestamp: number; + }; + eventSource: string; + eventVersion: string; + eventID: string; + eventName: string; + invokeIdentityArn: string; + awsRegion: string; + eventSourceARN: string; +} + +export interface KinesisRecords { + Records: KinesisRecord[]; +} + export interface StateHistoryModel { objectId: string; type: Category; diff --git a/source/packages/services/assetlibraryhistory/src/lambda_iot_rule.ts b/source/packages/services/assetlibraryhistory/src/lambda_iot_rule.ts index 6b3aa037b..4b7944262 100644 --- a/source/packages/services/assetlibraryhistory/src/lambda_iot_rule.ts +++ b/source/packages/services/assetlibraryhistory/src/lambda_iot_rule.ts @@ -12,19 +12,22 @@ *********************************************************************************************************************/ import 'reflect-metadata'; -import { container } from './di/inversify.config'; import { logger } from '@awssolutions/simple-cdf-logger'; +import { container } from './di/inversify.config'; import { TYPES } from './di/types'; +import { EventModel, KinesisRecords } from './events/events.models'; import { EventsService } from './events/events.service'; const eventsService: EventsService = container.get(TYPES.EventsService); -exports.iot_rule_handler = async (event: any, _context: unknown) => { +exports.iot_rule_handler = async (event: KinesisRecords, _context: unknown) => { logger.debug(`events.service create: in: event: ${JSON.stringify(event)}`); for (const record of event.Records) { // Kinesis record data is base64 encoded - const payload = JSON.parse(Buffer.from(record.kinesis.data, 'base64').toString('ascii')); + const payload: EventModel = JSON.parse( + Buffer.from(record.kinesis.data, 'base64').toString('ascii') + ); // TODO validation await eventsService.create(payload); logger.debug('events.service create: exit:'); From f8f7e2047248d1f5c865fa50fd4216ffdbf18e4c Mon Sep 17 00:00:00 2001 From: Aaron Pittenger Date: Wed, 20 Mar 2024 11:22:23 -0400 Subject: [PATCH 8/8] fix(assetlibraryhistory): up the events lambda timeout time to 120 seconds --- .../infrastructure/cfn-assetLibraryHistory.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml b/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml index 6fc0f9cd8..c427a6c83 100644 --- a/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml +++ b/source/packages/services/assetlibraryhistory/infrastructure/cfn-assetLibraryHistory.yml @@ -208,7 +208,6 @@ Resources: Resource: - !GetAtt HistoryTable.Arn - !Sub '${HistoryTable.Arn}/index/type-time-index' - - !GetAtt EventsKinesisStream.Arn - PolicyName: 'Kinesis' PolicyDocument: Version: '2012-10-17' @@ -339,7 +338,7 @@ Resources: MemorySize: 512 Role: !GetAtt LambdaExecutionRole.Arn Runtime: nodejs18.x - Timeout: 30 + Timeout: 120 Environment: Variables: APP_CONFIG_DIR: 'config'