From 36a0f32970064e69ae30145bca713316e711b8c2 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Wed, 26 Jan 2022 13:01:31 -0800 Subject: [PATCH 01/31] feat(assetlibrary) enhanced mode cloudformation templates and deploy scripts --- .../infrastructure/cfn-assetLibrary.yaml | 7 +- .../infrastructure/cfn-enhancedsearch.yaml | 403 ++++++++++++++++++ .../infrastructure/cfn-neptune.yaml | 18 +- 3 files changed, 423 insertions(+), 5 deletions(-) create mode 100644 source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-assetLibrary.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-assetLibrary.yaml index 6e4a92b98..33f1fc71e 100644 --- a/source/packages/services/assetlibrary/infrastructure/cfn-assetLibrary.yaml +++ b/source/packages/services/assetlibrary/infrastructure/cfn-assetLibrary.yaml @@ -43,12 +43,13 @@ Parameters: Type: CommaDelimitedList Mode: - Description: Run in 'lite' mode which includes device registry only, or 'full' mode which augments the device registry with an additional datastore + Description: Run in 'lite' mode which includes device registry only, 'full' mode which augments the device registry with an additional datastore, or 'enhanced' mode which adds enhanced search to full mode Type: String Default: full AllowedValues: - full - lite + - enhanced MinLength: 1 PrivateApiGatewayVPCEndpoint: @@ -131,7 +132,7 @@ Parameters: MinLength: 1 Conditions: - DeployFullMode: !Equals [!Ref Mode, 'full'] + DeployFullOrEnhancedMode: !Or [!Equals [!Ref Mode, 'full'], !Equals [!Ref Mode, 'enhanced']] DeployLiteMode: !Equals [!Ref Mode, 'lite'] DeployInVPC: !Not [!Equals [!Ref VpcId, 'N/A']] @@ -336,7 +337,7 @@ Resources: AssetLibraryInit: Type: Custom::AssetLibraryInit - Condition: DeployFullMode + Condition: DeployFullOrEnhancedMode Version: 1.0 Properties: ServiceToken: !Ref CustomResourceVPCLambdaArn diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml new file mode 100644 index 000000000..d5716f227 --- /dev/null +++ b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml @@ -0,0 +1,403 @@ +#----------------------------------------------------------------------------------------------------------------------- +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. +#----------------------------------------------------------------------------------------------------------------------- +AWSTemplateFormatVersion : '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: CDF Asset Library Service Neptune-to-ElasticSearch connection for enhanced search + +Parameters: + Environment: + Description: Name of environment. Used to name the created resources. + Type: String + MinLength: 1 + VpcId: + Description: ID of VPC to deploy the OpenSearch domain into + Type: AWS::EC2::VPC::Id + PrivateSubNetIds: + Description: Comma delimited list of private subnetIds to deploy Neptune into. Number of subnets must match the number of availability zones deployed into, i.e. only pass one subnet if operating in a single AZ. + Type: List + PrivateRouteTableIds: + Description: Comma delimited list of private route table ids to allow access to Neptune + Type: String + CDFSecurityGroupId: + Description: ID of an existing security group to allow access to ElasticSearch + Type: AWS::EC2::SecurityGroup::Id + NeptuneSecurityGroupId: + Description: ID of an existing security group that contains the Neptune nodes + Type: AWS::EC2::SecurityGroup::Id + KmsKeyId: + Description: The KMS key ID used to encrypt the ElasticSearch database + Type: String + MinLength: 1 + ElasticSearchInstanceType: + Description: ElasticSearch instance type. + Type: String + Default: m6g.large.search + ConstraintDescription: 'Must be a supported OpenSearch instance type in the region, support ElasticSearch, and must support at rest. See also: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html' + ElasticSearchInstanceCount: + Type: Number + Default: 2 + Description: The number of data nodes (instances) to use in the OpenSearch domain. Must be a multiple of the number of availability zones. + ElasticSearchDedicatedMasterCount: + Type: Number + Default: 0 + Description: The number of dedicated master nodes (instances) to use in the OpenSearch domain. + ElasticSearchEBSVolumeSize: + Type: Number + Default: 20 + Description: Size of EBS volumes attached to each ElasticSearch node, in GB. + ElasticSearchEBSVolumeType: + Type: String + Default: gp2 + Description: Type of the EBS volume attached to each ElasticSearch node. + AllowedValues: + - gp2 + - io1 + - standard + ConstraintDescription: | + See list of valid EBS volume types at https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html. + Not every instance type is compatible with every volume type. + ElasticSearchEncryptAtRest: + Type: String + Default: 'false' + AllowedValues: + - 'true' + - 'false' + Description: Enable Encryption at rest. + NeptuneClusterEndpoint: + Description: 'Neptune cluster endpoint. Format: :' + Type: String + NeptunePollerLambdaMemorySize: + Type: Number + Default: 2048 + Description: Neptune Poller Lambda memory size (in MB). + AllowedValues: + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 3008 + NeptunePollerLambdaLoggingLevel: + Type: String + Default: INFO + Description: Poller Lambda logging level. + AllowedValues: + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + NeptunePollerStreamRecordsBatchSize: + Type: Number + Default: 5000 + MaxValue: 50000 + MinValue: 1 + Description: "Number of records to be read from stream in each batch. Should be between 1 to 50000." + NeptunePollerMaxPollingWaitTime: + Type: Number + Default: 60 + MaxValue: 3600 + MinValue: 0 + Description: "Maximum wait time in seconds between two successive polling from stream. Set value to 0 sec for continuous polling. Maximum value can be 3600 sec (1 hour)." + NeptunePollerMaxPollingInterval: + Type: Number + Default: 600 + MaxValue: 900 + MinValue: 5 + Description: "Period for which we can continuously poll stream for records on one Lambda instance. Should be between 5 sec to 900 sec. This parameter is used to set Poller Lambda Timeout." + NeptunePollerStepFunctionFallbackPeriod: + Type: Number + Default: 5 + Description: "Period after which Step function is invoked using Cloud Watch Events to recover from failure. Unit for Step Function Fallback period is set separately." + NeptunePollerStepFunctionFallbackPeriodUnit: + Type: String + Default: minutes + AllowedValues: + - minutes + - minute + - hours + - hour + - days + - day + Description: "Step Function FallbackPeriod unit. Should be one of minutes, minute, hours, hour, days, day" + EnableNonStringIndexing: + Type: String + Default: 'true' + AllowedValues: + - 'true' + - 'false' + ConstraintDescription: "Must be either true or false" + Description: Flag to enable/disable indexing Non-String fields + NumberOfShards: + Type: Number + Default: 5 + Description: Number of Shards for Elastic Search Index. Default value is 5. + NumberOfReplica: + Type: Number + Default: 1 + Description: Number of replicas for Elastic Search Index. Default value is 1. + GeoLocationFields: + Type: String + Default: '' + Description: 'Comma Delimited list of Property Keys to be mapped to Geo Point Type in Elastic Search. For Example: location,area. Currently, for a field to be mapped to Geo Point type, value should be in the format "latitude,longitude" Ex: "41.33,-11.69"' + PropertiesToExclude: + Type: String + Default: '' + Description: Comma delimited list of Property Keys to exclude from being indexed into Elastic Search. Optional Parameter - If left blank, all property keys will be indexed." + DatatypesToExclude: + Type: String + Default: '' + Description: 'Comma delimited list of Property Value Data Types to exclude from being indexed into Elastic Search. Optional Parameter - If left blank, all valid property values will be indexed. Type inputs that are unsupported for the specified query language will be ignored. Valid inputs for Gremlin data: [string, date, bool, byte, short, int, long, float, double]' + IgnoreMissingDocument: + Type: String + Default: 'true' + AllowedValues: + - 'true' + - 'false' + ConstraintDescription: Must be a either true or false + Description: 'Flag to determine if missing document error in Elastic Search can be ignored. Missing document error can occur rarely but will need manual intervention if not ignored.' + CdfService: + Description: Service name to tag resources. + Type: String + Default: assetlibrary + CreateCloudWatchAlarm: + Type: String + Default: false + Description: Flag used to determine whether to create Cloud watch alarm or not. + AllowedValues: + - 'true' + - 'false' + ConstraintDescription: Must be a either true or false + NotificationEmail: + Type: String + Default: "" + Description: "Email Address for CloudWatch Alarm Notification. Optional Parameter - Only needed when selecting option to create CloudWatch Alarm." + + +Conditions: + ElasticSearchEncryptAtRest: !Equals [ !Ref ElasticSearchEncryptAtRest, 'true' ] + EnableDedicatedMasterNodes: !Not [ !Equals [ !Ref ElasticSearchDedicatedMasterCount, 0 ] ] + CreateCloudWatchAlarmCondition: !Equals [ !Ref CreateCloudWatchAlarm, 'true' ] + EnableNonStringIndexingCondition: !Equals [ !Ref EnableNonStringIndexing, 'true' ] + + +Resources: + + OpenSearchSG: + Type: 'AWS::EC2::SecurityGroup' + Properties: + VpcId: !Ref VpcId + GroupDescription: !Sub 'CDF Asset Library (${Environment}) OpenSearch Access' + Tags: + - Key: cdf_environment + Value: !Ref Environment + - Key: cdf_service + Value: !Ref CdfService + + OpenSearchSGIngressRule: + Type: 'AWS::EC2::SecurityGroupIngress' + Properties: + GroupId: !Ref OpenSearchSG + FromPort: 443 + ToPort: 443 + IpProtocol: tcp + SourceSecurityGroupId: !Ref NeptuneSecurityGroupId + Description: Access from Neptune to ElasticSearch + + # TODO: remove this, debugging use only, allows https access to ElasticSearch API and Kibana via bastion tunneling + OpenSearchSGIngressRule2DeleteMe: + Type: 'AWS::EC2::SecurityGroupIngress' + Properties: + GroupId: !Ref OpenSearchSG + FromPort: 443 + ToPort: 443 + IpProtocol: tcp + SourceSecurityGroupId: !Ref CDFSecurityGroupId + Description: Access from CDF default security group to ElasticSearch + + OpenSearchDomain: + Type: AWS::OpenSearchService::Domain + # TODO: copied deletion/update policies over from Neptune template, probably need something + # equivalent here... + DeletionPolicy: Snapshot + UpdateReplacePolicy: Snapshot + Properties: + AccessPolicies: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + AWS: '*' + Action: 'es:*' + # https://serverfault.com/questions/937008/how-can-one-configure-an-aws-elasticsearch-access-policy-using-cloudformation + Resource: '/*' + ClusterConfig: + DedicatedMasterCount: + Fn::If: + - EnableDedicatedMasterNodes + - !Ref ElasticSearchDedicatedMasterCount + - Ref: AWS::NoValue + DedicatedMasterEnabled: + Fn::If: + - EnableDedicatedMasterNodes + - true + - false + # DedicatedMasterType: + # Fn::If: + # - EnableDedicatedMasterNodes + # - TODO + # - Ref: AWS::NoValue + InstanceCount: !Ref ElasticSearchInstanceCount + InstanceType: !Ref ElasticSearchInstanceType + ZoneAwarenessEnabled: true + # CognitoOptions: + # CognitoOptions + DomainEndpointOptions: + # CustomEndpoint: String + # CustomEndpointCertificateArn: String + # CustomEndpointEnabled: Boolean + EnforceHTTPS: true + # TLSSecurityPolicy: String + EBSOptions: + EBSEnabled: true + VolumeSize: !Ref ElasticSearchEBSVolumeSize + VolumeType: !Ref ElasticSearchEBSVolumeType + # Iops: Integer + EncryptionAtRestOptions: + Enabled: !Ref ElasticSearchEncryptAtRest + KmsKeyId: + Fn::If: + - ElasticSearchEncryptAtRest + - !Ref KmsKeyId + - Ref: AWS::NoValue + # When integrating with Amazon OpenSearch Service, Neptune requires Elasticsearch version 7.1 or higher + # rather than OpenSearch version 1.0. Neptune is not currently compatible with OpenSearch version 1.0. + # https://docs.aws.amazon.com/neptune/latest/userguide/full-text-search-cfn-create.html + EngineVersion: 'Elasticsearch_7.10' + # LogPublishingOptions: + # # TODO, example values from https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-opensearchservice-domain.html#aws-resource-opensearchservice-domain--examples + # ES_APPLICATION_LOGS: + # CloudWatchLogsLogGroupArn: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/opensearch/domains/opensearch-application-logs' + # Enabled: true + # SEARCH_SLOW_LOGS: + # CloudWatchLogsLogGroupArn: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/opensearch/domains/opensearch-slow-logs' + # Enabled: true + # INDEX_SLOW_LOGS: + # CloudWatchLogsLogGroupArn: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/opensearch/domains/opensearch-index-slow-logs' + # Enabled: true + NodeToNodeEncryptionOptions: + Enabled: true + VPCOptions: + SecurityGroupIds: + - !Ref OpenSearchSG + SubnetIds: !Ref PrivateSubNetIds + Tags: + - Key: cdf_environment + Value: !Ref Environment + - Key: cdf_service + Value: !Ref CdfService + + ElasticSearchAccessPolicy: + Type: 'AWS::IAM::ManagedPolicy' + Properties: + ManagedPolicyName: ElasticSearchAccessPolicy + Description: "Policy for ElasticSearch Access for Neptune Lambda Poller" + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: "elasticsearchaccess" + Effect: Allow + Action: + - 'es:ESHttpDelete' + - 'es:ESHttpGet' + - 'es:ESHttpHead' + - 'es:ESHttpPost' + - 'es:ESHttpPut' + Resource: !Sub '${OpenSearchDomain.Arn}/*' + + NeptuneStreamPoller: + Type: 'AWS::CloudFormation::Stack' + Properties: + TemplateURL: 'https://s3.amazonaws.com/aws-neptune-customer-samples/neptune-stream/neptune_stream_poller_nested_full_stack.json' + Parameters: + AdditionalParams: + Fn::Sub: + - "{ \"ElasticSearchEndpoint\": \"${ElasticSearchEndpoint}\", \"NumberOfShards\": \"${NumberOfShards}\", \"NumberOfReplica\": \"${NumberOfReplica}\", \"IgnoreMissingDocument\": \"${IgnoreMissingDocument}\", \"ReplicationScope\": \"${ReplicationScope}\", \"GeoLocationFields\": \"${GeoLocationFields}\", \"DatatypesToExclude\": \"${DatatypesToExclude}\", \"PropertiesToExclude\": \"${PropertiesToExclude}\", \"EnableNonStringIndexing\": \"${EnableNonStringIndexing}\"}" + - ElasticSearchEndpoint: !GetAtt OpenSearchDomain.DomainEndpoint + NumberOfShards: !Ref "NumberOfShards" + NumberOfReplica: !Ref "NumberOfReplica" + GeoLocationFields: !Ref "GeoLocationFields" + PropertiesToExclude: !Ref "PropertiesToExclude" + DatatypesToExclude: !Ref "DatatypesToExclude" + IgnoreMissingDocument: !Ref "IgnoreMissingDocument" + ReplicationScope: "All" + EnableNonStringIndexing: !Ref EnableNonStringIndexing + ApplicationName: !Sub 'cdf-${CdfService}-neptune-elasticsearch-connector' + LambdaMemorySize: !Ref NeptunePollerLambdaMemorySize + LambdaRuntime: python3.6 + LambdaS3Bucket: !Join ["-", ["aws-neptune-customer-samples", Ref: "AWS::Region"]] + LambdaS3Key: "neptune-stream/lambda/python36/release_2021_08_23/neptune-to-es.zip" + ManagedPolicies: !Ref ElasticSearchAccessPolicy + LambdaLoggingLevel: !Ref NeptunePollerLambdaLoggingLevel + StreamRecordsHandler: + # The published template in the Neptune documentation uses a three-level mapping here. + # The mapping is flattened into a single If because Gremlin and Python are fixed. + Fn::If: + - EnableNonStringIndexingCondition + - "neptune_to_es.neptune_gremlin_es_handler.ElasticSearchGremlinHandler" + - "neptune_to_es.neptune_gremlin_es_handler.ElasticSearchStringOnlyGremlinHandler" + StreamRecordsBatchSize: !Ref NeptunePollerStreamRecordsBatchSize + StepFunctionFallbackPeriod: !Ref NeptunePollerStepFunctionFallbackPeriod + StepFunctionFallbackPeriodUnit: !Ref NeptunePollerStepFunctionFallbackPeriodUnit + MaxPollingWaitTime: !Ref NeptunePollerMaxPollingWaitTime + NeptuneStreamEndpoint: !Sub 'https://${NeptuneClusterEndpoint}:8182/gremlin/stream' + IAMAuthEnabledOnSourceStream: false + # StreamDBClusterResourceId: !Ref StreamDBClusterResourceId -- only used when IAM Auth is enabled + MaxPollingInterval: !Ref NeptunePollerMaxPollingInterval + VPC: !Ref VpcId + RouteTableIds: !Ref PrivateRouteTableIds + CreateDDBVPCEndPoint: false + CreateMonitoringEndPoint: true + CreateCloudWatchAlarm: !Ref CreateCloudWatchAlarm + NotificationEmail: !Ref NotificationEmail + SubnetIds: !Join [ ",", !Ref PrivateSubNetIds ] + SecurityGroupIds: !Ref CDFSecurityGroupId + +Outputs: + OpenSearchDomainEndpoint: + Description: HTTPS endpoint URL for ElasticSearch cluster + Value: !Sub 'https://${OpenSearchDomain.DomainEndpoint}' + Export: + Name: !Sub "cdf-assetlibrary-elasticsearch-${Environment}-OpenSearchDomainEndpoint" + HTTPSAccessSG: + Description: 'HTTPS Access Security Group Arn' + Value: !GetAtt NeptuneStreamPoller.Outputs.HTTPSAccessSG + LeaseDynamoDBTable: + Description: 'Neptune Stream Poller Lease Table' + Value: !GetAtt NeptuneStreamPoller.Outputs.LeaseDynamoDBTable + StateMachineArn: + Description: 'Neptune Stream Poller State Machine Arn' + Value: !GetAtt NeptuneStreamPoller.Outputs.StateMachineArn + CronArn: + Description: 'Neptune Stream Poller Scheduler Cron Arn' + Value: !GetAtt NeptuneStreamPoller.Outputs.CronArn + StateMachineAlarmArn: + Description: 'Neptune Stream Poller State Machine Alarm Arn' + Condition: CreateCloudWatchAlarmCondition + Value: !GetAtt NeptuneStreamPoller.Outputs.StateMachineAlarmArn + NeptuneStreamPollerLambdaArn: + Description: 'Neptune Stream Poller Lambda Arn' + Value: !GetAtt NeptuneStreamPoller.Outputs.NeptuneStreamPollerLambdaArn + CloudWatchMetricsDashboardURI: + Description: 'CloudWatch Metrics Dashboard URI' + Value: !GetAtt NeptuneStreamPoller.Outputs.CloudWatchMetricsDashboardURI diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-neptune.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-neptune.yaml index e73443b58..842da8450 100644 --- a/source/packages/services/assetlibrary/infrastructure/cfn-neptune.yaml +++ b/source/packages/services/assetlibrary/infrastructure/cfn-neptune.yaml @@ -30,7 +30,7 @@ Parameters: Type: List DbInstanceType: Description: > - Neptune DB instance type. The list of available instance types for your region can be found here: + Neptune DB instance type. The list of available instance types for your region can be found here: https://aws.amazon.com/neptune/pricing/ Type: String AllowedPattern: "^db\\.[tr]\\d+[a-z0-9]*\\.[a-z0-9]*$" @@ -53,6 +53,19 @@ Parameters: - 0 - 1 Description: Enable Audit Log. 0 means disable and 1 means enable. + NeptuneEnableStreams: + Type: Number + Default: 0 + AllowedValues: + - 0 + - 1 + Description: Enable Neptune Streams. 0 means disable and 1 means enable. Must be enabled for Asset Library Enhanced Search. + BackupRetentionDays: + Type: Number + Default: 1 + MinValue: 1 + MaxValue: 35 + Description: Days automated snapshots will be retained IamAuthEnabled: Type: String Default: 'false' @@ -133,7 +146,7 @@ Resources: ToPort: 8182 IpProtocol: tcp SourceSecurityGroupId: !Ref CDFSecurityGroupId - Description: Allow access from default securty group + Description: Allow access from default security group NeptuneEC2InstanceProfile: Type: 'AWS::IAM::InstanceProfile' @@ -238,6 +251,7 @@ Resources: Description: CDF parameters Parameters: neptune_enable_audit_log: !Ref NeptuneEnableAuditLog + neptune_streams: !Ref NeptuneEnableStreams Tags: - Key: cdf_environment Value: !Ref Environment From 5ba40a96eb63cd74c11aab99adbc91700c39f566 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Wed, 26 Jan 2022 13:07:46 -0800 Subject: [PATCH 02/31] refactor(assetlibrary) remove unused stack policy file --- .../infrastructure/cfn-neptune-stack-policy.json | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 source/packages/services/assetlibrary/infrastructure/cfn-neptune-stack-policy.json diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-neptune-stack-policy.json b/source/packages/services/assetlibrary/infrastructure/cfn-neptune-stack-policy.json deleted file mode 100644 index d9a67d079..000000000 --- a/source/packages/services/assetlibrary/infrastructure/cfn-neptune-stack-policy.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "Statement": [ - { - "Effect": "Allow", - "Action": "Update:*", - "Principal": "*", - "Resource": "*" - } - ] -} From f8fd5c560e28e347212a2ecdf3778a478f5b3ea3 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Wed, 26 Jan 2022 13:44:13 -0800 Subject: [PATCH 03/31] fix(assetlibrary) OpenSearch CFN todos --- .../infrastructure/cfn-enhancedsearch.yaml | 167 ++++++++++++------ 1 file changed, 109 insertions(+), 58 deletions(-) diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml index d5716f227..ce071495f 100644 --- a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml +++ b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml @@ -19,11 +19,15 @@ Parameters: Description: Name of environment. Used to name the created resources. Type: String MinLength: 1 + MaxLength: 24 + ConstraintDescription: Must be ≤24 chars to fit within 28 char limit for OpenSearch domain name with "cdf-" prefix. VpcId: Description: ID of VPC to deploy the OpenSearch domain into Type: AWS::EC2::VPC::Id PrivateSubNetIds: - Description: Comma delimited list of private subnetIds to deploy Neptune into. Number of subnets must match the number of availability zones deployed into, i.e. only pass one subnet if operating in a single AZ. + Description: > + Comma delimited list of private subnetIds to deploy Neptune into. Number of subnets must match the number of + availability zones deployed into, i.e. only pass one subnet if operating in a single AZ. Type: List PrivateRouteTableIds: Description: Comma delimited list of private route table ids to allow access to Neptune @@ -39,40 +43,79 @@ Parameters: Type: String MinLength: 1 ElasticSearchInstanceType: - Description: ElasticSearch instance type. + Description: > + ElasticSearch instance type. Must be a supported OpenSearch instance type in the region, support ElasticSearch, + and must support at rest. See also: + https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html Type: String - Default: m6g.large.search - ConstraintDescription: 'Must be a supported OpenSearch instance type in the region, support ElasticSearch, and must support at rest. See also: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html' + Default: t3.small.search ElasticSearchInstanceCount: Type: Number Default: 2 - Description: The number of data nodes (instances) to use in the OpenSearch domain. Must be a multiple of the number of availability zones. + Description: > + The number of data nodes (instances) to use in the OpenSearch domain. Must be a multiple of the number of + availability zones. ElasticSearchDedicatedMasterCount: Type: Number Default: 0 Description: The number of dedicated master nodes (instances) to use in the OpenSearch domain. ElasticSearchEBSVolumeSize: Type: Number - Default: 20 - Description: Size of EBS volumes attached to each ElasticSearch node, in GB. + Default: 10 + MinValue: 10 + Description: > + Size of EBS volumes attached to each ElasticSearch node, in GiB. Allowed ranges depend on the + instance type chosen in ElasticSearchInstanceType and are documented here: + https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html#ebsresource ElasticSearchEBSVolumeType: Type: String Default: gp2 Description: Type of the EBS volume attached to each ElasticSearch node. AllowedValues: - gp2 + - gp3 - io1 + - io2 - standard - ConstraintDescription: | + ConstraintDescription: > See list of valid EBS volume types at https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html. Not every instance type is compatible with every volume type. + ElasticSearchEBSProvisionedIOPS: + Type: Number + Default: 16000 + Description: > + The number of I/O operations per second (IOPS) that the volume supports. This property applies only to the + Provisioned IOPS EBS volume type (io1). ElasticSearchEncryptAtRest: Type: String - Default: 'false' + Default: 'true' AllowedValues: - 'true' - 'false' Description: Enable Encryption at rest. + ElasticSearchAuditLogsCloudWatchLogsLogGroupArn: + Type: String + Default: '' + Description: > + ARN of Cloudwatch Log Group where ElasticSearch audit Logs are sent. If left empty, logs will not be generated. + ElasticSearchApplicationLogsCloudWatchLogsLogGroupArn: + Type: String + Default: '' + Description: > + ARN of Cloudwatch Log Group where ElasticSearch application Logs are sent. If left empty, logs will not be + generated. + ElasticSearchIndexSlowLogsCloudWatchLogsLogGroupArn: + Type: String + Default: '' + Description: > + ARN of Cloudwatch Log Group where ElasticSearch Index Slow Logs are sent. If left empty, logs will not be + generated. + ElasticSearchSearchSlowLogsCloudWatchLogsLogGroupArn: + Type: String + Default: '' + Description: > + ARN of Cloudwatch Log Group where ElasticSearch Search Slow Logs are sent. If left empty, logs will not be + generated. NeptuneClusterEndpoint: Description: 'Neptune cluster endpoint. Format: :' Type: String @@ -189,6 +232,11 @@ Conditions: EnableDedicatedMasterNodes: !Not [ !Equals [ !Ref ElasticSearchDedicatedMasterCount, 0 ] ] CreateCloudWatchAlarmCondition: !Equals [ !Ref CreateCloudWatchAlarm, 'true' ] EnableNonStringIndexingCondition: !Equals [ !Ref EnableNonStringIndexing, 'true' ] + ElasticSearchAuditLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref ElasticSearchAuditLogsCloudWatchLogsLogGroupArn, '' ] ] + ElasticSearchApplicationLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref ElasticSearchApplicationLogsCloudWatchLogsLogGroupArn, '' ] ] + ElasticSearchIndexSlowLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref ElasticSearchIndexSlowLogsCloudWatchLogsLogGroupArn, '' ] ] + ElasticSearchSearchSlowLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref ElasticSearchSearchSlowLogsCloudWatchLogsLogGroupArn, '' ] ] + ElasticSearchEBSVolumeTypeIsIo1: !Equals [ !Ref ElasticSearchEBSVolumeType, 'io1' ] Resources: @@ -213,25 +261,13 @@ Resources: IpProtocol: tcp SourceSecurityGroupId: !Ref NeptuneSecurityGroupId Description: Access from Neptune to ElasticSearch - - # TODO: remove this, debugging use only, allows https access to ElasticSearch API and Kibana via bastion tunneling - OpenSearchSGIngressRule2DeleteMe: - Type: 'AWS::EC2::SecurityGroupIngress' - Properties: - GroupId: !Ref OpenSearchSG - FromPort: 443 - ToPort: 443 - IpProtocol: tcp - SourceSecurityGroupId: !Ref CDFSecurityGroupId - Description: Access from CDF default security group to ElasticSearch OpenSearchDomain: Type: AWS::OpenSearchService::Domain - # TODO: copied deletion/update policies over from Neptune template, probably need something - # equivalent here... DeletionPolicy: Snapshot UpdateReplacePolicy: Snapshot Properties: + DomainName: !Sub 'cdf-${Environment}' AccessPolicies: Version: '2012-10-17' Statement: @@ -239,62 +275,79 @@ Resources: Principal: AWS: '*' Action: 'es:*' - # https://serverfault.com/questions/937008/how-can-one-configure-an-aws-elasticsearch-access-policy-using-cloudformation - Resource: '/*' + Resource: !Join + - ':' + - - 'arn:aws:es' + - !Ref 'AWS::Region' + - !Ref 'AWS::AccountId' + - !Sub 'domain/cdf-${Environment}/*' ClusterConfig: + DedicatedMasterEnabled: + Fn::If: + - EnableDedicatedMasterNodes + - true + - false DedicatedMasterCount: Fn::If: - EnableDedicatedMasterNodes - !Ref ElasticSearchDedicatedMasterCount - - Ref: AWS::NoValue - DedicatedMasterEnabled: + - !Ref AWS::NoValue + DedicatedMasterType: Fn::If: - EnableDedicatedMasterNodes - - true - - false - # DedicatedMasterType: - # Fn::If: - # - EnableDedicatedMasterNodes - # - TODO - # - Ref: AWS::NoValue + - 3 + - !Ref AWS::NoValue InstanceCount: !Ref ElasticSearchInstanceCount InstanceType: !Ref ElasticSearchInstanceType ZoneAwarenessEnabled: true - # CognitoOptions: - # CognitoOptions DomainEndpointOptions: - # CustomEndpoint: String - # CustomEndpointCertificateArn: String - # CustomEndpointEnabled: Boolean EnforceHTTPS: true - # TLSSecurityPolicy: String + TLSSecurityPolicy: 'Policy-Min-TLS-1-2-2019-07' EBSOptions: EBSEnabled: true VolumeSize: !Ref ElasticSearchEBSVolumeSize VolumeType: !Ref ElasticSearchEBSVolumeType - # Iops: Integer + Iops: + Fn::If: + - ElasticSearchEBSVolumeTypeIsIo1 + - !Ref ElasticSearchEBSProvisionedIOPS + - !Ref AWS::NoValue EncryptionAtRestOptions: Enabled: !Ref ElasticSearchEncryptAtRest KmsKeyId: Fn::If: - ElasticSearchEncryptAtRest - !Ref KmsKeyId - - Ref: AWS::NoValue + - !Ref AWS::NoValue # When integrating with Amazon OpenSearch Service, Neptune requires Elasticsearch version 7.1 or higher # rather than OpenSearch version 1.0. Neptune is not currently compatible with OpenSearch version 1.0. # https://docs.aws.amazon.com/neptune/latest/userguide/full-text-search-cfn-create.html EngineVersion: 'Elasticsearch_7.10' - # LogPublishingOptions: - # # TODO, example values from https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-opensearchservice-domain.html#aws-resource-opensearchservice-domain--examples - # ES_APPLICATION_LOGS: - # CloudWatchLogsLogGroupArn: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/opensearch/domains/opensearch-application-logs' - # Enabled: true - # SEARCH_SLOW_LOGS: - # CloudWatchLogsLogGroupArn: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/opensearch/domains/opensearch-slow-logs' - # Enabled: true - # INDEX_SLOW_LOGS: - # CloudWatchLogsLogGroupArn: 'arn:aws:logs:us-east-1:123456789012:log-group:/aws/opensearch/domains/opensearch-index-slow-logs' - # Enabled: true + LogPublishingOptions: + ES_APPLICATION_LOGS: + Fn::If: + - ElasticSearchApplicationLogsCloudWatchLogsEnabled + - CloudWatchLogsLogGroupArn: !Ref ElasticSearchApplicationLogsCloudWatchLogsLogGroupArn + Enabled: true + - !Ref AWS::NoValue + SEARCH_SLOW_LOGS: + Fn::If: + - ElasticSearchSearchSlowLogsCloudWatchLogsEnabled + - CloudWatchLogsLogGroupArn: !Ref ElasticSearchSearchSlowLogsCloudWatchLogsLogGroupArn + Enabled: true + - !Ref AWS::NoValue + INDEX_SLOW_LOGS: + Fn::If: + - ElasticSearchIndexSlowLogsCloudWatchLogsEnabled + - CloudWatchLogsLogGroupArn: !Ref ElasticSearchIndexSlowLogsCloudWatchLogsLogGroupArn + Enabled: true + - !Ref AWS::NoValue + AUDIT_LOGS: + Fn::If: + - ElasticSearchAuditLogsCloudWatchLogsEnabled + - CloudWatchLogsLogGroupArn: !Ref ElasticSearchAuditLogsCloudWatchLogsLogGroupArn + Enabled: true + - !Ref AWS::NoValue NodeToNodeEncryptionOptions: Enabled: true VPCOptions: @@ -310,8 +363,7 @@ Resources: ElasticSearchAccessPolicy: Type: 'AWS::IAM::ManagedPolicy' Properties: - ManagedPolicyName: ElasticSearchAccessPolicy - Description: "Policy for ElasticSearch Access for Neptune Lambda Poller" + Description: "Allows ElasticSearch access for Neptune Lambda Poller" PolicyDocument: Version: '2012-10-17' Statement: @@ -342,10 +394,10 @@ Resources: IgnoreMissingDocument: !Ref "IgnoreMissingDocument" ReplicationScope: "All" EnableNonStringIndexing: !Ref EnableNonStringIndexing - ApplicationName: !Sub 'cdf-${CdfService}-neptune-elasticsearch-connector' + ApplicationName: !Sub 'cdf-${CdfService}-enhancedsearch-${Environment}' LambdaMemorySize: !Ref NeptunePollerLambdaMemorySize LambdaRuntime: python3.6 - LambdaS3Bucket: !Join ["-", ["aws-neptune-customer-samples", Ref: "AWS::Region"]] + LambdaS3Bucket: !Join ["-", ["aws-neptune-customer-samples", !Ref "AWS::Region"]] LambdaS3Key: "neptune-stream/lambda/python36/release_2021_08_23/neptune-to-es.zip" ManagedPolicies: !Ref ElasticSearchAccessPolicy LambdaLoggingLevel: !Ref NeptunePollerLambdaLoggingLevel @@ -362,7 +414,6 @@ Resources: MaxPollingWaitTime: !Ref NeptunePollerMaxPollingWaitTime NeptuneStreamEndpoint: !Sub 'https://${NeptuneClusterEndpoint}:8182/gremlin/stream' IAMAuthEnabledOnSourceStream: false - # StreamDBClusterResourceId: !Ref StreamDBClusterResourceId -- only used when IAM Auth is enabled MaxPollingInterval: !Ref NeptunePollerMaxPollingInterval VPC: !Ref VpcId RouteTableIds: !Ref PrivateRouteTableIds @@ -378,7 +429,7 @@ Outputs: Description: HTTPS endpoint URL for ElasticSearch cluster Value: !Sub 'https://${OpenSearchDomain.DomainEndpoint}' Export: - Name: !Sub "cdf-assetlibrary-elasticsearch-${Environment}-OpenSearchDomainEndpoint" + Name: !Sub "cdf-assetlibrary-enhancedsearch-${Environment}-OpenSearchDomainEndpoint" HTTPSAccessSG: Description: 'HTTPS Access Security Group Arn' Value: !GetAtt NeptuneStreamPoller.Outputs.HTTPSAccessSG From 5fef222d97ebbc9b5e30d2aea90b8549fafd2111 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Tue, 1 Feb 2022 21:20:37 -0800 Subject: [PATCH 04/31] docs(assetlibrary): add detail about ElasticSearchDedicatedMasterCount parameter --- .../assetlibrary/infrastructure/cfn-enhancedsearch.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml index ce071495f..78ca0f2f4 100644 --- a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml +++ b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml @@ -58,7 +58,10 @@ Parameters: ElasticSearchDedicatedMasterCount: Type: Number Default: 0 - Description: The number of dedicated master nodes (instances) to use in the OpenSearch domain. + Description: > + The number of dedicated master nodes (instances) to use in the OpenSearch domain. The number must be larger than 2 + and should never be an even number. See also: + https://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-dedicatedmasternodes.html ElasticSearchEBSVolumeSize: Type: Number Default: 10 From 68a21ed25722eff30fb7d9fe13fdb059679d51e1 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Wed, 2 Feb 2022 18:00:38 -0800 Subject: [PATCH 05/31] fix(assetlibrary): expand list of accepted Neptune instance types, validate OpenSearch instance type --- .../assetlibrary/infrastructure/cfn-enhancedsearch.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml index 78ca0f2f4..e56d62276 100644 --- a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml +++ b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml @@ -44,11 +44,13 @@ Parameters: MinLength: 1 ElasticSearchInstanceType: Description: > - ElasticSearch instance type. Must be a supported OpenSearch instance type in the region, support ElasticSearch, - and must support at rest. See also: + OpenSearch data instance type. Must be a supported OpenSearch instance type in the region, support ElasticSearch, + and must support encryption at rest. See also: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html Type: String Default: t3.small.search + AllowedPattern: "^[a-z][a-z0-9]+\\.[a-z0-9]+\\.search$" + ConstraintDescription: Must be a valid OpenSearch instance type. ElasticSearchInstanceCount: Type: Number Default: 2 From d21c22b263fde02cef24677d69c15edf7b2fbdc6 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Mon, 14 Mar 2022 14:02:50 -0700 Subject: [PATCH 06/31] feat(installer): Introduce "enhanced" as answer option for asset library mode --- .../commands/modules/service/assetLibrary.ts | 47 ++++++++++++------- .../services/installer/src/models/answers.ts | 15 +++++- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts b/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts index f0f2c04a2..798d306ed 100644 --- a/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts +++ b/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts @@ -14,7 +14,7 @@ import inquirer from 'inquirer'; import { ListrTask } from 'listr2'; import ow from 'ow'; import path from 'path'; -import { Answers } from '../../../models/answers'; +import { Answers, AssetLibraryMode } from '../../../models/answers'; import { ModuleName, PostmanEnvironment, RestModule } from '../../../models/modules'; import { applicationConfigurationPrompt } from '../../../prompts/applicationConfiguration.prompt'; import { @@ -44,6 +44,13 @@ const ASSUMED_NEPTUNE_ENGINE_VERSION = '1.1.0.0'; // AWS.RDS.DescribeOrderableDBInstanceOptions API. const DEFAULT_NEPTUNE_INSTANCE_TYPE = 'db.r5.xlarge'; +export function modeRequiresNeptune(mode: string): boolean { + return mode === AssetLibraryMode.Full || mode === AssetLibraryMode.Enhanced; +} + +export function modeRequiresOpenSearch(mode: string): boolean { + return mode === AssetLibraryMode.Enhanced; +} export class AssetLibraryInstaller implements RestModule { public readonly friendlyName = 'Asset Library'; public readonly name = 'assetLibrary'; @@ -79,11 +86,15 @@ export class AssetLibraryInstaller implements RestModule { updatedAnswers = await inquirer.prompt( [ { - message: `Run in 'full' mode (with Amazon Neptune), or 'lite' mode (using AWS IoT Device Registry only). Note that 'lite' mode supports a reduced set of Asset Library features (see documentation for further info).`, + message: `Asset library mode: 'full' uses Amazon Neptune as data store. 'enhanced' adds Amazon OpenSearch for enhanced search features. 'lite' uses only AWS IoT Device Registry and supports a reduced feature set. See documentation for details.`, type: 'list', - choices: ['full', 'lite'], + choices: [ + AssetLibraryMode.Full, + AssetLibraryMode.Lite, + AssetLibraryMode.Enhanced, + ], name: 'assetLibrary.mode', - default: answers.assetLibrary?.mode ?? 'full', + default: answers.assetLibrary?.mode ?? AssetLibraryMode.Full, askAnswered: true, validate(answer: string) { if (answer?.length === 0) { @@ -108,7 +119,7 @@ export class AssetLibraryInstaller implements RestModule { loop: false, pageSize: 10, when(answers: Answers) { - return answers.assetLibrary?.mode === 'full'; + return modeRequiresNeptune(answers.assetLibrary?.mode); }, validate(answer: string) { if ( @@ -129,7 +140,7 @@ export class AssetLibraryInstaller implements RestModule { default: answers.assetLibrary?.createDbReplicaInstance ?? false, askAnswered: true, when(answers: Answers) { - return answers.assetLibrary?.mode === 'full'; + return modeRequiresNeptune(answers.assetLibrary?.mode); }, validate(answer: string) { if (answer?.length === 0) { @@ -145,7 +156,7 @@ export class AssetLibraryInstaller implements RestModule { default: answers.assetLibrary?.restoreFromSnapshot ?? false, askAnswered: true, when(answers: Answers) { - return answers.assetLibrary?.mode === 'full'; + return modeRequiresNeptune(answers.assetLibrary?.mode); }, validate(answer: string) { if (answer?.length === 0) { @@ -162,7 +173,7 @@ export class AssetLibraryInstaller implements RestModule { askAnswered: true, when(answers: Answers) { return ( - answers.assetLibrary?.mode === 'full' && + modeRequiresNeptune(answers.assetLibrary?.mode) && answers.assetLibrary?.restoreFromSnapshot ); }, @@ -216,7 +227,7 @@ export class AssetLibraryInstaller implements RestModule { includeOptionalModule( 'vpc', updatedAnswers.modules, - updatedAnswers.assetLibrary.mode === 'full' + modeRequiresNeptune(updatedAnswers.assetLibrary.mode) ); return updatedAnswers; } @@ -290,7 +301,7 @@ export class AssetLibraryInstaller implements RestModule { ); addIfSpecified( 'CustomResourceVPCLambdaArn', - answers.assetLibrary.mode === 'full' + modeRequiresNeptune(answers.assetLibrary?.mode) ? answers.deploymentHelper.vpcLambdaArn : answers.deploymentHelper.lambdaArn ); @@ -335,7 +346,7 @@ export class AssetLibraryInstaller implements RestModule { return [answers, tasks]; } - if (answers.assetLibrary.mode === 'full') { + if (modeRequiresNeptune(answers.assetLibrary?.mode)) { tasks.push({ title: `Deploying stack '${this.neptuneStackName}'`, task: async () => { @@ -433,12 +444,14 @@ export class AssetLibraryInstaller implements RestModule { }, }); - tasks.push({ - title: `Deleting stack '${this.neptuneStackName}'`, - task: async () => { - await deleteStack(this.neptuneStackName, answers.region); - }, - }); + if (modeRequiresNeptune(answers.assetLibrary?.mode)) { + tasks.push({ + title: `Deleting stack '${this.neptuneStackName}'`, + task: async () => { + await deleteStack(this.neptuneStackName, answers.region); + }, + }); + } return tasks; } } diff --git a/source/packages/services/installer/src/models/answers.ts b/source/packages/services/installer/src/models/answers.ts index 18a9238f8..508288f9e 100644 --- a/source/packages/services/installer/src/models/answers.ts +++ b/source/packages/services/installer/src/models/answers.ts @@ -103,6 +103,7 @@ export interface ProvisionedConcurrencyModuleAttribues extends ServiceModuleAttr provisionedConcurrentExecutions?: number; enableAutoScaling?: boolean; } + export interface RestServiceModuleAttribues extends ServiceModuleAttributes { enableCustomDomain?: boolean; customDomainBasePath?: string; @@ -121,15 +122,27 @@ export interface OrganizationManager extends RestServiceModuleAttribues { artifactBucketPrefix?: string; } +export const enum AssetLibraryMode { + Lite = 'lite', + Full = 'full', + Enhanced = 'enhanced', +} + export interface AssetLibrary extends RestServiceModuleAttribues, ProvisionedConcurrencyModuleAttribues { - mode?: 'full' | 'lite'; + mode?: AssetLibraryMode; + // Neptune Configuration neptuneDbInstanceType?: string; createDbReplicaInstance?: boolean; neptuneSnapshotIdentifier?: string; restoreFromSnapshot?: boolean; neptuneUrl?: string; + // OpenSearch Configuration + openSearchDataNodeInstanceType?: string; + openSearchEBSVolumeSize?: number; + neptuneSecurityGroup?: string; + neptuneClusterReadEndpoint?: string; // Application Configuration defaultAnswer?: boolean; defaultDevicesParentRelationName?: string; From ff269990f8d40e1781510766827336546a05855a Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Tue, 15 Mar 2022 11:22:38 -0700 Subject: [PATCH 07/31] feat(installer): automatically create service-linked role for OpenSearch --- source/infrastructure/install-policy-3.json | 22 +++++++++++++--- .../commands/modules/service/assetLibrary.ts | 26 +++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/source/infrastructure/install-policy-3.json b/source/infrastructure/install-policy-3.json index d6e6645cd..fbe13318d 100644 --- a/source/infrastructure/install-policy-3.json +++ b/source/infrastructure/install-policy-3.json @@ -108,12 +108,28 @@ } } }, + { + "Sid": "OpenSearchServiceLinkedRoleCreate", + "Action": "iam:CreateServiceLinkedRole", + "Effect": "Allow", + "Resource": "arn:aws:iam::*:role/aws-service-role/es.amazonaws.com/AWSServiceRoleForAmazonElasticsearchService", + "Condition": { + "StringLike": { + "iam:AWSServiceName":"es.amazonaws.com" + } + } + }, + { + "Sid": "OpenSearchServiceLinkedRoleGet", + "Action": "iam:GetRole", + "Effect": "Allow", + "Resource": "arn:aws:iam::*:role/aws-service-role/es.amazonaws.com/AWSServiceRoleForAmazonElasticsearchService" + }, { "Sid": "ECS", "Action": "ecs:*", "Effect": "Allow", "Resource": "*" - } - + } ] -} \ No newline at end of file +} diff --git a/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts b/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts index 798d306ed..34cea2d65 100644 --- a/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts +++ b/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts @@ -10,6 +10,7 @@ * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions * * and limitations under the License. * *********************************************************************************************************************/ +import { CreateServiceLinkedRoleCommand, IAMClient } from '@aws-sdk/client-iam'; import inquirer from 'inquirer'; import { ListrTask } from 'listr2'; import ow from 'ow'; @@ -378,6 +379,31 @@ export class AssetLibraryInstaller implements RestModule { answers.assetLibrary.neptuneUrl = byOutputKey('GremlinEndpoint'); }, }); + if (modeRequiresOpenSearch(answers.assetLibrary.mode)) { + tasks.push({ + title: `Ensure service-linked role 'AWSServiceRoleForAmazonElasticsearchService' exists`, + task: async () => { + const iamClient = new IAMClient({}); + const command = new CreateServiceLinkedRoleCommand({ + AWSServiceName: 'es.amazonaws.com', + }); + try { + await iamClient.send(command); + } catch (err) { + console.error(`ERROR! ${JSON.stringify(err)}`); + throw err; + } + + // An error occurred (InvalidInput) when calling the CreateServiceLinkedRole operation: Service role name + // AWSServiceRoleForAmazonElasticsearchService has been taken in this account, please try a different suffix. + + // await execa('aws', [ + // 'iam', 'create-service-linked-role', + // '--aws-service-name', 'es.amazonaws.com' + // ]); + }, + }); + } } tasks.push({ From 38fbf10773ff370d136954a364fbd63215747356 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Tue, 15 Mar 2022 14:58:00 -0700 Subject: [PATCH 08/31] feat(installer): deploy cfn-enhancedsearch.yaml --- .../infrastructure/cfn-assetLibrary.yaml | 2 +- .../infrastructure/cfn-enhancedsearch.yaml | 113 +++++++----- .../commands/modules/service/assetLibrary.ts | 170 ++++++++++++++++-- .../services/installer/src/models/answers.ts | 1 + 4 files changed, 227 insertions(+), 59 deletions(-) diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-assetLibrary.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-assetLibrary.yaml index 33f1fc71e..b3322afef 100644 --- a/source/packages/services/assetlibrary/infrastructure/cfn-assetLibrary.yaml +++ b/source/packages/services/assetlibrary/infrastructure/cfn-assetLibrary.yaml @@ -95,7 +95,7 @@ Parameters: Default: 'N/A' ProvisionedConcurrentExecutions: - Description: The no. of desired concurrent executions to provision. Set to 0 to disable. + Description: The no. of desired concurrent executions to provision. Set to 0 to disable. Type: Number Default: 0 diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml index e56d62276..1e9560d06 100644 --- a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml +++ b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml @@ -29,9 +29,6 @@ Parameters: Comma delimited list of private subnetIds to deploy Neptune into. Number of subnets must match the number of availability zones deployed into, i.e. only pass one subnet if operating in a single AZ. Type: List - PrivateRouteTableIds: - Description: Comma delimited list of private route table ids to allow access to Neptune - Type: String CDFSecurityGroupId: Description: ID of an existing security group to allow access to ElasticSearch Type: AWS::EC2::SecurityGroup::Id @@ -42,7 +39,18 @@ Parameters: Description: The KMS key ID used to encrypt the ElasticSearch database Type: String MinLength: 1 - ElasticSearchInstanceType: + OpenSearchAvailabilityZoneCount: + Description: > + If deploying more than one data instance (defined in OpenSearchInstanceCount), the instances can be distributed + across availability zones. Default is 1, i.e. single-AZ. Choose 2 or 3 for multi-AZ. The list of subnets given in + PrivateSubNetIDs must have a length equal to the number given here. The number of data instances in + OpenSearchInstanceCount must be an integer multiple of the number given here. + Type: Number + Default: 1 + MinValue: 1 + MaxValue: 3 + ConstraintDescription: OpenSearchAvailabilityZoneCount must equal 1, 2, or 3. + OpenSearchInstanceType: Description: > OpenSearch data instance type. Must be a supported OpenSearch instance type in the region, support ElasticSearch, and must support encryption at rest. See also: @@ -51,28 +59,37 @@ Parameters: Default: t3.small.search AllowedPattern: "^[a-z][a-z0-9]+\\.[a-z0-9]+\\.search$" ConstraintDescription: Must be a valid OpenSearch instance type. - ElasticSearchInstanceCount: + OpenSearchInstanceCount: Type: Number - Default: 2 + Default: 1 Description: > - The number of data nodes (instances) to use in the OpenSearch domain. Must be a multiple of the number of - availability zones. - ElasticSearchDedicatedMasterCount: + The number of data nodes (instances) to use in the OpenSearch domain. Must be an integer multiple of the number of + availability zones specified in OpenSearchAvailabilityZoneCount. + OpenSearchDedicatedMasterCount: Type: Number Default: 0 Description: > The number of dedicated master nodes (instances) to use in the OpenSearch domain. The number must be larger than 2 and should never be an even number. See also: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-dedicatedmasternodes.html - ElasticSearchEBSVolumeSize: + OpenSearchDedicatedMasterInstanceType: + Description: > + OpenSearch master instance type. Must be a supported OpenSearch instance type in the region, support ElasticSearch, + and must support encryption at rest. See also: + https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html + Type: String + Default: t3.small.search + AllowedPattern: "^[a-z][a-z0-9]+\\.[a-z0-9]+\\.search$" + ConstraintDescription: Must be a valid OpenSearch instance type. + OpenSearchEBSVolumeSize: Type: Number Default: 10 MinValue: 10 Description: > Size of EBS volumes attached to each ElasticSearch node, in GiB. Allowed ranges depend on the - instance type chosen in ElasticSearchInstanceType and are documented here: + instance type chosen in OpenSearchInstanceType and are documented here: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html#ebsresource - ElasticSearchEBSVolumeType: + OpenSearchEBSVolumeType: Type: String Default: gp2 Description: Type of the EBS volume attached to each ElasticSearch node. @@ -85,37 +102,30 @@ Parameters: ConstraintDescription: > See list of valid EBS volume types at https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html. Not every instance type is compatible with every volume type. - ElasticSearchEBSProvisionedIOPS: + OpenSearchEBSProvisionedIOPS: Type: Number Default: 16000 Description: > The number of I/O operations per second (IOPS) that the volume supports. This property applies only to the Provisioned IOPS EBS volume type (io1). - ElasticSearchEncryptAtRest: - Type: String - Default: 'true' - AllowedValues: - - 'true' - - 'false' - Description: Enable Encryption at rest. - ElasticSearchAuditLogsCloudWatchLogsLogGroupArn: + OpenSearchAuditLogsCloudWatchLogsLogGroupArn: Type: String Default: '' Description: > ARN of Cloudwatch Log Group where ElasticSearch audit Logs are sent. If left empty, logs will not be generated. - ElasticSearchApplicationLogsCloudWatchLogsLogGroupArn: + OpenSearchApplicationLogsCloudWatchLogsLogGroupArn: Type: String Default: '' Description: > ARN of Cloudwatch Log Group where ElasticSearch application Logs are sent. If left empty, logs will not be generated. - ElasticSearchIndexSlowLogsCloudWatchLogsLogGroupArn: + OpenSearchIndexSlowLogsCloudWatchLogsLogGroupArn: Type: String Default: '' Description: > ARN of Cloudwatch Log Group where ElasticSearch Index Slow Logs are sent. If left empty, logs will not be generated. - ElasticSearchSearchSlowLogsCloudWatchLogsLogGroupArn: + OpenSearchSearchSlowLogsCloudWatchLogsLogGroupArn: Type: String Default: '' Description: > @@ -233,15 +243,16 @@ Parameters: Conditions: - ElasticSearchEncryptAtRest: !Equals [ !Ref ElasticSearchEncryptAtRest, 'true' ] - EnableDedicatedMasterNodes: !Not [ !Equals [ !Ref ElasticSearchDedicatedMasterCount, 0 ] ] + KmsKeyIdProvided: !Not [ !Equals [ !Ref KmsKeyId, "" ] ] + EnableDedicatedMasterNodes: !Not [ !Equals [ !Ref OpenSearchDedicatedMasterCount, 0 ] ] CreateCloudWatchAlarmCondition: !Equals [ !Ref CreateCloudWatchAlarm, 'true' ] EnableNonStringIndexingCondition: !Equals [ !Ref EnableNonStringIndexing, 'true' ] - ElasticSearchAuditLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref ElasticSearchAuditLogsCloudWatchLogsLogGroupArn, '' ] ] - ElasticSearchApplicationLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref ElasticSearchApplicationLogsCloudWatchLogsLogGroupArn, '' ] ] - ElasticSearchIndexSlowLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref ElasticSearchIndexSlowLogsCloudWatchLogsLogGroupArn, '' ] ] - ElasticSearchSearchSlowLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref ElasticSearchSearchSlowLogsCloudWatchLogsLogGroupArn, '' ] ] - ElasticSearchEBSVolumeTypeIsIo1: !Equals [ !Ref ElasticSearchEBSVolumeType, 'io1' ] + ElasticSearchAuditLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref OpenSearchAuditLogsCloudWatchLogsLogGroupArn, '' ] ] + ElasticSearchApplicationLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref OpenSearchApplicationLogsCloudWatchLogsLogGroupArn, '' ] ] + ElasticSearchIndexSlowLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref OpenSearchIndexSlowLogsCloudWatchLogsLogGroupArn, '' ] ] + ElasticSearchSearchSlowLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref OpenSearchSearchSlowLogsCloudWatchLogsLogGroupArn, '' ] ] + OpenSearchEBSVolumeTypeIsIo1: !Equals [ !Ref OpenSearchEBSVolumeType, 'io1' ] + EnableZoneAwareness: !Not [ !Equals [ !Ref OpenSearchAvailabilityZoneCount, 1 ] ] Resources: @@ -287,6 +298,8 @@ Resources: - !Ref 'AWS::AccountId' - !Sub 'domain/cdf-${Environment}/*' ClusterConfig: + InstanceCount: !Ref OpenSearchInstanceCount + InstanceType: !Ref OpenSearchInstanceType DedicatedMasterEnabled: Fn::If: - EnableDedicatedMasterNodes @@ -295,33 +308,40 @@ Resources: DedicatedMasterCount: Fn::If: - EnableDedicatedMasterNodes - - !Ref ElasticSearchDedicatedMasterCount + - !Ref OpenSearchDedicatedMasterCount - !Ref AWS::NoValue DedicatedMasterType: Fn::If: - EnableDedicatedMasterNodes - - 3 + - !Ref OpenSearchDedicatedMasterInstanceType + - !Ref AWS::NoValue + ZoneAwarenessEnabled: + Fn::If: + - EnableZoneAwareness + - true + - false + ZoneAwarenessConfig: + Fn::If: + - EnableZoneAwareness + - AvailabilityZoneCount: !Ref OpenSearchAvailabilityZoneCount - !Ref AWS::NoValue - InstanceCount: !Ref ElasticSearchInstanceCount - InstanceType: !Ref ElasticSearchInstanceType - ZoneAwarenessEnabled: true DomainEndpointOptions: EnforceHTTPS: true TLSSecurityPolicy: 'Policy-Min-TLS-1-2-2019-07' EBSOptions: EBSEnabled: true - VolumeSize: !Ref ElasticSearchEBSVolumeSize - VolumeType: !Ref ElasticSearchEBSVolumeType + VolumeSize: !Ref OpenSearchEBSVolumeSize + VolumeType: !Ref OpenSearchEBSVolumeType Iops: Fn::If: - - ElasticSearchEBSVolumeTypeIsIo1 - - !Ref ElasticSearchEBSProvisionedIOPS + - OpenSearchEBSVolumeTypeIsIo1 + - !Ref OpenSearchEBSProvisionedIOPS - !Ref AWS::NoValue EncryptionAtRestOptions: - Enabled: !Ref ElasticSearchEncryptAtRest + Enabled: true KmsKeyId: Fn::If: - - ElasticSearchEncryptAtRest + - KmsKeyIdProvided - !Ref KmsKeyId - !Ref AWS::NoValue # When integrating with Amazon OpenSearch Service, Neptune requires Elasticsearch version 7.1 or higher @@ -332,25 +352,25 @@ Resources: ES_APPLICATION_LOGS: Fn::If: - ElasticSearchApplicationLogsCloudWatchLogsEnabled - - CloudWatchLogsLogGroupArn: !Ref ElasticSearchApplicationLogsCloudWatchLogsLogGroupArn + - CloudWatchLogsLogGroupArn: !Ref OpenSearchApplicationLogsCloudWatchLogsLogGroupArn Enabled: true - !Ref AWS::NoValue SEARCH_SLOW_LOGS: Fn::If: - ElasticSearchSearchSlowLogsCloudWatchLogsEnabled - - CloudWatchLogsLogGroupArn: !Ref ElasticSearchSearchSlowLogsCloudWatchLogsLogGroupArn + - CloudWatchLogsLogGroupArn: !Ref OpenSearchSearchSlowLogsCloudWatchLogsLogGroupArn Enabled: true - !Ref AWS::NoValue INDEX_SLOW_LOGS: Fn::If: - ElasticSearchIndexSlowLogsCloudWatchLogsEnabled - - CloudWatchLogsLogGroupArn: !Ref ElasticSearchIndexSlowLogsCloudWatchLogsLogGroupArn + - CloudWatchLogsLogGroupArn: !Ref OpenSearchIndexSlowLogsCloudWatchLogsLogGroupArn Enabled: true - !Ref AWS::NoValue AUDIT_LOGS: Fn::If: - ElasticSearchAuditLogsCloudWatchLogsEnabled - - CloudWatchLogsLogGroupArn: !Ref ElasticSearchAuditLogsCloudWatchLogsLogGroupArn + - CloudWatchLogsLogGroupArn: !Ref OpenSearchAuditLogsCloudWatchLogsLogGroupArn Enabled: true - !Ref AWS::NoValue NodeToNodeEncryptionOptions: @@ -421,7 +441,6 @@ Resources: IAMAuthEnabledOnSourceStream: false MaxPollingInterval: !Ref NeptunePollerMaxPollingInterval VPC: !Ref VpcId - RouteTableIds: !Ref PrivateRouteTableIds CreateDDBVPCEndPoint: false CreateMonitoringEndPoint: true CreateCloudWatchAlarm: !Ref CreateCloudWatchAlarm diff --git a/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts b/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts index 34cea2d65..11d87c379 100644 --- a/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts +++ b/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts @@ -10,7 +10,7 @@ * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions * * and limitations under the License. * *********************************************************************************************************************/ -import { CreateServiceLinkedRoleCommand, IAMClient } from '@aws-sdk/client-iam'; +import { CreateServiceLinkedRoleCommand, GetRoleCommand, IAMClient } from '@aws-sdk/client-iam'; import inquirer from 'inquirer'; import { ListrTask } from 'listr2'; import ow from 'ow'; @@ -63,9 +63,11 @@ export class AssetLibraryInstaller implements RestModule { public readonly stackName: string; private readonly neptuneStackName: string; + private readonly enhancedSearchStackName: string; constructor(environment: string) { this.neptuneStackName = `cdf-assetlibrary-neptune-${environment}`; + this.enhancedSearchStackName = `cdf-assetlibrary-enhancedsearch-${environment}`; this.stackName = `cdf-assetlibrary-${environment}`; } includeOptionalModules: (answers: Answers) => Answers; @@ -166,6 +168,53 @@ export class AssetLibraryInstaller implements RestModule { return true; }, }, + { + message: `Select the OpenSearch cluster data node instance type:`, + type: 'input', + name: 'assetLibrary.openSearchDataNodeInstanceType', + default: + answers.assetLibrary?.openSearchDataNodeInstanceType ?? + 't3.small.search', + askAnswered: true, + when: (answers: Answers) => + modeRequiresOpenSearch(answers.assetLibrary?.mode), + validate(answer: string) { + if (answer?.length === 0) { + return 'You must enter the OpenSearch data node instance type.'; + } + return true; + }, + }, + { + message: `Enter the number of OpenSearch cluster data node instances. This number must either be 1 or a multiple of the number of private subnets in the VPC:`, + type: 'number', + name: 'assetLibrary.openSearchDataNodeInstanceCount', + default: answers.assetLibrary?.openSearchDataNodeInstanceCount ?? 1, + askAnswered: true, + when: (answers: Answers) => + modeRequiresOpenSearch(answers.assetLibrary?.mode), + validate(answer: number) { + if (answer < 1) { + return 'The number of OpenSearch data node instances must be a non-zero multiple of the number of availability zones.'; + } + return true; + }, + }, + { + message: `Size of the EBS volume attached to OpenSearch data nodes in GiB`, + type: 'number', + name: 'assetLibrary.openSearchEBSVolumeSize', + default: answers.assetLibrary?.openSearchEBSVolumeSize ?? 10, + askAnswered: true, + when: (answers: Answers) => + modeRequiresOpenSearch(answers.assetLibrary?.mode), + validate(answer: number) { + if (answer < 10) { + return `You must specify at least 10 GiB for EBS volume size. For detailed documentation, see: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html#ebsresource`; + } + return true; + }, + }, { message: `Enter the Neptune database snapshot identifier:`, type: 'input', @@ -379,28 +428,127 @@ export class AssetLibraryInstaller implements RestModule { answers.assetLibrary.neptuneUrl = byOutputKey('GremlinEndpoint'); }, }); + if (modeRequiresOpenSearch(answers.assetLibrary.mode)) { tasks.push({ title: `Ensure service-linked role 'AWSServiceRoleForAmazonElasticsearchService' exists`, task: async () => { const iamClient = new IAMClient({}); - const command = new CreateServiceLinkedRoleCommand({ - AWSServiceName: 'es.amazonaws.com', + + const getCommand1 = new GetRoleCommand({ + RoleName: 'AWSServiceRoleForAmazonOpenSearchService', }); try { - await iamClient.send(command); - } catch (err) { - console.error(`ERROR! ${JSON.stringify(err)}`); - throw err; + const data1 = await iamClient.send(getCommand1); + console.log( + `First attempt at finding SLR ${data1.$metadata.httpStatusCode}` + ); + if (data1.$metadata.httpStatusCode === 200) return; + } catch { + /* do nothing */ + } + // also probe for the legacy name of the role + const getCommand2 = new GetRoleCommand({ + RoleName: 'AWSServiceRoleForAmazonElasticsearchService', + }); + try { + const data2 = await iamClient.send(getCommand2); + console.log( + `Second attempt at finding SLR ${data2.$metadata.httpStatusCode}` + ); + if (data2.$metadata.httpStatusCode === 200) return; + } catch { + /* do nothing */ } + // if neither role exists, create it + const createCommand = new CreateServiceLinkedRoleCommand({ + AWSServiceName: 'es.amazonaws.com', + }); + await iamClient.send(createCommand); + // An error occurred (InvalidInput) when calling the CreateServiceLinkedRole operation: Service role name // AWSServiceRoleForAmazonElasticsearchService has been taken in this account, please try a different suffix. + }, + }); - // await execa('aws', [ - // 'iam', 'create-service-linked-role', - // '--aws-service-name', 'es.amazonaws.com' - // ]); + tasks.push({ + title: `Deploying stack '${this.enhancedSearchStackName}'`, + task: async () => { + const vpcSubnetIdsArr = answers.vpc.privateSubnetIds.split(','); + const instanceCount = answers.assetLibrary.openSearchDataNodeInstanceCount; + let subnetIds = answers.vpc.privateSubnetIds; + if (instanceCount < vpcSubnetIdsArr.length) { + subnetIds = vpcSubnetIdsArr.slice(0, instanceCount).join(','); + } else if (instanceCount % vpcSubnetIdsArr.length != 0) { + throw new Error( + `The chosen number of OpenSearch data nodes (${instanceCount}) is not an integer multiple of the number of VPC subnets given (${vpcSubnetIdsArr.length}: ${answers.vpc.privateSubnetIds}).` + ); + } + const availabilityZoneCount = subnetIds.split(',').length; + + const parameterOverrides = [ + `Environment=${answers.environment}`, + `VpcId=${answers.vpc.id}`, + `CDFSecurityGroupId=${answers.vpc.securityGroupId}`, + `PrivateSubNetIds=${subnetIds}`, + `CustomResourceVPCLambdaArn=${answers.deploymentHelper.vpcLambdaArn}`, + `KmsKeyId=${answers.kms.id}`, + `NeptuneSecurityGroupId=${answers.assetLibrary.neptuneSecurityGroup}`, + `NeptuneClusterEndpoint=${answers.assetLibrary.neptuneClusterReadEndpoint}`, + `OpenSearchInstanceType=${answers.assetLibrary.openSearchDataNodeInstanceType}`, + `OpenSearchInstanceCount=${answers.assetLibrary.openSearchDataNodeInstanceCount}`, + `OpenSearchEBSVolumeSize=${answers.assetLibrary.openSearchEBSVolumeSize}`, + `OpenSearchAvailabilityZoneCount=${availabilityZoneCount}`, + // # Parameters available in the Cloudformation template but not currently exposed in the installer are + // # listed as comments below. + // ## Unused Parameters for defining OpenSearch cluster setup: + // OpenSearchDedicatedMasterCount + // OpenSearchDedicatedMasterInstanceType + // OpenSearchEBSVolumeType + // OpenSearchEBSProvisionedIOPS + // NumberOfShards + // NumberOfReplica + // ## Unused Parameters for defining stream poller lambda behavior: + // NeptunePollerLambdaMemorySize + // NeptunePollerLambdaLoggingLevel + // NeptunePollerStreamRecordsBatchSize + // NeptunePollerMaxPollingWaitTime + // NeptunePollerMaxPollingInterval + // NeptunePollerStepFunctionFallbackPeriod + // NeptunePollerStepFunctionFallbackPeriodUnit + // ## Unused Parameters for defining data syncing behavior: + // GeoLocationFields + // PropertiesToExclude + // DatatypesToExclude + // IgnoreMissingDocument + // EnableNonStringIndexing + // ## Unused Parameters for defining monitoring and alerting: + // OpenSearchAuditLogsCloudWatchLogsLogGroupArn + // OpenSearchApplicationLogsCloudWatchLogsLogGroupArn + // OpenSearchIndexSlowLogsCloudWatchLogsLogGroupArn + // OpenSearchSearchSlowLogsCloudWatchLogsLogGroupArn + // CreateCloudWatchAlarm + // NotificationEmail + ]; + + await packageAndDeployStack({ + answers: answers, + stackName: this.enhancedSearchStackName, + serviceName: 'assetlibrary', + templateFile: 'infrastructure/cfn-enhancedsearch.yaml', + cwd: path.join( + monorepoRoot, + 'source', + 'packages', + 'services', + 'assetlibrary' + ), + parameterOverrides: parameterOverrides, + needsPackaging: false, + needsCapabilityNamedIAM: true, + needsCapabilityAutoExpand: false, + }); }, }); } diff --git a/source/packages/services/installer/src/models/answers.ts b/source/packages/services/installer/src/models/answers.ts index 508288f9e..6a1691435 100644 --- a/source/packages/services/installer/src/models/answers.ts +++ b/source/packages/services/installer/src/models/answers.ts @@ -140,6 +140,7 @@ export interface AssetLibrary neptuneUrl?: string; // OpenSearch Configuration openSearchDataNodeInstanceType?: string; + openSearchDataNodeInstanceCount?: number; openSearchEBSVolumeSize?: number; neptuneSecurityGroup?: string; neptuneClusterReadEndpoint?: string; From d47998c1a02efe673e4c0aaeedbd03e1a5f8f20e Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Tue, 15 Mar 2022 21:25:43 -0700 Subject: [PATCH 09/31] feat(installer): export enhancedsearch config values --- .../installer/src/commands/modules/service/assetLibrary.ts | 4 ++++ source/packages/services/installer/src/models/answers.ts | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts b/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts index 11d87c379..c9e909270 100644 --- a/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts +++ b/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts @@ -381,6 +381,10 @@ export class AssetLibraryInstaller implements RestModule { addIfSpecified('DbInstanceType', answers.assetLibrary.neptuneDbInstanceType); addIfSpecified('CreateDBReplicaInstance', answers.assetLibrary.createDbReplicaInstance); addIfSpecified('SnapshotIdentifier', answers.assetLibrary.neptuneSnapshotIdentifier); + // The Neptune-to-OpenSearch integration relies on Neptune Streams + if (modeRequiresOpenSearch(answers.assetLibrary.mode)) { + parameterOverrides.push('NeptuneEnableStreams=1'); + } return parameterOverrides; } diff --git a/source/packages/services/installer/src/models/answers.ts b/source/packages/services/installer/src/models/answers.ts index 6a1691435..0f97126c7 100644 --- a/source/packages/services/installer/src/models/answers.ts +++ b/source/packages/services/installer/src/models/answers.ts @@ -137,7 +137,6 @@ export interface AssetLibrary createDbReplicaInstance?: boolean; neptuneSnapshotIdentifier?: string; restoreFromSnapshot?: boolean; - neptuneUrl?: string; // OpenSearch Configuration openSearchDataNodeInstanceType?: string; openSearchDataNodeInstanceCount?: number; @@ -146,12 +145,14 @@ export interface AssetLibrary neptuneClusterReadEndpoint?: string; // Application Configuration defaultAnswer?: boolean; + neptuneUrl?: string; defaultDevicesParentRelationName?: string; defaultDevicesParentPath?: string; defaultDevicesState?: string; defaultGroupsValidateAllowedParentPath?: string; enableDfeOptimization?: boolean; authorizationEnabled?: boolean; + openSearchEndpoint?: string; } export interface AssetLibraryExport extends ServiceModuleAttributes { From 5bda0ac85e5014a6ea914d49ae7f1aca11d99629 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Wed, 26 Jan 2022 13:21:10 -0800 Subject: [PATCH 10/31] enhanced search parameters for assetlibary-client --- .../src/client/search.model.spec.ts | 3 ++ .../src/client/search.model.ts | 42 +++++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/source/packages/libraries/clients/assetlibrary-client/src/client/search.model.spec.ts b/source/packages/libraries/clients/assetlibrary-client/src/client/search.model.spec.ts index 4391f98f7..b73fb8d45 100644 --- a/source/packages/libraries/clients/assetlibrary-client/src/client/search.model.spec.ts +++ b/source/packages/libraries/clients/assetlibrary-client/src/client/search.model.spec.ts @@ -125,6 +125,9 @@ describe('SearchRequestModel', () => { 'startsWith', 'endsWith', 'contains', + 'fulltext', + 'regex', + 'lucene', 'exist', 'nexist', ]; diff --git a/source/packages/libraries/clients/assetlibrary-client/src/client/search.model.ts b/source/packages/libraries/clients/assetlibrary-client/src/client/search.model.ts index 670d25a47..45ceca910 100644 --- a/source/packages/libraries/clients/assetlibrary-client/src/client/search.model.ts +++ b/source/packages/libraries/clients/assetlibrary-client/src/client/search.model.ts @@ -48,6 +48,9 @@ export class SearchRequestModel { startsWith?: SearchRequestFilters; endsWith?: SearchRequestFilters; contains?: SearchRequestFilters; + fulltext?: SearchRequestFilters; + regex?: SearchRequestFilters; + lucene?: SearchRequestFilters; exist?: SearchRequestFilters; nexist?: SearchRequestFilters; @@ -72,6 +75,9 @@ export class SearchRequestModel { this.startsWith = other.startsWith; this.endsWith = other.endsWith; this.contains = other.contains; + this.fulltext = other.fulltext; + this.regex = other.regex; + this.lucene = other.lucene; this.exist = other.exist; this.nexist = other.nexist; this.facetField = other.facetField; @@ -169,6 +175,18 @@ export class SearchRequestModel { qs = qs.concat(this.buildQSValues('endsWith', this.endsWith, true)); } + if (this.fulltext) { + qs = qs.concat(this.buildQSValues('fulltext', this.fulltext, true)); + } + + if (this.regex) { + qs = qs.concat(this.buildQSValues('regex', this.regex, true)); + } + + if (this.lucene) { + qs = qs.concat(this.buildQSValues('lucene', this.lucene, true)); + } + if (this.contains) { qs = qs.concat(this.buildQSValues('contains', this.contains, true)); } @@ -239,6 +257,26 @@ export class SearchRequestModel { qs['lte'] = values.map((v) => v.split('=')[1]); } + if (this.fulltext) { + const values = this.buildQSValues('fulltext', this.fulltext); + qs['fulltext'] = values.map((v) => v.split('=')[1]); + } + + if (this.regex) { + const values = this.buildQSValues('regex', this.regex); + qs['regex'] = values.map((v) => v.split('=')[1]); + } + + if (this.lucene) { + const values = this.buildQSValues('lucene', this.lucene); + qs['lucene'] = values.map((v) => v.split('=')[1]); + } + + if (this.exist) { + const values = this.buildQSValues('exist', this.exist); + qs['exist'] = values.map((v) => v.split('=')[1]); + } + if (this.gt) { const values = this.buildQSValues('gt', this.gt); qs['gt'] = values.map((v) => v.split('=')[1]); @@ -263,10 +301,6 @@ export class SearchRequestModel { const values = this.buildQSValues('contains', this.contains); qs['contains'] = values.map((v) => v.split('=')[1]); } - if (this.exist) { - const values = this.buildQSValues('exist', this.exist); - qs['exist'] = values.map((v) => v.split('=')[1]); - } if (this.nexist) { const values = this.buildQSValues('nexist', this.nexist); From df9f10143efb1b25ebfdb33cbbf1c613274fffa1 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Wed, 26 Jan 2022 13:22:48 -0800 Subject: [PATCH 11/31] swagger for enhanced search new parameters --- .../services/assetlibrary/docs/swagger.yml | 134 ++++++++++++++---- 1 file changed, 103 insertions(+), 31 deletions(-) diff --git a/source/packages/services/assetlibrary/docs/swagger.yml b/source/packages/services/assetlibrary/docs/swagger.yml index 85fe2c6c8..0f96be63f 100644 --- a/source/packages/services/assetlibrary/docs/swagger.yml +++ b/source/packages/services/assetlibrary/docs/swagger.yml @@ -11,18 +11,6 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # and limitations under the License. #----------------------------------------------------------------------------------------------------------------------- -#----------------------------------------------------------------------------------------------------------------------- -# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance -# with the License. A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions -# and limitations under the License. -#----------------------------------------------------------------------------------------------------------------------- openapi: 3.0.0 info: title: 'AWS Connected Device Framework: Asset Library' @@ -48,7 +36,7 @@ info: Likewise, `Device Templates` can be created to represent the different types of devices within your fleet, each with their own attributes. - `Profiles` can be created and applied to device and groups to populate with default attirbutes and/or relations. + `Profiles` can be created and applied to device and groups to populate with default attributes and/or relations. `Policies` represent a document that can be attached to one or more groups within a hierarchy, and are automatically inherited by the devices and groups. @@ -96,7 +84,7 @@ tags: each with their own attributes and constraints. - Devices are identified by a unique `deviceId`, each have the following built-in attributes: + Devices are identified by a unique `deviceId`. Each device has the following built-in attributes: - `templateId`: a specific device template that represents what custom attributes the device can have @@ -120,7 +108,7 @@ tags: When a Device is created as a component of another Device, it has all the same built-in attributes as described above with the exception of `groups`. - Groups are identified by a unique `path`, and each have the following built-on attributes: + Groups are identified by a unique `path`. Each group has the following built-on attributes: - `templateId`: a specific group template that represents what custom attributes the group can have @@ -195,7 +183,7 @@ tags: A good use for policies is to look up appropriate documents or authorization levels based on a device or groups associations to specific hierarchies. As an example, let's say you need to apply different AWS IoT security policies when registering devices as Things depending upon their location. This would be handled by assigning a policy representing a provisoning template to different groups within a hierarchy representing the location. The appropriate provisioning template will be returned for the device/group depending on which and where in a hierarchy they are attached to. - name: Search description: > - The search api allows you to search across both devices and groups + The search API allows you to search across both devices and groups applying a variety of different filters. @@ -1376,25 +1364,27 @@ paths: - Search summary: Search for groups and devices. description: > - Search results can be filtered by type, ancestorPath, and an arbitrary number of + Search results can be filtered by type, ancestorPath, and an arbitrary number of additional filter parameters. Each filter can reference a field of the search - result, for example `eq=fieldname:value` or traverse the asset library graph to + result, for example `eq=fieldname:value` or traverse the asset library graph to reference a related entry, for example `eq=traversal1:out:traversal2:in:fieldname:value`. All parameters are combined with a logical AND. - For all search parameters that include a search key and search value separated - by a colon (:) character, the HTTP parameter must be assembled using the following + For all search filters that include a search key and search value separated + by a colon (:) character, the HTTP parameter must be assembled using the following sequence of steps: 1. URL-encode the search value. - 2. Concatenate the search key (incl. any traversals), the colon character, and + 2. Concatenate the search key (incl. any traversals), the colon character, and result of step 1. 3. URL-encode the output of step 2. Failure to do so can yield incorrect search results for any search values that include the colon character. - For example, a search for entries with an outgoing "manufactured_by" relation whose + For example, a search for entries with an outgoing "manufactured_by" relation whose name starts with "Mfg+Asy Inc" should be expressed as: `/search?startsWith=manufactured_by%3Aout%3Aname%3AMfg%252BAsy%2520Inc` + + Search filters are case-sensitive unless otherwise noted for the URL parameter. operationId: search parameters: - name: type @@ -1517,7 +1507,7 @@ paths: - name: exist in: query description: - Return a match if the device/group in context has a matching relation/atrribute. E.g. + Return a match if the device/group in context has a matching relation/attribute. E.g. `?exists=installed_in:out:groupPath:/vehicle/001` explode: true schema: @@ -1534,6 +1524,88 @@ paths: type: array items: type: string + - name: fulltext + in: query + description: > + Filter by an attribute based on an OpenSearch query string. This filter is only available + in the "enhanced" mode of Asset Library. This filter is case-insensitive. + + The fulltext filter matches individual words in a string as opposed to the complete string. + If a field contains more than one word, only one of the words needs to match, for example + `fulltext=widgets` matches "Widgets Incorporated". If the filter query contains more than one + word, they are combined using logical `or``, for example `fulltext=Widgets Gadgets Incorporated` + matches "Widgets Incorporated" and "Gadgets Incorporated" and "Gadgets Ltd". Field and search + strings are split into words using the rules of the ElasticSearch standard query analyzer: + https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-standard-analyzer.html + + Append the `~` character to any word to allow for fuzzy matches. Specify the edit distance + allowed by fuzzy match in the format `~n`, for example `Masachussets~3` matches "Massachusetts". + explode: true + schema: + type: array + items: + type: string + - name: regex + in: query + description: > + Filter by an attribute based on a regex pattern. This filter is only available in the + "enhanced" mode of Asset Library. + + For example: `?regex=serialNo:XY[A-Z]\-[0-4][^9].*`. Syntax and limitations are those of + the ElasticSearch DSL, documented here: + https://www.elastic.co/guide/en/elasticsearch/reference/7.10/regexp-syntax.html. + Notably, ElasticSearch does NOT support: + 1. anchor operators, such as ^ and $ for start and end of a string + 2. character class tokens such as \d (any digit) and \S (all non-whitespace character) + + Regex filters are performed against the original complete value of the field, not individual + words. For example, a device with attribute "mfg" set to "Widgets Incorporated" can be found with + the query `regex=mfg:[wW]idgets Inc.*` but not with the query `regex=mfg:[wW]idgets`. + + Regex filters are case-sensitive. Do not include slash delimiters at the beginning or + end of your regular expression, i.e. use `regex=abc[0-9]`, not `regex=/abc[0-9]/`. + + Correct URL-encoding is especially important with the regex parameter because regular expressions + can contain many special characters. Note that the aforementioned examples are shown before + URL-encoding for readability but may not work unless correctly URL-encoded. Examples of regex + parameter values before and after URL-encoding: + * `regex=mfg:[wW]idgets Inc.*` --> `regex=mfg%3A%255BwW%255Didgets%2520Inc.*` + * `regex=mpn:ABC\[[0-9]\]` --> `regex=mpn:ABC%255C%255B%255B0-9%255D%255C%255D` + * `regex=desc:back\\slash` (literal backslash) --> `regex=desc%3Aback%255C%255Cslash` + explode: true + schema: + type: array + items: + type: string + - name: lucene + in: query + description: > + Filter by one or more fields of the same node using Apache Lucene query syntax. This + filter is only available in the "enhanced" mode of Asset Library. + + This filter exposes low level access to Amazon Neptune full text search (FTS): + https://docs.aws.amazon.com/neptune/latest/userguide/full-text-search.html Parameter values + are passed to FTS using the `query_string` query type with minimal processing. + + Values can include any valid Lucene query syntax, for example: + `?lucene=fieldname:(fuzzyvalue~2 AND /regexval.*/) OR exactvalue` + + To reference more than one field in the same query, use `*` as the filter key and follow the + Amazon Neptune FTS documentation for how to reference attributes in the query. For example, + to query for a device/group by attributes `attr1` and `attr2`: + `?lucene=*:(predicates.attr1.value:foo AND predicates.attr2.value:bar)` + Traversals are support as follows: + `?lucene=relation:direction:*:(predicates.attr1.value:foo AND predicates.attr2.value:bar)` + It is not possible to reference fields from different nodes in a single lucene query parameter. + + Correct URL-encoding is especially important with the lucene parameter because Lucene queries + can contain many special characters. Note that the aforementioned examples are shown before + URL-encoding for readability but may not work unless correctly URL-encoded. + explode: true + schema: + type: array + items: + type: string - name: facetField in: query description: Perform a faceted query. Specify in the format of @@ -1572,23 +1644,23 @@ paths: - Search summary: Search for groups and devices, and delete the results. description: > - Search results can be filtered by type, ancestorPath, and an arbitrary number of + Search results can be filtered by type, ancestorPath, and an arbitrary number of additional filter parameters. Each filter can reference a field of the search - result, for example `eq=fieldname:value` or traverse the asset library graph to + result, for example `eq=fieldname:value` or traverse the asset library graph to reference a related entry, for example `eq=traversal1:out:traversal2:in:fieldname:value`. All parameters are combined with a logical AND. - For all search parameters that include a search key and search value separated - by a colon (:) character, the HTTP parameter must be assembled using the following + For all search parameters that include a search key and search value separated + by a colon (:) character, the HTTP parameter must be assembled using the following sequence of steps: 1. URL-encode the search value. - 2. Concatenate the search key (incl. any traversals), the colon character, and + 2. Concatenate the search key (incl. any traversals), the colon character, and result of step 1. 3. URL-encode the output of step 2. Failure to do so can yield incorrect search results for any search values that include the colon character. - For example, a search for entries with an outgoing "manufactured_by" relation whose + For example, a search for entries with an outgoing "manufactured_by" relation whose name starts with "Mfg+Asy Inc" should be expressed as: `/search?startsWith=manufactured_by%3Aout%3Aname%3AMfg%252BAsy%2520Inc` operationId: deleteSearch @@ -2447,7 +2519,7 @@ components: type: integer total: type: number - description: Total number of search results. Only returned by the search API's + description: Total number of search results. Only returned by the search API when `summarize` is set to true. DeviceList: @@ -2468,7 +2540,7 @@ components: type: integer total: type: number - description: Total number of search results. Only returned by the search API's + description: Total number of search results. Only returned by the search API when `summarize` is set to true. DeviceProfileList: From c6b6f834e9c52716208e31edb1cf905dfab0ca64 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Wed, 26 Jan 2022 13:06:56 -0800 Subject: [PATCH 12/31] feat(assetlibrary): v1 of typescript code for enhanced search --- .../src/di/inversify.config.enhanced.ts | 106 ++++++++ .../src/di/inversify.config.full.ts | 2 +- .../assetlibrary/src/di/inversify.config.ts | 3 + .../src/search/search.assembler.spec.ts | 35 ++- .../src/search/search.assembler.ts | 9 +- .../src/search/search.controller.ts | 18 +- .../src/search/search.enhanced.dao.ts | 248 ++++++++++++++++++ .../assetlibrary/src/search/search.models.ts | 3 + 8 files changed, 418 insertions(+), 6 deletions(-) create mode 100644 source/packages/services/assetlibrary/src/di/inversify.config.enhanced.ts create mode 100644 source/packages/services/assetlibrary/src/search/search.enhanced.dao.ts diff --git a/source/packages/services/assetlibrary/src/di/inversify.config.enhanced.ts b/source/packages/services/assetlibrary/src/di/inversify.config.enhanced.ts new file mode 100644 index 000000000..773584c81 --- /dev/null +++ b/source/packages/services/assetlibrary/src/di/inversify.config.enhanced.ts @@ -0,0 +1,106 @@ +/********************************************************************************************************************* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * + * * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance * + * with the License. A copy of the License is located at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES * + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions * + * and limitations under the License. * + *********************************************************************************************************************/ +import { ContainerModule, decorate, injectable, interfaces } from 'inversify'; +import { structure } from 'gremlin'; + +import { AuthzDaoFull } from '../authz/authz.full.dao'; +import { AuthzServiceFull } from '../authz/authz.full.service'; +import { CommonDaoFull } from '../data/common.full.dao'; +import { FullAssembler } from '../data/full.assembler'; +import { DevicesDaoFull } from '../devices/devices.full.dao'; +import { DevicesServiceFull } from '../devices/devices.full.service'; +import { DevicesService } from '../devices/devices.service'; +import { GroupsDaoFull } from '../groups/groups.full.dao'; +import { GroupsServiceFull } from '../groups/groups.full.service'; +import { GroupsService } from '../groups/groups.service'; +import { InitDaoFull } from '../init/init.full.dao'; +import { InitServiceFull } from '../init/init.full.service'; +import { InitService } from '../init/init.service'; +import { PoliciesAssembler } from '../policies/policies.assembler'; +import { PoliciesDaoFull } from '../policies/policies.full.dao'; +import { PoliciesServiceFull } from '../policies/policies.full.service'; +import { PoliciesService } from '../policies/policies.service'; +import { ProfilesAssembler } from '../profiles/profiles.assembler'; +import { ProfilesDaoFull } from '../profiles/profiles.full.dao'; +import { ProfilesServiceFull } from '../profiles/profiles.full.service'; +import { ProfilesService } from '../profiles/profiles.service'; +import { SearchDaoEnhanced } from '../search/search.enhanced.dao'; // TODO: changed compared to inversify.config.full.ts +import { SearchServiceFull } from '../search/search.full.service'; +import { SearchService } from '../search/search.service'; +import { TypesDaoFull } from '../types/types.full.dao'; +import { TypesServiceFull } from '../types/types.full.service'; +import { TypesService } from '../types/types.service'; +import { SchemaValidatorService } from '../utils/schemaValidator.service'; +import { TYPES } from './types'; + + + +export const EnhancedContainerModule = new ContainerModule ( + ( + bind: interfaces.Bind, + _unbind: interfaces.Unbind, + _isBound: interfaces.IsBound, + _rebind: interfaces.Rebind + ) => { + bind('neptuneUrl').toConstantValue(process.env.AWS_NEPTUNE_URL); + bind('enableDfeOptimization').toConstantValue(process.env.ENABLE_DFE_OPTIMIZATION === 'true'); + bind('openSearchEndpoint').toConstantValue(process.env.OPENSEARCH_ENDPOINT); + bind('defaults.devices.parent.relation').toConstantValue(process.env.DEFAULTS_DEVICES_PARENT_RELATION); + bind('defaults.devices.parent.groupPath').toConstantValue(process.env.DEFAULTS_DEVICES_PARENT_GROUPPATH); + bind('defaults.devices.state').toConstantValue(process.env.DEFAULTS_DEVICES_STATE); + bind('authorization.enabled').toConstantValue(process.env.AUTHORIZATION_ENABLED === 'true'); + bind('defaults.groups.validateAllowedParentPaths').toConstantValue(process.env.DEFAULTS_GROUPS_VALIDATEALLOWEDPARENTPATHS === 'true'); + + bind(TYPES.TypesService).to(TypesServiceFull).inSingletonScope(); + bind(TYPES.TypesDao).to(TypesDaoFull).inSingletonScope(); + + bind(TYPES.DevicesService).to(DevicesServiceFull).inSingletonScope(); + bind(TYPES.DevicesDao).to(DevicesDaoFull).inSingletonScope(); + + bind(TYPES.GroupsService).to(GroupsServiceFull).inSingletonScope(); + bind(TYPES.GroupsDao).to(GroupsDaoFull).inSingletonScope(); + + bind(TYPES.CommonDao).to(CommonDaoFull).inSingletonScope(); + + bind(TYPES.FullAssembler).to(FullAssembler).inSingletonScope(); + + bind(TYPES.ProfilesService).to(ProfilesServiceFull).inSingletonScope(); + bind(TYPES.ProfilesDao).to(ProfilesDaoFull).inSingletonScope(); + bind(TYPES.ProfilesAssembler).to(ProfilesAssembler).inSingletonScope(); + + bind(TYPES.SearchService).to(SearchServiceFull).inSingletonScope(); + bind(TYPES.SearchDao).to(SearchDaoEnhanced).inSingletonScope(); + + bind(TYPES.PoliciesService).to(PoliciesServiceFull).inSingletonScope(); + bind(TYPES.PoliciesDao).to(PoliciesDaoFull).inSingletonScope(); + bind(TYPES.PoliciesAssembler).to(PoliciesAssembler).inSingletonScope(); + + bind(TYPES.InitService).to(InitServiceFull).inSingletonScope(); + bind(TYPES.InitDao).to(InitDaoFull).inSingletonScope(); + + bind(TYPES.SchemaValidatorService).to(SchemaValidatorService).inSingletonScope(); + + bind(TYPES.AuthzDaoFull).to(AuthzDaoFull).inSingletonScope(); + bind(TYPES.AuthzServiceFull).to(AuthzServiceFull).inSingletonScope(); + + decorate(injectable(), structure.Graph); + bind>(TYPES.GraphSourceFactory) + .toFactory((_ctx: interfaces.Context) => { + return () => { + + return new structure.Graph(); + + }; + }); + } +); diff --git a/source/packages/services/assetlibrary/src/di/inversify.config.full.ts b/source/packages/services/assetlibrary/src/di/inversify.config.full.ts index 59d13ed54..b7350f544 100644 --- a/source/packages/services/assetlibrary/src/di/inversify.config.full.ts +++ b/source/packages/services/assetlibrary/src/di/inversify.config.full.ts @@ -1,4 +1,3 @@ -import { structure } from 'gremlin'; /********************************************************************************************************************* * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * * * @@ -12,6 +11,7 @@ import { structure } from 'gremlin'; * and limitations under the License. * *********************************************************************************************************************/ import { ContainerModule, decorate, injectable, interfaces } from 'inversify'; +import { structure } from 'gremlin'; import { AuthzDaoFull } from '../authz/authz.full.dao'; import { AuthzServiceFull } from '../authz/authz.full.service'; diff --git a/source/packages/services/assetlibrary/src/di/inversify.config.ts b/source/packages/services/assetlibrary/src/di/inversify.config.ts index 3d5faf8eb..31039ff02 100644 --- a/source/packages/services/assetlibrary/src/di/inversify.config.ts +++ b/source/packages/services/assetlibrary/src/di/inversify.config.ts @@ -25,6 +25,7 @@ import { HttpHeaderUtils } from '../utils/httpHeaders'; import { TypeUtils } from '../utils/typeUtils'; import * as full from './inversify.config.full'; import * as lite from './inversify.config.lite'; +import * as enhanced from './inversify.config.enhanced'; import { TYPES } from './types'; import AWS from 'aws-sdk'; @@ -33,6 +34,8 @@ export const container = new Container(); if (process.env.MODE === 'lite') { container.load(lite.LiteContainerModule); +} else if (process.env.MODE === 'enhanced') { + container.load(enhanced.EnhancedContainerModule); } else { container.load(full.FullContainerModule); } diff --git a/source/packages/services/assetlibrary/src/search/search.assembler.spec.ts b/source/packages/services/assetlibrary/src/search/search.assembler.spec.ts index e0c8dba4e..6a0b0f6b0 100644 --- a/source/packages/services/assetlibrary/src/search/search.assembler.spec.ts +++ b/source/packages/services/assetlibrary/src/search/search.assembler.spec.ts @@ -36,10 +36,13 @@ describe('SearchServiceAssembler', () => { neqs: string | string[] | undefined; lts: string | string[] | undefined; gtes: string | string[] | undefined; - facetField: string | undefined; startsWiths: string | string[] | undefined; endsWiths: string | string[] | undefined; containses: string | string[] | undefined; + fulltexts: string | string[] | undefined; + regexes: string | string[] | undefined; + lucenes: string | string[] | undefined; + facetField: string | undefined; }; beforeEach(() => { mockedDeviceAssembler = createMockInstance(DevicesAssembler); @@ -66,6 +69,9 @@ describe('SearchServiceAssembler', () => { startsWiths: undefined, endsWiths: undefined, containses: undefined, + fulltexts: undefined, + regexes: undefined, + lucenes: undefined, exists: undefined, nexists: undefined, facetField: undefined, @@ -85,6 +91,9 @@ describe('SearchServiceAssembler', () => { mockedSearchRequest.startsWiths, mockedSearchRequest.endsWiths, mockedSearchRequest.containses, + mockedSearchRequest.fulltexts, + mockedSearchRequest.regexes, + mockedSearchRequest.lucenes, mockedSearchRequest.exists, mockedSearchRequest.nexists, mockedSearchRequest.facetField @@ -118,6 +127,9 @@ describe('SearchServiceAssembler', () => { startsWiths: 'swfield:abc', endsWiths: 'ewfield:xyz', containses: 'confield:opq', + fulltexts: 'ftfield:*abc*', + regexes: 'refield:AB[CD]12++', + lucenes: undefined, exists: 'exfield:exval', nexists: 'nexfield:nexval', facetField: undefined, @@ -137,6 +149,9 @@ describe('SearchServiceAssembler', () => { mockedSearchRequest.startsWiths, mockedSearchRequest.endsWiths, mockedSearchRequest.containses, + mockedSearchRequest.fulltexts, + mockedSearchRequest.regexes, + mockedSearchRequest.lucenes, mockedSearchRequest.exists, mockedSearchRequest.nexists, mockedSearchRequest.facetField @@ -174,6 +189,12 @@ describe('SearchServiceAssembler', () => { expect(searchRequestModel.contains).toHaveLength(1); expect(searchRequestModel.contains[0].field).toEqual('confield'); expect(searchRequestModel.contains[0].value).toEqual('opq'); + expect(searchRequestModel.fulltext).toHaveLength(1); + expect(searchRequestModel.fulltext[0].field).toEqual('ftfield'); + expect(searchRequestModel.fulltext[0].value).toEqual('*abc*'); + expect(searchRequestModel.regex).toHaveLength(1); + expect(searchRequestModel.regex[0].field).toEqual('refield'); + expect(searchRequestModel.regex[0].value).toEqual('AB[CD]12++'); expect(searchRequestModel.exists).toHaveLength(1); expect(searchRequestModel.exists[0].field).toEqual('exfield'); expect(searchRequestModel.exists[0].value).toEqual('exval'); @@ -197,6 +218,9 @@ describe('SearchServiceAssembler', () => { startsWiths: undefined, endsWiths: undefined, containses: undefined, + fulltexts: undefined, + regexes: undefined, + lucenes: undefined, exists: undefined, nexists: undefined, facetField: undefined, @@ -216,6 +240,9 @@ describe('SearchServiceAssembler', () => { mockedSearchRequest.startsWiths, mockedSearchRequest.endsWiths, mockedSearchRequest.containses, + mockedSearchRequest.fulltexts, + mockedSearchRequest.regexes, + mockedSearchRequest.lucenes, mockedSearchRequest.exists, mockedSearchRequest.nexists, mockedSearchRequest.facetField @@ -249,6 +276,9 @@ describe('SearchServiceAssembler', () => { startsWiths: undefined, endsWiths: undefined, containses: undefined, + fulltexts: undefined, + regexes: undefined, + lucenes: undefined, exists: undefined, nexists: undefined, facetField: undefined, @@ -268,6 +298,9 @@ describe('SearchServiceAssembler', () => { mockedSearchRequest.startsWiths, mockedSearchRequest.endsWiths, mockedSearchRequest.containses, + mockedSearchRequest.fulltexts, + mockedSearchRequest.regexes, + mockedSearchRequest.lucenes, mockedSearchRequest.exists, mockedSearchRequest.nexists, mockedSearchRequest.facetField diff --git a/source/packages/services/assetlibrary/src/search/search.assembler.ts b/source/packages/services/assetlibrary/src/search/search.assembler.ts index af3584806..73ca4e642 100644 --- a/source/packages/services/assetlibrary/src/search/search.assembler.ts +++ b/source/packages/services/assetlibrary/src/search/search.assembler.ts @@ -48,9 +48,12 @@ export class SearchAssembler { startsWiths: string | string[], endsWiths: string | string[], containses: string | string[], + fulltexts: string | string[], + regexes: string | string[], + lucenes: string | string[], exists: string | string[], nexists: string | string[], - facetField?: string, + facetField: string, offset?: number, count?: number, sort?: string @@ -82,7 +85,9 @@ export class SearchAssembler { req.startsWith = this.toSearchRequestFilters(startsWiths); req.endsWith = this.toSearchRequestFilters(endsWiths); req.contains = this.toSearchRequestFilters(containses); - + req.fulltext = this.toSearchRequestFilters(fulltexts); + req.regex = this.toSearchRequestFilters(regexes); + req.lucene = this.toSearchRequestFilters(lucenes); req.exists = this.toSearchRequestFilters(exists); req.nexists = this.toSearchRequestFilters(nexists); diff --git a/source/packages/services/assetlibrary/src/search/search.controller.ts b/source/packages/services/assetlibrary/src/search/search.controller.ts index 8f8fdae63..26d34bd04 100644 --- a/source/packages/services/assetlibrary/src/search/search.controller.ts +++ b/source/packages/services/assetlibrary/src/search/search.controller.ts @@ -50,6 +50,9 @@ export class SearchController implements interfaces.Controller { @queryParam('startsWith') startsWiths: string | string[], @queryParam('endsWith') endsWiths: string | string[], @queryParam('contains') containses: string | string[], + @queryParam('fulltext') fulltexts: string | string[], + @queryParam('regex') regexes: string | string[], + @queryParam('lucene') lucenes: string | string[], @queryParam('exist') exists: string | string[], @queryParam('nexist') nexists: string | string[], @queryParam('facetField') facetField: string, @@ -61,7 +64,7 @@ export class SearchController implements interfaces.Controller { @response() res: Response ): Promise { logger.debug( - `search.controller search: in: types:${types}, ancestorPath:${ancestorPath}, eqs:${eqs}, neqs:${neqs}, lts:${lts}, ltes:${ltes}, gts:${gts}, gtes:${gtes}, startsWiths:${startsWiths}, endsWiths:${endsWiths}, containses:${containses}, exists:${exists}, nexists:${nexists}, facetField:${facetField}, summarize:${summarize}, offset:${offset}, count:${count}, sort:${sort}` + `search.controller search: in: types:${types}, ancestorPath:${ancestorPath}, eqs:${eqs}, neqs:${neqs}, lts:${lts}, ltes:${ltes}, gts:${gts}, gtes:${gtes}, startsWiths:${startsWiths}, endsWiths:${endsWiths}, containses:${containses}, fulltexts:${fulltexts}, regexes:${regexes}, lucenes:${lucenes}, exists:${exists}, nexists:${nexists}, facetField:${facetField}, summarize:${summarize}, offset:${offset}, count:${count}, sort:${sort}` ); const r: SearchResultsResource = { results: [] }; @@ -81,6 +84,9 @@ export class SearchController implements interfaces.Controller { startsWiths, endsWiths, containses, + fulltexts, + regexes, + lucenes, exists, nexists, facetField, @@ -135,8 +141,12 @@ export class SearchController implements interfaces.Controller { @queryParam('startsWith') startsWiths: string | string[], @queryParam('endsWith') endsWiths: string | string[], @queryParam('contains') containses: string | string[], + @queryParam('fulltext') fulltexts: string | string[], + @queryParam('regex') regexes: string | string[], + @queryParam('lucene') lucenes: string | string[], @queryParam('exist') exists: string | string[], @queryParam('nexist') nexists: string | string[], + @queryParam('facetField') facetField: string, @response() res: Response ): Promise { logger.debug( @@ -157,8 +167,12 @@ export class SearchController implements interfaces.Controller { startsWiths, endsWiths, containses, + fulltexts, + regexes, + lucenes, exists, - nexists + nexists, + facetField ); try { diff --git a/source/packages/services/assetlibrary/src/search/search.enhanced.dao.ts b/source/packages/services/assetlibrary/src/search/search.enhanced.dao.ts new file mode 100644 index 000000000..1d1d346be --- /dev/null +++ b/source/packages/services/assetlibrary/src/search/search.enhanced.dao.ts @@ -0,0 +1,248 @@ +/********************************************************************************************************************* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * + * * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance * + * with the License. A copy of the License is located at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES * + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions * + * and limitations under the License. * + *********************************************************************************************************************/ +import { process, structure } from 'gremlin'; +import { injectable, inject } from 'inversify'; +import {logger} from '../utils/logger'; +import {TYPES} from '../di/types'; +import { SearchRequestModel } from './search.models'; +import {NodeAssembler} from '../data/assembler'; +import {NeptuneConnection} from '../data/base.full.dao'; +import { SearchDaoFull } from './search.full.dao'; +import { TypeUtils } from '../utils/typeUtils'; + +const __ = process.statics; + +@injectable() +export class SearchDaoEnhanced extends SearchDaoFull { + + public constructor( + @inject('neptuneUrl') neptuneUrl: string, + @inject('enableDfeOptimization') enableDfeOptimization: boolean, + @inject('openSearchEndpoint') private openSearchEndpoint: boolean, + @inject(TYPES.TypeUtils) typeUtils: TypeUtils, + @inject(TYPES.NodeAssembler) assembler: NodeAssembler, + @inject(TYPES.GraphSourceFactory) graphSourceFactory: () => structure.Graph + ) { + super(neptuneUrl, enableDfeOptimization, typeUtils, assembler, graphSourceFactory); + } + + protected buildSearchTraverser(conn: NeptuneConnection, request: SearchRequestModel, authorizedPaths:string[]) : process.GraphTraversal { + + logger.debug(`search.enhanced.dao buildSearchTraverser: in: request: ${JSON.stringify(request)}, authorizedPaths:${authorizedPaths}`); + + let source: process.GraphTraversalSource = conn.traversal; + if (this.enableDfeOptimization) { + source = source.withSideEffect('Neptune#useDFE', true); + } + source = source.withSideEffect("Neptune#fts.endpoint", this.openSearchEndpoint); + source = source.withSideEffect("Neptune#fts.queryType", "query_string"); + + // if a path is provided, that becomes the starting point + let traverser: process.GraphTraversal; + if (request.ancestorPath!==undefined) { + const ancestorId = `group___${request.ancestorPath}`; + traverser = source.V(ancestorId). + repeat(__.in_().simplePath().dedup()).emit().as('a'); + } else { + traverser = source.V().as('a'); + } + + // construct Gremlin traverser from request parameters + + if (request.types!==undefined) { + request.types.forEach(t=> traverser.select('a').hasLabel(t)); + } + + if (request.eq!==undefined) { + request.eq.forEach(f=> { + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has(f.field, f.value); + }); + } + if (request.neq!==undefined) { + request.neq.forEach(f=> { + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.not(__.has(f.field, f.value)); + }); + } + if (request.lt!==undefined) { + request.lt.forEach(f=> { + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has(f.field, process.P.lt(Number(f.value))); + }); + } + if (request.lte!==undefined) { + request.lte.forEach(f=> { + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has(f.field, process.P.lte(Number(f.value))); + }); + } + if (request.gt!==undefined) { + request.gt.forEach(f=> { + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has(f.field, process.P.gt(Number(f.value))); + }); + } + if (request.gte!==undefined) { + request.gte.forEach(f=> { + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has(f.field, process.P.gte(Number(f.value))); + }); + } + if (request.startsWith!==undefined) { + request.startsWith.forEach(f=> { + const luceneQueryKey = this.buildLuceneQueryKey(f.field, true); + let luceneQueryVal = f.value.toString(); + [':', '/', ' '].forEach( char => { + luceneQueryVal = luceneQueryVal.replace(new RegExp(char, 'g'), `\\${char}`); + }); + ['[', ']'].forEach( char => { + luceneQueryVal = luceneQueryVal.replace(new RegExp(`\\${char}`, 'g'), `\\${char}`); + }); + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has('*', `Neptune#fts ${luceneQueryKey}:${luceneQueryVal}*`); + }); + } + + if (request.endsWith!==undefined) { + request.endsWith.forEach(f=> { + const luceneQueryKey = this.buildLuceneQueryKey(f.field, true); + let luceneQueryVal = f.value.toString(); + [':', '/', ' '].forEach( char => { + luceneQueryVal = luceneQueryVal.replace(new RegExp(char, 'g'), `\\${char}`); + }); + ['[', ']'].forEach( char => { + luceneQueryVal = luceneQueryVal.replace(new RegExp(`\\${char}`, 'g'), `\\${char}`); + }); + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has('*', `Neptune#fts ${luceneQueryKey}:*${luceneQueryVal}`); + }); + } + + if (request.contains!==undefined) { + request.contains.forEach(f=> { + const luceneQueryKey = this.buildLuceneQueryKey(f.field, true); + let luceneQueryVal = f.value.toString(); + [':', '/', ' '].forEach( char => { + luceneQueryVal = luceneQueryVal.replace(new RegExp(char, 'g'), `\\${char}`); + }); + ['[', ']'].forEach( char => { + luceneQueryVal = luceneQueryVal.replace(new RegExp(`\\${char}`, 'g'), `\\${char}`); + }); + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has('*', `Neptune#fts ${luceneQueryKey}:*${luceneQueryVal}*`); + }); + } + + if (request.exists!==undefined) { + request.exists.forEach(f=> { + traverser.select('a'); + this.buildSearchFilterEBase(f, traverser); + traverser.has(f.field, f.value); + }); + } + + if (request.nexists!==undefined) { + request.nexists.forEach(f=> { + traverser.select('a'); + this.buildSearchFilterEBaseNegated(f, traverser, f.field, f.value); + }); + } + + if (request.fulltext!==undefined) { + request.fulltext.forEach(f=> { + // Remove any characters that would be recognized by Lucene as control characters. + // Alternatively, could escape them but the standard query string analyzer will + // replace them with spaces anyway. + let luceneQueryVal = f.value.toString(); + [':', '/', '\\[', '\\]'].forEach( char => { + luceneQueryVal = luceneQueryVal.replace(new RegExp(char, 'g'), ' '); + }); + // RegExp('\\\\') = regex for single backslash, '\\\\' = string with two backslashes + luceneQueryVal = luceneQueryVal.replace(new RegExp('\\\\', 'g'), `\\\\`); + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has(f.field, `Neptune#fts ${luceneQueryVal}`); + }); + } + + if (request.regex!==undefined) { + request.regex.forEach(f=> { + const luceneQueryKey = this.buildLuceneQueryKey(f.field, true); + // Escape characters that can appear, escaped or unescaped, in regex but are also + // control characters for Lucene. For example, "abc/def" is a valid regex but the contained + // slash is interpreted by Lucene as the end of the regex. Square brackets [] must not + // be escaped even though they denote range queries in Lucene because Lucene ignores + // them inside of regexes. + let luceneQueryVal = f.value.toString(); + [':', '/', ' '].forEach( char => { + luceneQueryVal = luceneQueryVal.replace(new RegExp(char, 'g'), `\\${char}`); + }); + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + traverser.has('*', `Neptune#fts ${luceneQueryKey}:/${luceneQueryVal}/`); + }); + } + + if (request.lucene!==undefined) { + request.lucene.forEach(f=> { + traverser.select('a'); + this.buildSearchFilterVBase(f, traverser); + // no escaping for Opensearch, user can send control characters and is responsible for + // escaping them in the search request when necessary + traverser.has(f.field, `Neptune#fts ${f.value}`); + }); + } + + // must reset all traversals so far as we may meed to use simplePath if FGAC is enabled to prevent cyclic checks + traverser.select('a').dedup().fold().unfold().as('matched'); + + // if authz is enabled, only return results that the user is authorized to view + if (authorizedPaths!==undefined && authorizedPaths.length>0) { + + const authorizedPathIds = authorizedPaths.map(path=>`group___${path}`); + traverser. + local( + __.until( + __.hasId(process.P.within(authorizedPathIds)) + ).repeat( + __.out().simplePath().dedup() + ) + ).as('authorization'); + } + + logger.debug(`search.enhanced.dao buildSearchTraverser: traverser: ${traverser.toString()}`); + + return traverser.select('matched').dedup(); + + } + + private buildLuceneQueryKey(field: string, keyword?: boolean) : string { + if (field === 'id') return 'entity_id'; + if (field === 'label') return 'entity_type'; + + let components: string[] = []; + components = ['predicates', field, 'value']; + if (keyword) components.push('keyword'); + return components.join('.'); + } +} diff --git a/source/packages/services/assetlibrary/src/search/search.models.ts b/source/packages/services/assetlibrary/src/search/search.models.ts index 850999a7d..9ac9f2ca4 100644 --- a/source/packages/services/assetlibrary/src/search/search.models.ts +++ b/source/packages/services/assetlibrary/src/search/search.models.ts @@ -48,6 +48,9 @@ export class SearchRequestModel { startsWith?: SearchRequestFilters; endsWith?: SearchRequestFilters; contains?: SearchRequestFilters; + fulltext?: SearchRequestFilters; + regex?: SearchRequestFilters; + lucene?: SearchRequestFilters; exists?: SearchRequestFilters; nexists?: SearchRequestFilters; From d93284c8f91084ca2f4566dd80250e4dba4874fb Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Fri, 18 Mar 2022 11:15:41 -0700 Subject: [PATCH 13/31] docs(assetlibrary): swagger: add pagination field to search results --- .../services/assetlibrary/docs/swagger.yml | 52 ++++++------------- 1 file changed, 16 insertions(+), 36 deletions(-) diff --git a/source/packages/services/assetlibrary/docs/swagger.yml b/source/packages/services/assetlibrary/docs/swagger.yml index 0f96be63f..421106f46 100644 --- a/source/packages/services/assetlibrary/docs/swagger.yml +++ b/source/packages/services/assetlibrary/docs/swagger.yml @@ -2230,6 +2230,14 @@ components: required: true schemas: + Pagination: + type: object + properties: + offset: + type: integer + count: + type: integer + Entity: type: object properties: @@ -2511,12 +2519,7 @@ components: - $ref: '#/components/schemas/Group_1_0' - $ref: '#/components/schemas/Group_2_0' pagination: - type: object - properties: - offset: - type: integer - count: - type: integer + $ref: '#/components/schemas/Pagination' total: type: number description: Total number of search results. Only returned by the search API @@ -2532,12 +2535,7 @@ components: - $ref: '#/components/schemas/Device_1_0' - $ref: '#/components/schemas/Device_2_0' pagination: - type: object - properties: - offset: - type: integer - count: - type: integer + $ref: '#/components/schemas/Pagination' total: type: number description: Total number of search results. Only returned by the search API @@ -2553,12 +2551,7 @@ components: - $ref: '#/components/schemas/DeviceProfile_1_0' - $ref: '#/components/schemas/DeviceProfile_2_0' pagination: - type: object - properties: - offset: - type: integer - count: - type: integer + $ref: '#/components/schemas/Pagination' GroupProfileList: type: object @@ -2570,12 +2563,7 @@ components: - $ref: '#/components/schemas/GroupProfile_1_0' - $ref: '#/components/schemas/GroupProfile_2_0' pagination: - type: object - properties: - offset: - type: integer - count: - type: integer + $ref: '#/components/schemas/Pagination' SearchResults: type: object @@ -2590,6 +2578,8 @@ components: - anyOf: - $ref: '#/components/schemas/Device_2_0' - $ref: '#/components/schemas/Group_2_0' + pagination: + $ref: '#/components/schemas/Pagination' TemplateInfoProperties: type: object @@ -2698,12 +2688,7 @@ components: items: $ref: '#/components/schemas/TemplateInfo' pagination: - type: object - properties: - offset: - type: integer - count: - type: integer + $ref: '#/components/schemas/Pagination' TemplateDefinition: type: object @@ -2791,12 +2776,7 @@ components: items: $ref: '#/components/schemas/Policy' pagination: - type: object - properties: - offset: - type: integer - count: - type: integer + $ref: '#/components/schemas/Pagination' Error: type: object From 34235abb0b6ef540b277365e533d98cac0ba28db Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Fri, 18 Mar 2022 11:47:28 -0700 Subject: [PATCH 14/31] fix(assetlibrary): remove duplication in full/enhanced inversify configs --- .../src/di/inversify.config.enhanced.ts | 90 ++----------------- .../assetlibrary/src/di/inversify.config.ts | 2 + 2 files changed, 8 insertions(+), 84 deletions(-) diff --git a/source/packages/services/assetlibrary/src/di/inversify.config.enhanced.ts b/source/packages/services/assetlibrary/src/di/inversify.config.enhanced.ts index 773584c81..f7d5e27a6 100644 --- a/source/packages/services/assetlibrary/src/di/inversify.config.enhanced.ts +++ b/source/packages/services/assetlibrary/src/di/inversify.config.enhanced.ts @@ -10,97 +10,19 @@ * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions * * and limitations under the License. * *********************************************************************************************************************/ -import { ContainerModule, decorate, injectable, interfaces } from 'inversify'; -import { structure } from 'gremlin'; - -import { AuthzDaoFull } from '../authz/authz.full.dao'; -import { AuthzServiceFull } from '../authz/authz.full.service'; -import { CommonDaoFull } from '../data/common.full.dao'; -import { FullAssembler } from '../data/full.assembler'; -import { DevicesDaoFull } from '../devices/devices.full.dao'; -import { DevicesServiceFull } from '../devices/devices.full.service'; -import { DevicesService } from '../devices/devices.service'; -import { GroupsDaoFull } from '../groups/groups.full.dao'; -import { GroupsServiceFull } from '../groups/groups.full.service'; -import { GroupsService } from '../groups/groups.service'; -import { InitDaoFull } from '../init/init.full.dao'; -import { InitServiceFull } from '../init/init.full.service'; -import { InitService } from '../init/init.service'; -import { PoliciesAssembler } from '../policies/policies.assembler'; -import { PoliciesDaoFull } from '../policies/policies.full.dao'; -import { PoliciesServiceFull } from '../policies/policies.full.service'; -import { PoliciesService } from '../policies/policies.service'; -import { ProfilesAssembler } from '../profiles/profiles.assembler'; -import { ProfilesDaoFull } from '../profiles/profiles.full.dao'; -import { ProfilesServiceFull } from '../profiles/profiles.full.service'; -import { ProfilesService } from '../profiles/profiles.service'; -import { SearchDaoEnhanced } from '../search/search.enhanced.dao'; // TODO: changed compared to inversify.config.full.ts -import { SearchServiceFull } from '../search/search.full.service'; -import { SearchService } from '../search/search.service'; -import { TypesDaoFull } from '../types/types.full.dao'; -import { TypesServiceFull } from '../types/types.full.service'; -import { TypesService } from '../types/types.service'; -import { SchemaValidatorService } from '../utils/schemaValidator.service'; +// import { ContainerModule, decorate, injectable, interfaces } from 'inversify'; +import { ContainerModule, interfaces } from 'inversify'; +import { SearchDaoEnhanced } from '../search/search.enhanced.dao'; import { TYPES } from './types'; - - -export const EnhancedContainerModule = new ContainerModule ( +export const EnhancedContainerModule = new ContainerModule( ( bind: interfaces.Bind, _unbind: interfaces.Unbind, _isBound: interfaces.IsBound, - _rebind: interfaces.Rebind + rebind: interfaces.Rebind ) => { - bind('neptuneUrl').toConstantValue(process.env.AWS_NEPTUNE_URL); - bind('enableDfeOptimization').toConstantValue(process.env.ENABLE_DFE_OPTIMIZATION === 'true'); bind('openSearchEndpoint').toConstantValue(process.env.OPENSEARCH_ENDPOINT); - bind('defaults.devices.parent.relation').toConstantValue(process.env.DEFAULTS_DEVICES_PARENT_RELATION); - bind('defaults.devices.parent.groupPath').toConstantValue(process.env.DEFAULTS_DEVICES_PARENT_GROUPPATH); - bind('defaults.devices.state').toConstantValue(process.env.DEFAULTS_DEVICES_STATE); - bind('authorization.enabled').toConstantValue(process.env.AUTHORIZATION_ENABLED === 'true'); - bind('defaults.groups.validateAllowedParentPaths').toConstantValue(process.env.DEFAULTS_GROUPS_VALIDATEALLOWEDPARENTPATHS === 'true'); - - bind(TYPES.TypesService).to(TypesServiceFull).inSingletonScope(); - bind(TYPES.TypesDao).to(TypesDaoFull).inSingletonScope(); - - bind(TYPES.DevicesService).to(DevicesServiceFull).inSingletonScope(); - bind(TYPES.DevicesDao).to(DevicesDaoFull).inSingletonScope(); - - bind(TYPES.GroupsService).to(GroupsServiceFull).inSingletonScope(); - bind(TYPES.GroupsDao).to(GroupsDaoFull).inSingletonScope(); - - bind(TYPES.CommonDao).to(CommonDaoFull).inSingletonScope(); - - bind(TYPES.FullAssembler).to(FullAssembler).inSingletonScope(); - - bind(TYPES.ProfilesService).to(ProfilesServiceFull).inSingletonScope(); - bind(TYPES.ProfilesDao).to(ProfilesDaoFull).inSingletonScope(); - bind(TYPES.ProfilesAssembler).to(ProfilesAssembler).inSingletonScope(); - - bind(TYPES.SearchService).to(SearchServiceFull).inSingletonScope(); - bind(TYPES.SearchDao).to(SearchDaoEnhanced).inSingletonScope(); - - bind(TYPES.PoliciesService).to(PoliciesServiceFull).inSingletonScope(); - bind(TYPES.PoliciesDao).to(PoliciesDaoFull).inSingletonScope(); - bind(TYPES.PoliciesAssembler).to(PoliciesAssembler).inSingletonScope(); - - bind(TYPES.InitService).to(InitServiceFull).inSingletonScope(); - bind(TYPES.InitDao).to(InitDaoFull).inSingletonScope(); - - bind(TYPES.SchemaValidatorService).to(SchemaValidatorService).inSingletonScope(); - - bind(TYPES.AuthzDaoFull).to(AuthzDaoFull).inSingletonScope(); - bind(TYPES.AuthzServiceFull).to(AuthzServiceFull).inSingletonScope(); - - decorate(injectable(), structure.Graph); - bind>(TYPES.GraphSourceFactory) - .toFactory((_ctx: interfaces.Context) => { - return () => { - - return new structure.Graph(); - - }; - }); + rebind(TYPES.SearchDao).to(SearchDaoEnhanced).inSingletonScope(); } ); diff --git a/source/packages/services/assetlibrary/src/di/inversify.config.ts b/source/packages/services/assetlibrary/src/di/inversify.config.ts index 31039ff02..85cdae5b6 100644 --- a/source/packages/services/assetlibrary/src/di/inversify.config.ts +++ b/source/packages/services/assetlibrary/src/di/inversify.config.ts @@ -35,6 +35,8 @@ export const container = new Container(); if (process.env.MODE === 'lite') { container.load(lite.LiteContainerModule); } else if (process.env.MODE === 'enhanced') { + // EnhancedContainerModule extends, not replaces, FullContainerModule + container.load(full.FullContainerModule); container.load(enhanced.EnhancedContainerModule); } else { container.load(full.FullContainerModule); From b85cfa8593c17f597c9b4d9203738893854892f8 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Fri, 18 Mar 2022 15:05:12 -0700 Subject: [PATCH 15/31] docs(assetlibrary): add enhanced mode to "Assetlibrary Modes" documentation --- .../packages/services/assetlibrary/README.md | 2 +- .../services/assetlibrary/docs/modes.md | 139 ++++++++++-------- 2 files changed, 80 insertions(+), 61 deletions(-) diff --git a/source/packages/services/assetlibrary/README.md b/source/packages/services/assetlibrary/README.md index cce4c9732..4958146ea 100644 --- a/source/packages/services/assetlibrary/README.md +++ b/source/packages/services/assetlibrary/README.md @@ -192,7 +192,7 @@ In the example above, retrieving the list of policies for `device001`, `device00 - [Application configuration](docs/configuration.md) - [Events](docs/events.md) - [Fine-gained access controll](docs/fine-grained-access-control.md) -- [Full vs lite mode](docs/modes.md) +- [Asset Library modes: lite, full, enhanced](docs/modes.md) - [Profiles](docs/profiles.md) - [Swagger](docs/swagger.yml) - [Templates user guide](docs/templates-user.md) diff --git a/source/packages/services/assetlibrary/docs/modes.md b/source/packages/services/assetlibrary/docs/modes.md index 214fabf65..4c4662a5a 100644 --- a/source/packages/services/assetlibrary/docs/modes.md +++ b/source/packages/services/assetlibrary/docs/modes.md @@ -2,9 +2,10 @@ ## Introduction -The Asset Library is capable of running in one of two modes: `full` and `lite`. +The Asset Library is capable of running in one of three modes: `full`, `enhanced`, and `lite`. -The `lite` version uses The AWS IoT Device Registry to store all devices and groups data, whereas the `full` version utilizes [AWS Neptune](https://aws.amazon.com/neptune/) to provide more advanced data modelling features. +The `lite` version uses The AWS IoT Device Registry to store all devices and groups data, whereas the `full` version utilizes [AWS Neptune](https://aws.amazon.com/neptune/) to provide more advanced data modeling features. +In `enhanced` mode, an OpenSearch cluster is deployed as secondary data store and provides enhanced search functionality. The mode is determined via a configuration property at the time of deployment. The following describes the differences in functionality between the two modes. @@ -14,7 +15,7 @@ The following table indicates which REST API's are available in which mode: ### Devices -| Endpoint | Description | `full` mode | `lite` mode | +| Endpoint | Description | `full` and `enhanced` modes | `lite` mode | | ------------------------------------------------------------------- | ------------------------------------------------------------------------- | ------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | | `POST /devices` | Adds a new device to the Asset Library | ✅ (adding to a default parent group if none provided) | ✅ (creating components not supported, and no default parent group set if none provided) | | `POST /bulkdevices` | Adds a batch of devices to the Asset Library | ✅ | ✅ (see `POST /devices`) | @@ -32,75 +33,91 @@ The following table indicates which REST API's are available in which mode: ### Groups -| Endpoint | Description | `full` mode | `lite` mode | -| -------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------ | -| `POST /groups` | Adds a new group to the device library as a child of the `parentPath` as specified in the request body | ✅ | ✅ (specifying a parent is optional, specifying a template is not supported, and linking groups to other groups not supported) | -| `POST /bulkgroups` | Adds a batch of new group to the asset library as a child of the `parentPath` as specified in the request body | ✅ | ✅ (see `POST /groups`) | -| `GET /groups/{groupPath}` | Find group by Group's path | ✅ | ✅ | -| `DELETE /groups/{groupPath}` | Delete group with supplied path | ✅ | ✅ | -| `PATCH /groups/{groupPath}` | Update an existing group's attributes, including changing its parent group | ✅ | ✅ (see `POST /groups`) | -| `GET /groups/{groupPath}/members/devices` | List device members of group for supplied Group name | ✅ | ✅ (filtering by template or state not supported) | -| `GET /groups/{groupPath}/members/groups` | List group members of group for supplied Group name | ✅ | ✅ (filtering by template not supported) | -| `GET /groups/{groupPath}/memberships` | List all ancestor groups of a specific group | ✅ | ✅ | -| `PUT /groups/{sourceGroupPath}/{relationship}/groups/{targetGroupPath}` | Associates a group with another group, giving context to its relationship | ✅ | ⛔ | -| `DELETE /groups/{sourceGroupPath}/{relationship}/groups/{targetGroupPath}` | Removes a group from an associated group | ✅ | ⛔ | +| Endpoint | Description | `full` and `enhanced` modes | `lite` mode | +| -------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `POST /groups` | Adds a new group to the device library as a child of the `parentPath` as specified in the request body | ✅ | ✅ (specifying a parent is optional, specifying a template is not supported, and linking groups to other groups not supported) | +| `POST /bulkgroups` | Adds a batch of new group to the asset library as a child of the `parentPath` as specified in the request body | ✅ | ✅ (see `POST /groups`) | +| `GET /groups/{groupPath}` | Find group by Group's path | ✅ | ✅ | +| `DELETE /groups/{groupPath}` | Delete group with supplied path | ✅ | ✅ | +| `PATCH /groups/{groupPath}` | Update an existing group's attributes, including changing its parent group | ✅ | ✅ (see `POST /groups`) | +| `GET /groups/{groupPath}/members/devices` | List device members of group for supplied Group name | ✅ | ✅ (filtering by template or state not supported) | +| `GET /groups/{groupPath}/members/groups` | List group members of group for supplied Group name | ✅ | ✅ (filtering by template not supported) | +| `GET /groups/{groupPath}/memberships` | List all ancestor groups of a specific group | ✅ | ✅ | +| `PUT /groups/{sourceGroupPath}/{relationship}/groups/{targetGroupPath}` | Associates a group with another group, giving context to its relationship | ✅ | ⛔ | +| `DELETE /groups/{sourceGroupPath}/{relationship}/groups/{targetGroupPath}` | Removes a group from an associated group | ✅ | ⛔ | ### Device Templates -| Endpoint | Description | `full` mode | `lite` mode | -| -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | ----------- | -------------------------------------------------------------------------------------------------------------------------------------- | -| `POST /templates/device/{templateId}` | Registers a new device template within the system, using the JSON Schema standard to define the device template attributes and constraints | ✅ | ✅ (string types supported only, defining allowed relations to other group types not supported, and required attributes not supported) | -| `GET /templates/device/{templateId}` | Find device template by ID | ✅ | ✅ | -| `PATCH /templates/device/{templateId}` | Update an existing device template | ✅ | ✅ (see `POST /templates/devices/{templateId}`) | -| `DELETE /templates/device/{templateId}` | Deletes an existing device template | ✅ | ✅ (deleting a template will deprecate the Thing Type, not delete it) | -| `PUT /templates/device/{templateId}/publish` | Publishes an existing device template | ✅ | ⛔ | +| Endpoint | Description | `full` and `enhanced` modes | `lite` mode | +| -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| `POST /templates/device/{templateId}` | Registers a new device template within the system, using the JSON Schema standard to define the device template attributes and constraints | ✅ | ✅ (string types supported only, defining allowed relations to other group types not supported, and required attributes not supported) | +| `GET /templates/device/{templateId}` | Find device template by ID | ✅ | ✅ | +| `PATCH /templates/device/{templateId}` | Update an existing device template | ✅ | ✅ (see `POST /templates/devices/{templateId}`) | +| `DELETE /templates/device/{templateId}` | Deletes an existing device template | ✅ | ✅ (deleting a template will deprecate the Thing Type, not delete it) | +| `PUT /templates/device/{templateId}/publish` | Publishes an existing device template | ✅ | ⛔ | ### Group Templates -| Endpoint | Description | `full` mode | `lite` mode | -| ------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | ----------- | ----------- | -| `POST /templates/group/{templateId}` | Registers a new group template within the system, using the JSON Schema standard to define the group template attributes and constraints | ✅ | ⛔ | -| `GET /templates/group/{templateId}` | Find group template by ID | ✅ | ⛔ | -| `PATCH /templates/group/{templateId}` | Update an existing group template | ✅ | ⛔ | -| `DELETE /templates/group/{templateId}` | Deletes an existing group template | ✅ | ⛔ | -| `PUT /templates/group/{templateId}/publish` | Publishes an existing group template | ✅ | ⛔ | +| Endpoint | Description | `full` and `enhanced` modes | `lite` mode | +| ------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | --------------------------- | ----------- | +| `POST /templates/group/{templateId}` | Registers a new group template within the system, using the JSON Schema standard to define the group template attributes and constraints | ✅ | ⛔ | +| `GET /templates/group/{templateId}` | Find group template by ID | ✅ | ⛔ | +| `PATCH /templates/group/{templateId}` | Update an existing group template | ✅ | ⛔ | +| `DELETE /templates/group/{templateId}` | Deletes an existing group template | ✅ | ⛔ | +| `PUT /templates/group/{templateId}/publish` | Publishes an existing group template | ✅ | ⛔ | ### Device Profiles -| Endpoint | Description | `full` mode | `lite` mode | -| -------------------------------------------------- | -------------------------------------------------- | ----------- | ----------- | -| `POST /profiles/device/{templateId}` | Adds a new device profile for a specific template | ✅ | ⛔ | -| `GET /profiles/device/{templateId}` | Return all device profiles for a specific template | ✅ | ⛔ | -| `GET /profiles/device/{templateId}/{profileId}` | Retrieve a device profile | ✅ | ⛔ | -| `DELETE /profiles/device/{templateId}/{profileId}` | Delete a specific device profile | ✅ | ⛔ | -| `PATCH /profiles/device/{templateId}/{profileId}` | Update an existing device profile | ✅ | ⛔ | +| Endpoint | Description | `full` and `enhanced` modes | `lite` mode | +| -------------------------------------------------- | -------------------------------------------------- | --------------------------- | ----------- | +| `POST /profiles/device/{templateId}` | Adds a new device profile for a specific template | ✅ | ⛔ | +| `GET /profiles/device/{templateId}` | Return all device profiles for a specific template | ✅ | ⛔ | +| `GET /profiles/device/{templateId}/{profileId}` | Retrieve a device profile | ✅ | ⛔ | +| `DELETE /profiles/device/{templateId}/{profileId}` | Delete a specific device profile | ✅ | ⛔ | +| `PATCH /profiles/device/{templateId}/{profileId}` | Update an existing device profile | ✅ | ⛔ | ### Group Profiles -| Endpoint | Description | `full` mode | `lite` mode | -| ------------------------------------------------- | ------------------------------------------------- | ----------- | ----------- | -| `POST /profiles/group/{templateId}` | Adds a new group profile for a specific template | ✅ | ⛔ | -| `GET /profiles/group/{templateId}` | Return all group profiles for a specific template | ✅ | ⛔ | -| `GET /profiles/group/{templateId}/{profileId}` | Retrieve a group profile | ✅ | ⛔ | -| `DELETE /profiles/group/{templateId}/{profileId}` | Delete a specific group profile | ✅ | ⛔ | -| `PATCH /profiles/group/{templateId}/{profileId}` | Update an existing group profile | ✅ | ⛔ | +| Endpoint | Description | `full` and `enhanced` modes | `lite` mode | +| ------------------------------------------------- | ------------------------------------------------- | --------------------------- | ----------- | +| `POST /profiles/group/{templateId}` | Adds a new group profile for a specific template | ✅ | ⛔ | +| `GET /profiles/group/{templateId}` | Return all group profiles for a specific template | ✅ | ⛔ | +| `GET /profiles/group/{templateId}/{profileId}` | Retrieve a group profile | ✅ | ⛔ | +| `DELETE /profiles/group/{templateId}/{profileId}` | Delete a specific group profile | ✅ | ⛔ | +| `PATCH /profiles/group/{templateId}/{profileId}` | Update an existing group profile | ✅ | ⛔ | ### Policies -| Endpoint | Description | `full` mode | `lite` mode | -| ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | ----------- | -| `POST /policies` | Creates a new `Policy`, and applies it to the provided `Groups` | ✅ | ⛔ | -| `GET /policies` | List policies, optionally filtered by policy type | ✅ | ⛔ | -| `GET /policies/inherited` | Returns all inherited `Policies` for a `Device` or set of `Groups` where the `Device`/`Groups` are associated with all the hierarchies that the `Policy` applies to. Either `deviceId` or `groupPath` must be provided | ✅ | ⛔ | -| `PATCH /policies/{policyId}` | Update the attributes of an existing policy | ✅ | ⛔ | -| `DELETE /policies/{policyId}` | Delete an existing policy | ✅ | ⛔ | -| `GET /policies/{policyId}` | Retrieve a specific policy | ✅ | ⛔ | +| Endpoint | Description | `full` and `enhanced` modes | `lite` mode | +| ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------- | ----------- | +| `POST /policies` | Creates a new `Policy`, and applies it to the provided `Groups` | ✅ | ⛔ | +| `GET /policies` | List policies, optionally filtered by policy type | ✅ | ⛔ | +| `GET /policies/inherited` | Returns all inherited `Policies` for a `Device` or set of `Groups` where the `Device`/`Groups` are associated with all the hierarchies that the `Policy` applies to. Either `deviceId` or `groupPath` must be provided | ✅ | ⛔ | +| `PATCH /policies/{policyId}` | Update the attributes of an existing policy | ✅ | ⛔ | +| `DELETE /policies/{policyId}` | Delete an existing policy | ✅ | ⛔ | +| `GET /policies/{policyId}` | Retrieve a specific policy | ✅ | ⛔ | ### Search -| Endpoint | Description | `full` mode | `lite` mode | -| ------------- | ----------------------------- | ----------- | ----------- | -| `GET /search` | Search for groups and devices | ✅ | ✅ | +| Endpoint | Description | `full` mode | `enhanced` mode | `lite` mode | +| ---------------------------------- | ----------------------------- | ----------- | --------------- | ----------- | +| `GET /search` | Search for groups and devices | ✅ | ✅ | +| `GET /search?type={filter} | ✅ | ✅ | ✅ | +| `GET /search?ancestorPath={filter} | ✅ | ✅ | ✅ | +| `GET /search?eq={filter} | ✅ | ✅ | ✅ | +| `GET /search?neq={filter} | ✅ | ✅ | ✅ | +| `GET /search?lt={filter} | ✅ | ✅ | ✅ | +| `GET /search?lte={filter} | ✅ | ✅ | ✅ | +| `GET /search?gt={filter} | ✅ | ✅ | ✅ | +| `GET /search?gte={filter} | ✅ | ✅ | ✅ | +| `GET /search?exists={filter} | ✅ | ✅ | ✅ | +| `GET /search?nexists={filter} | ✅ | ✅ | ✅ | +| `GET /search?startsWith={filter} | ✅ | ✅ (faster) | ✅ | +| `GET /search?endsWith={filter} | ✅ (since version 5.4.0) | ✅ (faster) | ⛔ | +| `GET /search?contains={filter} | ✅ (since version 5.4.0) | ✅ (faster) | ⛔ | +| `GET /search?fulltext={filter} | ✅ | ⛔ | ⛔ | +| `GET /search?regex={filter} | ✅ | ⛔ | ⛔ | +| `GET /search?lucene={filter} | ✅ | ⛔ | ⛔ | ## Supported Functionality by Area @@ -157,9 +174,11 @@ Not supported in `lite` mode. ### Search -| Description | `full` mode | `lite` mode | -| ---------------------------- | ----------------------- | ------------------------------------------------------------ | -| No. query terms | Maximum 2048 characters | Maximum 2048 characters, and maximum 5 query terms per query | -| No. results | Unlimited | Maximm 500 per query | -| Aggregation | Supported | Not supported | -| Searching by group ancestors | Supported | Supports filtering by directly linked groups only | +| Description | `full` mode | `enhanced` mode | `lite` mode | +| --------------------------------------- | -------------------------------------- | --------------------------- | ------------------------------------------------------------ | +| No. query terms | Maximum 2048 characters | Maximum 2048 characters | Maximum 2048 characters, and maximum 5 query terms per query | +| No. results | Unlimited | Unlimited | Maximm 500 per query | +| Aggregation | Supported | Supported | Not supported | +| Searching by group ancestors | Supported | Supported | Supports filtering by directly linked groups only | +| `endsWith` and `contains` operators | Supported, using Neptune string search | Supported, using OpenSearch | Not supported | +| `fulltext`, `regex`, `lucene` operators | Not supported | Supported | Not supported | From 7d043a3e6bbfd77f91bec7ec462d59c3e88dffe0 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Wed, 26 Jan 2022 09:56:22 -0800 Subject: [PATCH 16/31] fix(assetlibrary): SearchServiceLite implements SearchService --- .../services/assetlibrary/src/search/search.lite.service.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/packages/services/assetlibrary/src/search/search.lite.service.ts b/source/packages/services/assetlibrary/src/search/search.lite.service.ts index c0e36f338..a86f0ac9a 100644 --- a/source/packages/services/assetlibrary/src/search/search.lite.service.ts +++ b/source/packages/services/assetlibrary/src/search/search.lite.service.ts @@ -22,9 +22,10 @@ import { TypeCategory } from '../types/constants'; import { NotSupportedError } from '../utils/errors'; import { SearchDaoLite } from './search.lite.dao'; import { FacetResults, SearchRequestModel } from './search.models'; +import { SearchService } from './search.service'; @injectable() -export class SearchServiceLite { +export class SearchServiceLite implements SearchService { private readonly DEFAULT_SEARCH_COUNT = 200; constructor( From 38db572ec12b4724eb06178150108823e38cd4e9 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Sun, 20 Mar 2022 12:37:41 -0700 Subject: [PATCH 17/31] fix(assetlibrary): tighten up security groups for neptune poller --- .../infrastructure/cfn-enhancedsearch.yaml | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml index 1e9560d06..87092f25e 100644 --- a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml +++ b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml @@ -29,9 +29,6 @@ Parameters: Comma delimited list of private subnetIds to deploy Neptune into. Number of subnets must match the number of availability zones deployed into, i.e. only pass one subnet if operating in a single AZ. Type: List - CDFSecurityGroupId: - Description: ID of an existing security group to allow access to ElasticSearch - Type: AWS::EC2::SecurityGroup::Id NeptuneSecurityGroupId: Description: ID of an existing security group that contains the Neptune nodes Type: AWS::EC2::SecurityGroup::Id @@ -261,14 +258,15 @@ Resources: Type: 'AWS::EC2::SecurityGroup' Properties: VpcId: !Ref VpcId - GroupDescription: !Sub 'CDF Asset Library (${Environment}) OpenSearch Access' - Tags: - - Key: cdf_environment - Value: !Ref Environment - - Key: cdf_service - Value: !Ref CdfService + GroupDescription: !Sub 'CDF Asset Library (${Environment}) OpenSearch Security Group' - OpenSearchSGIngressRule: + NeptuneStreamPollerSG: + Type: 'AWS::EC2::SecurityGroup' + Properties: + VpcId: !Ref VpcId + GroupDescription: !Sub 'CDF Asset Library (${Environment}) Neptune Stream Poller Security Group' + + OpenSearchSGIngressRule1: Type: 'AWS::EC2::SecurityGroupIngress' Properties: GroupId: !Ref OpenSearchSG @@ -276,7 +274,27 @@ Resources: ToPort: 443 IpProtocol: tcp SourceSecurityGroupId: !Ref NeptuneSecurityGroupId - Description: Access from Neptune to ElasticSearch + Description: Access from Neptune to OpenSearch + + OpenSearchSGIngressRule2: + Type: 'AWS::EC2::SecurityGroupIngress' + Properties: + GroupId: !Ref OpenSearchSG + FromPort: 443 + ToPort: 443 + IpProtocol: tcp + SourceSecurityGroupId: !Ref NeptuneStreamPollerSG + Description: Access from Neptune Stream Poller to OpenSearch + + NeptuneSGIngressRule: + Type: 'AWS::EC2::SecurityGroupIngress' + Properties: + GroupId: !Ref NeptuneSecurityGroupId + FromPort: 8182 + ToPort: 8182 + IpProtocol: tcp + SourceSecurityGroupId: !Ref NeptuneStreamPollerSG + Description: Access from Neptune Stream Poller to Neptune OpenSearchDomain: Type: AWS::OpenSearchService::Domain @@ -385,7 +403,7 @@ Resources: - Key: cdf_service Value: !Ref CdfService - ElasticSearchAccessPolicy: + OpenSearchAccessPolicy: Type: 'AWS::IAM::ManagedPolicy' Properties: Description: "Allows ElasticSearch access for Neptune Lambda Poller" @@ -424,7 +442,7 @@ Resources: LambdaRuntime: python3.6 LambdaS3Bucket: !Join ["-", ["aws-neptune-customer-samples", !Ref "AWS::Region"]] LambdaS3Key: "neptune-stream/lambda/python36/release_2021_08_23/neptune-to-es.zip" - ManagedPolicies: !Ref ElasticSearchAccessPolicy + ManagedPolicies: !Ref OpenSearchAccessPolicy LambdaLoggingLevel: !Ref NeptunePollerLambdaLoggingLevel StreamRecordsHandler: # The published template in the Neptune documentation uses a three-level mapping here. @@ -446,7 +464,7 @@ Resources: CreateCloudWatchAlarm: !Ref CreateCloudWatchAlarm NotificationEmail: !Ref NotificationEmail SubnetIds: !Join [ ",", !Ref PrivateSubNetIds ] - SecurityGroupIds: !Ref CDFSecurityGroupId + SecurityGroupIds: !Ref NeptuneStreamPollerSG Outputs: OpenSearchDomainEndpoint: From 48afa839c8ac1537949fcdf7986eb800bff45d72 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Sun, 20 Mar 2022 12:56:49 -0700 Subject: [PATCH 18/31] feat(assetlibrary): Neptune FTS now works with OpenSearch 1.0 --- .../assetlibrary/infrastructure/cfn-enhancedsearch.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml index 87092f25e..92e8aeeba 100644 --- a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml +++ b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml @@ -362,10 +362,7 @@ Resources: - KmsKeyIdProvided - !Ref KmsKeyId - !Ref AWS::NoValue - # When integrating with Amazon OpenSearch Service, Neptune requires Elasticsearch version 7.1 or higher - # rather than OpenSearch version 1.0. Neptune is not currently compatible with OpenSearch version 1.0. - # https://docs.aws.amazon.com/neptune/latest/userguide/full-text-search-cfn-create.html - EngineVersion: 'Elasticsearch_7.10' + EnableVersionUpgrade: true LogPublishingOptions: ES_APPLICATION_LOGS: Fn::If: From 318db95d98b25ab56374e94ef4ea35dc4369e507 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Mon, 21 Mar 2022 08:35:41 -0700 Subject: [PATCH 19/31] WIP(assetlibrary): Reduce Neptune Poller memory, increase polling frequency --- .../infrastructure/cfn-enhancedsearch.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml index 92e8aeeba..cb5230e4b 100644 --- a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml +++ b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml @@ -133,8 +133,8 @@ Parameters: Type: String NeptunePollerLambdaMemorySize: Type: Number - Default: 2048 - Description: Neptune Poller Lambda memory size (in MB). + Default: 512 + Description: Neptune Poller Lambda memory size (in MB). AllowedValues: - 128 - 256 @@ -160,7 +160,7 @@ Parameters: Description: "Number of records to be read from stream in each batch. Should be between 1 to 50000." NeptunePollerMaxPollingWaitTime: Type: Number - Default: 60 + Default: 10 MaxValue: 3600 MinValue: 0 Description: "Maximum wait time in seconds between two successive polling from stream. Set value to 0 sec for continuous polling. Maximum value can be 3600 sec (1 hour)." @@ -169,7 +169,7 @@ Parameters: Default: 600 MaxValue: 900 MinValue: 5 - Description: "Period for which we can continuously poll stream for records on one Lambda instance. Should be between 5 sec to 900 sec. This parameter is used to set Poller Lambda Timeout." + Description: "Number of seconds for which we can continuously poll stream for records on one Lambda instance. Should be between 5 sec to 900 sec. This parameter is used to set Poller Lambda Timeout." NeptunePollerStepFunctionFallbackPeriod: Type: Number Default: 5 @@ -300,6 +300,8 @@ Resources: Type: AWS::OpenSearchService::Domain DeletionPolicy: Snapshot UpdateReplacePolicy: Snapshot + UpdatePolicy: + EnableVersionUpgrade: true Properties: DomainName: !Sub 'cdf-${Environment}' AccessPolicies: @@ -362,7 +364,6 @@ Resources: - KmsKeyIdProvided - !Ref KmsKeyId - !Ref AWS::NoValue - EnableVersionUpgrade: true LogPublishingOptions: ES_APPLICATION_LOGS: Fn::If: @@ -438,7 +439,7 @@ Resources: LambdaMemorySize: !Ref NeptunePollerLambdaMemorySize LambdaRuntime: python3.6 LambdaS3Bucket: !Join ["-", ["aws-neptune-customer-samples", !Ref "AWS::Region"]] - LambdaS3Key: "neptune-stream/lambda/python36/release_2021_08_23/neptune-to-es.zip" + LambdaS3Key: "neptune-stream/lambda/python36/release_2022_03_14/neptune-to-es.zip" ManagedPolicies: !Ref OpenSearchAccessPolicy LambdaLoggingLevel: !Ref NeptunePollerLambdaLoggingLevel StreamRecordsHandler: From edd74d2423b2f4a1e7725337f4c67a862742dc3d Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Tue, 5 Apr 2022 17:53:43 -0700 Subject: [PATCH 20/31] style(assetlibrary): remove leftover commented out code --- .../services/assetlibrary/src/di/inversify.config.enhanced.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/source/packages/services/assetlibrary/src/di/inversify.config.enhanced.ts b/source/packages/services/assetlibrary/src/di/inversify.config.enhanced.ts index f7d5e27a6..daf3bae81 100644 --- a/source/packages/services/assetlibrary/src/di/inversify.config.enhanced.ts +++ b/source/packages/services/assetlibrary/src/di/inversify.config.enhanced.ts @@ -10,7 +10,6 @@ * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions * * and limitations under the License. * *********************************************************************************************************************/ -// import { ContainerModule, decorate, injectable, interfaces } from 'inversify'; import { ContainerModule, interfaces } from 'inversify'; import { SearchDaoEnhanced } from '../search/search.enhanced.dao'; import { TYPES } from './types'; From 66f48833d4ad9bb4cbfa765bb5b10a107ef0c95a Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Tue, 5 Apr 2022 18:08:57 -0700 Subject: [PATCH 21/31] docs(assetlibrary): fix search modes comparison table markdown --- .../services/assetlibrary/docs/modes.md | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/source/packages/services/assetlibrary/docs/modes.md b/source/packages/services/assetlibrary/docs/modes.md index 4c4662a5a..8fc6bc4cb 100644 --- a/source/packages/services/assetlibrary/docs/modes.md +++ b/source/packages/services/assetlibrary/docs/modes.md @@ -99,25 +99,24 @@ The following table indicates which REST API's are available in which mode: ### Search -| Endpoint | Description | `full` mode | `enhanced` mode | `lite` mode | -| ---------------------------------- | ----------------------------- | ----------- | --------------- | ----------- | -| `GET /search` | Search for groups and devices | ✅ | ✅ | -| `GET /search?type={filter} | ✅ | ✅ | ✅ | -| `GET /search?ancestorPath={filter} | ✅ | ✅ | ✅ | -| `GET /search?eq={filter} | ✅ | ✅ | ✅ | -| `GET /search?neq={filter} | ✅ | ✅ | ✅ | -| `GET /search?lt={filter} | ✅ | ✅ | ✅ | -| `GET /search?lte={filter} | ✅ | ✅ | ✅ | -| `GET /search?gt={filter} | ✅ | ✅ | ✅ | -| `GET /search?gte={filter} | ✅ | ✅ | ✅ | -| `GET /search?exists={filter} | ✅ | ✅ | ✅ | -| `GET /search?nexists={filter} | ✅ | ✅ | ✅ | -| `GET /search?startsWith={filter} | ✅ | ✅ (faster) | ✅ | -| `GET /search?endsWith={filter} | ✅ (since version 5.4.0) | ✅ (faster) | ⛔ | -| `GET /search?contains={filter} | ✅ (since version 5.4.0) | ✅ (faster) | ⛔ | -| `GET /search?fulltext={filter} | ✅ | ⛔ | ⛔ | -| `GET /search?regex={filter} | ✅ | ⛔ | ⛔ | -| `GET /search?lucene={filter} | ✅ | ⛔ | ⛔ | +| Endpoint/Parameter | `full` mode | `enhanced` mode | `lite` mode | +| ----------------------------------- | ------------------------ | --------------- | ----------- | +| `GET /search?type={filter}` | ✅ | ✅ | ✅ | +| `GET /search?ancestorPath={filter}` | ✅ | ✅ | ✅ | +| `GET /search?eq={filter}` | ✅ | ✅ | ✅ | +| `GET /search?neq={filter}` | ✅ | ✅ | ✅ | +| `GET /search?lt={filter}` | ✅ | ✅ | ✅ | +| `GET /search?lte={filter}` | ✅ | ✅ | ✅ | +| `GET /search?gt={filter}` | ✅ | ✅ | ✅ | +| `GET /search?gte={filter}` | ✅ | ✅ | ✅ | +| `GET /search?exists={filter}` | ✅ | ✅ | ✅ | +| `GET /search?nexists={filter}` | ✅ | ✅ | ✅ | +| `GET /search?startsWith={filter}` | ✅ | ✅ (faster) | ✅ | +| `GET /search?endsWith={filter}` | ✅ (since version 5.4.0) | ✅ (faster) | ⛔ | +| `GET /search?contains={filter}` | ✅ (since version 5.4.0) | ✅ (faster) | ⛔ | +| `GET /search?fulltext={filter}` | ⛔ | ✅ | ⛔ | +| `GET /search?regex={filter}` | ⛔ | ✅ | ⛔ | +| `GET /search?lucene={filter}` | ⛔ | ✅ | ⛔ | ## Supported Functionality by Area From 31a171cbc473bdf8f9874d8865f4947ea57a8a16 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Tue, 5 Apr 2022 19:43:02 -0700 Subject: [PATCH 22/31] docs(assetlibrary): update remaining references to OpenSearch --- source/packages/services/assetlibrary/docs/swagger.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/source/packages/services/assetlibrary/docs/swagger.yml b/source/packages/services/assetlibrary/docs/swagger.yml index 421106f46..01f72fd6a 100644 --- a/source/packages/services/assetlibrary/docs/swagger.yml +++ b/source/packages/services/assetlibrary/docs/swagger.yml @@ -1551,12 +1551,11 @@ paths: Filter by an attribute based on a regex pattern. This filter is only available in the "enhanced" mode of Asset Library. - For example: `?regex=serialNo:XY[A-Z]\-[0-4][^9].*`. Syntax and limitations are those of - the ElasticSearch DSL, documented here: - https://www.elastic.co/guide/en/elasticsearch/reference/7.10/regexp-syntax.html. - Notably, ElasticSearch does NOT support: + For example: `?regex=serialNo:XY[A-Z]\-[0-4][^9].*`. Regular expressions use the Lucene syntax, which + differs from more standardized implementations. Notably, the Lucene syntax does NOT support: 1. anchor operators, such as ^ and $ for start and end of a string 2. character class tokens such as \d (any digit) and \S (all non-whitespace character) + See also: https://opensearch.org/docs/latest/opensearch/query-dsl/term/#regex. Regex filters are performed against the original complete value of the field, not individual words. For example, a device with attribute "mfg" set to "Widgets Incorporated" can be found with From 98636fdef8bb8f2fb1afc7ed7edb5ba6c7e67392 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Fri, 8 Apr 2022 17:11:10 -0700 Subject: [PATCH 23/31] docs(assetlibrary): consistently use opensearch over now legacy es names --- source/infrastructure/install-policy-3.json | 13 +++++- .../services/assetlibrary/docs/swagger.yml | 4 +- .../infrastructure/cfn-enhancedsearch.yaml | 46 +++++++++---------- 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/source/infrastructure/install-policy-3.json b/source/infrastructure/install-policy-3.json index fbe13318d..0332df6e9 100644 --- a/source/infrastructure/install-policy-3.json +++ b/source/infrastructure/install-policy-3.json @@ -112,10 +112,13 @@ "Sid": "OpenSearchServiceLinkedRoleCreate", "Action": "iam:CreateServiceLinkedRole", "Effect": "Allow", - "Resource": "arn:aws:iam::*:role/aws-service-role/es.amazonaws.com/AWSServiceRoleForAmazonElasticsearchService", + "Resource": "arn:aws:iam::*:role/aws-service-role/es.amazonaws.com/AWSServiceRoleForAmazonOpenSearchService", "Condition": { "StringLike": { - "iam:AWSServiceName":"es.amazonaws.com" + "iam:AWSServiceName": [ + "opensearchservice.amazonaws.com", + "es.amazonaws.com" + ] } } }, @@ -123,6 +126,12 @@ "Sid": "OpenSearchServiceLinkedRoleGet", "Action": "iam:GetRole", "Effect": "Allow", + "Resource": "arn:aws:iam::*:role/aws-service-role/es.amazonaws.com/AWSServiceRoleForAmazonOpenSearchService" + }, + { + "Sid": "OpenSearchLegacyServiceLinkedRoleGet", + "Action": "iam:GetRole", + "Effect": "Allow", "Resource": "arn:aws:iam::*:role/aws-service-role/es.amazonaws.com/AWSServiceRoleForAmazonElasticsearchService" }, { diff --git a/source/packages/services/assetlibrary/docs/swagger.yml b/source/packages/services/assetlibrary/docs/swagger.yml index 01f72fd6a..ebaca1345 100644 --- a/source/packages/services/assetlibrary/docs/swagger.yml +++ b/source/packages/services/assetlibrary/docs/swagger.yml @@ -1535,8 +1535,8 @@ paths: `fulltext=widgets` matches "Widgets Incorporated". If the filter query contains more than one word, they are combined using logical `or``, for example `fulltext=Widgets Gadgets Incorporated` matches "Widgets Incorporated" and "Gadgets Incorporated" and "Gadgets Ltd". Field and search - strings are split into words using the rules of the ElasticSearch standard query analyzer: - https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis-standard-analyzer.html + strings are split into words using the rules of the OpenSearch standard query analyzer: + https://opensearch.org/docs/latest/opensearch/query-dsl/full-text/ Append the `~` character to any word to allow for fuzzy matches. Specify the edit distance allowed by fuzzy match in the format `~n`, for example `Masachussets~3` matches "Massachusetts". diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml index cb5230e4b..27605b341 100644 --- a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml +++ b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml @@ -12,7 +12,7 @@ #----------------------------------------------------------------------------------------------------------------------- AWSTemplateFormatVersion : '2010-09-09' Transform: AWS::Serverless-2016-10-31 -Description: CDF Asset Library Service Neptune-to-ElasticSearch connection for enhanced search +Description: CDF Asset Library Service Neptune-to-OpenSearch connection for enhanced search Parameters: Environment: @@ -33,7 +33,7 @@ Parameters: Description: ID of an existing security group that contains the Neptune nodes Type: AWS::EC2::SecurityGroup::Id KmsKeyId: - Description: The KMS key ID used to encrypt the ElasticSearch database + Description: The KMS key ID used to encrypt the OpenSearch database Type: String MinLength: 1 OpenSearchAvailabilityZoneCount: @@ -49,8 +49,8 @@ Parameters: ConstraintDescription: OpenSearchAvailabilityZoneCount must equal 1, 2, or 3. OpenSearchInstanceType: Description: > - OpenSearch data instance type. Must be a supported OpenSearch instance type in the region, support ElasticSearch, - and must support encryption at rest. See also: + OpenSearch data instance type. Must be a supported OpenSearch instance type in the region and must support + encryption at rest. See also: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html Type: String Default: t3.small.search @@ -71,8 +71,8 @@ Parameters: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/managedomains-dedicatedmasternodes.html OpenSearchDedicatedMasterInstanceType: Description: > - OpenSearch master instance type. Must be a supported OpenSearch instance type in the region, support ElasticSearch, - and must support encryption at rest. See also: + OpenSearch master instance type. Must be a supported OpenSearch instance type in the region and must support + encryption at rest. See also: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/supported-instance-types.html Type: String Default: t3.small.search @@ -83,13 +83,13 @@ Parameters: Default: 10 MinValue: 10 Description: > - Size of EBS volumes attached to each ElasticSearch node, in GiB. Allowed ranges depend on the + Size of EBS volumes attached to each OpenSearch node, in GiB. Allowed ranges depend on the instance type chosen in OpenSearchInstanceType and are documented here: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/limits.html#ebsresource OpenSearchEBSVolumeType: Type: String Default: gp2 - Description: Type of the EBS volume attached to each ElasticSearch node. + Description: Type of the EBS volume attached to each OpenSearch node. AllowedValues: - gp2 - gp3 @@ -109,24 +109,24 @@ Parameters: Type: String Default: '' Description: > - ARN of Cloudwatch Log Group where ElasticSearch audit Logs are sent. If left empty, logs will not be generated. + ARN of Cloudwatch Log Group where OpenSearch audit Logs are sent. If left empty, logs will not be generated. OpenSearchApplicationLogsCloudWatchLogsLogGroupArn: Type: String Default: '' Description: > - ARN of Cloudwatch Log Group where ElasticSearch application Logs are sent. If left empty, logs will not be + ARN of Cloudwatch Log Group where OpenSearch application Logs are sent. If left empty, logs will not be generated. OpenSearchIndexSlowLogsCloudWatchLogsLogGroupArn: Type: String Default: '' Description: > - ARN of Cloudwatch Log Group where ElasticSearch Index Slow Logs are sent. If left empty, logs will not be + ARN of Cloudwatch Log Group where OpenSearch Index Slow Logs are sent. If left empty, logs will not be generated. OpenSearchSearchSlowLogsCloudWatchLogsLogGroupArn: Type: String Default: '' Description: > - ARN of Cloudwatch Log Group where ElasticSearch Search Slow Logs are sent. If left empty, logs will not be + ARN of Cloudwatch Log Group where OpenSearch Search Slow Logs are sent. If left empty, logs will not be generated. NeptuneClusterEndpoint: Description: 'Neptune cluster endpoint. Format: :' @@ -244,10 +244,10 @@ Conditions: EnableDedicatedMasterNodes: !Not [ !Equals [ !Ref OpenSearchDedicatedMasterCount, 0 ] ] CreateCloudWatchAlarmCondition: !Equals [ !Ref CreateCloudWatchAlarm, 'true' ] EnableNonStringIndexingCondition: !Equals [ !Ref EnableNonStringIndexing, 'true' ] - ElasticSearchAuditLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref OpenSearchAuditLogsCloudWatchLogsLogGroupArn, '' ] ] - ElasticSearchApplicationLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref OpenSearchApplicationLogsCloudWatchLogsLogGroupArn, '' ] ] - ElasticSearchIndexSlowLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref OpenSearchIndexSlowLogsCloudWatchLogsLogGroupArn, '' ] ] - ElasticSearchSearchSlowLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref OpenSearchSearchSlowLogsCloudWatchLogsLogGroupArn, '' ] ] + OpenSearchAuditLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref OpenSearchAuditLogsCloudWatchLogsLogGroupArn, '' ] ] + OpenSearchApplicationLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref OpenSearchApplicationLogsCloudWatchLogsLogGroupArn, '' ] ] + OpenSearchIndexSlowLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref OpenSearchIndexSlowLogsCloudWatchLogsLogGroupArn, '' ] ] + OpenSearchSearchSlowLogsCloudWatchLogsEnabled: !Not [ !Equals [ !Ref OpenSearchSearchSlowLogsCloudWatchLogsLogGroupArn, '' ] ] OpenSearchEBSVolumeTypeIsIo1: !Equals [ !Ref OpenSearchEBSVolumeType, 'io1' ] EnableZoneAwareness: !Not [ !Equals [ !Ref OpenSearchAvailabilityZoneCount, 1 ] ] @@ -367,25 +367,25 @@ Resources: LogPublishingOptions: ES_APPLICATION_LOGS: Fn::If: - - ElasticSearchApplicationLogsCloudWatchLogsEnabled + - OpenSearchApplicationLogsCloudWatchLogsEnabled - CloudWatchLogsLogGroupArn: !Ref OpenSearchApplicationLogsCloudWatchLogsLogGroupArn Enabled: true - !Ref AWS::NoValue SEARCH_SLOW_LOGS: Fn::If: - - ElasticSearchSearchSlowLogsCloudWatchLogsEnabled + - OpenSearchSearchSlowLogsCloudWatchLogsEnabled - CloudWatchLogsLogGroupArn: !Ref OpenSearchSearchSlowLogsCloudWatchLogsLogGroupArn Enabled: true - !Ref AWS::NoValue INDEX_SLOW_LOGS: Fn::If: - - ElasticSearchIndexSlowLogsCloudWatchLogsEnabled + - OpenSearchIndexSlowLogsCloudWatchLogsEnabled - CloudWatchLogsLogGroupArn: !Ref OpenSearchIndexSlowLogsCloudWatchLogsLogGroupArn Enabled: true - !Ref AWS::NoValue AUDIT_LOGS: Fn::If: - - ElasticSearchAuditLogsCloudWatchLogsEnabled + - OpenSearchAuditLogsCloudWatchLogsEnabled - CloudWatchLogsLogGroupArn: !Ref OpenSearchAuditLogsCloudWatchLogsLogGroupArn Enabled: true - !Ref AWS::NoValue @@ -404,11 +404,11 @@ Resources: OpenSearchAccessPolicy: Type: 'AWS::IAM::ManagedPolicy' Properties: - Description: "Allows ElasticSearch access for Neptune Lambda Poller" + Description: "Allows OpenSearch access for Neptune Lambda Poller" PolicyDocument: Version: '2012-10-17' Statement: - - Sid: "elasticsearchaccess" + - Sid: "opensearchaccess" Effect: Allow Action: - 'es:ESHttpDelete' @@ -466,7 +466,7 @@ Resources: Outputs: OpenSearchDomainEndpoint: - Description: HTTPS endpoint URL for ElasticSearch cluster + Description: HTTPS endpoint URL for OpenSearch cluster Value: !Sub 'https://${OpenSearchDomain.DomainEndpoint}' Export: Name: !Sub "cdf-assetlibrary-enhancedsearch-${Environment}-OpenSearchDomainEndpoint" From 668f52b6445eaec2fb0ef5fb4ee383ca32ba9314 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Mon, 13 Jun 2022 09:33:15 -0700 Subject: [PATCH 24/31] fix(assetlibrary): upgrade enhanced search lambda to python3.9 runtime --- .../assetlibrary/infrastructure/cfn-enhancedsearch.yaml | 6 +++--- .../installer/src/commands/modules/service/assetLibrary.ts | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml index 27605b341..fb7de9561 100644 --- a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml +++ b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml @@ -437,14 +437,14 @@ Resources: EnableNonStringIndexing: !Ref EnableNonStringIndexing ApplicationName: !Sub 'cdf-${CdfService}-enhancedsearch-${Environment}' LambdaMemorySize: !Ref NeptunePollerLambdaMemorySize - LambdaRuntime: python3.6 + LambdaRuntime: python3.9 LambdaS3Bucket: !Join ["-", ["aws-neptune-customer-samples", !Ref "AWS::Region"]] - LambdaS3Key: "neptune-stream/lambda/python36/release_2022_03_14/neptune-to-es.zip" + LambdaS3Key: "neptune-stream/lambda/python39/release_2022_06_06/neptune-to-es.zip" ManagedPolicies: !Ref OpenSearchAccessPolicy LambdaLoggingLevel: !Ref NeptunePollerLambdaLoggingLevel StreamRecordsHandler: # The published template in the Neptune documentation uses a three-level mapping here. - # The mapping is flattened into a single If because Gremlin and Python are fixed. + # The mapping is flattened into a single "If" because Gremlin and Python are fixed. Fn::If: - EnableNonStringIndexingCondition - "neptune_to_es.neptune_gremlin_es_handler.ElasticSearchGremlinHandler" diff --git a/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts b/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts index c9e909270..9d664eb47 100644 --- a/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts +++ b/source/packages/services/installer/src/commands/modules/service/assetLibrary.ts @@ -59,7 +59,7 @@ export class AssetLibraryInstaller implements RestModule { public readonly type = 'SERVICE'; public readonly dependsOnMandatory: ModuleName[] = ['apigw', 'deploymentHelper']; - public readonly dependsOnOptional: ModuleName[] = ['vpc']; + public readonly dependsOnOptional: ModuleName[] = ['vpc', 'kms']; public readonly stackName: string; private readonly neptuneStackName: string; @@ -279,6 +279,11 @@ export class AssetLibraryInstaller implements RestModule { updatedAnswers.modules, modeRequiresNeptune(updatedAnswers.assetLibrary.mode) ); + includeOptionalModule( + 'kms', + updatedAnswers.modules, + modeRequiresOpenSearch(updatedAnswers.assetLibrary.mode) + ); return updatedAnswers; } From 9ac1f65c9bc01da43aa5c1e40af244d81edede91 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Wed, 15 Jun 2022 17:16:20 -0700 Subject: [PATCH 25/31] docs(assetlibrary): integration tests --- .../features/assetlibrary/README.md | 53 ++-- .../deviceSearch.feature | 249 ++++++++++++++++++ .../assetlibrary/search.steps.ts | 4 +- .../src/support/assetLibrary_hooks.ts | 113 ++++++++ .../packages/services/assetlibrary/README.md | 1 + .../assetlibrary/docs/configuration.md | 4 +- .../services/assetlibrary/docs/modes.md | 2 +- .../command-and-control/docs/configuration.md | 5 +- 8 files changed, 401 insertions(+), 30 deletions(-) create mode 100644 source/packages/integration-tests/features/assetlibrary/full-with-enhancedsearch/deviceSearch.feature diff --git a/source/packages/integration-tests/features/assetlibrary/README.md b/source/packages/integration-tests/features/assetlibrary/README.md index 1005c7c0d..d6b8ef3ce 100644 --- a/source/packages/integration-tests/features/assetlibrary/README.md +++ b/source/packages/integration-tests/features/assetlibrary/README.md @@ -1,38 +1,47 @@ -# Asset Library integration tests +# Asset Library Integration Tests -Note: only the _full_ mode is tested as part of the CI/CD pipeline. We need to add both the _full (witht FGAC)_ and _lite_ modes to it. But for now, these need testing manually... +After following the [general steps for setting up an environment for integration testing](../README.md), the commands below can be used to run integration tests for Asset Library in various configurations. +The tests are executed locally on your development environment and send HTTP requests to an Asset Library API deployed in an AWS account. -### Testing full-with-authz mode +The subset of tests must match the mode and configuration of the Asset Library deployment at the URL configured in `ASSETLIBRARY_BASE_URL` in `path/to/local/.env`. +For example, integration tests in the `full-with-authz` folder only pass with an Asset Library deployment that is configured with `AUTHORIZATION_ENABLED=true`. -- Run asset library with FGAC enabled (only supported in _full_ mode): -```sh -$ assetlibrary> CONFIG_LOCATION="path/to/local/.env" ASSETLIBRARY_AUTHORIZATION_ENABLED=true npm run start -``` -- Run FGAC specific integration tests: -```sh -$ integration-tests> CONFIG_LOCATION="path/to/integrationtest/.env" pnpm run integration-test -- features/assetlibrary/full-with-authz/* -``` +Note: Only the _full_ mode is currently tested as part of the CI/CD pipeline. -### Testing lite mode +## Testing full/enhanced mode -See the [special notes for running lite mode tests](./lite/README.md). +To run integration tests for Asset Library in _full_ or _enhanced_ mode, irrespective of `AUTHORIZATION_ENABLED` setting: -- Run asset library in _lite_ mode: ```sh -$ assetlibrary> CONFIG_LOCATION="path/to/local/.env" ASSETLIBRARY_MODE=lite npm run start +$ source/packages/integration-tests> CONFIG_LOCATION="path/to/integrationtest/.env" pnpm run integration-test -- features/assetlibrary/full/* ``` -- Run FGAC specific integration tests: + +## Testing full-with-authz mode + +To run integration tests for Asset Library in _full_ mode, when `AUTHORIZATION_ENABLED` is set to `true`: + ```sh -$ integration-tests> CONFIG_LOCATION="path/to/integrationtest/.env" pnpm run integration-test -- features/assetlibrary/lite/* +$ source/packages/integration-tests> CONFIG_LOCATION="path/to/integrationtest/.env" npm run integration-test -- features/assetlibrary/full-with-authz/* ``` -### Testing full mode +These tests are an addition to, not a superset of, "Testing full/enhanced mode". + +## Testing enhanced mode + +To run integration tests for Asset Library in _enhanced_ mode, irrespective of `AUTHORIZATION_ENABLED` setting: -- Run asset library in the default _full_ mode (requires a Neptune tunnel of running loc) ```sh -$ assetlibrary> CONFIG_LOCATION="path/to/local/.env" npm run start +$ source/packages/integration-tests> CONFIG_LOCATION="path/to/integrationtest/.env" npm run integration-test -- features/assetlibrary/full-with-enhancedsearch/* ``` -- Run FGAC specific integration tests: + +These tests are an addition to, not a superset of, "Testing full/enhanced mode". + +## Testing lite mode + +See the [special notes for running lite mode tests](lite/README.md). + +To run integration tests for Asset Library in _lite_ mode: + ```sh -$ integration-tests> CONFIG_LOCATION="path/to/integrationtest/.env" pnpm run integration-test -- features/assetlibrary/full/* +$ source/packages/integration-tests> CONFIG_LOCATION="path/to/integrationtest/.env" npm run integration-test -- features/assetlibrary/lite/* ``` diff --git a/source/packages/integration-tests/features/assetlibrary/full-with-enhancedsearch/deviceSearch.feature b/source/packages/integration-tests/features/assetlibrary/full-with-enhancedsearch/deviceSearch.feature new file mode 100644 index 000000000..87e17cd08 --- /dev/null +++ b/source/packages/integration-tests/features/assetlibrary/full-with-enhancedsearch/deviceSearch.feature @@ -0,0 +1,249 @@ +#----------------------------------------------------------------------------------------------------------------------- +# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. +#----------------------------------------------------------------------------------------------------------------------- + +Feature: Device enhanced search + + @setup_deviceSearch_enhanced_feature + Scenario: Setup + Given published assetlibrary device template "test-enhancedsearch-deviceTpl" exists + And group "/enhancedSearchGroup_all" exists + And group "/enhancedSearchGroup_xxyy" exists + And group "/enhancedSearchGroup_xyyx" exists + And device "test-enhancedsearch-aaaa" exists + And device "test-enhancedsearch-aaab" exists + + Scenario: Baseline search for all devices + When I search with summary with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + Then search result contains 16 total + + Scenario: Startswith search using enhanced search 1 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | startsWith | characters:a | + Then search result contains 8 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-aaab" + And search result contains device "test-enhancedsearch-aaba" + And search result contains device "test-enhancedsearch-aabb" + And search result contains device "test-enhancedsearch-abaa" + And search result contains device "test-enhancedsearch-abab" + And search result contains device "test-enhancedsearch-abba" + And search result contains device "test-enhancedsearch-abbb" + + Scenario: Startswith search using enhanced search 2 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | startsWith | characters:aa | + Then search result contains 4 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-aaab" + And search result contains device "test-enhancedsearch-aaba" + And search result contains device "test-enhancedsearch-aabb" + + Scenario: Startswith search using enhanced search 3 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | startsWith | characters:aaa | + Then search result contains 2 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-aaab" + + Scenario: Endswith search using enhanced search 1 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | endsWith | characters:a | + Then search result contains 8 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-aaba" + And search result contains device "test-enhancedsearch-abaa" + And search result contains device "test-enhancedsearch-abba" + And search result contains device "test-enhancedsearch-baaa" + And search result contains device "test-enhancedsearch-baba" + And search result contains device "test-enhancedsearch-bbaa" + And search result contains device "test-enhancedsearch-bbba" + + Scenario: Endswith search using enhanced search 2 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | endsWith | characters:aa | + Then search result contains 4 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-abaa" + And search result contains device "test-enhancedsearch-baaa" + And search result contains device "test-enhancedsearch-bbaa" + + Scenario: Endswith search using enhanced search 3 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | endsWith | characters:aaa | + Then search result contains 2 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-baaa" + + Scenario: Contains search using enhanced search 1 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | contains | characters:aaa | + Then search result contains 3 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-baaa" + And search result contains device "test-enhancedsearch-aaab" + + Scenario: Contains search using enhanced search 2 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | contains | characters:aba | + Then search result contains 4 total + And search result contains device "test-enhancedsearch-aaba" + And search result contains device "test-enhancedsearch-abaa" + And search result contains device "test-enhancedsearch-baba" + And search result contains device "test-enhancedsearch-abab" + + Scenario: Regex search 1 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | regex | characters:a[ab]aa | + Then search result contains 2 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-abaa" + + Scenario: Regex search 2 + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | regex | characters:a.*a | + Then search result contains 4 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-aaba" + And search result contains device "test-enhancedsearch-abaa" + And search result contains device "test-enhancedsearch-abba" + + Scenario: Fulltext search Apple + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | fulltext | words:apple | + Then search result contains 5 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-abaa" + And search result contains device "test-enhancedsearch-abab" + And search result contains device "test-enhancedsearch-baba" + And search result contains device "test-enhancedsearch-bbba" + + Scenario: Fulltext search wildcard match for Apple and Pineapple + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | fulltext | words:*apple | + Then search result contains 9 total + And search result contains device "test-enhancedsearch-aaaa" + And search result contains device "test-enhancedsearch-aaba" + And search result contains device "test-enhancedsearch-abaa" + And search result contains device "test-enhancedsearch-abab" + And search result contains device "test-enhancedsearch-baaa" + And search result contains device "test-enhancedsearch-baba" + And search result contains device "test-enhancedsearch-babb" + And search result contains device "test-enhancedsearch-bbaa" + And search result contains device "test-enhancedsearch-bbba" + + Scenario: Fulltext fuzzy search Cherry misspelling + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | fulltext | words:chery~ | + Then search result contains 4 total + And search result contains device "test-enhancedsearch-aaab" + And search result contains device "test-enhancedsearch-abaa" + And search result contains device "test-enhancedsearch-baba" + And search result contains device "test-enhancedsearch-bbbb" + + Scenario: Fulltext two words without operator + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | fulltext | words:kiwi banana | + Then search result contains 10 total + + Scenario: Fulltext two words with AND operator + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | fulltext | words:kiwi AND banana | + Then search result contains 1 total + And search result contains device "test-enhancedsearch-bbab" + + Scenario: Lucene query with simple equality + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + # important: must URL-encode the filter value, otherwise the step definition will incorrectly parse it because of + # the ":" characters in the lucene query + | lucene | *:predicates.characters.value%3Abbba | + Then search result contains 1 total + And search result contains device "test-enhancedsearch-bbba" + + Scenario: Lucene query with boolean OR + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | lucene | *:predicates.characters.value%3Abbba%20OR%20predicates.characters.value%3Abbbb | + Then search result contains 2 total + And search result contains device "test-enhancedsearch-bbba" + And search result contains device "test-enhancedsearch-bbbb" + + Scenario: Lucene query with boolean AND + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | lucene | *:predicates.characters.value%3Abbba%20AND%20predicates.characters.value%3Abbbb | + Then search result contains 0 total + + Scenario: Lucene query with fulltext exact match + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | lucene | *:predicates.words.value%3Agrapefruit | + Then search result contains 3 total + + Scenario: Lucene query with fulltext fuzzy match + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | lucene | *:predicates.words.value%3Agripefruit~ | + Then search result contains 3 total + + Scenario: Lucene query with fulltext regex match + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | lucene | *:predicates.words.value%3A/g.+fruit/ | + Then search result contains 3 total + + Scenario: Lucene query with regex fuzzy and boolean operators + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | lucene | *:predicates.characters.value%3Ababa%20AND%20predicates.words.value%3Agrapefruit | + Then search result contains 1 total + And search result contains device "test-enhancedsearch-baba" + + Scenario: Regex search with traversal + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | regex | part_of_group:out:name:enhancedSearchGroup_xy.* | + Then search result contains 8 total + + Scenario: Regex search with traversal and non-traversal startsWith + When I search with following attributes: + | ancestorPath | /enhancedSearchGroup_all | + | regex | part_of_group:out:name:enhancedSearchGroup_x[xy]{2}x | + | startsWith | characters:a | + Then search result contains 0 total + + @teardown_deviceSearch_enhanced_feature + Scenario: Teardown + Given draft assetlibrary device template "test-enhancedsearch-deviceTpl" does not exist + And published assetlibrary device template "test-enhancedsearch-deviceTpl" does not exist + And group "/enhancedSearchGroup_all" does not exist + And group "/enhancedSearchGroup_xxyy" does not exist + And group "/enhancedSearchGroup_xyyx" does not exist + And device "test-enhancedsearch-aaaa" does not exist + And device "test-enhancedsearch-aaab" does not exist diff --git a/source/packages/integration-tests/src/step_definitions/assetlibrary/search.steps.ts b/source/packages/integration-tests/src/step_definitions/assetlibrary/search.steps.ts index b41980a5e..6cc9ed390 100644 --- a/source/packages/integration-tests/src/step_definitions/assetlibrary/search.steps.ts +++ b/source/packages/integration-tests/src/step_definitions/assetlibrary/search.steps.ts @@ -80,7 +80,9 @@ function buildSearchRequest(data: DataTable): SearchRequestModel { const filter: SearchRequestFilter = { field: attrs[attrs.length - 2], - value: attrs[attrs.length - 1], + // test cases can optionally URL-encode the filter value, for example for lucene search operator + // where the filter value contains ":" characters + value: decodeURIComponent(attrs[attrs.length - 1]), }; // do we have traversals defined? if (attrs.length > 2) { diff --git a/source/packages/integration-tests/src/support/assetLibrary_hooks.ts b/source/packages/integration-tests/src/support/assetLibrary_hooks.ts index 59486472e..43d7d0238 100644 --- a/source/packages/integration-tests/src/support/assetLibrary_hooks.ts +++ b/source/packages/integration-tests/src/support/assetLibrary_hooks.ts @@ -210,6 +210,27 @@ const templatesService: TemplatesService = container.get( ASSETLIBRARY_CLIENT_TYPES.TemplatesService ); const profilesService: ProfilesService = container.get(ASSETLIBRARY_CLIENT_TYPES.ProfilesService); +const ENHANCEDSEARCH_FEATURES_DEVICE_TEMPLATE_ID = 'TEST-enhancedSearch-deviceTpl'; +const ENHANCEDSEARCH_FEATURES_GROUP_ALL = 'enhancedSearchGroup_all'; +const ENHANCEDSEARCH_FEATURES_GROUP_SUFFIXES = ['xxyy', 'xyyx']; +const ENHANCEDSEARCH_FEATURES_DEVICES = [ + { characters: 'aaaa', groups: ['xxyy'], words: 'apple orange kiwi' }, + { characters: 'aaab', groups: ['xxyy'], words: 'cherry blackberry grapefruit' }, + { characters: 'aaba', groups: ['xxyy'], words: 'pineapple pear orange' }, + { characters: 'aabb', groups: ['xxyy'], words: 'pear kiwi orange' }, + { characters: 'abaa', groups: ['xxyy'], words: 'cherry apple blackberry' }, + { characters: 'abab', groups: ['xxyy'], words: 'orange pineapple apple' }, + { characters: 'abba', groups: ['xxyy'], words: 'kiwi orange blackberry' }, + { characters: 'abbb', groups: ['xxyy'], words: 'blackberry orange kiwi' }, + { characters: 'baaa', groups: ['xyyx'], words: 'pineapple kiwi peach' }, + { characters: 'baab', groups: ['xyyx'], words: 'banana pear grapefruit' }, + { characters: 'baba', groups: ['xyyx'], words: 'grapefruit cherry apple' }, + { characters: 'babb', groups: ['xyyx'], words: 'blackberry orange pineapple' }, + { characters: 'bbaa', groups: ['xyyx'], words: 'orange banana pineapple' }, + { characters: 'bbab', groups: ['xyyx'], words: 'kiwi strawberry banana' }, + { characters: 'bbba', groups: ['xyyx'], words: 'orange apple banana' }, + { characters: 'bbbb', groups: ['xyyx'], words: 'cherry banana potato' }, +]; /* Cucumber describes current scenario context as “World”. It can be used to store the state of the scenario @@ -1464,3 +1485,95 @@ Before({ tags: '@setup_groupSearch_lite_feature' }, async function () { Before({ tags: '@teardown_groupSearch_lite_feature' }, async function () { await teardown_groupSearch_lite_feature(); }); + +async function teardown_deviceSearch_enhanced_feature() { + await deleteAssetLibraryDevices( + ENHANCEDSEARCH_FEATURES_DEVICES.map( + ({ characters }) => `TEST-enhancedSearch-${characters}` + ) + ); + await deleteAssetLibraryGroups([ + `/${ENHANCEDSEARCH_FEATURES_GROUP_ALL}`, + ...ENHANCEDSEARCH_FEATURES_GROUP_SUFFIXES.map( + (suffix) => `/enhancedSearchGroup_${suffix}` + ), + ]); + await deleteAssetLibraryTemplates(CategoryEnum.device, [ + ENHANCEDSEARCH_FEATURES_DEVICE_TEMPLATE_ID, + ]); +} + +Before({ tags: '@setup_deviceSearch_enhanced_feature' }, async function () { + // teardown first just in case + await teardown_deviceSearch_enhanced_feature(); + + // device template + const deviceType: TypeResource = { + templateId: ENHANCEDSEARCH_FEATURES_DEVICE_TEMPLATE_ID, + category: 'device', + properties: { + words: { type: ['string'] }, + characters: { type: ['string'] }, + }, + relations: { + out: { + part_of_group: ['root'], + }, + }, + }; + await templatesService.createTemplate(deviceType, additionalHeaders); + await templatesService.publishTemplate( + CategoryEnum.device, + ENHANCEDSEARCH_FEATURES_DEVICE_TEMPLATE_ID, + additionalHeaders + ); + + // create group that all devices are part of + await groupsService.createGroup({ + templateId: 'root', + parentPath: '/', + name: ENHANCEDSEARCH_FEATURES_GROUP_ALL, + attributes: {}, + }); + + // create sub-groups that only a subset of devices is part of + await Promise.all( + ENHANCEDSEARCH_FEATURES_GROUP_SUFFIXES.map((suffix) => + groupsService.createGroup({ + templateId: 'root', + parentPath: '/', + name: `enhancedSearchGroup_${suffix}`, + attributes: {}, + }) + ) + ); + + // create devices + await Promise.all( + ENHANCEDSEARCH_FEATURES_DEVICES.map(({ characters, words, groups }) => + devicesService.createDevice({ + templateId: ENHANCEDSEARCH_FEATURES_DEVICE_TEMPLATE_ID, + deviceId: `TEST-enhancedSearch-${characters}`, + attributes: { + words: words, + characters: characters, + }, + groups: { + out: { + part_of_group: [ + `/${ENHANCEDSEARCH_FEATURES_GROUP_ALL}`, + ...groups.map((suffix) => `/enhancedSearchGroup_${suffix}`), + ], + }, + }, + }) + ) + ); + + // wait a short while for data to arrive in the OpenSearch index + return new Promise((resolve) => setTimeout(resolve, 10000)); +}); + +Before({ tags: '@teardown_deviceSearch_enhanced_feature' }, async function () { + await teardown_deviceSearch_enhanced_feature(); +}); diff --git a/source/packages/services/assetlibrary/README.md b/source/packages/services/assetlibrary/README.md index 4958146ea..aaa83bf16 100644 --- a/source/packages/services/assetlibrary/README.md +++ b/source/packages/services/assetlibrary/README.md @@ -197,3 +197,4 @@ In the example above, retrieving the list of policies for `device001`, `device00 - [Swagger](docs/swagger.yml) - [Templates user guide](docs/templates-user.md) - [Templates developer guide](docs/templates-developer.md) +- [Integration Testing Asset Library](../../../packages/integration-tests/features/assetlibrary/README.md) diff --git a/source/packages/services/assetlibrary/docs/configuration.md b/source/packages/services/assetlibrary/docs/configuration.md index 2ececfdc6..2fde9a459 100644 --- a/source/packages/services/assetlibrary/docs/configuration.md +++ b/source/packages/services/assetlibrary/docs/configuration.md @@ -52,9 +52,7 @@ CORS_EXPOSED_HEADERS=content-type,location # the base path from the request to allow the module to map the incoming request to the correct lambda handler CUSTOMDOMAIN_BASEPATH= -# The Asset Library mode. `full` (default) will enable the full feature set and -# use Neptune as its datastore, whereas `lite` will offer a reduced feature set -# (see documentation) and use the AWS IoT Device Registry as its datastore. +# The Asset Library mode: `full`, `enhanced`, or `lite`. See docs/modes.md for details. MODE=full # If true, fine-grained access control will be enabled. Refer to documentation diff --git a/source/packages/services/assetlibrary/docs/modes.md b/source/packages/services/assetlibrary/docs/modes.md index 8fc6bc4cb..7d41bc800 100644 --- a/source/packages/services/assetlibrary/docs/modes.md +++ b/source/packages/services/assetlibrary/docs/modes.md @@ -4,7 +4,7 @@ The Asset Library is capable of running in one of three modes: `full`, `enhanced`, and `lite`. -The `lite` version uses The AWS IoT Device Registry to store all devices and groups data, whereas the `full` version utilizes [AWS Neptune](https://aws.amazon.com/neptune/) to provide more advanced data modeling features. +The `lite` version uses the AWS IoT Device Registry to store all devices and groups data, whereas the `full` version utilizes [Amazon Neptune](https://aws.amazon.com/neptune/) to provide more advanced data modeling features. In `enhanced` mode, an OpenSearch cluster is deployed as secondary data store and provides enhanced search functionality. The mode is determined via a configuration property at the time of deployment. The following describes the differences in functionality between the two modes. diff --git a/source/packages/services/command-and-control/docs/configuration.md b/source/packages/services/command-and-control/docs/configuration.md index af2ea5a7a..84bb8ed03 100644 --- a/source/packages/services/command-and-control/docs/configuration.md +++ b/source/packages/services/command-and-control/docs/configuration.md @@ -22,9 +22,8 @@ CORS_EXPOSED_HEADERS=content-type,location # the base path from the request to allow the module to map the incoming request to the correct lambda handler CUSTOMDOMAIN_BASEPATH= -# The Asset Library mode. `full` (default) will enable the full feature set and -# use Neptune as its datastore, whereas `lite` will offer a reduced feature set -# (see documentation) and use the AWS IoT Device Registry as its datastore. +# The Asset Library mode: `full`, `enhanced`, or `lite`. See source/packages/services/assetlibrary/docs/modes.md +# for details. MODE=full #Application logging level. Set to (in order) error, warn, info, verbose, debug or silly. From 45fe683119981055eafabd5f6d19ad4bab8fd454 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Thu, 30 Jun 2022 09:48:53 -0700 Subject: [PATCH 26/31] changelog --- .../feature-enhanced-search_2022-06-30-16-47.json | 11 +++++++++++ .../feature-enhanced-search_2022-06-30-16-47.json | 11 +++++++++++ .../feature-enhanced-search_2022-06-30-16-47.json | 11 +++++++++++ .../feature-enhanced-search_2022-06-30-16-47.json | 11 +++++++++++ 4 files changed, 44 insertions(+) create mode 100644 source/common/changes/@cdf/assetlibrary-client/feature-enhanced-search_2022-06-30-16-47.json create mode 100644 source/common/changes/@cdf/assetlibrary/feature-enhanced-search_2022-06-30-16-47.json create mode 100644 source/common/changes/@cdf/installer/feature-enhanced-search_2022-06-30-16-47.json create mode 100644 source/common/changes/@cdf/integration-tests/feature-enhanced-search_2022-06-30-16-47.json diff --git a/source/common/changes/@cdf/assetlibrary-client/feature-enhanced-search_2022-06-30-16-47.json b/source/common/changes/@cdf/assetlibrary-client/feature-enhanced-search_2022-06-30-16-47.json new file mode 100644 index 000000000..6b0cd5da4 --- /dev/null +++ b/source/common/changes/@cdf/assetlibrary-client/feature-enhanced-search_2022-06-30-16-47.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@cdf/assetlibrary-client", + "comment": "New \"enhanced mode\" for Asset Library", + "type": "patch" + } + ], + "packageName": "@cdf/assetlibrary-client", + "email": "jonasneu@amazon.com" +} \ No newline at end of file diff --git a/source/common/changes/@cdf/assetlibrary/feature-enhanced-search_2022-06-30-16-47.json b/source/common/changes/@cdf/assetlibrary/feature-enhanced-search_2022-06-30-16-47.json new file mode 100644 index 000000000..e72483f9c --- /dev/null +++ b/source/common/changes/@cdf/assetlibrary/feature-enhanced-search_2022-06-30-16-47.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@cdf/assetlibrary", + "comment": "New \"enhanced mode\" for Asset Library", + "type": "minor" + } + ], + "packageName": "@cdf/assetlibrary", + "email": "jonasneu@amazon.com" +} \ No newline at end of file diff --git a/source/common/changes/@cdf/installer/feature-enhanced-search_2022-06-30-16-47.json b/source/common/changes/@cdf/installer/feature-enhanced-search_2022-06-30-16-47.json new file mode 100644 index 000000000..c8aca0620 --- /dev/null +++ b/source/common/changes/@cdf/installer/feature-enhanced-search_2022-06-30-16-47.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@cdf/installer", + "comment": "New \"enhanced mode\" for Asset Library", + "type": "patch" + } + ], + "packageName": "@cdf/installer", + "email": "jonasneu@amazon.com" +} \ No newline at end of file diff --git a/source/common/changes/@cdf/integration-tests/feature-enhanced-search_2022-06-30-16-47.json b/source/common/changes/@cdf/integration-tests/feature-enhanced-search_2022-06-30-16-47.json new file mode 100644 index 000000000..5013916e0 --- /dev/null +++ b/source/common/changes/@cdf/integration-tests/feature-enhanced-search_2022-06-30-16-47.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@cdf/integration-tests", + "comment": "New \"enhanced mode\" for Asset Library", + "type": "patch" + } + ], + "packageName": "@cdf/integration-tests", + "email": "jonasneu@amazon.com" +} \ No newline at end of file From 370856cd72a4baa4839a891ffe3b8056dfad86d2 Mon Sep 17 00:00:00 2001 From: Jonas Neubert Date: Thu, 30 Jun 2022 09:50:15 -0700 Subject: [PATCH 27/31] docs(assetlibrary): migrating an existing asset library deployment --- source/docs/migration.md | 4 + .../packages/services/assetlibrary/README.md | 1 + .../assetlibrary/docs/enhanced-search.md | 110 ++++++++++++++++++ .../infrastructure/cfn-enhancedsearch.yaml | 2 +- 4 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 source/packages/services/assetlibrary/docs/enhanced-search.md diff --git a/source/docs/migration.md b/source/docs/migration.md index 8934bdddb..e8aecb795 100644 --- a/source/docs/migration.md +++ b/source/docs/migration.md @@ -2,6 +2,10 @@ While we endeavor to always make backward compatible changes, there may be times when we need to make changes that are not backward compatible. If these changes are made at the API level then the affected modules REST API vendor mime types will be versioned supporting both new and old versions, as well as the modules minor version bumped. But if the change affect something else such as how configuration is handled, or how applications are deployed, then the major versions of the modules will be bumped with migration notes added here. +## Migrating an existing Asset Library deployment to Enhanced Search + +Starting with CDF Asset Library version 6.0.10 (part of CDF version 1.0.13), a new "enhanced" mode is available for CDF Asset Library. See the section [Migrating from full mode to enhanced mode](../packages/services/assetlibrary/docs/enhanced-search.md#migrating-from-full-mode-to-enhanced-mode) for guidance on migrating to enanced mode. + ## Migrating from Release <=1.0.10 to 1.0.11 ### Asset Library is now optional modules diff --git a/source/packages/services/assetlibrary/README.md b/source/packages/services/assetlibrary/README.md index aaa83bf16..4cec3e51a 100644 --- a/source/packages/services/assetlibrary/README.md +++ b/source/packages/services/assetlibrary/README.md @@ -193,6 +193,7 @@ In the example above, retrieving the list of policies for `device001`, `device00 - [Events](docs/events.md) - [Fine-gained access controll](docs/fine-grained-access-control.md) - [Asset Library modes: lite, full, enhanced](docs/modes.md) +- [Enhanced Search](docs/enhanced-search.md) - [Profiles](docs/profiles.md) - [Swagger](docs/swagger.yml) - [Templates user guide](docs/templates-user.md) diff --git a/source/packages/services/assetlibrary/docs/enhanced-search.md b/source/packages/services/assetlibrary/docs/enhanced-search.md new file mode 100644 index 000000000..546d2787b --- /dev/null +++ b/source/packages/services/assetlibrary/docs/enhanced-search.md @@ -0,0 +1,110 @@ +# ASSET LIBRARY ENHANCED SEARCH + +## Migrating from `full` mode to `enhanced` mode + +Creating a new CDF Asset Library deployment is readily achieved by running the [CDF Installer](../../installer/README.md) which creates a Neptune database, an OpenSearch cluster, and serverless components that synchronize changes from Neptune to OpenSearch. +The migration of an _existing_ CDF Asset Library from `full` mode to `enhanced` mode, requires additional steps to import the existing data first. + +There is no migration path from `lite` mode to `enhanced` mode. + +The following instructions use the [Export Neptune to ElasticSearch](https://github.com/awslabs/amazon-neptune-tools/tree/master/export-neptune-to-elasticsearch) solution in a way that should be sufficient for most CDF Asset Library deployments. +Users are encouraged to review the solution's documentation for additional customization options and considerations related to large databases. + +### Step 1: Update the CDF configuration file + +1. Ensure that the [CDF Installer](../../installer/README.md) configuration file for the CDF deployment you want to migrate is available at the location where the CDF Installer expects it. For example, on Linux and macOS the CDF Installer expects the file to be located at a path formatted like: `~/aws-connected-device-framework/config///.json`. Recommended best practice is to check the configuration file(s) into source control. +2. [Configure your terminal/shell](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) so that the AWS CLI has permissions to deploy CDF in the AWS account where the CDF deployment you wish to migrate is located. For example, you can set the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_DEFAULT_REGION` environment variables. +3. Run the CDF Installer's configuration wizard: `cdf-cli deploy --dryrun`. The `--dryrun` option ensures that the CDF configuration file gets modified but the deployment is _not_ performed yet. + +Confirm that the modified config file specifies `enhanced` mode and contains the configuration settings required to deploy enhanced search: + +```json +{ + "environment": "", + "region": "", + "accountId": "", + "assetLibrary": { + "mode": "enhanced", + "openSearchDataNodeInstanceType": "**.******.search", + "openSearchDataNodeInstanceCount": x, + "openSearchEBSVolumeSize": xx + } +} +``` + +Note: In the configuration file snippet above, only relevant fields are shown for illustration. The actual file is much longer. + +### Step 2: Pause Asset Library database changes + +No changes should be made to the Asset Library database until the initial data import is complete. +Take the appropriate steps to temporarily stop write traffic to the Asset Library database, for example by switching your CDF Facade to maintenance mode. + +### Step 3: Deploy the updated configuration file + +Deploy the updated CDF configuration with the command + +```sh +cdf-cli deploy -c ~/aws-connected-device-framework/config///.json +``` + +Confirm that this created a new CloudFormation stack named `cdf-assetlibrary-enhancedsearch-`. + +### Step 4: Deploy the "Export Neptune to ElasticSearch" solution + +1. Log into the AWS Console account that contains the CDF deployment you wish to migrate. +This ensures that clicking the link in the next step opens in the correct account. +2. Launch the Neptune-to-ElasticSearch CloudFormation stack. +You can do so by clicking the link for the AWS region that contains the CDF deployment you wish to migrate in the [installation instructions for the "Export Neptune to ElasticSearch" solution](https://github.com/awslabs/amazon-neptune-tools/blob/master/export-neptune-to-elasticsearch/readme.md#installation). The table below shows suggested values for the stack parameters. Keep the stack name as the default `neptune-index`. +3. Acknowledge the CloudFormation templates' use of IAM capabilities at the bottom of the form. + +Stack parameters for the Neptune-to-ElasticSearch CloudFormation stack: + +| Stack parameter | Value | +| --- | --- | +| _Network Configuration_ | +| VPC | The `VpcId` output of the stack `cdf-network-` | +| Subnet1 | The first comma-separated value in the `PrivateSubnetIds` of the stack `cdf-network-` | +| _Neptune Configuration_ | +| NeptuneEndpoint | The `DBClusterReadEndpoint` output of the stack `cdf-assetlibrary-neptune-` | +| NeptunePort | Keep default `8182` +| NeptuneEngine | Select `gremlin` | +| ExportScope | Select `all` | +| CloneCluster | Select `yes` | +| NeptuneClientSecurityGroup | The `CDFSecurityGroupId` output of the stack `cdf-network-` | +| AdditionalParams | Leave empty | +| _ElasticSearch Configuration_ | +| ElasticSearchEndpoint | The `OpenSearchDomainEndpoint` output of the stack `cdf-assetlibrary-enhancedsearch-` _without the `https://` prefix_. | +| NumberOfShards | Keep default. Only change if you modified the `NumberOfReplica` parameter of the stack `cdf-assetlibrary-enhancedsearch-`. | +| NumberOfReplica | Keep default. Only change if you modified the `NumberOfShards` parameter of the stack `cdf-assetlibrary-enhancedsearch-`. | +| GeoLocationFields | Leave empty | +| ElasticSearchClientSecurityGroup | The `HTTPSAccessSG` output of the stack `cdf-assetlibrary-enhancedsearch-` | +| _Advanced_ | +| Concurrency | Keep default | +| KinesisShardCount | Keep default | +| BatchSize | Keep default | + +Confirm that the CloudFormation stack `neptune-index` exists and has status `CREATE_COMPLETE`. + +Note that the `neptune-index` CloudFormation stack contains a nested stack named `neptune-index-EbsVolumeSizeStack-xxxxxxxxxxxx`. +You will not interact with this nested stack directly. + +### Step 5: Use the "Export Neptune to ElasticSearch" solution + +Invoke the AWS Lambda function created as part of the `neptune-index` CloudFormation stack. + +You can do either by navigating to the AWS Lambda console or the AWS CLI. + +Using the AWS CLI, copy the invoke command from the `StartExportCommand` stack output of the `neptune-index` stack and run it from the terminal/shell configured in [Step 1](#step-1-update-the-cdf-configuration-file). + +Alternatively, in the console, navigate to the function named `export-neptune-to-kinesis-xxxx` and invoke it. + +### Step 6: Resume Asset Library database changes + +At this point, all existing Asset Library data has been synchronized into the OpenSearch cluster and is available for search queries that include enhanced search operators (see [swagger.yml](./swagger.yml) for examples). +Incremental changes to the Amazon Neptune database content can resume. + +Reverse the steps you have taken in [Step 2](#step-2-pause-asset-library-database-changes). + +### Step 7: Clean Up + +Delete the "Export Neptune to ElasticSearch" solution by removing the CloudFormation stack created in [Step 4](#step-4-deploy-the-export-neptune-to-elasticsearch-solution). diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml index fb7de9561..a28791d3a 100644 --- a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml +++ b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml @@ -26,7 +26,7 @@ Parameters: Type: AWS::EC2::VPC::Id PrivateSubNetIds: Description: > - Comma delimited list of private subnetIds to deploy Neptune into. Number of subnets must match the number of + Comma delimited list of private subnetIds to deploy OpenSearch into. Number of subnets must match the number of availability zones deployed into, i.e. only pass one subnet if operating in a single AZ. Type: List NeptuneSecurityGroupId: From 4ee44485b3bb90780ea344c302d91ce5491b421e Mon Sep 17 00:00:00 2001 From: Aaron Pittenger Date: Wed, 28 Sep 2022 11:32:28 -0400 Subject: [PATCH 28/31] docs(assetlibrary): updated deployment parameters in enhanced AL migration documentation --- source/packages/services/assetlibrary/docs/enhanced-search.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/packages/services/assetlibrary/docs/enhanced-search.md b/source/packages/services/assetlibrary/docs/enhanced-search.md index 546d2787b..a00e9cbbf 100644 --- a/source/packages/services/assetlibrary/docs/enhanced-search.md +++ b/source/packages/services/assetlibrary/docs/enhanced-search.md @@ -70,7 +70,7 @@ Stack parameters for the Neptune-to-ElasticSearch CloudFormation stack: | NeptuneEngine | Select `gremlin` | | ExportScope | Select `all` | | CloneCluster | Select `yes` | -| NeptuneClientSecurityGroup | The `CDFSecurityGroupId` output of the stack `cdf-network-` | +| NeptuneClientSecurityGroup | The `NeptuneSecurityGroupID` output of the stack `cdf-assetlibrary-neptune-` | | AdditionalParams | Leave empty | | _ElasticSearch Configuration_ | | ElasticSearchEndpoint | The `OpenSearchDomainEndpoint` output of the stack `cdf-assetlibrary-enhancedsearch-` _without the `https://` prefix_. | From e21f443ee128c64a2130426c5cfe014aa0c5568a Mon Sep 17 00:00:00 2001 From: Aaron Pittenger Date: Wed, 28 Sep 2022 11:48:00 -0400 Subject: [PATCH 29/31] docs(assetlibrary): added a note about needing to reboot neptune during enhanced AL migration --- source/packages/services/assetlibrary/docs/enhanced-search.md | 1 + 1 file changed, 1 insertion(+) diff --git a/source/packages/services/assetlibrary/docs/enhanced-search.md b/source/packages/services/assetlibrary/docs/enhanced-search.md index a00e9cbbf..4add7097d 100644 --- a/source/packages/services/assetlibrary/docs/enhanced-search.md +++ b/source/packages/services/assetlibrary/docs/enhanced-search.md @@ -49,6 +49,7 @@ cdf-cli deploy -c ~/aws-connected-devic Confirm that this created a new CloudFormation stack named `cdf-assetlibrary-enhancedsearch-`. +> :warning: Note: This solution includes adding a Neptune DB Cluster parameter group to enable Neptune streams. In order for those parameter group changes to take effect, all instances of the Neptune DB cluster need to be rebooted. ### Step 4: Deploy the "Export Neptune to ElasticSearch" solution 1. Log into the AWS Console account that contains the CDF deployment you wish to migrate. From b46963eba40bf74d22559d6a191116771fb5d7f6 Mon Sep 17 00:00:00 2001 From: Aaron Pittenger Date: Wed, 28 Sep 2022 12:03:53 -0400 Subject: [PATCH 30/31] fix(assetlibrary): added extra security group for neptune stream poller access to open search --- .../infrastructure/cfn-enhancedsearch.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml index a28791d3a..f5c03dd69 100644 --- a/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml +++ b/source/packages/services/assetlibrary/infrastructure/cfn-enhancedsearch.yaml @@ -286,6 +286,16 @@ Resources: SourceSecurityGroupId: !Ref NeptuneStreamPollerSG Description: Access from Neptune Stream Poller to OpenSearch + OpenSearchSGIngressRule3: + Type: 'AWS::EC2::SecurityGroupIngress' + Properties: + GroupId: !Ref OpenSearchSG + FromPort: 443 + ToPort: 443 + IpProtocol: tcp + SourceSecurityGroupId: !GetAtt NeptuneStreamPoller.Outputs.HTTPSAccessSG + Description: Access for the Kinesis-to-opensearch lambda + NeptuneSGIngressRule: Type: 'AWS::EC2::SecurityGroupIngress' Properties: From 16aabfe44c32f227176f3930a8e7e6be97bc4f7c Mon Sep 17 00:00:00 2001 From: Aaron Pittenger Date: Sun, 17 Sep 2023 20:06:58 -0400 Subject: [PATCH 31/31] fix(assetlibrary): make search dao extendable using protected functions/variables --- .../src/search/search.enhanced.dao.ts | 161 ++++++++++-------- .../src/search/search.full.dao.ts | 10 +- 2 files changed, 92 insertions(+), 79 deletions(-) diff --git a/source/packages/services/assetlibrary/src/search/search.enhanced.dao.ts b/source/packages/services/assetlibrary/src/search/search.enhanced.dao.ts index 1d1d346be..e4776eaf7 100644 --- a/source/packages/services/assetlibrary/src/search/search.enhanced.dao.ts +++ b/source/packages/services/assetlibrary/src/search/search.enhanced.dao.ts @@ -10,110 +10,118 @@ * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions * * and limitations under the License. * *********************************************************************************************************************/ +import { logger } from '@awssolutions/simple-cdf-logger'; import { process, structure } from 'gremlin'; -import { injectable, inject } from 'inversify'; -import {logger} from '../utils/logger'; -import {TYPES} from '../di/types'; -import { SearchRequestModel } from './search.models'; -import {NodeAssembler} from '../data/assembler'; -import {NeptuneConnection} from '../data/base.full.dao'; -import { SearchDaoFull } from './search.full.dao'; +import { inject, injectable } from 'inversify'; +import { NodeAssembler } from '../data/assembler'; +import { NeptuneConnection } from '../data/base.full.dao'; +import { TYPES } from '../di/types'; import { TypeUtils } from '../utils/typeUtils'; +import { SearchDaoFull } from './search.full.dao'; +import { SearchRequestModel } from './search.models'; const __ = process.statics; @injectable() export class SearchDaoEnhanced extends SearchDaoFull { - public constructor( @inject('neptuneUrl') neptuneUrl: string, @inject('enableDfeOptimization') enableDfeOptimization: boolean, @inject('openSearchEndpoint') private openSearchEndpoint: boolean, @inject(TYPES.TypeUtils) typeUtils: TypeUtils, @inject(TYPES.NodeAssembler) assembler: NodeAssembler, - @inject(TYPES.GraphSourceFactory) graphSourceFactory: () => structure.Graph + @inject(TYPES.GraphSourceFactory) graphSourceFactory: () => structure.Graph ) { super(neptuneUrl, enableDfeOptimization, typeUtils, assembler, graphSourceFactory); } - protected buildSearchTraverser(conn: NeptuneConnection, request: SearchRequestModel, authorizedPaths:string[]) : process.GraphTraversal { - - logger.debug(`search.enhanced.dao buildSearchTraverser: in: request: ${JSON.stringify(request)}, authorizedPaths:${authorizedPaths}`); + protected buildSearchTraverser( + conn: NeptuneConnection, + request: SearchRequestModel, + authorizedPaths: string[] + ): process.GraphTraversal { + logger.debug( + `search.enhanced.dao buildSearchTraverser: in: request: ${JSON.stringify( + request + )}, authorizedPaths:${authorizedPaths}` + ); let source: process.GraphTraversalSource = conn.traversal; if (this.enableDfeOptimization) { source = source.withSideEffect('Neptune#useDFE', true); } - source = source.withSideEffect("Neptune#fts.endpoint", this.openSearchEndpoint); - source = source.withSideEffect("Neptune#fts.queryType", "query_string"); - + source = source.withSideEffect('Neptune#fts.endpoint', this.openSearchEndpoint); + source = source.withSideEffect('Neptune#fts.queryType', 'query_string'); + // if a path is provided, that becomes the starting point let traverser: process.GraphTraversal; - if (request.ancestorPath!==undefined) { + if (request.ancestorPath !== undefined) { const ancestorId = `group___${request.ancestorPath}`; - traverser = source.V(ancestorId). - repeat(__.in_().simplePath().dedup()).emit().as('a'); + traverser = source.V(ancestorId).repeat(__.in_().simplePath().dedup()).emit().as('a'); } else { traverser = source.V().as('a'); } // construct Gremlin traverser from request parameters - if (request.types!==undefined) { - request.types.forEach(t=> traverser.select('a').hasLabel(t)); + if (request.types !== undefined) { + request.types.forEach((t) => traverser.select('a').hasLabel(t)); } - if (request.eq!==undefined) { - request.eq.forEach(f=> { + if (request.eq !== undefined) { + request.eq.forEach((f) => { traverser.select('a'); this.buildSearchFilterVBase(f, traverser); traverser.has(f.field, f.value); }); } - if (request.neq!==undefined) { - request.neq.forEach(f=> { + if (request.neq !== undefined) { + request.neq.forEach((f) => { traverser.select('a'); this.buildSearchFilterVBase(f, traverser); traverser.not(__.has(f.field, f.value)); }); } - if (request.lt!==undefined) { - request.lt.forEach(f=> { + if (request.lt !== undefined) { + request.lt.forEach((f) => { traverser.select('a'); this.buildSearchFilterVBase(f, traverser); traverser.has(f.field, process.P.lt(Number(f.value))); }); } - if (request.lte!==undefined) { - request.lte.forEach(f=> { + if (request.lte !== undefined) { + request.lte.forEach((f) => { traverser.select('a'); this.buildSearchFilterVBase(f, traverser); traverser.has(f.field, process.P.lte(Number(f.value))); }); } - if (request.gt!==undefined) { - request.gt.forEach(f=> { + if (request.gt !== undefined) { + request.gt.forEach((f) => { traverser.select('a'); this.buildSearchFilterVBase(f, traverser); traverser.has(f.field, process.P.gt(Number(f.value))); }); } - if (request.gte!==undefined) { - request.gte.forEach(f=> { + if (request.gte !== undefined) { + request.gte.forEach((f) => { traverser.select('a'); this.buildSearchFilterVBase(f, traverser); traverser.has(f.field, process.P.gte(Number(f.value))); }); } - if (request.startsWith!==undefined) { - request.startsWith.forEach(f=> { + if (request.startsWith !== undefined) { + request.startsWith.forEach((f) => { const luceneQueryKey = this.buildLuceneQueryKey(f.field, true); let luceneQueryVal = f.value.toString(); - [':', '/', ' '].forEach( char => { + [':', '/', ' '].forEach((char) => { luceneQueryVal = luceneQueryVal.replace(new RegExp(char, 'g'), `\\${char}`); }); - ['[', ']'].forEach( char => { - luceneQueryVal = luceneQueryVal.replace(new RegExp(`\\${char}`, 'g'), `\\${char}`); + ['[', ']'].forEach((char) => { + luceneQueryVal = luceneQueryVal.replace( + new RegExp(`\\${char}`, 'g'), + `\\${char}` + ); }); traverser.select('a'); this.buildSearchFilterVBase(f, traverser); @@ -121,15 +129,18 @@ export class SearchDaoEnhanced extends SearchDaoFull { }); } - if (request.endsWith!==undefined) { - request.endsWith.forEach(f=> { + if (request.endsWith !== undefined) { + request.endsWith.forEach((f) => { const luceneQueryKey = this.buildLuceneQueryKey(f.field, true); let luceneQueryVal = f.value.toString(); - [':', '/', ' '].forEach( char => { + [':', '/', ' '].forEach((char) => { luceneQueryVal = luceneQueryVal.replace(new RegExp(char, 'g'), `\\${char}`); }); - ['[', ']'].forEach( char => { - luceneQueryVal = luceneQueryVal.replace(new RegExp(`\\${char}`, 'g'), `\\${char}`); + ['[', ']'].forEach((char) => { + luceneQueryVal = luceneQueryVal.replace( + new RegExp(`\\${char}`, 'g'), + `\\${char}` + ); }); traverser.select('a'); this.buildSearchFilterVBase(f, traverser); @@ -137,15 +148,18 @@ export class SearchDaoEnhanced extends SearchDaoFull { }); } - if (request.contains!==undefined) { - request.contains.forEach(f=> { + if (request.contains !== undefined) { + request.contains.forEach((f) => { const luceneQueryKey = this.buildLuceneQueryKey(f.field, true); let luceneQueryVal = f.value.toString(); - [':', '/', ' '].forEach( char => { + [':', '/', ' '].forEach((char) => { luceneQueryVal = luceneQueryVal.replace(new RegExp(char, 'g'), `\\${char}`); }); - ['[', ']'].forEach( char => { - luceneQueryVal = luceneQueryVal.replace(new RegExp(`\\${char}`, 'g'), `\\${char}`); + ['[', ']'].forEach((char) => { + luceneQueryVal = luceneQueryVal.replace( + new RegExp(`\\${char}`, 'g'), + `\\${char}` + ); }); traverser.select('a'); this.buildSearchFilterVBase(f, traverser); @@ -153,28 +167,28 @@ export class SearchDaoEnhanced extends SearchDaoFull { }); } - if (request.exists!==undefined) { - request.exists.forEach(f=> { + if (request.exists !== undefined) { + request.exists.forEach((f) => { traverser.select('a'); this.buildSearchFilterEBase(f, traverser); traverser.has(f.field, f.value); }); } - if (request.nexists!==undefined) { - request.nexists.forEach(f=> { + if (request.nexists !== undefined) { + request.nexists.forEach((f) => { traverser.select('a'); this.buildSearchFilterEBaseNegated(f, traverser, f.field, f.value); }); } - if (request.fulltext!==undefined) { - request.fulltext.forEach(f=> { + if (request.fulltext !== undefined) { + request.fulltext.forEach((f) => { // Remove any characters that would be recognized by Lucene as control characters. - // Alternatively, could escape them but the standard query string analyzer will + // Alternatively, could escape them but the standard query string analyzer will // replace them with spaces anyway. let luceneQueryVal = f.value.toString(); - [':', '/', '\\[', '\\]'].forEach( char => { + [':', '/', '\\[', '\\]'].forEach((char) => { luceneQueryVal = luceneQueryVal.replace(new RegExp(char, 'g'), ' '); }); // RegExp('\\\\') = regex for single backslash, '\\\\' = string with two backslashes @@ -185,8 +199,8 @@ export class SearchDaoEnhanced extends SearchDaoFull { }); } - if (request.regex!==undefined) { - request.regex.forEach(f=> { + if (request.regex !== undefined) { + request.regex.forEach((f) => { const luceneQueryKey = this.buildLuceneQueryKey(f.field, true); // Escape characters that can appear, escaped or unescaped, in regex but are also // control characters for Lucene. For example, "abc/def" is a valid regex but the contained @@ -194,7 +208,7 @@ export class SearchDaoEnhanced extends SearchDaoFull { // be escaped even though they denote range queries in Lucene because Lucene ignores // them inside of regexes. let luceneQueryVal = f.value.toString(); - [':', '/', ' '].forEach( char => { + [':', '/', ' '].forEach((char) => { luceneQueryVal = luceneQueryVal.replace(new RegExp(char, 'g'), `\\${char}`); }); traverser.select('a'); @@ -203,11 +217,11 @@ export class SearchDaoEnhanced extends SearchDaoFull { }); } - if (request.lucene!==undefined) { - request.lucene.forEach(f=> { + if (request.lucene !== undefined) { + request.lucene.forEach((f) => { traverser.select('a'); this.buildSearchFilterVBase(f, traverser); - // no escaping for Opensearch, user can send control characters and is responsible for + // no escaping for Opensearch, user can send control characters and is responsible for // escaping them in the search request when necessary traverser.has(f.field, `Neptune#fts ${f.value}`); }); @@ -217,29 +231,28 @@ export class SearchDaoEnhanced extends SearchDaoFull { traverser.select('a').dedup().fold().unfold().as('matched'); // if authz is enabled, only return results that the user is authorized to view - if (authorizedPaths!==undefined && authorizedPaths.length>0) { - - const authorizedPathIds = authorizedPaths.map(path=>`group___${path}`); - traverser. - local( - __.until( - __.hasId(process.P.within(authorizedPathIds)) - ).repeat( + if (authorizedPaths !== undefined && authorizedPaths.length > 0) { + const authorizedPathIds = authorizedPaths.map((path) => `group___${path}`); + traverser + .local( + __.until(__.hasId(process.P.within(authorizedPathIds))).repeat( __.out().simplePath().dedup() ) - ).as('authorization'); + ) + .as('authorization'); } - logger.debug(`search.enhanced.dao buildSearchTraverser: traverser: ${traverser.toString()}`); + logger.debug( + `search.enhanced.dao buildSearchTraverser: traverser: ${traverser.toString()}` + ); return traverser.select('matched').dedup(); - } - private buildLuceneQueryKey(field: string, keyword?: boolean) : string { + private buildLuceneQueryKey(field: string, keyword?: boolean): string { if (field === 'id') return 'entity_id'; if (field === 'label') return 'entity_type'; - + let components: string[] = []; components = ['predicates', field, 'value']; if (keyword) components.push('keyword'); diff --git a/source/packages/services/assetlibrary/src/search/search.full.dao.ts b/source/packages/services/assetlibrary/src/search/search.full.dao.ts index 1939266c9..4a3ac90e9 100644 --- a/source/packages/services/assetlibrary/src/search/search.full.dao.ts +++ b/source/packages/services/assetlibrary/src/search/search.full.dao.ts @@ -32,7 +32,7 @@ const __ = process.statics; export class SearchDaoFull extends BaseDaoFull { public constructor( @inject('neptuneUrl') neptuneUrl: string, - @inject('enableDfeOptimization') private enableDfeOptimization: boolean, + @inject('enableDfeOptimization') protected enableDfeOptimization: boolean, @inject(TYPES.TypeUtils) private typeUtils: TypeUtils, @inject(TYPES.NodeAssembler) private assembler: NodeAssembler, @inject(TYPES.GraphSourceFactory) graphSourceFactory: () => structure.Graph @@ -40,7 +40,7 @@ export class SearchDaoFull extends BaseDaoFull { super(neptuneUrl, graphSourceFactory); } - private buildSearchTraverser( + protected buildSearchTraverser( conn: NeptuneConnection, request: SearchRequestModel, authorizedPaths: string[] @@ -184,7 +184,7 @@ export class SearchDaoFull extends BaseDaoFull { return traverser.select('a').dedup(); } - private buildSearchFilterVBase( + protected buildSearchFilterVBase( filter: SearchRequestFilter | SearchRequestFacet, traverser: process.GraphTraversal ): void { @@ -199,7 +199,7 @@ export class SearchDaoFull extends BaseDaoFull { } } - private buildSearchFilterEBase( + protected buildSearchFilterEBase( filter: SearchRequestFilter | SearchRequestFacet, traverser: process.GraphTraversal ): void { @@ -215,7 +215,7 @@ export class SearchDaoFull extends BaseDaoFull { } } - private buildSearchFilterEBaseNegated( + protected buildSearchFilterEBaseNegated( filter: SearchRequestFilter | SearchRequestFacet, traverser: process.GraphTraversal, field: unknown,