From 3452301d2fcaa356c0f08db68f036580a512d63b Mon Sep 17 00:00:00 2001 From: Stewart Wallace Date: Mon, 5 Sep 2022 15:04:31 +0100 Subject: [PATCH 1/8] Initial commit for adding in an event bus --- .../configure_account_alias.py | 5 +++ .../configure_account_tags.py | 8 +++- .../account_processing/create_account.py | 9 +++- .../account_processing/delete_default_vpc.py | 8 ++++ .../register_account_for_support.py | 5 +++ .../adf-build/shared/python/events.py | 41 +++++++++++++++++++ src/template.yml | 9 ++++ 7 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py diff --git a/src/lambda_codebase/account_processing/configure_account_alias.py b/src/lambda_codebase/account_processing/configure_account_alias.py index 243497166..1d466f0d2 100644 --- a/src/lambda_codebase/account_processing/configure_account_alias.py +++ b/src/lambda_codebase/account_processing/configure_account_alias.py @@ -6,15 +6,19 @@ """ import os +import json +import boto3 from sts import STS from aws_xray_sdk.core import patch_all from logger import configure_logger +from events import ADFEvents patch_all() LOGGER = configure_logger(__name__) ADF_ROLE_NAME = os.getenv("ADF_ROLE_NAME") AWS_PARTITION = os.getenv("AWS_PARTITION") +EVENTS = ADFEvents(boto3.client("events"), "AccountManagement.Alias") def delete_account_aliases(account, iam_client, current_aliases): @@ -76,6 +80,7 @@ def lambda_handler(event, _): "adf_account_alias_config", ) ensure_account_has_alias(event, role.client("iam")) + EVENTS.put_event(detail=json.dumps(event), detailType="ACCOUNT_ALIAS_CONFIGURED", resources=[account_id]) else: LOGGER.info( "Account: %s does not need an alias", diff --git a/src/lambda_codebase/account_processing/configure_account_tags.py b/src/lambda_codebase/account_processing/configure_account_tags.py index b0c3bcec4..aeda75683 100644 --- a/src/lambda_codebase/account_processing/configure_account_tags.py +++ b/src/lambda_codebase/account_processing/configure_account_tags.py @@ -8,13 +8,18 @@ in the config file. """ -from organizations import Organizations + +import json import boto3 + +from organizations import Organizations from aws_xray_sdk.core import patch_all from logger import configure_logger +from events import ADFEvents patch_all() +EVENTS = ADFEvents(boto3.client("events"), "AccountManagement.Tags") LOGGER = configure_logger(__name__) @@ -35,6 +40,7 @@ def lambda_handler(event, _): event.get("tags"), organizations, ) + EVENTS.put_event(detail=json.dumps(event), detailType="ACCOUNT_TAGS_CONFIGURED", resources=[event.get('account_id')]) else: LOGGER.info( "Account: %s does not need tags configured", diff --git a/src/lambda_codebase/account_processing/create_account.py b/src/lambda_codebase/account_processing/create_account.py index 5d7963abd..6c2b80e6e 100644 --- a/src/lambda_codebase/account_processing/create_account.py +++ b/src/lambda_codebase/account_processing/create_account.py @@ -6,14 +6,20 @@ """ import os +import json from aws_xray_sdk.core import patch_all import boto3 + from logger import configure_logger +from events import ADFEvents + patch_all() LOGGER = configure_logger(__name__) ADF_ROLE_NAME = os.getenv("ADF_ROLE_NAME") +EVENTS = ADFEvents(boto3.client("events"), "AccountManagement.AccountProvisioning") + def create_account(account, adf_role_name, org_client): @@ -42,4 +48,5 @@ def create_account(account, adf_role_name, org_client): def lambda_handler(event, _): org_client = boto3.client("organizations") - return create_account(event, ADF_ROLE_NAME, org_client) + details = create_account(event, ADF_ROLE_NAME, org_client) + EVENTS.put_event(detail=json.dumps(details), detailType="ACCOUNT_PROVISIONED", resources=[details.get("account_id")]) diff --git a/src/lambda_codebase/account_processing/delete_default_vpc.py b/src/lambda_codebase/account_processing/delete_default_vpc.py index 1e223c3ef..ae5f32c2c 100644 --- a/src/lambda_codebase/account_processing/delete_default_vpc.py +++ b/src/lambda_codebase/account_processing/delete_default_vpc.py @@ -5,15 +5,20 @@ Deletes the default VPC in a particular region """ import os +import json +import boto3 from sts import STS from aws_xray_sdk.core import patch_all from logger import configure_logger +from events import ADFEvents patch_all() LOGGER = configure_logger(__name__) ADF_ROLE_NAME = os.getenv("ADF_ROLE_NAME") AWS_PARTITION = os.getenv("AWS_PARTITION") +EVENTS = ADFEvents(boto3.client("events"), "AccountManagement.VPC") + def assume_role(account_id): @@ -62,6 +67,7 @@ def delete_default_vpc(ec2_resource, ec2_client, default_vpc_id): + def lambda_handler(event, _): event = event.get("Payload") LOGGER.info("Checking for default VPC: %s", event.get('account_full_name')) @@ -78,5 +84,7 @@ def lambda_handler(event, _): ) ec2_resource = role.resource("ec2", region_name=event.get("region")) delete_default_vpc(ec2_resource, ec2_client, default_vpc_id) + EVENTS.put_event(detail=json.dumps(event), detailType="DEFAULT_VPC_DELETED", resources=[event.get("account_id"), default_vpc_id]) + return {"Payload": event} diff --git a/src/lambda_codebase/account_processing/register_account_for_support.py b/src/lambda_codebase/account_processing/register_account_for_support.py index 34f50c399..e967a59bf 100644 --- a/src/lambda_codebase/account_processing/register_account_for_support.py +++ b/src/lambda_codebase/account_processing/register_account_for_support.py @@ -8,14 +8,17 @@ """ from enum import Enum +import json import boto3 from botocore.exceptions import ClientError, BotoCoreError from botocore.config import Config from logger import configure_logger +from events import ADFEvents from aws_xray_sdk.core import patch_all LOGGER = configure_logger(__name__) +EVENTS = ADFEvents(boto3.client("events"), "AccountManagement.Support") patch_all() @@ -191,6 +194,8 @@ def _enable_support_for_account( account_id, account.get("email"), ) + EVENTS.put_event(detail=json.dumps(account), detailType="ENTERPRISE_SUPPORT_REQUESTED", resources=[account.get("account_id")]) + except (ClientError, BotoCoreError): LOGGER.error( diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py new file mode 100644 index 000000000..11969ecbc --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py @@ -0,0 +1,41 @@ +""" +Standardised class for pushing events from within the ADF Namespace +""" + + +import os +import boto3 + + +class ADFEvents: + def __init__( + self, client: boto3.client, service, namespace="ADF", eventbus_arn=None + ) -> None: + """ + Client: Any Boto3 EventBridge client + Service: The name of the Service e.g AccountManagement.EnableSupport + namespace: Defaults to ADF + eventbus_arn: Optionally specify a custom EventBridge ARN. If no ARN is specified, and no ENV variable set, will default to ADF-Event-Bus + + """ + self.events = client + self.source = f"{namespace}.{service}" + self.eventbus_arn = ( + os.environ.get("ADF_EVENTBUS_ARN", "ADF-Event-Bus") + if eventbus_arn is None + else eventbus_arn + ) + + # This dict isn't mutated. So it's safe to default to this + def put_event(self, detailType, detail, resources=[]): # pylint: disable=W0102 + payload = { + "Source": self.source, + "Resources": resources, + "DetailType": detailType, + "Detail": detail, + "EventBusName": self.eventbus_arn, + } + trace_id = os.getenv("_X_AMZN_TRACE_ID") + if trace_id: + payload["TraceHeader"] = trace_id + self.events.put_events(Entries=[payload]) diff --git a/src/template.yml b/src/template.yml index 016a5ddd8..cfe5ec7a0 100644 --- a/src/template.yml +++ b/src/template.yml @@ -233,6 +233,9 @@ Resources: - "xray:PutTelemetryRecords" - "xray:PutTraceSegments" Resource: "*" + - Effect: "Allow" + Action: "events:PutEvents" + Resource: !GetAtt ADFEventBus.Arn Roles: - !Ref AccountProcessingLambdaRole - !Ref GetAccountRegionsFunctionRole @@ -363,6 +366,7 @@ Resources: ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] ADF_LOG_LEVEL: !Ref LogLevel ADF_ROLE_NAME: !Ref CrossAccountAccessRoleName + ADF_EVENTBUS_ARN: !GetAtt ADFEventBus.Arn FunctionName: AccountAliasConfigurationFunction Role: !GetAtt AccountAliasConfigFunctionRole.Arn @@ -1881,6 +1885,11 @@ Resources: RoleArn: !GetAtt PipelineCloudWatchEventRole.Arn Id: adf-codepipeline-trigger-bootstrap + ADFEventBus: + Type: AWS::Events::EventBus + Properties: + Name: ADF-Event-Bus + Outputs: ADFVersionNumber: Value: !FindInMap ["Metadata", "ADF", "Version"] From e0110082945b19dce58dd3ef6a25871d0458bd29 Mon Sep 17 00:00:00 2001 From: Stewart Wallace Date: Mon, 5 Sep 2022 18:23:40 +0100 Subject: [PATCH 2/8] Adding in creation complete event to state machine Ignoring check I3042 for cfn-lint Adding in EventBus to deployment account Adding in x-ray tracing to pipeline management lambda functions Changing event content, adding in xray layer to pipeline management lambda functions Documentation Mega Lint Fixes Forgot to hit save :( --- README.md | 1 + docs/integrations-guide.md | 38 ++++++++++++++++ .../configure_account_alias.py | 4 +- .../configure_account_tags.py | 4 +- .../account_processing/create_account.py | 2 +- .../account_processing/delete_default_vpc.py | 4 +- .../adf-bootstrap/deployment/global.yml | 8 ++++ .../create_or_update_rule.py | 6 +++ .../pipeline_management/create_repository.py | 14 ++++++ .../generate_pipeline_inputs.py | 2 + .../identify_out_of_date_pipelines.py | 2 + .../{ => lambda_layer}/requirements.txt | 0 .../process_deployment_map.py | 2 + .../store_pipeline_definition.py | 2 + .../deployment/pipeline_management.yml | 25 +++++++++++ .../adf-build/shared/python/events.py | 2 +- .../adf-build/shared/requirements.txt | 1 + src/template.yml | 43 ++++++++++++++++++- 18 files changed, 150 insertions(+), 10 deletions(-) create mode 100644 docs/integrations-guide.md rename src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/{ => lambda_layer}/requirements.txt (100%) diff --git a/README.md b/README.md index 64fcba76f..5ed7fe45e 100644 --- a/README.md +++ b/README.md @@ -36,3 +36,4 @@ within the AWS Console. - Refer to the [User Guide](docs/user-guide.md) for using ADF once it is setup. - Refer to the [Samples Guide](docs/samples-guide.md) for a detailed walk through of the provided samples. +- Refer to the [Integrations Guide](docs/integrations-guide.md) for information on events produced by the ADF. diff --git a/docs/integrations-guide.md b/docs/integrations-guide.md new file mode 100644 index 000000000..549c75cdd --- /dev/null +++ b/docs/integrations-guide.md @@ -0,0 +1,38 @@ +# Integrations Guide +## Introduction +The AWS Deployment Framework enables integrations with external workflows via an Event Bus deployed into the organisational root account. + +## Account Management Events +The account management events are emitted at various stages during an execution of the Account Management State Machine. +Currently - events are emitted for the following states: +- ACCOUNT_PROVISIONED + Emitted when an AWS account is created. + Contains the account definition from the .yml file as well as the account_id. +- ENTERPRISE_SUPPORT_REQUESTED + Emitted when the support ticket to AWS Support is raised. + Contains the account definition from the .yml file as well as the account_id. +- ACCOUNT_ALIAS_CONFIGURED + Emitted when the accounts alias is configured by ADF. + The details section contains the account id and the alias value. The resource field also contains the account id +- ACCOUNT_TAGS_CONFIGURED + Emitted when the accounts tags are updated by ADF. + The details section contains the account id and the tags. The resource field also contains the account id +- DEFAULT_VPC_DELETED + Emitted when the default VPC in a region is deleted. + The details section contains the account id and the region of the VPC. The resource field contains the deleted VPC id. +- ACCOUNT_CREATION_COMPLETE + Emitted when the state machine completes successfully. + Contains the account definition from the .yml file as well as the account_id in the resource field. + + + + +## Pipeline Management Events +- CROSS_ACCOUNT_RULE_CREATED_OR_UPDATED + Emitted when a rule is created to trigger pipelines from a different account. + The details sections contains the source_account_id (The account where the CodeCommit repository is located) and the resource sections contains the deployment account Id (The account where the CodePipeline is located) +- REPOSITORY_CREATED_OR_UPDATED + Emitted when a codecommit repository is created in a different account than the deployment account. + The details sections contains the repository_account_id (The account where the CodeCommit repository is located) as well as the stack_name (The CloudFormation stack that creates the repository) and the resource sections contains the repository account Id and the pipeline name + + diff --git a/src/lambda_codebase/account_processing/configure_account_alias.py b/src/lambda_codebase/account_processing/configure_account_alias.py index 1d466f0d2..1b0ab124e 100644 --- a/src/lambda_codebase/account_processing/configure_account_alias.py +++ b/src/lambda_codebase/account_processing/configure_account_alias.py @@ -18,7 +18,7 @@ LOGGER = configure_logger(__name__) ADF_ROLE_NAME = os.getenv("ADF_ROLE_NAME") AWS_PARTITION = os.getenv("AWS_PARTITION") -EVENTS = ADFEvents(boto3.client("events"), "AccountManagement.Alias") +EVENTS = ADFEvents(boto3.client("events"), "AccountManagement") def delete_account_aliases(account, iam_client, current_aliases): @@ -80,7 +80,7 @@ def lambda_handler(event, _): "adf_account_alias_config", ) ensure_account_has_alias(event, role.client("iam")) - EVENTS.put_event(detail=json.dumps(event), detailType="ACCOUNT_ALIAS_CONFIGURED", resources=[account_id]) + EVENTS.put_event(detail=json.dumps({"account_id": account_id, "alias_value": event.get("alias")}), detailType="ACCOUNT_ALIAS_CONFIGURED", resources=[account_id]) else: LOGGER.info( "Account: %s does not need an alias", diff --git a/src/lambda_codebase/account_processing/configure_account_tags.py b/src/lambda_codebase/account_processing/configure_account_tags.py index aeda75683..d3d744bdc 100644 --- a/src/lambda_codebase/account_processing/configure_account_tags.py +++ b/src/lambda_codebase/account_processing/configure_account_tags.py @@ -19,7 +19,7 @@ from events import ADFEvents patch_all() -EVENTS = ADFEvents(boto3.client("events"), "AccountManagement.Tags") +EVENTS = ADFEvents(boto3.client("events"), "AccountManagement") LOGGER = configure_logger(__name__) @@ -40,7 +40,7 @@ def lambda_handler(event, _): event.get("tags"), organizations, ) - EVENTS.put_event(detail=json.dumps(event), detailType="ACCOUNT_TAGS_CONFIGURED", resources=[event.get('account_id')]) + EVENTS.put_event(detail=json.dumps({"tags": event.get("tags"), "account_id": event.get("account_id")}), detailType="ACCOUNT_TAGS_CONFIGURED", resources=[event.get('account_id')]) else: LOGGER.info( "Account: %s does not need tags configured", diff --git a/src/lambda_codebase/account_processing/create_account.py b/src/lambda_codebase/account_processing/create_account.py index 6c2b80e6e..353f47188 100644 --- a/src/lambda_codebase/account_processing/create_account.py +++ b/src/lambda_codebase/account_processing/create_account.py @@ -18,7 +18,7 @@ LOGGER = configure_logger(__name__) ADF_ROLE_NAME = os.getenv("ADF_ROLE_NAME") -EVENTS = ADFEvents(boto3.client("events"), "AccountManagement.AccountProvisioning") +EVENTS = ADFEvents(boto3.client("events"), "AccountManagement") diff --git a/src/lambda_codebase/account_processing/delete_default_vpc.py b/src/lambda_codebase/account_processing/delete_default_vpc.py index ae5f32c2c..761ce8c04 100644 --- a/src/lambda_codebase/account_processing/delete_default_vpc.py +++ b/src/lambda_codebase/account_processing/delete_default_vpc.py @@ -17,7 +17,7 @@ LOGGER = configure_logger(__name__) ADF_ROLE_NAME = os.getenv("ADF_ROLE_NAME") AWS_PARTITION = os.getenv("AWS_PARTITION") -EVENTS = ADFEvents(boto3.client("events"), "AccountManagement.VPC") +EVENTS = ADFEvents(boto3.client("events"), "AccountManagement") @@ -84,7 +84,7 @@ def lambda_handler(event, _): ) ec2_resource = role.resource("ec2", region_name=event.get("region")) delete_default_vpc(ec2_resource, ec2_client, default_vpc_id) - EVENTS.put_event(detail=json.dumps(event), detailType="DEFAULT_VPC_DELETED", resources=[event.get("account_id"), default_vpc_id]) + EVENTS.put_event(detail=json.dumps({"region": event.get("region"), "account_id":event.get("account_id")}), detailType="DEFAULT_VPC_DELETED", resources=[default_vpc_id]) return {"Payload": event} diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml index 0d6de4340..fb366a5a2 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml @@ -69,6 +69,13 @@ Globals: CodeUri: lambda_codebase Runtime: python3.9 +Mappings: + OrganisationPartitionRegionMapping: + aws: + region: "us-east-1" + aws-us-gov: + region: "us-gov-west-1" + Resources: LambdaLayerVersion: Type: "AWS::Serverless::LayerVersion" @@ -183,6 +190,7 @@ Resources: CrossAccountAccessRole: !Ref CrossAccountAccessRole PipelineBucket: !Ref PipelineBucket RootAccountId: !Ref MasterAccountId + RootAccountRegion: !FindInMap [OrganisationPartitionRegionMapping, !Ref "AWS::Partition", "region"] CodeBuildImage: !Ref Image CodeBuildComputeType: !Ref ComputeType SharedModulesBucket: !Ref SharedModulesBucket diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py index b8f6ef550..180bb6fed 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py @@ -5,20 +5,25 @@ """ import os +import json import boto3 from cache import Cache from rule import Rule from logger import configure_logger from cloudwatch import ADFMetrics +from events import ADFEvents +from aws_xray_sdk.core import patch_all +patch_all() LOGGER = configure_logger(__name__) DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"] DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"] PIPELINE_MANAGEMENT_STATEMACHINE = os.getenv("PIPELINE_MANAGEMENT_STATEMACHINE_ARN") CLOUDWATCH = boto3.client("cloudwatch") METRICS = ADFMetrics(CLOUDWATCH, "PIPELINE_MANAGEMENT/RULE") +EVENTS = ADFEvents(boto3.client("events", region_name=os.getenv("ADF_EVENTBUS_REGION")), "PipelineManagement") _cache = None @@ -56,5 +61,6 @@ def lambda_handler(pipeline, _): METRICS.put_metric_data( {"MetricName": "CreateOrUpdate", "Value": 1, "Unit": "Count"} ) + EVENTS.put_event(detail=json.dumps({"source_account_id": _source_account_id}), detailType="CROSS_ACCOUNT_RULE_CREATED_OR_UPDATED", resources=[DEPLOYMENT_ACCOUNT_ID]) return pipeline diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py index fcfcfa376..aea638893 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py @@ -4,12 +4,14 @@ """ import os +import json import boto3 from repo import Repo from logger import configure_logger from cloudwatch import ADFMetrics from parameter_store import ParameterStore +from events import ADFEvents CLOUDWATCH = boto3.client("cloudwatch") @@ -17,6 +19,8 @@ LOGGER = configure_logger(__name__) DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"] DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"] +EVENTS = ADFEvents(boto3.client("events", region_name=os.getenv("ADF_EVENTBUS_REGION")), "PipelineManagement") + def lambda_handler(pipeline, _): @@ -52,5 +56,15 @@ def lambda_handler(pipeline, _): METRICS.put_metric_data( {"MetricName": "CreateOrUpdate", "Value": 1, "Unit": "Count"} ) + EVENTS.put_event( + detail=json.dumps({ + "repository_account_id": code_account_id, + "stack_name": repo.stack_name + }), + detailType="REPOSITORY_CREATED_OR_UPDATED", + resources=[ + f'{code_account_id}:{pipeline.get("name")}' + ] + ) return pipeline diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/generate_pipeline_inputs.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/generate_pipeline_inputs.py index 2a9d34090..ff00aaf31 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/generate_pipeline_inputs.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/generate_pipeline_inputs.py @@ -13,8 +13,10 @@ from sts import STS from logger import configure_logger from partition import get_partition +from aws_xray_sdk.core import patch_all +patch_all() LOGGER = configure_logger(__name__) DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"] DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"] diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/identify_out_of_date_pipelines.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/identify_out_of_date_pipelines.py index 29400b6d5..a86123484 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/identify_out_of_date_pipelines.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/identify_out_of_date_pipelines.py @@ -14,8 +14,10 @@ from logger import configure_logger from deployment_map import DeploymentMap from parameter_store import ParameterStore +from aws_xray_sdk.core import patch_all +patch_all() LOGGER = configure_logger(__name__) S3_BUCKET_NAME = os.environ["S3_BUCKET_NAME"] DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"] diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/lambda_layer/requirements.txt similarity index 100% rename from src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/requirements.txt rename to src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/lambda_layer/requirements.txt diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py index b13e9356b..c7a8db9d3 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py @@ -14,8 +14,10 @@ import boto3 from botocore.exceptions import ClientError from logger import configure_logger +from aws_xray_sdk.core import patch_all +patch_all() LOGGER = configure_logger(__name__) DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"] DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"] diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/store_pipeline_definition.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/store_pipeline_definition.py index bc4c5c347..c32f2e82d 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/store_pipeline_definition.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/store_pipeline_definition.py @@ -9,8 +9,10 @@ import boto3 from logger import configure_logger +from aws_xray_sdk.core import patch_all +patch_all() LOGGER = configure_logger(__name__) DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"] DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"] diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml index 3e0983c63..8859bfa9d 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml @@ -27,6 +27,10 @@ Parameters: Type: String MinLength: "1" + RootAccountRegion: + Type: String + MinLength: "1" + CodeBuildImage: Type: String MinLength: "1" @@ -61,8 +65,19 @@ Globals: Tracing: Active Layers: - !Ref LambdaLayer + - !Ref PipelineManagementLayerVersion Resources: + PipelineManagementLayerVersion: + Type: "AWS::Serverless::LayerVersion" + Properties: + ContentUri: "../../adf-build/shared/" + CompatibleRuntimes: + - python3.9 + Description: "Common dependencies for ADF Pipeline Management Functions" + LayerName: pipeline_management_layer + Metadata: + BuildMethod: python3.9 ADFPipelineMangementLambdaBasePolicy: Type: "AWS::IAM::ManagedPolicy" Properties: @@ -79,6 +94,10 @@ Resources: - "xray:PutTraceSegments" - "cloudwatch:PutMetricData" Resource: "*" + - Effect: Allow + Action: + - "events:PutEvents" + Resource: !Sub "arn:${AWS::Partition}:events:*:${RootAccountId}:event-bus/ADF-Event-Bus" Roles: - !Ref DeploymentMapProcessingLambdaRole - !Ref CreateOrUpdateRuleLambdaRole @@ -823,6 +842,8 @@ Resources: ADF_LOG_LEVEL: !Ref ADFLogLevel PIPELINE_MANAGEMENT_STATE_MACHINE: !Sub "arn:${AWS::Partition}:states:${AWS::Region}:${AWS::AccountId}:stateMachine:ADFPipelineManagementStateMachine" ADF_ROLE_NAME: !Ref CrossAccountAccessRole + ADF_EVENTBUS_ARN: !Sub "arn:${AWS::Partition}:events:${RootAccountRegion}:${RootAccountId}:event-bus/ADF-Event-Bus" + ADF_EVENTBUS_REGION: !Ref RootAccountRegion FunctionName: DeploymentMapProcessorFunction Role: !GetAtt DeploymentMapProcessingLambdaRole.Arn Events: @@ -861,6 +882,8 @@ Resources: ADF_LOG_LEVEL: !Ref ADFLogLevel ADF_ROLE_NAME: !Ref CrossAccountAccessRole S3_BUCKET_NAME: !Ref PipelineBucket + ADF_EVENTBUS_ARN: !Sub "arn:${AWS::Partition}:events:${RootAccountRegion}:${RootAccountId}:event-bus/ADF-Event-Bus" + ADF_EVENTBUS_REGION: !Ref RootAccountRegion FunctionName: ADFPipelineCreateOrUpdateRuleFunction Role: !GetAtt CreateOrUpdateRuleLambdaRole.Arn @@ -877,6 +900,8 @@ Resources: ADF_LOG_LEVEL: !Ref ADFLogLevel ADF_ROLE_NAME: !Ref CrossAccountAccessRole S3_BUCKET_NAME: !Ref PipelineBucket + ADF_EVENTBUS_ARN: !Sub "arn:${AWS::Partition}:events:${RootAccountRegion}:${RootAccountId}:event-bus/ADF-Event-Bus" + ADF_EVENTBUS_REGION: !Ref RootAccountRegion FunctionName: ADFPipelineCreateRepositoryFunction Role: !GetAtt CreateRepositoryLambdaRole.Arn diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py index 11969ecbc..185abb9ab 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py @@ -37,5 +37,5 @@ def put_event(self, detailType, detail, resources=[]): # pylint: disable=W0102 } trace_id = os.getenv("_X_AMZN_TRACE_ID") if trace_id: - payload["TraceHeader"] = trace_id + payload["TraceHeader"] = trace_id.split(";")[0] self.events.put_events(Entries=[payload]) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt index f58289651..339cb4b3b 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt @@ -59,3 +59,4 @@ pyyaml>=5.4.1 schema~=0.7.5 tenacity==8.0.1 urllib3~=1.26.12 +aws-xray-sdk==2.10.0 diff --git a/src/template.yml b/src/template.yml index cfe5ec7a0..3bc8de1a2 100644 --- a/src/template.yml +++ b/src/template.yml @@ -263,6 +263,9 @@ Resources: PolicyDocument: Version: "2012-10-17" Statement: + - Effect: "Allow" + Action: "events:PutEvents" + Resource: !GetAtt ADFEventBus.Arn - Effect: Allow Action: - "xray:PutTelemetryRecords" @@ -636,6 +639,11 @@ Resources: AccountManagementStateMachine: Type: "AWS::StepFunctions::StateMachine" + Metadata: + cfn-lint: + config: + ignore_checks: + - I3042 # Seems to be incorrectly thinking there's a hard-coded ARN in the definition Properties: RoleArn: !GetAtt StateMachineExecutionRole.Arn TracingConfiguration: @@ -791,7 +799,7 @@ Resources: "Next": "GetAccountDefaultRegionsFunction" } ], - "Default": "Success" + "Default": "PublishCompleteEvent" }, "GetAccountDefaultRegionsFunction": { "Type": "Task", @@ -819,7 +827,7 @@ Resources: }, "DeleteDefaultVPCMap": { "Type": "Map", - "Next": "Success", + "Next": "PublishCompleteEvent", "Iterator": { "StartAt": "DeleteDefaultVPC", "States": { @@ -856,6 +864,21 @@ Resources: }, "ResultPath": null }, + "PublishCompleteEvent": { + "Type": "Task", + "Resource": "arn:aws:states:::events:putEvents", + "Parameters": { + "Entries": [ + { + "Detail.$": "$", + "DetailType": "ACCOUNT_CREATION_COMPLETE", + "EventBusName": "ADF-Event-Bus", + "Source": "ADF.AccountManagement.AccountCreation" + } + ] + }, + "Next": "Success" + }, "Success": { "Type": "Succeed" } @@ -1890,6 +1913,22 @@ Resources: Properties: Name: ADF-Event-Bus + DeploymentAcountPutEventsPolicy: + Type: AWS::Events::EventBusPolicy + Properties: + StatementId: "DeploymentAccountPutEventStatement" + Statement: + Effect: "Allow" + Principal: "*" + Action: "events:PutEvents" + Resource: !GetAtt ADFEventBus.Arn + Condition: + ArnLike: + aws:PrincipalArn: + - !Sub "arn:${AWS::Partition}:iam::${DeploymentAccount.AccountId}:role/adf-automation/*" + EventBusName: + Ref: ADFEventBus + Outputs: ADFVersionNumber: Value: !FindInMap ["Metadata", "ADF", "Version"] From c2e97d4d49b60cf7094aefef251e4197ae7b45a5 Mon Sep 17 00:00:00 2001 From: Stewart Wallace Date: Thu, 15 Sep 2022 09:26:53 +0100 Subject: [PATCH 3/8] Linting and addressing code review comments --- docs/integrations-guide.md | 26 +++++++------- .../configure_account_alias.py | 34 ++++++++++++------- .../configure_account_tags.py | 13 ++++--- .../account_processing/create_account.py | 11 +++--- .../account_processing/delete_default_vpc.py | 19 ++++++----- .../register_account_for_support.py | 2 +- .../create_or_update_rule.py | 2 +- .../pipeline_management/create_repository.py | 23 ++++++------- .../deployment/pipeline_management.yml | 2 +- .../adf-build/shared/python/events.py | 13 +++++-- 10 files changed, 84 insertions(+), 61 deletions(-) diff --git a/docs/integrations-guide.md b/docs/integrations-guide.md index 549c75cdd..ca998debe 100644 --- a/docs/integrations-guide.md +++ b/docs/integrations-guide.md @@ -1,37 +1,37 @@ # Integrations Guide + ## Introduction -The AWS Deployment Framework enables integrations with external workflows via an Event Bus deployed into the organisational root account. + +The AWS Deployment Framework enables integrations with external workflows via an Event Bus deployed into the organizations management account. ## Account Management Events -The account management events are emitted at various stages during an execution of the Account Management State Machine. Currently - events are emitted for the following states: -- ACCOUNT_PROVISIONED + +- `ACCOUNT_PROVISIONED` Emitted when an AWS account is created. Contains the account definition from the .yml file as well as the account_id. -- ENTERPRISE_SUPPORT_REQUESTED +- `ENTERPRISE_SUPPORT_REQUESTED` Emitted when the support ticket to AWS Support is raised. Contains the account definition from the .yml file as well as the account_id. -- ACCOUNT_ALIAS_CONFIGURED +- `ACCOUNT_ALIAS_CONFIGURED` Emitted when the accounts alias is configured by ADF. The details section contains the account id and the alias value. The resource field also contains the account id -- ACCOUNT_TAGS_CONFIGURED +- `ACCOUNT_TAGS_CONFIGURED` Emitted when the accounts tags are updated by ADF. The details section contains the account id and the tags. The resource field also contains the account id -- DEFAULT_VPC_DELETED +- `DEFAULT_VPC_DELETED` Emitted when the default VPC in a region is deleted. The details section contains the account id and the region of the VPC. The resource field contains the deleted VPC id. -- ACCOUNT_CREATION_COMPLETE +- `ACCOUNT_CREATION_COMPLETE` Emitted when the state machine completes successfully. Contains the account definition from the .yml file as well as the account_id in the resource field. - - - ## Pipeline Management Events -- CROSS_ACCOUNT_RULE_CREATED_OR_UPDATED + +- `CROSS_ACCOUNT_RULE_CREATED_OR_UPDATED` Emitted when a rule is created to trigger pipelines from a different account. The details sections contains the source_account_id (The account where the CodeCommit repository is located) and the resource sections contains the deployment account Id (The account where the CodePipeline is located) -- REPOSITORY_CREATED_OR_UPDATED +- `REPOSITORY_CREATED_OR_UPDATED` Emitted when a codecommit repository is created in a different account than the deployment account. The details sections contains the repository_account_id (The account where the CodeCommit repository is located) as well as the stack_name (The CloudFormation stack that creates the repository) and the resource sections contains the repository account Id and the pipeline name diff --git a/src/lambda_codebase/account_processing/configure_account_alias.py b/src/lambda_codebase/account_processing/configure_account_alias.py index 1b0ab124e..39b20214a 100644 --- a/src/lambda_codebase/account_processing/configure_account_alias.py +++ b/src/lambda_codebase/account_processing/configure_account_alias.py @@ -7,7 +7,6 @@ import os import json -import boto3 from sts import STS from aws_xray_sdk.core import patch_all from logger import configure_logger @@ -18,14 +17,14 @@ LOGGER = configure_logger(__name__) ADF_ROLE_NAME = os.getenv("ADF_ROLE_NAME") AWS_PARTITION = os.getenv("AWS_PARTITION") -EVENTS = ADFEvents(boto3.client("events"), "AccountManagement") +EVENTS = ADFEvents("AccountManagement") def delete_account_aliases(account, iam_client, current_aliases): for alias in current_aliases: LOGGER.info( "Account %s, removing alias %s", - account.get('account_full_name'), + account.get("account_full_name"), alias, ) iam_client.delete_account_alias(AccountAlias=alias) @@ -34,8 +33,8 @@ def delete_account_aliases(account, iam_client, current_aliases): def create_account_alias(account, iam_client): LOGGER.info( "Adding alias to: %s alias %s", - account.get('account_full_name'), - account.get('alias'), + account.get("account_full_name"), + account.get("alias"), ) try: iam_client.create_account_alias(AccountAlias=account.get("alias")) @@ -53,15 +52,15 @@ def create_account_alias(account, iam_client): def ensure_account_has_alias(account, iam_client): LOGGER.info( "Ensuring Account: %s has alias %s", - account.get('account_full_name'), - account.get('alias'), + account.get("account_full_name"), + account.get("alias"), ) - current_aliases = iam_client.list_account_aliases().get('AccountAliases') - if account.get('alias') in current_aliases: + current_aliases = iam_client.list_account_aliases().get("AccountAliases") + if account.get("alias") in current_aliases: LOGGER.info( "Account: %s already has alias %s", - account.get('account_full_name'), - account.get('alias'), + account.get("account_full_name"), + account.get("alias"), ) return @@ -80,10 +79,19 @@ def lambda_handler(event, _): "adf_account_alias_config", ) ensure_account_has_alias(event, role.client("iam")) - EVENTS.put_event(detail=json.dumps({"account_id": account_id, "alias_value": event.get("alias")}), detailType="ACCOUNT_ALIAS_CONFIGURED", resources=[account_id]) + EVENTS.put_event( + detail=json.dumps( + { + "account_id": account_id, + "alias_value": event.get("alias"), + } + ), + detailType="ACCOUNT_ALIAS_CONFIGURED", + resources=[account_id], + ) else: LOGGER.info( "Account: %s does not need an alias", - event.get('account_full_name'), + event.get("account_full_name"), ) return event diff --git a/src/lambda_codebase/account_processing/configure_account_tags.py b/src/lambda_codebase/account_processing/configure_account_tags.py index d3d744bdc..ab1fcb369 100644 --- a/src/lambda_codebase/account_processing/configure_account_tags.py +++ b/src/lambda_codebase/account_processing/configure_account_tags.py @@ -9,7 +9,6 @@ """ - import json import boto3 @@ -19,7 +18,7 @@ from events import ADFEvents patch_all() -EVENTS = ADFEvents(boto3.client("events"), "AccountManagement") +EVENTS = ADFEvents("AccountManagement") LOGGER = configure_logger(__name__) @@ -40,10 +39,16 @@ def lambda_handler(event, _): event.get("tags"), organizations, ) - EVENTS.put_event(detail=json.dumps({"tags": event.get("tags"), "account_id": event.get("account_id")}), detailType="ACCOUNT_TAGS_CONFIGURED", resources=[event.get('account_id')]) + EVENTS.put_event( + detail=json.dumps( + {"tags": event.get("tags"), "account_id": event.get("account_id")} + ), + detailType="ACCOUNT_TAGS_CONFIGURED", + resources=[event.get("account_id")], + ) else: LOGGER.info( "Account: %s does not need tags configured", - event.get('account_full_name'), + event.get("account_full_name"), ) return event diff --git a/src/lambda_codebase/account_processing/create_account.py b/src/lambda_codebase/account_processing/create_account.py index 353f47188..89964788d 100644 --- a/src/lambda_codebase/account_processing/create_account.py +++ b/src/lambda_codebase/account_processing/create_account.py @@ -18,12 +18,11 @@ LOGGER = configure_logger(__name__) ADF_ROLE_NAME = os.getenv("ADF_ROLE_NAME") -EVENTS = ADFEvents(boto3.client("events"), "AccountManagement") - +EVENTS = ADFEvents("AccountManagement") def create_account(account, adf_role_name, org_client): - LOGGER.info("Creating account %s", account.get('account_full_name')) + LOGGER.info("Creating account %s", account.get("account_full_name")) allow_billing = "ALLOW" if account.get("allow_billing", False) else "DENY" response = org_client.create_account( Email=account.get("email"), @@ -49,4 +48,8 @@ def create_account(account, adf_role_name, org_client): def lambda_handler(event, _): org_client = boto3.client("organizations") details = create_account(event, ADF_ROLE_NAME, org_client) - EVENTS.put_event(detail=json.dumps(details), detailType="ACCOUNT_PROVISIONED", resources=[details.get("account_id")]) + EVENTS.put_event( + detail=json.dumps(details), + detailType="ACCOUNT_PROVISIONED", + resources=[details.get("account_id")], + ) diff --git a/src/lambda_codebase/account_processing/delete_default_vpc.py b/src/lambda_codebase/account_processing/delete_default_vpc.py index 761ce8c04..edcd1da40 100644 --- a/src/lambda_codebase/account_processing/delete_default_vpc.py +++ b/src/lambda_codebase/account_processing/delete_default_vpc.py @@ -6,7 +6,6 @@ """ import os import json -import boto3 from sts import STS from aws_xray_sdk.core import patch_all from logger import configure_logger @@ -17,8 +16,7 @@ LOGGER = configure_logger(__name__) ADF_ROLE_NAME = os.getenv("ADF_ROLE_NAME") AWS_PARTITION = os.getenv("AWS_PARTITION") -EVENTS = ADFEvents(boto3.client("events"), "AccountManagement") - +EVENTS = ADFEvents("AccountManagement") def assume_role(account_id): @@ -66,11 +64,9 @@ def delete_default_vpc(ec2_resource, ec2_client, default_vpc_id): ec2_client.delete_vpc(VpcId=default_vpc_id) - - def lambda_handler(event, _): event = event.get("Payload") - LOGGER.info("Checking for default VPC: %s", event.get('account_full_name')) + LOGGER.info("Checking for default VPC: %s", event.get("account_full_name")) role = assume_role(account_id=event.get("account_id")) ec2_client = role.client("ec2", region_name=event.get("region")) @@ -80,11 +76,16 @@ def lambda_handler(event, _): LOGGER.info( "Default VPC found: %s in %s", default_vpc_id, - event.get('account_full_name'), + event.get("account_full_name"), ) ec2_resource = role.resource("ec2", region_name=event.get("region")) delete_default_vpc(ec2_resource, ec2_client, default_vpc_id) - EVENTS.put_event(detail=json.dumps({"region": event.get("region"), "account_id":event.get("account_id")}), detailType="DEFAULT_VPC_DELETED", resources=[default_vpc_id]) - + EVENTS.put_event( + detail=json.dumps( + {"region": event.get("region"), "account_id": event.get("account_id")} + ), + detailType="DEFAULT_VPC_DELETED", + resources=[default_vpc_id], + ) return {"Payload": event} diff --git a/src/lambda_codebase/account_processing/register_account_for_support.py b/src/lambda_codebase/account_processing/register_account_for_support.py index e967a59bf..97dd9e2f6 100644 --- a/src/lambda_codebase/account_processing/register_account_for_support.py +++ b/src/lambda_codebase/account_processing/register_account_for_support.py @@ -18,7 +18,7 @@ LOGGER = configure_logger(__name__) -EVENTS = ADFEvents(boto3.client("events"), "AccountManagement.Support") +EVENTS = ADFEvents("AccountManagement") patch_all() diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py index 180bb6fed..6dbf464f3 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py @@ -23,7 +23,7 @@ PIPELINE_MANAGEMENT_STATEMACHINE = os.getenv("PIPELINE_MANAGEMENT_STATEMACHINE_ARN") CLOUDWATCH = boto3.client("cloudwatch") METRICS = ADFMetrics(CLOUDWATCH, "PIPELINE_MANAGEMENT/RULE") -EVENTS = ADFEvents(boto3.client("events", region_name=os.getenv("ADF_EVENTBUS_REGION")), "PipelineManagement") +EVENTS = ADFEvents("PipelineManagement") _cache = None diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py index aea638893..87f170cda 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py @@ -19,16 +19,15 @@ LOGGER = configure_logger(__name__) DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"] DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"] -EVENTS = ADFEvents(boto3.client("events", region_name=os.getenv("ADF_EVENTBUS_REGION")), "PipelineManagement") - +EVENTS = ADFEvents("PipelineManagement") def lambda_handler(pipeline, _): """Main Lambda Entry point""" parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3) auto_create_repositories = parameter_store.fetch_parameter( - "auto_create_repositories" - ) + "auto_create_repositories" + ) LOGGER.info(auto_create_repositories) if auto_create_repositories == "enabled": code_account_id = ( @@ -57,14 +56,14 @@ def lambda_handler(pipeline, _): {"MetricName": "CreateOrUpdate", "Value": 1, "Unit": "Count"} ) EVENTS.put_event( - detail=json.dumps({ - "repository_account_id": code_account_id, - "stack_name": repo.stack_name - }), + detail=json.dumps( + { + "repository_account_id": code_account_id, + "stack_name": repo.stack_name, + } + ), detailType="REPOSITORY_CREATED_OR_UPDATED", - resources=[ - f'{code_account_id}:{pipeline.get("name")}' - ] - ) + resources=[f'{code_account_id}:{pipeline.get("name")}'], + ) return pipeline diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml index 8859bfa9d..acf53f3a1 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml @@ -97,7 +97,7 @@ Resources: - Effect: Allow Action: - "events:PutEvents" - Resource: !Sub "arn:${AWS::Partition}:events:*:${RootAccountId}:event-bus/ADF-Event-Bus" + Resource: !Sub "arn:${AWS::Partition}:events:${RootAccountRegion}:${RootAccountId}:event-bus/ADF-Event-Bus" Roles: - !Ref DeploymentMapProcessingLambdaRole - !Ref CreateOrUpdateRuleLambdaRole diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py index 185abb9ab..cc07b1f63 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py @@ -9,7 +9,7 @@ class ADFEvents: def __init__( - self, client: boto3.client, service, namespace="ADF", eventbus_arn=None + self, service, namespace="adf", eventbus_arn=None, client: boto3.client = None ) -> None: """ Client: Any Boto3 EventBridge client @@ -18,7 +18,14 @@ def __init__( eventbus_arn: Optionally specify a custom EventBridge ARN. If no ARN is specified, and no ENV variable set, will default to ADF-Event-Bus """ - self.events = client + self.events = ( + client + if client + else boto3.client( + "events", + region_name=os.getenv("ADF_EVENTBUS_REGION", os.getenv("AWS_REGION")), + ) + ) self.source = f"{namespace}.{service}" self.eventbus_arn = ( os.environ.get("ADF_EVENTBUS_ARN", "ADF-Event-Bus") @@ -27,7 +34,7 @@ def __init__( ) # This dict isn't mutated. So it's safe to default to this - def put_event(self, detailType, detail, resources=[]): # pylint: disable=W0102 + def put_event(self, detailType, detail, resources=[]): # pylint: disable=W0102 payload = { "Source": self.source, "Resources": resources, From 5f6e243a02c79ce7729ea08d429457fdb65a5fbb Mon Sep 17 00:00:00 2001 From: Stewart Wallace Date: Fri, 16 Sep 2022 08:43:18 +0100 Subject: [PATCH 4/8] Addressing megalint feedback on variables.tf --- README.md | 3 ++- docs/integrations-guide.md | 31 +++++++++++++++++---------- samples/sample-terraform/variables.tf | 12 ++++++++--- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 5ed7fe45e..8aca1ef9f 100644 --- a/README.md +++ b/README.md @@ -36,4 +36,5 @@ within the AWS Console. - Refer to the [User Guide](docs/user-guide.md) for using ADF once it is setup. - Refer to the [Samples Guide](docs/samples-guide.md) for a detailed walk through of the provided samples. -- Refer to the [Integrations Guide](docs/integrations-guide.md) for information on events produced by the ADF. +- Refer to the [Integrations Guide](docs/integrations-guide.md) for information + on events produced by the ADF. diff --git a/docs/integrations-guide.md b/docs/integrations-guide.md index ca998debe..266ea75e3 100644 --- a/docs/integrations-guide.md +++ b/docs/integrations-guide.md @@ -2,37 +2,46 @@ ## Introduction -The AWS Deployment Framework enables integrations with external workflows via an Event Bus deployed into the organizations management account. +The AWS Deployment Framework enables integrations with external workflows via an +Event Bus deployed into the organizations management account. ## Account Management Events + Currently - events are emitted for the following states: - `ACCOUNT_PROVISIONED` Emitted when an AWS account is created. - Contains the account definition from the .yml file as well as the account_id. + Contains the account definition as well as the account_id. - `ENTERPRISE_SUPPORT_REQUESTED` Emitted when the support ticket to AWS Support is raised. - Contains the account definition from the .yml file as well as the account_id. + Contains the account definition as well as the account_id. - `ACCOUNT_ALIAS_CONFIGURED` Emitted when the accounts alias is configured by ADF. - The details section contains the account id and the alias value. The resource field also contains the account id + The details section contains the account id and the alias value. + The resource field also contains the account id - `ACCOUNT_TAGS_CONFIGURED` Emitted when the accounts tags are updated by ADF. - The details section contains the account id and the tags. The resource field also contains the account id + The details section contains the account id and the tags. + The resource field also contains the account id - `DEFAULT_VPC_DELETED` Emitted when the default VPC in a region is deleted. - The details section contains the account id and the region of the VPC. The resource field contains the deleted VPC id. + The details section contains the account id and the region of the VPC. + he resource field contains the deleted VPC id. - `ACCOUNT_CREATION_COMPLETE` Emitted when the state machine completes successfully. - Contains the account definition from the .yml file as well as the account_id in the resource field. + Contains the account definition and the account id in the resource field. ## Pipeline Management Events - `CROSS_ACCOUNT_RULE_CREATED_OR_UPDATED` Emitted when a rule is created to trigger pipelines from a different account. - The details sections contains the source_account_id (The account where the CodeCommit repository is located) and the resource sections contains the deployment account Id (The account where the CodePipeline is located) + The details sections contains the source account id. + The resource sections contains the deployment account Id. - `REPOSITORY_CREATED_OR_UPDATED` - Emitted when a codecommit repository is created in a different account than the deployment account. - The details sections contains the repository_account_id (The account where the CodeCommit repository is located) as well as the stack_name (The CloudFormation stack that creates the repository) and the resource sections contains the repository account Id and the pipeline name - + Emitted when a codecommit repository is created. + The details sections contains: + - the repository_account_id + - the CloudFormation stack name + The resource section contains an almagamation of the account_id and the + pipeline name. diff --git a/samples/sample-terraform/variables.tf b/samples/sample-terraform/variables.tf index dfe4dff2a..1d143131c 100644 --- a/samples/sample-terraform/variables.tf +++ b/samples/sample-terraform/variables.tf @@ -1,3 +1,9 @@ -variable "TARGET_ACCOUNT_ID" {} -variable "TARGET_ACCOUNT_ROLE" {} -variable my_bucket_name {} +variable "TARGET_ACCOUNT_ID" { + type = string +} +variable "TARGET_ACCOUNT_ROLE" { + type = string +} +variable my_bucket_name { + type = string +} From 3da63af018ce98c155713b32674f47521d75f6e9 Mon Sep 17 00:00:00 2001 From: Stewart Wallace Date: Fri, 16 Sep 2022 08:53:27 +0100 Subject: [PATCH 5/8] Removing unordered list --- docs/integrations-guide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations-guide.md b/docs/integrations-guide.md index 266ea75e3..2ac762bde 100644 --- a/docs/integrations-guide.md +++ b/docs/integrations-guide.md @@ -40,8 +40,8 @@ Currently - events are emitted for the following states: - `REPOSITORY_CREATED_OR_UPDATED` Emitted when a codecommit repository is created. The details sections contains: - - the repository_account_id - - the CloudFormation stack name + the repository_account_id + the CloudFormation stack name The resource section contains an almagamation of the account_id and the pipeline name. From 304dd2c9ce7b7839ab3c77a4cd85bf7a56f041ee Mon Sep 17 00:00:00 2001 From: Stewart Wallace Date: Wed, 25 Jan 2023 11:13:28 +0000 Subject: [PATCH 6/8] Fixing missed merge conflict --- .../create_or_update_rule.py | 8 +++-- .../pipeline_management/create_repository.py | 31 +++++++------------ 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py index 81aff1996..5e4402d65 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py @@ -50,7 +50,7 @@ def lambda_handler(event, _): LOGGER.info(event) - pipeline = event['pipeline_definition'] + pipeline = event["pipeline_definition"] source_provider = ( pipeline.get("default_providers", {}) @@ -84,6 +84,10 @@ def lambda_handler(event, _): METRICS.put_metric_data( {"MetricName": "CreateOrUpdate", "Value": 1, "Unit": "Count"} ) - EVENTS.put_event(detail=json.dumps({"source_account_id": _source_account_id}), detailType="CROSS_ACCOUNT_RULE_CREATED_OR_UPDATED", resources=[DEPLOYMENT_ACCOUNT_ID]) + EVENTS.put_event( + detail=json.dumps({"source_account_id": source_account_id}), + detailType="CROSS_ACCOUNT_RULE_CREATED_OR_UPDATED", + resources=[DEPLOYMENT_ACCOUNT_ID], + ) return event diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py index 928ffa7c5..b124c354a 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py @@ -34,7 +34,7 @@ def lambda_handler(event, _): Returns: dict: The input event. """ - pipeline = event.get('pipeline_definition') + pipeline = event.get("pipeline_definition") source_provider = ( pipeline.get("default_providers", {}) .get("source", {}) @@ -42,8 +42,7 @@ def lambda_handler(event, _): ) if source_provider != "codecommit": LOGGER.debug( - "This pipeline is not a CodeCommit source provider. " - "No actions required." + "This pipeline is not a CodeCommit source provider. " "No actions required." ) return event @@ -51,7 +50,6 @@ def lambda_handler(event, _): auto_create_repositories = parameter_store.fetch_parameter( "auto_create_repositories" ) -<<<<<<< HEAD LOGGER.info(auto_create_repositories) if auto_create_repositories == "enabled": code_account_id = ( @@ -73,11 +71,17 @@ def lambda_handler(event, _): and not has_custom_repo ): repo = Repo( - code_account_id, pipeline.get("name"), pipeline.get("description") + code_account_id, + pipeline.get("name"), + pipeline.get("description"), ) repo.create_update() METRICS.put_metric_data( - {"MetricName": "CreateOrUpdate", "Value": 1, "Unit": "Count"} + { + "MetricName": "CreateOrUpdate", + "Value": 1, + "Unit": "Count", + } ) EVENTS.put_event( detail=json.dumps( @@ -89,15 +93,6 @@ def lambda_handler(event, _): detailType="REPOSITORY_CREATED_OR_UPDATED", resources=[f'{code_account_id}:{pipeline.get("name")}'], ) -======= - LOGGER.debug("Auto create repositories is: %s", auto_create_repositories) - if auto_create_repositories != "enabled": - LOGGER.debug( - "ADF is not configured to automatically create CodeCommit " - "repositories if they don't exist yet." - ) - return event ->>>>>>> 84e7831bae60acd72eef7d5bd546771074bfbd02 code_account_id = ( pipeline.get("default_providers", {}) @@ -111,11 +106,7 @@ def lambda_handler(event, _): .get("properties", {}) .get("repository", {}) ) - if ( - code_account_id - and str(code_account_id).isdigit() - and not has_custom_repo - ): + if code_account_id and str(code_account_id).isdigit() and not has_custom_repo: repo = Repo( code_account_id, pipeline.get("name"), From 062a5185fb23d9265ce364364c353862c78ccb15 Mon Sep 17 00:00:00 2001 From: Stewart Wallace Date: Wed, 25 Jan 2023 12:40:34 +0000 Subject: [PATCH 7/8] Fixing lint issues and tox requirements path --- docs/integrations-guide.md | 2 +- .../adf-bootstrap/deployment/global.yml | 4 ++-- .../pipeline_management/lambda_layer/requirements.txt | 2 ++ .../adf-bootstrap/deployment/pipeline_management.yml | 2 +- .../bootstrap_repository/adf-build/shared/python/events.py | 2 +- .../bootstrap_repository/adf-build/shared/requirements.txt | 2 +- src/template.yml | 6 +++--- tox.ini | 2 +- 8 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/integrations-guide.md b/docs/integrations-guide.md index 2ac762bde..9905c748c 100644 --- a/docs/integrations-guide.md +++ b/docs/integrations-guide.md @@ -43,5 +43,5 @@ Currently - events are emitted for the following states: the repository_account_id the CloudFormation stack name - The resource section contains an almagamation of the account_id and the + The resource section contains an amalgamation of the account_id and the pipeline name. diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml index 31e15b918..19c7b69e6 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml @@ -70,7 +70,7 @@ Globals: Runtime: python3.9 Mappings: - OrganisationPartitionRegionMapping: + OrganizationPartitionRegionMapping: aws: region: "us-east-1" aws-us-gov: @@ -191,7 +191,7 @@ Resources: CrossAccountAccessRole: !Ref CrossAccountAccessRole PipelineBucket: !Ref PipelineBucket RootAccountId: !Ref MasterAccountId - RootAccountRegion: !FindInMap [OrganisationPartitionRegionMapping, !Ref "AWS::Partition", "region"] + RootAccountRegion: !FindInMap [OrganizationPartitionRegionMapping, !Ref "AWS::Partition", "region"] CodeBuildImage: !Ref Image CodeBuildComputeType: !Ref ComputeType SharedModulesBucket: !Ref SharedModulesBucket diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/lambda_layer/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/lambda_layer/requirements.txt index 272e34c72..dbe0bbe57 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/lambda_layer/requirements.txt +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/lambda_layer/requirements.txt @@ -2,3 +2,5 @@ pyyaml==5.4.1 schema==0.7.5 tenacity==8.1.0 wrapt==1.14.1 # https://github.com/aws/aws-lambda-builders/issues/302 +aws-xray-sdk==2.11.0 + diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml index 7807ed721..4164de4f4 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml @@ -79,7 +79,7 @@ Resources: Metadata: BuildMethod: python3.9 - ADFPipelineMangementLambdaBasePolicy: + ADFPipelineManagementLambdaBasePolicy: Type: "AWS::IAM::ManagedPolicy" Properties: Description: "Base policy for all ADF pipeline management lambdas" diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py index cc07b1f63..089ec5fca 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/events.py @@ -1,5 +1,5 @@ """ -Standardised class for pushing events from within the ADF Namespace +Standardized class for pushing events from within the ADF Namespace """ diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt index c288c6da9..0717ec7c4 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt @@ -60,4 +60,4 @@ schema~=0.7.5 tenacity==8.1.0 typing-extensions~=4.4.0 urllib3~=1.26.14 -aws-xray-sdk==2.10.0 +aws-xray-sdk==2.11.0 diff --git a/src/template.yml b/src/template.yml index f61998229..104086145 100644 --- a/src/template.yml +++ b/src/template.yml @@ -18,13 +18,13 @@ Metadata: Labels: ["adf", "aws-deployment-framework", "multi-account", "cicd", "devops"] HomePageUrl: https://github.com/awslabs/aws-deployment-framework - SemanticVersion: 3.2.0 + SemanticVersion: 3.2.2 SourceCodeUrl: https://github.com/awslabs/aws-deployment-framework Mappings: Metadata: ADF: - Version: 3.2.0 + Version: 3.2.2 Parameters: CrossAccountAccessRoleName: @@ -1989,7 +1989,7 @@ Resources: Properties: Name: ADF-Event-Bus - DeploymentAcountPutEventsPolicy: + DeploymentAccountPutEventsPolicy: Type: AWS::Events::EventBusPolicy Properties: StatementId: "DeploymentAccountPutEventStatement" diff --git a/tox.ini b/tox.ini index a7d5c8957..dcfd44f31 100644 --- a/tox.ini +++ b/tox.ini @@ -35,7 +35,7 @@ deps = -r{toxinidir}/src/lambda_codebase/cross_region_bucket/requirements.txt -r{toxinidir}/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/determine_default_branch/requirements.txt -r{toxinidir}/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/initial_commit/requirements.txt - -r{toxinidir}/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/requirements.txt + -r{toxinidir}/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/lambda_layer/requirements.txt -r{toxinidir}/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/requirements.txt -r{toxinidir}/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements.txt -r{toxinidir}/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/requirements.txt From 9cf3ee3b00301b28123f75d7c4ab99efc6d5b6d3 Mon Sep 17 00:00:00 2001 From: Stewart Wallace Date: Wed, 25 Jan 2023 12:48:47 +0000 Subject: [PATCH 8/8] More xray tracing in pipeline management --- .../pipeline_management/create_or_update_rule.py | 3 ++- .../lambda_codebase/pipeline_management/create_repository.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py index 5e4402d65..8eda9d78a 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py @@ -16,12 +16,13 @@ from aws_xray_sdk.core import patch_all -patch_all() LOGGER = configure_logger(__name__) DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"] CLOUDWATCH = boto3.client("cloudwatch") METRICS = ADFMetrics(CLOUDWATCH, "PIPELINE_MANAGEMENT/RULE") EVENTS = ADFEvents("PipelineManagement") +patch_all() + _CACHE = None diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py index b124c354a..e7b15ccea 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py @@ -12,6 +12,7 @@ from cloudwatch import ADFMetrics from parameter_store import ParameterStore from events import ADFEvents +from aws_xray_sdk.core import patch_all CLOUDWATCH = boto3.client("cloudwatch") @@ -20,6 +21,7 @@ DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"] DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"] EVENTS = ADFEvents("PipelineManagement") +patch_all() def lambda_handler(event, _):