From a75014b9b25e0a7d594ae44e13eaa85a0778a8e3 Mon Sep 17 00:00:00 2001 From: aiportel Date: Tue, 9 Jan 2024 18:18:11 +0100 Subject: [PATCH 1/4] feat: Adding support for AWS-CN partition on cn-north-1 region. --- src/lambda_codebase/account_bootstrap.py | 3 +- .../adf-bootstrap/deployment/global.yml | 5 + .../deployment/pipeline_management.yml | 5 + .../china-forward-function/handler.py | 29 ++++ .../china-forward-function/requirements.txt | 0 .../stepfunction_helper.py | 55 +++++++ .../adf-build/cn_northwest_bucket.yml | 26 +++ .../adf-build/cn_northwest_deploy.yml | 83 ++++++++++ .../adf-build/create_s3_cn.py | 50 ++++++ .../bootstrap_repository/adf-build/main.py | 111 ++++++++++--- .../adf-build/organization_policy.py | 10 ++ .../cdk/cdk_constructs/adf_codepipeline.py | 3 + .../helpers/retrieve_organization_accounts.py | 5 +- .../adf-build/shared/python/cloudformation.py | 59 +++++-- .../adf-build/shared/python/partition.py | 16 +- .../adf-build/shared/python/s3.py | 22 ++- .../shared/python/tests/test_partition.py | 12 +- src/lambda_codebase/organization/main.py | 6 +- src/template.yml | 149 ++++++++++++------ 19 files changed, 555 insertions(+), 94 deletions(-) create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/china-forward-function/handler.py create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/china-forward-function/requirements.txt create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/china-forward-function/stepfunction_helper.py create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/cn_northwest_bucket.yml create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/cn_northwest_deploy.yml create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/create_s3_cn.py diff --git a/src/lambda_codebase/account_bootstrap.py b/src/lambda_codebase/account_bootstrap.py index a7f2ac4f5..f8800b64e 100644 --- a/src/lambda_codebase/account_bootstrap.py +++ b/src/lambda_codebase/account_bootstrap.py @@ -83,7 +83,8 @@ def configure_generic_account(sts, event, region, role): def configure_master_account_parameters(event): """ - Update the management account parameter store in us-east-1 with the + Update the management account parameter store in the base region + of the partition (us-east-1, us-gov-west-1 or cn-north-1) with the deployment_account_id then updates the main deployment region with that same value """ diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml index dd4167e39..4a432ba38 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml @@ -687,6 +687,11 @@ Resources: python: 3.12 nodejs: 20 commands: + - | + if [ "${AWS::Region}" = "cn-north-1" ]; then + pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple + npm config set registry https://registry.npmmirror.com + fi - aws s3 cp s3://$SHARED_MODULES_BUCKET/adf-build/ ./adf-build/ --recursive --quiet - pip install -r adf-build/requirements.txt -r adf-build/helpers/requirements.txt -q -t ./adf-build pre_build: diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml index 412c1a214..5da91921c 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml @@ -674,6 +674,11 @@ Resources: python: 3.12 nodejs: 20 commands: + - | + if [ "${AWS::Region}" = "cn-north-1" ]; then + pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple + npm config set registry https://registry.npmmirror.com + fi - npm install aws-cdk@2.119.0 -g -y --quiet --no-progress - aws s3 cp s3://$SHARED_MODULES_BUCKET/adf-build/ ./adf-build/ --recursive --quiet - pip install -r adf-build/requirements.txt -q -t ./adf-build diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/china-forward-function/handler.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/china-forward-function/handler.py new file mode 100644 index 000000000..a800bb488 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/china-forward-function/handler.py @@ -0,0 +1,29 @@ +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +""" +The forward function will forward events to the target SFN. +""" + +import logging +import os + +import boto3 +from stepfunction_helper import Stepfunction + +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(os.environ.get("ADF_LOG_LEVEL", logging.INFO)) +SFN_ARN = os.getenv("SFN_ARN", "") +sfn_name = SFN_ARN.split(':')[-1] + + +def lambda_handler(event, context): + LOGGER.debug(event) + if "source" in event and event["source"] == "aws.organizations": + session = boto3.session.Session(region_name="cn-north-1") + sfn_instance = Stepfunction(session, LOGGER) + _, state_name = sfn_instance.invoke_sfn_execution( + sfn_arn=SFN_ARN, + input=event, + ) + LOGGER.info(f"Successfully invoke sfn {sfn_name} with statemachine name {state_name}.") diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/china-forward-function/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/china-forward-function/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/china-forward-function/stepfunction_helper.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/china-forward-function/stepfunction_helper.py new file mode 100644 index 000000000..3fa956992 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/china-forward-function/stepfunction_helper.py @@ -0,0 +1,55 @@ +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +import json +import uuid +from decimal import Decimal + + +def convert_decimals(obj): + if isinstance(obj, Decimal): + return str(obj) + elif isinstance(obj, list): + return [convert_decimals(item) for item in obj] + elif isinstance(obj, dict): + return {key: convert_decimals(value) for key, value in obj.items()} + else: + return obj + + +class Stepfunction: + """Class to handle Custom Stepfunction methods""" + + def __init__( + self, + session, + LOGGER + ): + self.logger = LOGGER + self.session = session + + def get_stepfunction_client(self): + return self.session.client("stepfunctions") + + def invoke_sfn_execution( + self, + sfn_arn, + input: dict, + execution_name=None): + try: + state_machine_arn = sfn_arn + sfn_client = self.get_stepfunction_client() + + if not execution_name: + execution_name = str(uuid.uuid4()) + event_body = json.dumps(convert_decimals(input), indent=2) + response = sfn_client.start_execution( + stateMachineArn=state_machine_arn, + name=execution_name, + input=event_body + ) + except Exception as e: + msg = f"Couldn't invoke stepfunction {sfn_arn}, error: {e}." + self.logger.error(msg) + raise + return response, execution_name diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/cn_northwest_bucket.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/cn_northwest_bucket.yml new file mode 100644 index 000000000..4c8aefef2 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/cn_northwest_bucket.yml @@ -0,0 +1,26 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: >- + ADF CloudFormation Template - Create S3 bucket for cn-northwest-1 + +Parameters: + BucketName: + Type: String + +Resources: + BootstrapArtifactStorageBucket: + DeletionPolicy: Delete + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref BucketName + AccessControl: BucketOwnerFullControl + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + VersioningConfiguration: + Status: Enabled + PublicAccessBlockConfiguration: + BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true \ No newline at end of file diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/cn_northwest_deploy.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/cn_northwest_deploy.yml new file mode 100644 index 000000000..116fc6a93 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/cn_northwest_deploy.yml @@ -0,0 +1,83 @@ +# // Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# // SPDX-License-Identifier: Apache-2.0 + +AWSTemplateFormatVersion: '2010-09-09' +Transform: 'AWS::Serverless-2016-10-31' +Description: ADF CloudFormation Stack for deploy extra resources in China cn-northwest-1. + +Parameters: + AcoountBootstrapingStateMachineArn: + Type: String + AdfLogLevel: + Type: String + +Globals: + Function: + Architectures: + - arm64 + CodeUri: china-forward-function + Runtime: python3.9 + Timeout: 300 + Tracing: Active + +Resources: + ForwardStateMachineFunctionRole: + Type: "AWS::IAM::Role" + Properties: + Path: "/adf-china-extra/" + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - "lambda.amazonaws.com" + Action: + - "sts:AssumeRole" + + ForwardStateMachineFunctionRolePolicy: + Type: AWS::IAM::Policy + Properties: + PolicyName: "forward-state-machine-function-policy" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: "states:StartExecution" + Resource: !Ref AcoountBootstrapingStateMachineArn + - Effect: Allow + Action: + - "logs:CreateLogGroup" + - "logs:CreateLogStream" + - "logs:PutLogEvents" + - "xray:PutTelemetryRecords" + - "xray:PutTraceSegments" + - "cloudwatch:PutMetricData" + Resource: "*" + Roles: + - !Ref ForwardStateMachineFunctionRole + + + ForwardStateMachineFunction: + Type: 'AWS::Serverless::Function' + Properties: + Handler: handler.lambda_handler + Description: "ADF Lambda Function - Forward events to statemachine" + Environment: + Variables: + SFN_ARN: !Ref AcoountBootstrapingStateMachineArn + ADF_LOG_LEVEL: !Ref AdfLogLevel + FunctionName: ForwardStateMachineFunction + Role: !GetAtt ForwardStateMachineFunctionRole.Arn + Events: + RuleEvent: + Type: EventBridgeRule + Properties: + Pattern: + source: + - aws.organizations + detail: + eventSource: + - organizations.amazonaws.com + eventName: + - MoveAccount \ No newline at end of file diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/create_s3_cn.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/create_s3_cn.py new file mode 100644 index 000000000..a1de4ece9 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/create_s3_cn.py @@ -0,0 +1,50 @@ +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +""" +Main entry point for create_s3_cn.py execution which +is executed from within AWS CodeBuild in the management account +""" +import os +import boto3 +from logger import configure_logger +from cloudformation import CloudFormation + +REGION_DEFAULT = os.environ["AWS_REGION"] +ACCOUNT_ID = os.environ["MASTER_ACCOUNT_ID"] +LOGGER = configure_logger(__name__) + +def _create_s3_bucket(bucket_name): + try: + LOGGER.info(f"Deploy S3 bucket {bucket_name}...") + extra_deploy_region = "cn-northwest-1" + template_path = "adf-build/cn_northwest_bucket.yml" + stack_name = 'adf-regional-base-china-bucket' + parameters= [ + { + 'ParameterKey': 'BucketName', + 'ParameterValue': bucket_name, + 'UsePreviousValue': False, + }, + ] + cloudformation = CloudFormation( + region=extra_deploy_region, + deployment_account_region=extra_deploy_region, + role=boto3, + wait=True, + stack_name=stack_name, + account_id=ACCOUNT_ID, + parameters = parameters, + local_template_path=template_path + ) + cloudformation.create_stack() + except Exception as error: + LOGGER.error(f"Failed to process _create_s3_bucket, error:\n {error}") + exit(1) + +def main(): + bucket_name = f"adf-china-bootstrap-cn-northwest-1-{ACCOUNT_ID}" + _create_s3_bucket(bucket_name) + +if __name__ == '__main__': + main() diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py index 61a3ff3bf..af92c23f9 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py @@ -14,21 +14,20 @@ from thread import PropagatingThread import boto3 - from botocore.exceptions import ClientError -from logger import configure_logger from cache import Cache from cloudformation import CloudFormation -from parameter_store import ParameterStore +from config import Config +from errors import GenericAccountConfigureError, ParameterNotFoundError +from logger import configure_logger +from organization_policy import OrganizationPolicy from organizations import Organizations +from parameter_store import ParameterStore +from partition import get_partition +from s3 import S3 from stepfunctions import StepFunctions -from errors import GenericAccountConfigureError, ParameterNotFoundError from sts import STS -from s3 import S3 -from partition import get_partition -from config import Config -from organization_policy import OrganizationPolicy - +from thread import PropagatingThread S3_BUCKET_NAME = os.environ["S3_BUCKET"] REGION_DEFAULT = os.environ["AWS_REGION"] @@ -148,9 +147,18 @@ def prepare_deployment_account(sts, deployment_account_id, config): ) deployment_account_parameter_store.put_parameter( 'default_scm_branch', - config.config.get('scm', {}).get( - 'default-scm-branch', - ADF_DEFAULT_SCM_FALLBACK_BRANCH, + ( + config.config + .get('scm', {}) + .get('default-scm-branch', ADF_DEFAULT_SCM_FALLBACK_BRANCH) + ) + ) + deployment_account_parameter_store.put_parameter( + '/adf/scm/default-scm-codecommit-account-id', + ( + config.config + .get('scm', {}) + .get('default-scm-codecommit-account-id', deployment_account_id) ) ) deployment_account_parameter_store.put_parameter( @@ -211,7 +219,6 @@ def _store_extension_parameters(parameter_store, config): ) -# pylint: disable=too-many-locals def worker_thread( account_id, sts, @@ -261,6 +268,15 @@ def worker_thread( 'bucket_name', updated_kms_bucket_dict[region]['s3_regional_bucket'], ) + + # Ensuring the stage parameter on the target account is up-to-date + parameter_store.put_parameter( + '/adf/org/stage', + config.config.get('org', {}).get( + 'stage', + ADF_DEFAULT_ORG_STAGE, + ) + ) cloudformation = CloudFormation( region=region, deployment_account_region=config.deployment_account_region, @@ -285,7 +301,7 @@ def worker_thread( 'Unit within AWS Organizations.', account_id, ) - raise LookupError from error + raise Exception from error except GenericAccountConfigureError as generic_account_error: LOGGER.info(generic_account_error) @@ -316,14 +332,14 @@ def await_sfn_executions(sfn_client): ), status_filter=None, ): + DOMAIN = "amazonaws.cn" if PARTITION == "aws-cn" else "aws.amazon.com" LOGGER.error( "Account Management State Machine encountered a failed, " "timed out, or aborted execution. Please look into this problem " "before retrying the bootstrap pipeline. You can navigate to: " - "https://%s.console.aws.amazon.com/states/home?region=%s#/statemachines/view/%s", - REGION_DEFAULT, - REGION_DEFAULT, - ACCOUNT_MANAGEMENT_STATE_MACHINE_ARN, + f"https://{REGION_DEFAULT}.console.{DOMAIN}/states/home?" + f"region={REGION_DEFAULT}#/statemachines/" + f"view/{ACCOUNT_MANAGEMENT_STATE_MACHINE_ARN}" ) sys.exit(1) if _sfn_execution_exists_with( @@ -342,15 +358,14 @@ def await_sfn_executions(sfn_client): "Account Bootstrapping State Machine encountered a failed, " "timed out, or aborted execution. Please look into this problem " "before retrying the bootstrap pipeline. You can navigate to: " - "https://%(region)s.console.aws.amazon.com/states/home" + "https://%(region)s.console.%(domain)s/states/home" "?region=%(region)s#/statemachines/view/%(sfn_arn)s", { "region": REGION_DEFAULT, "sfn_arn": ACCOUNT_BOOTSTRAPPING_STATE_MACHINE_ARN, + "domain": "amazonaws.cn" if PARTITION == "aws-cn" else "aws.amazon.com" }, ) - sys.exit(2) - def _await_running_sfn_executions( sfn_client, @@ -401,6 +416,57 @@ def _sfn_execution_exists_with( return False +def _china_region_extra_deploy(region: str): + if region != "cn-north-1": + return + else: + extra_deploy_region = "cn-northwest-1" + + parameters = [ + { + 'ParameterKey': 'AcoountBootstrapingStateMachineArn', + 'ParameterValue': ACCOUNT_BOOTSTRAPPING_STATE_MACHINE_ARN, + 'UsePreviousValue': False, + }, + { + 'ParameterKey': 'AdfLogLevel', + 'ParameterValue': ADF_LOG_LEVEL, + 'UsePreviousValue': False, + }, + ] + + try: + s3_china = S3( + region=REGION_DEFAULT, + bucket=S3_BUCKET_NAME + ) + cloudformation = CloudFormation( + region=extra_deploy_region, + deployment_account_region=extra_deploy_region, + role=boto3, + wait=True, + stack_name='adf-regional-base-china-extra', + s3=s3_china, + s3_key_path='adf-build', + account_id=ACCOUNT_ID, + template_file_prefix='cn_northwest_deploy', + parameters=parameters + + ) + cloudformation.create_stack() + + except Exception as error: + LOGGER.error( + "China extra stack adf-regional-base-china-extra deployment failed in region %(region)s, please check following error: " + "%(error)s", + { + "region": extra_deploy_region, + "error": str(error), + }, + ) + sys.exit(2) + + def main(): # pylint: disable=R0915 LOGGER.info("ADF Version %s", ADF_VERSION) LOGGER.info("ADF Log Level is %s", ADF_LOG_LEVEL) @@ -409,7 +475,8 @@ def main(): # pylint: disable=R0915 policies = OrganizationPolicy() config = Config() - + # fix the china org service endpoint + _china_region_extra_deploy(REGION_DEFAULT) try: parameter_store = ParameterStore(REGION_DEFAULT, boto3) deployment_account_id = parameter_store.fetch_parameter( diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy.py index 483eda0e6..4a8f45278 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy.py @@ -57,6 +57,16 @@ def _is_govcloud(region: str) -> bool: """ return region.startswith("us-gov") + @staticmethod + def _is_china(region: str) -> bool: + """ + Evaluates the region to determine if it is part of China Partition. + + :param region: a region (cn-north-1) + :return: Returns True if the region is China Partition, False otherwise. + """ + return region.startswith("cn-") + @staticmethod def set_scp_attachment(access_identifier, organization_mapping, path, organizations): if access_identifier: diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py index 4e5775e72..b7cac6b5f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py @@ -40,6 +40,9 @@ def get_partition(region_name: str) -> str: if region_name.startswith('us-gov'): return 'aws-us-gov' + elif region_name.startswith("cn-"): + return "aws-cn" + return 'aws' diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/retrieve_organization_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/retrieve_organization_accounts.py index e96c7aec1..9edb79b1d 100755 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/retrieve_organization_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/retrieve_organization_accounts.py @@ -156,13 +156,16 @@ def main(): def _get_partition(region_name: str) -> str: """Given the region, this function will return the appropriate partition. - :param region_name: The name of the region (us-east-1, us-gov-west-1) + :param region_name: The name of the region (us-east-1, us-gov-west-1 or cn-north-1) :return: Returns the partition name as a string. """ if region_name.startswith("us-gov"): return "aws-us-gov" + elif region_name.startswith("cn-"): + return "aws-cn" + return "aws" diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/cloudformation.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/cloudformation.py index 39d466838..1d113bbde 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/cloudformation.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/cloudformation.py @@ -20,9 +20,9 @@ LOGGER = configure_logger(__name__) STACK_TERMINATION_PROTECTION = os.environ.get('TERMINATION_PROTECTION', False) CFN_CONFIG = Config( - retries={ - "max_attempts": 10, - }, + retries=dict( + max_attempts=10 + ) ) # A stack name can contain only alphanumeric characters (case sensitive) # and hyphens. @@ -118,7 +118,6 @@ class WaitException(Exception): class CloudFormation(StackProperties): - # pylint: disable=too-many-arguments def __init__( self, region, @@ -132,6 +131,8 @@ def __init__( parameters=None, account_id=None, # Used for logging visibility role_arn=None, + template_file_prefix=None, # define a custom template file + local_template_path=None, # support local tempplate path ): self.client = role.client( 'cloudformation', @@ -151,7 +152,12 @@ def __init__( s3=s3, s3_key_path=s3_key_path ) - + self.template_url_from_template_file_prefix = self.s3.fetch_s3_url( + self._create_template_path(self.s3_key_path, template_file_prefix) + ) \ + if template_file_prefix else None + self.template_url = template_url or self.template_url_from_template_file_prefix + self.local_template_path = local_template_path def validate_template(self): try: return self.client.validate_template(TemplateURL=self.template_url) @@ -167,6 +173,20 @@ def validate_template(self): f"{self.template_url}: {error}", ) from None + def _handle_template_path( + self, + template_path + ): + try: + # Read the CloudFormation template from a file + with open(template_path, 'r') as template_file: + template_body = template_file.read() + + return template_body + except Exception as error: + LOGGER.error(f"Process _handle_template_path function error:\n {error}.") + return None + def _wait_if_in_progress(self): status = self.get_stack_status() if status not in StackProperties.in_progress_state_waiters: @@ -316,16 +336,26 @@ def _create_change_set(self): self.stack_name, ) try: - self.template_url = ( - self.template_url - if self.template_url is not None - else self.get_template_url() - ) - if self.template_url: - self.validate_template() + cfn_template_map = None + if self.local_template_path: + cfn_template_map = { + "TemplateBody": self._handle_template_path(self.local_template_path) + } + else: + self.template_url = ( + self.template_url + if self.template_url is not None + else self.get_template_url() + ) + if self.template_url: + self.validate_template() + cfn_template_map = { + "TemplateURL": self.template_url + } + + if cfn_template_map: change_set_params = { "StackName": self.stack_name, - "TemplateURL": self.template_url, "Parameters": ( self.parameters if self.parameters is not None @@ -342,6 +372,7 @@ def _create_change_set(self): "ChangeSetName": self.stack_name, "ChangeSetType": self._get_change_set_type() } + change_set_params.update(cfn_template_map) if self.role_arn: change_set_params["RoleARN"] = self.role_arn self._clean_up_when_required() @@ -516,7 +547,7 @@ def get_stack_output(self, value): ) return [item.get('OutputValue') for item in response.get('Stacks') [0].get('Outputs') if item.get('OutputKey') == value][0] - except BaseException: # pylint: disable=broad-exception-caught + except BaseException: LOGGER.warning( "%s in %s - Attempted to get stack output from %s " "but it failed.", diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/partition.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/partition.py index d1bb2a0cb..43c82b441 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/partition.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/partition.py @@ -10,7 +10,7 @@ from boto3.session import Session -COMPATIBLE_PARTITIONS = ['aws-us-gov', 'aws'] +COMPATIBLE_PARTITIONS = ['aws-us-gov', 'aws', 'aws-cn'] class IncompatiblePartitionError(Exception): @@ -44,4 +44,18 @@ def get_organization_api_region(region_name: str) -> str: if get_partition(region_name) == 'aws-us-gov': return 'us-gov-west-1' + elif region_name.startswith("cn-"): + return "aws-cn" + return 'us-east-1' + +def get_aws_domain(region_name: str) -> str: + """ + Get AWS domain suffix + """ + import re + _partition = get_partition(region_name) + if re.match("^aws-.*", _partition): + return "amazonaws.com.{0}".format(_partition.split("-")[-1]) + else: + return "amazonaws.com" diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/s3.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/s3.py index 0dc264739..117c6ff27 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/s3.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/s3.py @@ -9,6 +9,10 @@ from logger import configure_logger +from partition import ( + get_aws_domain, + get_partition +) LOGGER = configure_logger(__name__) @@ -23,6 +27,8 @@ def __init__(self, region, bucket): self.client = boto3.client('s3', region_name=region) self.resource = boto3.resource('s3', region_name=region) self.bucket = bucket + self.domain_suffix = get_aws_domain(region) + self.partition = get_partition(region) @staticmethod def supported_path_styles(): @@ -49,8 +55,8 @@ def build_pathing_style(self, style, key): 's3-url' returns: 's3://{bucket}/{key}' 's3-uri' returns: '{bucket}/{key}' 's3-key-only' return: '{key}' - 'path': returns: 'https://{s3-region}.amazonaws.com/{bucket}/{key}' - 'virtual-hosted' returns: 'https://{buycket}.{s3-region}.amazonaws.com/{key}' + 'path': returns: 'https://{s3-region}.{self.domain_suffix}/{bucket}/{key}' + 'virtual-hosted' returns: 'https://{buycket}.{s3-region}.{self.domain_suffix}/{key}' key (str): The object key to include in the path. @@ -69,9 +75,11 @@ def build_pathing_style(self, style, key): s3_region_name = f"s3-{self.region}" if style == 'path': - return f"https://{s3_region_name}.amazonaws.com/{self.bucket}/{key}" + if self.partition == "aws-cn": + return f"https://{self.bucket}.s3.{self.region}.{self.domain_suffix}/{key}" + return f"https://{s3_region_name}.{self.domain_suffix}/{self.bucket}/{key}" if style == 'virtual-hosted': - return f"https://{self.bucket}.{s3_region_name}.amazonaws.com/{key}" + return f"https://{self.bucket}.{s3_region_name}.{self.domain_suffix}/{key}" raise ValueError( f"Unknown upload style syntax: {style}. " @@ -190,8 +198,10 @@ def fetch_s3_url(self, key): s3_object.get() LOGGER.debug('Found Template at: %s', s3_object.key) if self.region == 'us-east-1': - return f"https://s3.amazonaws.com/{self.bucket}/{key}" - return f"https://s3-{self.region}.amazonaws.com/{self.bucket}/{key}" + return f"https://s3.{self.domain_suffix}/{self.bucket}/{key}" + if self.partition == 'aws-cn': + return self.build_pathing_style("path", key) + return f"https://s3-{self.region}.{self.domain_suffix}/{self.bucket}/{key}" except self.client.exceptions.NoSuchKey: # Split the path to remove the last key entry from the string key_level_up = key.split('/') diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_partition.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_partition.py index 9b514a9b3..ee8e1457d 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_partition.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_partition.py @@ -19,8 +19,11 @@ 'us-gov-east-1' ] -_incompatible_regions = [ - 'cn-north-1', +_china_region = [ + 'cn-north-1' + ] + +_incompatible_region = [ 'cn-northwest-1' ] @@ -34,8 +37,11 @@ def test_partition_govcloud_regions(region): def test_partition_us_commercial_regions(region): assert get_partition(region) == 'aws' +@pytest.mark.parametrize('region', _china_region) +def test_partition_china_regions(region): + assert get_partition(region) == 'aws-cn' -@pytest.mark.parametrize('region', _incompatible_regions) +@pytest.mark.parametrize('region', _incompatible_region) def test_partition_incompatible_regions(region): with pytest.raises(IncompatiblePartitionError) as excinfo: get_partition(region) diff --git a/src/lambda_codebase/organization/main.py b/src/lambda_codebase/organization/main.py index a02f632c2..012ad8572 100644 --- a/src/lambda_codebase/organization/main.py +++ b/src/lambda_codebase/organization/main.py @@ -71,14 +71,16 @@ def as_cfn_response(self) -> Tuple[PhysicalResourceId, Data]: def create_(_event: Mapping[str, Any], _context: Any) -> CloudFormationResponse: approved_regions = [ 'us-east-1', - 'us-gov-west-1' + 'us-gov-west-1', + 'cn-north-1' ] region = os.getenv('AWS_REGION') if region not in approved_regions: raise ValueError( "Deployment of ADF is only available via the us-east-1 " - "and us-gov-west-1 regions." + "us-gov-west-1 " + "and cn-north-1 regions." ) organization_id, created = ensure_organization() organization_root_id = get_organization_root_id() diff --git a/src/template.yml b/src/template.yml index 84034c32b..15dc8bf79 100644 --- a/src/template.yml +++ b/src/template.yml @@ -3,7 +3,7 @@ AWSTemplateFormatVersion: "2010-09-09" Transform: "AWS::Serverless-2016-10-31" -Description: ADF CloudFormation Initial Base Stack for the Management Account in the us-east-1 region. +Description: ADF CloudFormation Initial Base Stack for the Management Account in the cn-north-1 region. Metadata: AWS::ServerlessRepo::Application: @@ -77,9 +77,9 @@ Parameters: DeploymentAccountMainRegion: Type: String - AllowedPattern: "(us(-gov)?|ap|ca|eu|sa)-(central|(north|south)?(east|west)?)-\\d" + AllowedPattern: "(us(-gov)?|ap|ca|eu|sa|cn)-(central|(north|south)?(east|west)?)-\\d" MinLength: 6 - Description: "Example -> us-east-1, us-gov-west-1, eu-west-1" + Description: "Example -> us-east-1, us-gov-west-1, eu-west-1 or cn-north-1" DeploymentAccountTargetRegions: Type: CommaDelimitedList @@ -119,6 +119,14 @@ Globals: Runtime: python3.12 Timeout: 300 +Conditions: + IsChinaMainRegion: + "Fn::Equals": + - !Sub "${AWS::Region}" + - "cn-north-1" + NotChinaMainRegion: + "Fn::Not": + - Condition: IsChinaMainRegion Resources: BootstrapTemplatesBucketPolicy: Type: AWS::S3::BucketPolicy @@ -268,7 +276,6 @@ Resources: - !Ref GetAccountRegionsFunctionRole - !Ref DeleteDefaultVPCFunctionRole - !Ref AccountAliasConfigFunctionRole - - !Ref AccountRegionConfigFunctionRole - !Ref AccountTagConfigFunctionRole - !Ref AccountOUConfigFunctionRole - !Ref CreateAccountFunctionRole @@ -307,7 +314,6 @@ Resources: - !GetAtt AccountOUConfigFunction.Arn - !GetAtt GetAccountRegionsFunction.Arn - !GetAtt DeleteDefaultVPCFunction.Arn - - !GetAtt AccountRegionConfigFunction.Arn AccountFileProcessingFunction: Type: 'AWS::Serverless::Function' @@ -366,6 +372,15 @@ Resources: - lambda.amazonaws.com Action: "sts:AssumeRole" Path: "/aws-deployment-framework/account-management/" + Policies: + - PolicyName: "adf-lambda-create-account-alias-policy" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - "iam:CreateAccountAlias" + Resource: "*" AccountAliasConfigFunction: Type: 'AWS::Serverless::Function' @@ -409,7 +424,6 @@ Resources: - Effect: Allow Action: - "organizations:TagResource" - - "organizations:UntagResource" Resource: "*" AccountTagConfigFunction: @@ -726,7 +740,7 @@ Resources: "Next": "CreateAccount" } ], - "Default": "ConfigureAccountRegions" + "Default": "ConfigureAccountAlias" }, "ConfigureAccountAlias": { "Type": "Task", @@ -803,41 +817,7 @@ Resources: "MaxAttempts": 6 } ], - "Next": "ConfigureAccountRegions" - }, - "ConfigureAccountRegions": { - "Type": "Task", - "Resource": "${AccountRegionConfigFunction.Arn}", - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 2, - "MaxAttempts": 6, - "BackoffRate": 2 - } - ], - "Next": "AreRegionsConfigured" - }, - "AreRegionsConfigured": { - "Type": "Choice", - "Choices": [ - { - "Variable": "$.all_regions_enabled", - "BooleanEquals": true, - "Next": "ConfigureAccountAlias" - } - ], - "Default": "Wait 15 seconds" - }, - "Wait 15 seconds": { - "Type": "Wait", - "Seconds": 15, - "Next": "ConfigureAccountRegions" + "Next": "ConfigureAccountAlias" }, "ConfigureAccountTags": { "Type": "Task", @@ -1175,6 +1155,7 @@ Resources: CloudWatchEventsRule: Type: "AWS::Events::Rule" + Condition: NotChinaMainRegion Properties: Description: Triggers StateMachine on Move OU EventPattern: @@ -1190,6 +1171,39 @@ Resources: RoleArn: !GetAtt AccountBootstrapStartExecutionRole.Arn Id: CreateStackLinkedAccountV1 + CodeCommitRole: + Type: AWS::IAM::Role + Properties: + RoleName: "adf-codecommit-role-base" + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: codecommit.amazonaws.com + Action: + - sts:AssumeRole + Path: / + + CodeCommitPolicy: + Type: AWS::IAM::Policy + Properties: + PolicyName: "adf-organizations-codecommit-role-policy" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - "codecommit:BatchGetRepositories" + - "codecommit:Get*" + - "codecommit:GitPull" + - "codecommit:List*" + - "codecommit:CancelUploadArchive" + - "codecommit:UploadArchive" + Resource: "*" + Roles: + - !Ref CodeCommitRole + CodeBuildRole: Type: AWS::IAM::Role Properties: @@ -1300,6 +1314,7 @@ Resources: - !Sub "arn:${AWS::Partition}:cloudformation:*:*:stack/adf-global-base-*/*" - !Sub "arn:${AWS::Partition}:cloudformation:*:*:stack/adf-regional-base-*/*" - !Sub "arn:${AWS::Partition}:cloudformation:*:${AWS::AccountId}:stack/adf-global-base-adf-build/*" + - !Sub "arn:${AWS::Partition}:cloudformation:*:aws:transform/Serverless-2016-10-31" - Effect: "Allow" Action: - "s3:DeleteObject" @@ -1337,6 +1352,42 @@ Resources: - !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${CrossAccountAccessRoleName}" - !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${CrossAccountAccessRoleName}-readonly" + CodeBuildPolicyChina: + Type: "AWS::IAM::ManagedPolicy" + Condition: IsChinaMainRegion + Properties: + Description: "Policy to allow codebuild to perform actions for china region" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: + - "lambda:*" + - "iam:PassRole" + - "iam:CreatePolicy" + - "iam:CreateRole" + - "iam:DeleteRole" + - "iam:DeleteRolePolicy" + - "iam:GetRole" + - "iam:PutRolePolicy" + - "iam:UpdateAssumeRolePolicy" + Resource: + - !Sub "arn:${AWS::Partition}:lambda:*:${AWS::AccountId}:function:ForwardStateMachineFunction" + - !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/adf-china-extra/adf-regional-base-*" + - Effect: "Allow" + Action: + - "s3:*" + Resource: + - !Sub "arn:${AWS::Partition}:s3:::adf-china-bootstrap-cn-northwest-1-${AWS::AccountId}" + - !Sub "arn:${AWS::Partition}:s3:::adf-china-bootstrap-cn-northwest-1-${AWS::AccountId}/*" + - Effect: "Allow" + Action: + - "events:*" + Resource: + - !Sub "arn:${AWS::Partition}:events:cn-northwest-1:${AWS::AccountId}:rule/adf-regional-base-china*" + Roles: + - !Ref BootstrapCodeBuildRole + CodeCommitRepository: Type: AWS::CodeCommit::Repository Properties: @@ -1352,8 +1403,8 @@ Resources: Type: CODEPIPELINE Environment: ComputeType: "BUILD_GENERAL1_LARGE" - PrivilegedMode: false - Image: "aws/codebuild/standard:7.0" + PrivilegedMode: true + Image: "aws/codebuild/standard:5.0" EnvironmentVariables: - Name: ADF_VERSION Value: !FindInMap ["Metadata", "ADF", "Version"] @@ -1389,6 +1440,10 @@ Resources: python: 3.12 pre_build: commands: + - | + if [ "${AWS_REGION}" = "cn-north-1" ] || [ "${AWS_REGION}" = "cn-northwest-1" ]; then + pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple + fi - >- pip install -r requirements-dev.txt @@ -1416,6 +1471,12 @@ Resources: --s3-prefix adf-bootstrap/deployment --s3-bucket $DEPLOYMENT_ACCOUNT_BUCKET - python adf-build/store_config.py + - | + if [ "${AWS_REGION}" = "cn-north-1" ] || [ "${AWS_REGION}" = "cn-northwest-1" ]; then + python adf-build/create_s3_cn.py + sam build -t adf-build/cn_northwest_deploy.yml --region cn-northwest-1 + sam package --output-template-file adf-build/cn_northwest_deploy.yml --s3-prefix adf-bootstrap --s3-bucket adf-china-bootstrap-cn-northwest-1-${MASTER_ACCOUNT_ID} --region cn-northwest-1 + fi # Shared Modules to be used with AWS CodeBuild: - >- aws s3 sync From b47b810f356d9574891d1971e42bab84ad5c7c58 Mon Sep 17 00:00:00 2001 From: aiportel Date: Wed, 10 Jan 2024 21:36:48 +0100 Subject: [PATCH 2/4] doc: Adding documentation and expanding tests for china partition --- docs/admin-guide.md | 43 ++++++---- docs/installation-guide.md | 3 +- docs/samples-guide.md | 3 +- .../lambda_codebase/tests/stubs/slack.py | 84 +++++++++++++++++++ .../tests/stubs/stub_iam_cn.py | 41 +++++++++ .../tests/test_iam_cfn_deploy_role_policy.py | 37 +++++--- .../lambda_codebase/tests/test_slack.py | 22 +++-- .../china-forward-function/handler.py | 2 +- src/template.yml | 9 +- 9 files changed, 204 insertions(+), 40 deletions(-) create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/stubs/stub_iam_cn.py diff --git a/docs/admin-guide.md b/docs/admin-guide.md index 4f9b1a5e9..17b9a82f3 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -63,6 +63,9 @@ The `adfconfig.yml` file resides on the and defines the general high-level configuration for the AWS Deployment Framework. +For Govcloud and China deployments, `adfconfig.yml` file resides on the +[management account](#management-account) CodeCommit Repository (in us-gov-west-1 and cn-north-1 respectively) + The configuration properties are synced into AWS Systems Manager Parameter Store and are used for certain orchestration options throughout your Organization. @@ -917,8 +920,9 @@ To determine the current version, follow these steps: ### ADF version you have deployed To check the current version of ADF that you have deployed, go to the management -account in us-east-1. Check the CloudFormation stack output or tag of the -`serverlessrepo-aws-deployment-framework` Stack. +account in us-east-1 for global partition deployments. For Govcloud and China +deployments go to us-gov-west-1 and cn-north-1 respectively. Check the CloudFormation +stack output or tag of the `serverlessrepo-aws-deployment-framework` Stack. - In the outputs tab, it will show the version as the `ADFVersionNumber`. - In the tags on the CloudFormation stack, it is presented as @@ -938,8 +942,8 @@ releases](https://github.com/awslabs/aws-deployment-framework/releases). The `serverlessrepo-aws-deployment-framework` stack is updated through this process with new changes that were included in that release of ADF. -To check the progress in the management account in `us-east-1`, follow these -steps: +To check the progress in the management account in `us-east-1` for global partition deployments; for Govcloud and China +deployments go to us-gov-west-1 or cn-north-1 respectively, follow these steps: 1. Go to the [CloudFormation console](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks?filteringStatus=active&filteringText=serverlessrepo-aws-deployment-framework&viewNested=true&hideStacks=false) @@ -981,7 +985,8 @@ Which branch is used is determined by: Alternatively, you can also perform the update using the AWS CLI. -In the management account in `us-east-1`: +In the management account in `us-east-1` for global partition deployments; For Govcloud and China +deployments in us-gov-west-1 or cn-north-1 respectively: 1. Go to the Pull Request section of the `aws-deployment-framework-bootstrap` [CodeCommit @@ -996,7 +1001,8 @@ In the management account in `us-east-1`: changes that it proposes. Once reviewed, merge the pull request to continue. Confirm the `aws-deployment-framework-bootstrap` pipeline in the management -account in `us-east-1`: +account in `us-east-1` for global partition deployments; For Govcloud and China +deployments go to us-gov-west-1 or cn-north-1 respectively: 1. Go to the [CodePipeline console for the aws-deployment-framework-bootstrap pipeline](https://console.aws.amazon.com/codesuite/codepipeline/pipelines/aws-deployment-framework-bootstrap-pipeline/view?region=us-east-1). @@ -1091,7 +1097,9 @@ Alternatively, you can also perform the update using the AWS CLI. If you wish to remove ADF you can delete the CloudFormation stack named `serverlessrepo-aws-deployment-framework` in the management account in -the `us-east-1` region. This will move into a `DELETE_FAILED` at some stage because +the `us-east-1` region for global partition deployments; For Govcloud and China +deployments go to us-gov-west-1 or cn-north-1 respectively. +This will move into a `DELETE_FAILED` at some stage because there is an S3 Bucket that is created via a custom resource _(cross region)_. After it moves into `DELETE_FAILED`, you can right-click on the stack and hit delete again while selecting to skip the Bucket the stack will successfully @@ -1108,11 +1116,12 @@ the base stack when the account is moved to the Root of the AWS Organization. One thing to keep in mind if you are planning to re-install ADF is that you will want to clean up the parameter from SSM Parameter Store named -_deployment_account_id_ in `us-east-1` on the management account. AWS Step -Functions uses this parameter to determine if ADF has already got a deployment -account setup. If you re-install ADF with this parameter set to a value, -ADF will attempt an assume role to the account to do some work, which will fail -since that role will not be on the account at that point. +_deployment_account_id_ in `us-east-1` on the management account for global +partition deployments; For Govcloud and China deployments go to us-gov-west-1 +or cn-north-1 respectively. AWS Step Functions uses this parameter to determine +if ADF has already got a deployment account setup. If you re-install ADF with +this parameter set to a value, ADF will attempt an assume role to the account +to do some work, which will fail since that role will not be on the account at that point. There is also a CloudFormation stack named `adf-global-base-adf-build` which lives on the management account in your main deployment region. This stack @@ -1147,7 +1156,9 @@ There are two ways to enable this: to deploy the latest version again, set the `Log Level` to `DEBUG` to get extra logging information about the issue you are experiencing. 2. If you are running an older version of ADF, please navigate to the - CloudFormation Console in `us-east-1` of the AWS Management account. + CloudFormation Console in `us-east-1` of the AWS Management account for global + partition deployments; For Govcloud and China deployments go to us-gov-west-1 + or cn-north-1 respectively. 3. Update the stack. 4. For any ADF deployment of `v3.2.0` and later, please change the `Log Level` parameter and set it to `DEBUG`. Deploy those changes and revert them after @@ -1162,7 +1173,8 @@ Please trace the failed component and dive into/report the debug information. The main components to look at are: -1. In the AWS Management Account in `us-east-1`: +1. In the AWS Management Account in `us-east-1` for global partition deployments; For Govcloud and China +deployments go to us-gov-west-1 or cn-north-1 respectively: 2. The [CloudFormation aws-deployment-framework stack](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks?filteringStatus=active&filteringText=aws-deployment-framework&viewNested=true&hideStacks=false). 3. The [CloudWatch Logs for the Lambda functions deployed by ADF](https://console.aws.amazon.com/lambda/home?region=us-east-1#/functions?f0=true&n0=false&op=and&v0=ADF). 4. Check if the [CodeCommit pull @@ -1171,7 +1183,8 @@ The main components to look at are: branch for the `aws-deployment-framework-bootstrap` (ADF Bootstrap) repository. 5. The [CodePipeline execution of the AWS Bootstrap pipeline](https://console.aws.amazon.com/codesuite/codepipeline/pipelines/aws-deployment-framework-bootstrap-pipeline/view?region=us-east-1). 6. Navigate to the [AWS Step Functions service](https://us-east-1.console.aws.amazon.com/states/home?region=us-east-1#/statemachines) - in the management account in `us-east-1`. Check the state machines named + in the management account in `us-east-1`for global partition deployments; For Govcloud and China + deployments go to us-gov-west-1 or cn-north-1 respectively, check the state machines named `AccountManagementStateMachine...` and `AccountBootstrappingStateMachine...`. Look at recent executions only. - When you find one that has a failed execution, check the components that diff --git a/docs/installation-guide.md b/docs/installation-guide.md index 9669bd06c..8b8ab6f43 100644 --- a/docs/installation-guide.md +++ b/docs/installation-guide.md @@ -661,7 +661,8 @@ automatically in the background, to follow its progress: that started the bootstrap process for the deployment account. You can view the progress of this in the management account in the AWS Step Functions console for the step function `AccountBootstrappingStateMachine-` in the - `us-east-1` region. + `us-east-1` region for global partition deployments; For Govcloud and China + deployments go to us-gov-west-1 or cn-north-1 respectively. 3. Once the Step Function has completed, switch roles over to the newly bootstrapped deployment account in the region you defined as your main diff --git a/docs/samples-guide.md b/docs/samples-guide.md index bc5e0d39b..937fe66d3 100644 --- a/docs/samples-guide.md +++ b/docs/samples-guide.md @@ -70,7 +70,8 @@ Management Account. By default, there is a `global.yml` in the root of the be appended to as required. If we look at AWS Step Functions in the management account in `us-east-1` -we can see the progress of the bootstrap process. +we can see the progress of the bootstrap process for global partition deployments; +For Govcloud and China deployments go to us-gov-west-1 or cn-north-1 respectively. ![run-state-machine](./images/run-state-machine.png) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/stubs/slack.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/stubs/slack.py index 499f4a3ae..a28677513 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/stubs/slack.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/stubs/slack.py @@ -87,3 +87,87 @@ } }] } + +stub_approval_event_cn = { + 'Records': [{ + 'EventSource': 'aws:sns', + 'EventVersion': '1.0', + 'EventSubscriptionArn': 'arn:aws-cn:sns:cn-north-1:9999999:adf-pipeline-sample-vpc-PipelineSNSTopic-example', + 'Sns': { + 'Type': 'Notification', + 'MessageId': '1', + 'TopicArn': 'arn:aws-cn:sns:cn-north-1:9999999:adf-pipeline-sample-vpc-PipelineSNSTopic-example', + 'Subject': 'APPROVAL NEEDED: AWS CodePipeline adf-pipeline-sample-vpc for action Approve', + 'Message': '{"region":"cn-north-1","consoleLink":"https://console.amazonaws.cn","approval":{"pipelineName":"adf-pipeline-sample-vpc","stageName":"approval-stage-1","actionName":"Approve","token":"fa777887-41dc-4ac4-8455-a209a93c76b9","expires":"2019-03-17T11:08Z","externalEntityLink":null,"approvalReviewLink":"https://console.amazonaws.cn/codepipeline/"}}', + 'Timestamp': '3000-03-10T11:08:34.673Z', + 'SignatureVersion': '1', + 'Signature': '1', + 'SigningCertUrl': 'https://sns.opportunities/initiatives.amazonaws.com/SimpleNotificationService', + 'UnsubscribeUrl': 'https://sns.cn-north-1.amazonaws.com', + 'MessageAttributes': {} + } + }] +} + +stub_bootstrap_event_cn = { + 'Records': [{ + 'EventSource': 'aws:sns', + 'EventVersion': '1.0', + 'EventSubscriptionArn': 'arn:aws-cn:sns:cn-north-1:9999999:adf-pipeline-sample-vpc-PipelineSNSTopic-example', + 'Sns': { + 'Type': 'Notification', + 'MessageId': '1', + 'TopicArn': 'arn:aws-cn:sns:cn-north-1:9999999:adf-pipeline-sample-vpc-PipelineSNSTopic-example', + 'Subject': 'AWS Deployment Framework Bootstrap', + 'Message': 'Account 1111111 has now been bootstrapped into banking/production', + 'Timestamp': '3000-03-10T11:08:34.673Z', + 'SignatureVersion': '1', + 'Signature': '1', + 'SigningCertUrl': 'https://sns.cn-north-1.amazonaws.com/SimpleNotificationService', + 'UnsubscribeUrl': 'https://sns.cn-north-1.amazonaws.com', + 'MessageAttributes': {} + } + }] +} + +stub_failed_pipeline_event_cn = { + 'Records': [{ + 'EventSource': 'aws:sns', + 'EventVersion': '1.0', + 'EventSubscriptionArn': 'arn:aws-cn:sns:cn-north-1:9999999:adf-pipeline-sample-vpc-PipelineSNSTopic-example', + 'Sns': { + 'Type': 'Notification', + 'MessageId': '1', + 'TopicArn': 'arn:aws-cn:sns:cn-north-11:9999999:adf-pipeline-sample-vpc-PipelineSNSTopic-example', + 'Subject': None, + 'Message': '{"version":"0","id":"1","detail-type":"CodePipeline Pipeline Execution State Change","source":"aws.codepipeline","account":"2","time":"3000-03-10T11:09:38Z","region":"eu-central-1","resources":["arn:aws:codepipeline:eu-central-1:999999:adf-pipeline-sample-vpc"],"detail":{"pipeline":"adf-pipeline-sample-vpc","execution-id":"1","state":"FAILED","version":9.0}}', + 'Timestamp': '2019-03-10T11:09:49.953Z', + 'SignatureVersion': '1', + 'Signature': '2', + 'SigningCertUrl': 'https://sns.cn-north-1.amazonaws.com/SimpleNotificationService', + 'UnsubscribeUrl': 'https://sns.cn-north-1.amazonaws.com', + 'MessageAttributes': {} + } + }] +} + +stub_failed_bootstrap_event_cn = { + 'Records': [{ + 'EventSource': 'aws:sns', + 'EventVersion': '1.0', + 'EventSubscriptionArn': 'arn:aws-cn:sns:cn-north-1:9999999:adf-pipeline-sample-vpc-PipelineSNSTopic-example', + 'Sns': { + 'Type': 'Notification', + 'MessageId': '1', + 'TopicArn': 'arn:aws-cn:sns:cn-north-1:9999999:adf-pipeline-sample-vpc-PipelineSNSTopic-example', + 'Subject': 'Failure - AWS Deployment Framework Bootstrap', + 'Message': '{"Error":"Exception","Cause":"{\\"errorMessage\\": \\"CloudFormation Stack Failed - Account: 111 Region: eu-central-1 Status: ROLLBACK_IN_PROGRESS\\", \\"errorType\\": \\"Exception\\", \\"stackTrace\\": [[\\"/var/task/wait_until_complete.py\\", 99, \\"lambda_handler\\", \\"status))\\"]]}"}', + 'Timestamp': '2019-03-10T11:09:49.953Z', + 'SignatureVersion': '1', + 'Signature': '2', + 'SigningCertUrl': 'https://sns.cn-north-1.amazonaws.com/SimpleNotificationService', + 'UnsubscribeUrl': 'https://sns.cn-north-1.amazonaws.com', + 'MessageAttributes': {} + } + }] +} diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/stubs/stub_iam_cn.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/stubs/stub_iam_cn.py new file mode 100644 index 000000000..81496f1a4 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/stubs/stub_iam_cn.py @@ -0,0 +1,41 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +# pylint: skip-file + +""" +Stubs for testing iam.py +""" + +get_role_policy = { + 'RoleName': 'string', + 'PolicyName': 'string', + 'PolicyDocument': { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "KMS", + "Effect": "Allow", + "Action": ["iam:ChangePassword"], + "Resource": ( + "arn:aws-cn:kms:cn-north-1:111111111111:key/existing_key" + ), + }, + { + "Sid": "S3", + "Effect": "Allow", + "Action": "s3:ListAllMyBuckets", + "Resource": [ + "arn:aws-cn:s3:::existing_bucket", + "arn:aws-cn:s3:::existing_bucket/*", + ], + }, + { + "Sid": "AssumeRole", + "Effect": "Allow", + "Action": "sts:AssumeRole", + "Resource": ['something'], + }, + ] + } +} \ No newline at end of file diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/test_iam_cfn_deploy_role_policy.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/test_iam_cfn_deploy_role_policy.py index fd399e15c..f06975d9f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/test_iam_cfn_deploy_role_policy.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/test_iam_cfn_deploy_role_policy.py @@ -7,15 +7,23 @@ from pytest import fixture, raises from mock import call, Mock from copy import deepcopy -from .stubs import stub_iam from lambda_codebase.iam_cfn_deploy_role_policy import IAMCfnDeployRolePolicy +import os +from boto3.session import Session +REGION = os.getenv("AWS_REGION", "us-east-1") +PARTITION = Session().get_partition_for_region(REGION) + +if PARTITION == "aws-cn": + from .stubs import stub_iam_cn as stub_iam +else: + from .stubs import stub_iam @fixture def iam_client(): client = Mock() client.get_role_policy.side_effect = ( - lambda **kwargs: deepcopy(stub_iam.get_role_policy) + lambda **kwargs: deepcopy(stub_iam.get_role_policy) if PARTITION == "aws-cn" else deepcopy(stub_iam.get_role_policy) ) return client @@ -114,8 +122,8 @@ def test_grant_access_to_s3_buckets_new_bucket_single_resource(iam_client): ) assert instance.policy_document['Statement'][1]['Resource'] == [ policy_doc_before['Statement'][1]['Resource'], - 'arn:aws:s3:::new_bucket', - 'arn:aws:s3:::new_bucket/*', + f'arn:{PARTITION}:s3:::new_bucket', + f'arn:{PARTITION}:s3:::new_bucket/*', ] assert instance.policy_document['Statement'][2] == ( policy_doc_before['Statement'][2] @@ -149,10 +157,10 @@ def test_grant_access_to_s3_buckets_new_buckets(iam_client): assert instance.policy_document['Statement'][1]['Resource'] == [ policy_doc_before['Statement'][1]['Resource'][0], policy_doc_before['Statement'][1]['Resource'][1], - 'arn:aws:s3:::new_bucket', - 'arn:aws:s3:::new_bucket/*', - 'arn:aws:s3:::another_new_bucket', - 'arn:aws:s3:::another_new_bucket/*', + f'arn:{PARTITION}:s3:::new_bucket', + f'arn:{PARTITION}:s3:::new_bucket/*', + f'arn:{PARTITION}:s3:::another_new_bucket', + f'arn:{PARTITION}:s3:::another_new_bucket/*', ] assert instance.policy_document['Statement'][2] == ( policy_doc_before['Statement'][2] @@ -188,7 +196,10 @@ def test_grant_access_to_kms_keys_new_key_single_resource(iam_client): ) policy_doc_before = deepcopy(instance.policy_document) - new_key_arn = 'arn:aws:kms:eu-west-1:111111111111:key/new_key' + + test_region = "eu-west-1" if PARTITION == "aws-cn" else "cn-north-1" + new_key_arn = f'arn:{PARTITION}:kms:{test_region}:111111111111:key/new_key' + instance.grant_access_to_kms_keys([ new_key_arn, ]) @@ -226,8 +237,8 @@ def test_grant_access_to_kms_keys_new_keys(iam_client): ] policy_doc_before = deepcopy(instance.policy_document) - new_key_arn_1 = 'arn:aws:kms:eu-west-1:111111111111:key/new_key_no_1' - new_key_arn_2 = 'arn:aws:kms:eu-west-1:111111111111:key/new_key_no_2' + new_key_arn_1 = 'arn:{PARTITION}:kms:eu-west-1:111111111111:key/new_key_no_1' + new_key_arn_2 = 'arn:{PARTITION}:kms:eu-west-1:111111111111:key/new_key_no_2' instance.grant_access_to_kms_keys([ new_key_arn_1, existing_key_arn_1, @@ -350,8 +361,8 @@ def test_update_iam_role_policies_updated(iam_client): policy_doc['Statement'][1]['Resource'] = [ policy_doc['Statement'][1]['Resource'][0], policy_doc['Statement'][1]['Resource'][1], - 'arn:aws:s3:::new_bucket', - 'arn:aws:s3:::new_bucket/*', + f'arn:{PARTITION}:s3:::new_bucket', + f'arn:{PARTITION}:s3:::new_bucket/*', ] policy_doc_json = json.dumps(policy_doc) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/test_slack.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/test_slack.py index b2156d9b4..4d36d2d2c 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/test_slack.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/test_slack.py @@ -2,14 +2,26 @@ # SPDX-License-Identifier: MIT-0 # pylint: skip-file +import os +from boto3.session import Session from pytest import fixture from ..slack import * -from .stubs.slack import ( - stub_approval_event, - stub_failed_pipeline_event, - stub_bootstrap_event, - stub_failed_bootstrap_event, + +REGION = os.getenv("AWS_REGION", "us-east-1") +PARTITION = Session().get_partition_for_region(REGION) + +if PARTITION == "aws-cn": + from .stubs.slack import stub_approval_event_cn as stub_approval_event + from .stubs.slack import stub_failed_pipeline_event_cn as stub_failed_pipeline_event + from .stubs.slack import stub_bootstrap_event_cn as stub_bootstrap_event + from .stubs.slack import stub_failed_bootstrap_event_cn as stub_failed_bootstrap_event +else: + from .stubs.slack import ( + stub_approval_event, + stub_failed_pipeline_event, + stub_bootstrap_event, + stub_failed_bootstrap_event, ) @fixture diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/china-forward-function/handler.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/china-forward-function/handler.py index a800bb488..a78de13c4 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/china-forward-function/handler.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/china-forward-function/handler.py @@ -20,7 +20,7 @@ def lambda_handler(event, context): LOGGER.debug(event) if "source" in event and event["source"] == "aws.organizations": - session = boto3.session.Session(region_name="cn-north-1") + session = boto3.session.Session(region_name="cn-northwest-1") sfn_instance = Stepfunction(session, LOGGER) _, state_name = sfn_instance.invoke_sfn_execution( sfn_arn=SFN_ARN, diff --git a/src/template.yml b/src/template.yml index 15dc8bf79..d36b76f58 100644 --- a/src/template.yml +++ b/src/template.yml @@ -3,7 +3,7 @@ AWSTemplateFormatVersion: "2010-09-09" Transform: "AWS::Serverless-2016-10-31" -Description: ADF CloudFormation Initial Base Stack for the Management Account in the cn-north-1 region. +Description: ADF CloudFormation Initial Base Stack for the Management Account in the base region of the partition (us-east-1, us-gov-west-1 or cn-north-1). Metadata: AWS::ServerlessRepo::Application: @@ -1404,7 +1404,7 @@ Resources: Environment: ComputeType: "BUILD_GENERAL1_LARGE" PrivilegedMode: true - Image: "aws/codebuild/standard:5.0" + Image: "aws/codebuild/standard:7.0" EnvironmentVariables: - Name: ADF_VERSION Value: !FindInMap ["Metadata", "ADF", "Version"] @@ -1441,8 +1441,9 @@ Resources: pre_build: commands: - | - if [ "${AWS_REGION}" = "cn-north-1" ] || [ "${AWS_REGION}" = "cn-northwest-1" ]; then + if [ "${AWS_REGION}" = "cn-north-1" ]; then pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple + npm config set registry https://registry.npmmirror.com fi - >- pip install @@ -1472,7 +1473,7 @@ Resources: --s3-bucket $DEPLOYMENT_ACCOUNT_BUCKET - python adf-build/store_config.py - | - if [ "${AWS_REGION}" = "cn-north-1" ] || [ "${AWS_REGION}" = "cn-northwest-1" ]; then + if [ "${AWS_REGION}" = "cn-north-1" ]; then python adf-build/create_s3_cn.py sam build -t adf-build/cn_northwest_deploy.yml --region cn-northwest-1 sam package --output-template-file adf-build/cn_northwest_deploy.yml --s3-prefix adf-bootstrap --s3-bucket adf-china-bootstrap-cn-northwest-1-${MASTER_ACCOUNT_ID} --region cn-northwest-1 From dfb5483f3885f35a35412266f5ddd6020e01d38a Mon Sep 17 00:00:00 2001 From: Alejandro Portela <46484987+Alexpngal@users.noreply.github.com> Date: Fri, 19 Jan 2024 13:26:02 +0100 Subject: [PATCH 3/4] Fix linting --- docs/admin-guide.md | 80 ++++++++++--------- docs/samples-guide.md | 5 +- .../tests/stubs/stub_iam_cn.py | 2 +- .../adf-build/cn_northwest_bucket.yml | 4 +- .../adf-build/cn_northwest_deploy.yml | 8 +- .../adf-build/create_s3_cn.py | 2 +- .../bootstrap_repository/adf-build/main.py | 2 +- .../adf-build/shared/python/cloudformation.py | 4 +- .../shared/python/tests/test_partition.py | 2 +- src/template.yml | 4 +- 10 files changed, 60 insertions(+), 53 deletions(-) diff --git a/docs/admin-guide.md b/docs/admin-guide.md index 17b9a82f3..c39f1d3b0 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -64,7 +64,8 @@ and defines the general high-level configuration for the AWS Deployment Framework. For Govcloud and China deployments, `adfconfig.yml` file resides on the -[management account](#management-account) CodeCommit Repository (in us-gov-west-1 and cn-north-1 respectively) +[management account](#management-account) CodeCommit Repository +(in us-gov-west-1 and cn-north-1 respectively) The configuration properties are synced into AWS Systems Manager Parameter Store and are used for certain orchestration options throughout your @@ -778,8 +779,9 @@ accounts stay within your organization’s access control guidelines. ADF allows SCPs to be applied in a similar fashion as base stacks. You can define your SCP definition in a file named `scp.json` and place it in a folder that represents your Organizational Unit (or OU/AccountName path if you are -wanting to apply an account-specific SCP) within the `adf-bootstrap` folder from -the `aws-deployment-framework-bootstrap` repository on the management account. +wanting to apply an account-specific SCP) within the `adf-bootstrap` folder +from the `aws-deployment-framework-bootstrap` repository on the management +account. For example, if you have an account named `my_banking_account` under the `banking/dev` OU that needs a specific SCP, and another SCP defined for the @@ -823,8 +825,8 @@ You can define your Tagging Policy definition in a file named `tagging-policy.json` and place it in a folder that represents your Organizational Unit within the `adf-bootstrap` folder from the `aws-deployment-framework-bootstrap` repository on the management account. -Tagging policies can also be applied to a single account using the same approach -described above for SCPs. +Tagging policies can also be applied to a single account using the same +approach described above for SCPs. Tag Policies are available only in an organization that has [all features enabled](https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_org_support-all-features.html). @@ -919,10 +921,11 @@ To determine the current version, follow these steps: ### ADF version you have deployed -To check the current version of ADF that you have deployed, go to the management -account in us-east-1 for global partition deployments. For Govcloud and China -deployments go to us-gov-west-1 and cn-north-1 respectively. Check the CloudFormation -stack output or tag of the `serverlessrepo-aws-deployment-framework` Stack. +To check the current version of ADF that you have deployed, go to the +management account in us-east-1 for global partition deployments. For Govcloud +and China deployments go to us-gov-west-1 and cn-north-1 respectively. Check +the CloudFormation stack output or tag of the +`serverlessrepo-aws-deployment-framework` Stack. - In the outputs tab, it will show the version as the `ADFVersionNumber`. - In the tags on the CloudFormation stack, it is presented as @@ -942,8 +945,9 @@ releases](https://github.com/awslabs/aws-deployment-framework/releases). The `serverlessrepo-aws-deployment-framework` stack is updated through this process with new changes that were included in that release of ADF. -To check the progress in the management account in `us-east-1` for global partition deployments; for Govcloud and China -deployments go to us-gov-west-1 or cn-north-1 respectively, follow these steps: +To check the progress in the management account in `us-east-1` for global +partition deployments; for Govcloud and China deployments go to us-gov-west-1 +or cn-north-1 respectively, follow these steps: 1. Go to the [CloudFormation console](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks?filteringStatus=active&filteringText=serverlessrepo-aws-deployment-framework&viewNested=true&hideStacks=false) @@ -953,14 +957,14 @@ deployments go to us-gov-west-1 or cn-north-1 respectively, follow these steps: with a recent `Updated time` is what you want to see. 4. If it is in progress or if it has not applied the update yet, you can go to the `Events` tab to see what is happening and if any error happened. Use the - refresh button on the top right of the table to retrieve updates on the stack - deployment. + refresh button on the top right of the table to retrieve updates on the + stack deployment. - Once finished, you need to merge the pull request after reviewing the changes - if any are present. Since there might be changes to some of the foundational - aspects of ADF and how it works _(eg CDK Constructs)_. These changes might - need to be applied to the files that live within the _bootstrap_ repository - in your AWS management account too. + Once finished, you need to merge the pull request after reviewing the + changes if any are present. Since there might be changes to some of the + foundational aspects of ADF and how it works _(eg CDK Constructs)_. + These changes might need to be applied to the files that live within + the _bootstrap_ repository in your AWS management account too. To ease this process, the AWS CloudFormation stack will run the _InitialCommit_ Custom CloudFormation resource when updating ADF. @@ -985,8 +989,8 @@ Which branch is used is determined by: Alternatively, you can also perform the update using the AWS CLI. -In the management account in `us-east-1` for global partition deployments; For Govcloud and China -deployments in us-gov-west-1 or cn-north-1 respectively: +In the management account in `us-east-1` for global partition deployments; +For Govcloud and China deployments in us-gov-west-1 or cn-north-1 respectively: 1. Go to the Pull Request section of the `aws-deployment-framework-bootstrap` [CodeCommit @@ -1010,8 +1014,8 @@ deployments go to us-gov-west-1 or cn-north-1 respectively: pull request in the prior step, feel free to 'Release changes' on the pipeline to test it. 3. If any of these steps fail, you can click on the `Details` link to get more - insights into the failure. Please report the step where it failed and include - a copy of the logs when it fails here. + insights into the failure. Please report the step where it failed and + include a copy of the logs when it fails here. The `aws-deployment-framework-bootstrap` pipeline will trigger the account creation and on-boarding process in parallel. @@ -1033,8 +1037,8 @@ trigger the `aws-deployment-framework-pipelines` pipeline in the _deployment account_ in _your main region_: 1. Open your deployment account. -2. Make sure you are in the main deployment region, where all your pipelines are - located. +2. Make sure you are in the main deployment region, where all your pipelines + are located. 3. Go to the CodePipeline console and search for `aws-deployment-framework-pipelines`. 4. This should progress and turn up as green. If any of these steps fail, it @@ -1116,11 +1120,11 @@ the base stack when the account is moved to the Root of the AWS Organization. One thing to keep in mind if you are planning to re-install ADF is that you will want to clean up the parameter from SSM Parameter Store named -_deployment_account_id_ in `us-east-1` on the management account for global -partition deployments; For Govcloud and China deployments go to us-gov-west-1 -or cn-north-1 respectively. AWS Step Functions uses this parameter to determine -if ADF has already got a deployment account setup. If you re-install ADF with -this parameter set to a value, ADF will attempt an assume role to the account +_deployment_account_id_ in `us-east-1` on the management account for global +partition deployments; For Govcloud and China deployments go to us-gov-west-1 +or cn-north-1 respectively. AWS Step Functions uses this parameter to determine +if ADF has already got a deployment account setup. If you re-install ADF with +this parameter set to a value, ADF will attempt an assume role to the account to do some work, which will fail since that role will not be on the account at that point. There is also a CloudFormation stack named `adf-global-base-adf-build` which @@ -1156,9 +1160,9 @@ There are two ways to enable this: to deploy the latest version again, set the `Log Level` to `DEBUG` to get extra logging information about the issue you are experiencing. 2. If you are running an older version of ADF, please navigate to the - CloudFormation Console in `us-east-1` of the AWS Management account for global - partition deployments; For Govcloud and China deployments go to us-gov-west-1 - or cn-north-1 respectively. + CloudFormation Console in `us-east-1` of the AWS Management account for + global partition deployments; For Govcloud and China deployments go to + us-gov-west-1 or cn-north-1 respectively. 3. Update the stack. 4. For any ADF deployment of `v3.2.0` and later, please change the `Log Level` parameter and set it to `DEBUG`. Deploy those changes and revert them after @@ -1173,18 +1177,20 @@ Please trace the failed component and dive into/report the debug information. The main components to look at are: -1. In the AWS Management Account in `us-east-1` for global partition deployments; For Govcloud and China -deployments go to us-gov-west-1 or cn-north-1 respectively: +1. In the AWS Management Account in `us-east-1` for global partition deployments; +For Govcloud and Chinadeployments go to us-gov-west-1 or cn-north-1 respectively: 2. The [CloudFormation aws-deployment-framework stack](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks?filteringStatus=active&filteringText=aws-deployment-framework&viewNested=true&hideStacks=false). 3. The [CloudWatch Logs for the Lambda functions deployed by ADF](https://console.aws.amazon.com/lambda/home?region=us-east-1#/functions?f0=true&n0=false&op=and&v0=ADF). 4. Check if the [CodeCommit pull request](https://console.aws.amazon.com/codesuite/codecommit/repositories/aws-deployment-framework-bootstrap/pull-requests?region=us-east-1&status=OPEN) to install the latest version changes of ADF is merged into your default - branch for the `aws-deployment-framework-bootstrap` (ADF Bootstrap) repository. + branch for the `aws-deployment-framework-bootstrap` (ADF Bootstrap) + repository. 5. The [CodePipeline execution of the AWS Bootstrap pipeline](https://console.aws.amazon.com/codesuite/codepipeline/pipelines/aws-deployment-framework-bootstrap-pipeline/view?region=us-east-1). 6. Navigate to the [AWS Step Functions service](https://us-east-1.console.aws.amazon.com/states/home?region=us-east-1#/statemachines) - in the management account in `us-east-1`for global partition deployments; For Govcloud and China - deployments go to us-gov-west-1 or cn-north-1 respectively, check the state machines named + in the management account in `us-east-1`for global partition deployments; + For Govcloud and China deployments go to us-gov-west-1 or cn-north-1 + respectively, check the state machines named `AccountManagementStateMachine...` and `AccountBootstrappingStateMachine...`. Look at recent executions only. - When you find one that has a failed execution, check the components that diff --git a/docs/samples-guide.md b/docs/samples-guide.md index 937fe66d3..e229f3328 100644 --- a/docs/samples-guide.md +++ b/docs/samples-guide.md @@ -70,8 +70,9 @@ Management Account. By default, there is a `global.yml` in the root of the be appended to as required. If we look at AWS Step Functions in the management account in `us-east-1` -we can see the progress of the bootstrap process for global partition deployments; -For Govcloud and China deployments go to us-gov-west-1 or cn-north-1 respectively. +we can see the progress of the bootstrap process for global partition +deployments; For Govcloud and China deployments go to us-gov-west-1 or +cn-north-1 respectively. ![run-state-machine](./images/run-state-machine.png) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/stubs/stub_iam_cn.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/stubs/stub_iam_cn.py index 81496f1a4..fcdfb9bb2 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/stubs/stub_iam_cn.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/tests/stubs/stub_iam_cn.py @@ -38,4 +38,4 @@ }, ] } -} \ No newline at end of file +} diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/cn_northwest_bucket.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/cn_northwest_bucket.yml index 4c8aefef2..ae0d2bbc5 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/cn_northwest_bucket.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/cn_northwest_bucket.yml @@ -11,7 +11,7 @@ Resources: DeletionPolicy: Delete Type: AWS::S3::Bucket Properties: - BucketName: !Ref BucketName + BucketName: !Ref BucketName AccessControl: BucketOwnerFullControl BucketEncryption: ServerSideEncryptionConfiguration: @@ -23,4 +23,4 @@ Resources: BlockPublicAcls: true BlockPublicPolicy: true IgnorePublicAcls: true - RestrictPublicBuckets: true \ No newline at end of file + RestrictPublicBuckets: true diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/cn_northwest_deploy.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/cn_northwest_deploy.yml index 116fc6a93..31a1d949f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/cn_northwest_deploy.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/cn_northwest_deploy.yml @@ -6,7 +6,7 @@ Transform: 'AWS::Serverless-2016-10-31' Description: ADF CloudFormation Stack for deploy extra resources in China cn-northwest-1. Parameters: - AcoountBootstrapingStateMachineArn: + AccountBootstrapingStateMachineArn: Type: String AdfLogLevel: Type: String @@ -44,7 +44,7 @@ Resources: Statement: - Effect: "Allow" Action: "states:StartExecution" - Resource: !Ref AcoountBootstrapingStateMachineArn + Resource: !Ref AccountBootstrapingStateMachineArn - Effect: Allow Action: - "logs:CreateLogGroup" @@ -65,7 +65,7 @@ Resources: Description: "ADF Lambda Function - Forward events to statemachine" Environment: Variables: - SFN_ARN: !Ref AcoountBootstrapingStateMachineArn + SFN_ARN: !Ref AccountBootstrapingStateMachineArn ADF_LOG_LEVEL: !Ref AdfLogLevel FunctionName: ForwardStateMachineFunction Role: !GetAtt ForwardStateMachineFunctionRole.Arn @@ -80,4 +80,4 @@ Resources: eventSource: - organizations.amazonaws.com eventName: - - MoveAccount \ No newline at end of file + - MoveAccount diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/create_s3_cn.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/create_s3_cn.py index a1de4ece9..8f9822082 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/create_s3_cn.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/create_s3_cn.py @@ -25,7 +25,7 @@ def _create_s3_bucket(bucket_name): 'ParameterKey': 'BucketName', 'ParameterValue': bucket_name, 'UsePreviousValue': False, - }, + }, ] cloudformation = CloudFormation( region=extra_deploy_region, diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py index af92c23f9..4048969ea 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py @@ -424,7 +424,7 @@ def _china_region_extra_deploy(region: str): parameters = [ { - 'ParameterKey': 'AcoountBootstrapingStateMachineArn', + 'ParameterKey': 'AccountBootstrapingStateMachineArn', 'ParameterValue': ACCOUNT_BOOTSTRAPPING_STATE_MACHINE_ARN, 'UsePreviousValue': False, }, diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/cloudformation.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/cloudformation.py index 1d113bbde..2a2d95acf 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/cloudformation.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/cloudformation.py @@ -341,7 +341,7 @@ def _create_change_set(self): cfn_template_map = { "TemplateBody": self._handle_template_path(self.local_template_path) } - else: + else: self.template_url = ( self.template_url if self.template_url is not None @@ -351,7 +351,7 @@ def _create_change_set(self): self.validate_template() cfn_template_map = { "TemplateURL": self.template_url - } + } if cfn_template_map: change_set_params = { diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_partition.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_partition.py index ee8e1457d..2ab0fe045 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_partition.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_partition.py @@ -21,7 +21,7 @@ _china_region = [ 'cn-north-1' - ] +] _incompatible_region = [ 'cn-northwest-1' diff --git a/src/template.yml b/src/template.yml index d36b76f58..18a974d37 100644 --- a/src/template.yml +++ b/src/template.yml @@ -125,8 +125,8 @@ Conditions: - !Sub "${AWS::Region}" - "cn-north-1" NotChinaMainRegion: - "Fn::Not": - - Condition: IsChinaMainRegion + "Fn::Not": + - Condition: IsChinaMainRegion Resources: BootstrapTemplatesBucketPolicy: Type: AWS::S3::BucketPolicy From 99a5466d64d6fb10f937ab5579cd78b78107c9ca Mon Sep 17 00:00:00 2001 From: Alejandro Portela <46484987+Alexpngal@users.noreply.github.com> Date: Fri, 19 Jan 2024 13:32:46 +0100 Subject: [PATCH 4/4] Fix linting --- docs/admin-guide.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/admin-guide.md b/docs/admin-guide.md index c39f1d3b0..d486b82a8 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -1125,7 +1125,8 @@ partition deployments; For Govcloud and China deployments go to us-gov-west-1 or cn-north-1 respectively. AWS Step Functions uses this parameter to determine if ADF has already got a deployment account setup. If you re-install ADF with this parameter set to a value, ADF will attempt an assume role to the account -to do some work, which will fail since that role will not be on the account at that point. +to do some work, which will fail since that role will not be on the account at +that point. There is also a CloudFormation stack named `adf-global-base-adf-build` which lives on the management account in your main deployment region. This stack