From 572d497dd31def3c3366c18a08f6b6c441b69db3 Mon Sep 17 00:00:00 2001 From: pjnkumar Date: Tue, 10 Nov 2020 04:30:56 -0600 Subject: [PATCH 1/6] Issue #232 --- .../CFNUpdateStackSetForAccountsStep.java | 197 +++++++++ .../CFNUpdateStackSetForAccountsStepTest.java | 375 ++++++++++++++++++ 2 files changed, 572 insertions(+) create mode 100644 src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/CFNUpdateStackSetForAccountsStep.java create mode 100644 src/test/java/de/taimos/pipeline/aws/cloudformation/stacksets/CFNUpdateStackSetForAccountsStepTest.java diff --git a/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/CFNUpdateStackSetForAccountsStep.java b/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/CFNUpdateStackSetForAccountsStep.java new file mode 100644 index 00000000..a8e7c212 --- /dev/null +++ b/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/CFNUpdateStackSetForAccountsStep.java @@ -0,0 +1,197 @@ +/* + * - + * #%L + * Pipeline: AWS Steps + * %% + * Copyright (C) 2017 Taimos GmbH + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +package de.taimos.pipeline.aws.cloudformation.stacksets; + +//import com.amazonaws.services.cloudformation.model.*; +import com.amazonaws.services.cloudformation.model.DescribeStackSetResult; +import com.amazonaws.services.cloudformation.model.Parameter; +import com.amazonaws.services.cloudformation.model.StackInstanceSummary; +import com.amazonaws.services.cloudformation.model.StackSetOperationPreferences; +import com.amazonaws.services.cloudformation.model.StackSetStatus; +import com.amazonaws.services.cloudformation.model.Tag; +import com.amazonaws.services.cloudformation.model.UpdateStackSetRequest; +import com.amazonaws.services.cloudformation.model.UpdateStackSetResult; +import com.amazonaws.services.cloudformation.model.CreateStackInstancesResult; +import de.taimos.pipeline.aws.utils.StepUtils; +import hudson.Extension; +import org.jenkinsci.plugins.workflow.steps.StepContext; +import org.jenkinsci.plugins.workflow.steps.StepDescriptor; +import org.jenkinsci.plugins.workflow.steps.StepExecution; +import org.kohsuke.stapler.DataBoundConstructor; +import org.kohsuke.stapler.DataBoundSetter; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class CFNUpdateStackSetForAccountsStep extends AbstractCFNCreateStackSetStep { + + @DataBoundConstructor + public CFNUpdateStackSetForAccountsStep(String stackSet) { + super(stackSet); + } + + private StackSetOperationPreferences operationPreferences; + private BatchingOptions batchingOptions; + private Collection accounts; + private Collection regions; + + public StackSetOperationPreferences getOperationPreferences() { + return operationPreferences; + } + + @DataBoundSetter + public void setOperationPreferences(JenkinsStackSetOperationPreferences operationPreferences) { + this.operationPreferences = operationPreferences; + } + + @DataBoundSetter + public void setBatchingOptions(BatchingOptions batchingOptions) { + this.batchingOptions = batchingOptions; + } + + @DataBoundSetter + public void setAccounts(Collection accounts) { + this.accounts = accounts; + } + public Collection getAccounts () {return this.accounts;} + @DataBoundSetter + public void setRegions(Collection regions) { + this.regions = regions; + } + public Collection getRegions () {return this.regions; } + + @Extension + public static class DescriptorImpl extends StepDescriptor { + + @Override + public String getFunctionName() { + return "cfnUpdateStackSetForAccounts"; + } + + @Override + public String getDisplayName() { + return "Create or Update CloudFormation StackSet for specific accounts and regions input"; + } + + @Override + public Set> getRequiredContext() { + return StepUtils.requiresDefault(); + } + } + + @Override + public StepExecution start(StepContext context) throws Exception { + return new CFNUpdateStackSetForAccountsStep.Execution(this, context); + } + + public static class Execution extends AbstractCFNCreateStackSetStep.Execution { + + protected Execution(CFNUpdateStackSetForAccountsStep step, @Nonnull StepContext context) { + super(step, context); + } + + @Override + public void checkPreconditions() { + // Nothing to check here + } + + @Override + public String getThreadName() { + return "cfnUpdateStackSetForAccounts-" + getStep().getStackSet(); + } + + @Override + public DescribeStackSetResult whenStackSetExists(Collection parameters, Collection tags) throws Exception { + final String url = this.getStep().getUrl(); + CloudFormationStackSet cfnStackSet = this.getCfnStackSet(); + UpdateStackSetRequest req = new UpdateStackSetRequest() + .withParameters(parameters) + .withAdministrationRoleARN(this.getStep().getAdministratorRoleArn()) + .withExecutionRoleName(this.getStep().getExecutionRoleName()) + .withOperationPreferences(this.getStep().getOperationPreferences()) + .withTags(tags); + if (this.getStep().getRegions().isEmpty() || this.getStep().getAccounts().isEmpty()) { + throw new Exception("Either region list or accounts list are empty. Update stackset by account must supply both "); + } + + if (this.getStep().batchingOptions != null && this.getStep().batchingOptions.isRegions()) { + this.getListener().getLogger().println("Batching updates by region"); + List summaries = cfnStackSet.findStackSetInstances(); + Map> batches = summaries.stream().collect(Collectors.groupingBy(StackInstanceSummary::getRegion)); + for (Map.Entry> entry : batches.entrySet()) { + if (this.getStep().getRegions().contains(entry.getKey())) //Naresh + { + this.getListener().getLogger().format("Updating stack set update batch for region=%s %n", entry.getKey()); + this.getListener().getLogger().format("Making sure all accounts passed are present in StackSet"); + List accountsInStackSet = entry.getValue().stream().map(StackInstanceSummary::getAccount).collect(Collectors.toList()); + List accountToPass = new ArrayList(); + for (String account:this.getStep().getAccounts()) + { + if (accountsInStackSet.contains(account)){ + accountToPass.add(account); + } + } + if (!accountToPass.isEmpty()) { + UpdateStackSetResult operation = cfnStackSet.update(this.getStep().readTemplate(this), url, req.clone() + .withRegions(entry.getKey()) + .withAccounts(accountToPass) + ); + cfnStackSet.waitForOperationToComplete(operation.getOperationId(), getStep().getPollConfiguration().getPollInterval()); + this.getListener().getLogger().format("Updated stack set update batch for region=%s %n", entry.getKey()); + } + } + } + } else { + UpdateStackSetResult operation = cfnStackSet.update(this.getStep().readTemplate(this), url, req.clone() + .withAccounts(this.getStep().getAccounts()) + .withRegions(this.getStep().getRegions())); + cfnStackSet.waitForOperationToComplete(operation.getOperationId(), this.getStep().getPollConfiguration().getPollInterval()); + } + return cfnStackSet.describe(); + } + + + @Override + public DescribeStackSetResult whenStackSetMissing(Collection parameters, Collection tags) throws Exception { + final String url = getStep().getUrl(); + CloudFormationStackSet cfnStackSet = this.getCfnStackSet(); + cfnStackSet.create(this.getStep().readTemplate(this), url, parameters, tags, + this.getStep().getAdministratorRoleArn(), this.getStep().getExecutionRoleName()); + DescribeStackSetResult describeResult = cfnStackSet.waitForStackState(StackSetStatus.ACTIVE, getStep().getPollConfiguration().getPollInterval()); + this.getListener().getLogger().format("Creation of Stackset - %s", describeResult.toString()); + CreateStackInstancesResult stackInstanceCreate = cfnStackSet.createStackInstances(this.getStep().getAccounts(), this.getStep().getRegions()); + String getOpsId = stackInstanceCreate.getOperationId(); + java.time.Duration dur = getStep().getPollConfiguration().getPollInterval(); + cfnStackSet.waitForOperationToComplete(getOpsId,dur); + return cfnStackSet.describe(); + } + + private static final long serialVersionUID = 1L; + + } + +} diff --git a/src/test/java/de/taimos/pipeline/aws/cloudformation/stacksets/CFNUpdateStackSetForAccountsStepTest.java b/src/test/java/de/taimos/pipeline/aws/cloudformation/stacksets/CFNUpdateStackSetForAccountsStepTest.java new file mode 100644 index 00000000..5d0ce868 --- /dev/null +++ b/src/test/java/de/taimos/pipeline/aws/cloudformation/stacksets/CFNUpdateStackSetForAccountsStepTest.java @@ -0,0 +1,375 @@ +package de.taimos.pipeline.aws.cloudformation.stacksets; + +import com.amazonaws.client.builder.AwsSyncClientBuilder; +import com.amazonaws.services.cloudformation.AmazonCloudFormation; +import com.amazonaws.services.cloudformation.model.*; +import de.taimos.pipeline.aws.AWSClientFactory; +import hudson.EnvVars; +import hudson.model.TaskListener; +import lombok.Builder; +import lombok.Value; +import org.assertj.core.api.Assertions; +import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; +import org.jenkinsci.plugins.workflow.job.WorkflowJob; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.jvnet.hudson.test.JenkinsRule; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatcher; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.internal.stubbing.BaseStubbing; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.io.PrintWriter; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.BiConsumer; +import java.util.function.BinaryOperator; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collector; +import java.util.stream.Collectors; + +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; + +@RunWith(PowerMockRunner.class) +@PrepareForTest( + value = AWSClientFactory.class, + fullyQualifiedNames = "de.taimos.pipeline.aws.cloudformation.stacksets.*" +) +@PowerMockIgnore("javax.crypto.*") +public class CFNUpdateStackSetForAccountsStepTest { + + @Rule + private JenkinsRule jenkinsRule = new JenkinsRule(); + private CloudFormationStackSet stackSet; + + @Before + public void setupSdk() throws Exception { + stackSet = Mockito.mock(CloudFormationStackSet.class); + PowerMockito.mockStatic(AWSClientFactory.class); + PowerMockito.whenNew(CloudFormationStackSet.class) + .withAnyArguments() + .thenReturn(stackSet); + AmazonCloudFormation cloudFormation = Mockito.mock(AmazonCloudFormation.class); + PowerMockito.when(AWSClientFactory.create(Mockito.any(AwsSyncClientBuilder.class), Mockito.any(EnvVars.class))) + .thenReturn(cloudFormation); + } + + @Test + public void createNonExistantStack() throws Exception { + WorkflowJob job = jenkinsRule.jenkins.createProject(WorkflowJob.class, "createNonExistantStack"); + Mockito.when(stackSet.exists()).thenReturn(false); + DescribeStackSetResult stackSetResult = Mockito.mock(DescribeStackSetResult.class); + Mockito.when(stackSet.waitForStackState(StackSetStatus.ACTIVE,java.time.Duration.ofMillis(25))).thenReturn(stackSetResult); + CreateStackInstancesResult createResult = Mockito.mock(CreateStackInstancesResult.class); + Mockito.when(createResult.getOperationId()).thenReturn("SomeId"); + StackSetOperation stackSetOps = Mockito.mock(StackSetOperation.class); + DescribeStackSetOperationResult mockResult = Mockito.mock(DescribeStackSetOperationResult.class); + Mockito.when(stackSet.createStackInstances(Mockito.anyCollectionOf(String.class), Mockito.anyCollectionOf(String.class))).thenReturn(new CreateStackInstancesResult().withOperationId("Mockito")); + Mockito.when(stackSet.waitForOperationToComplete("Mockito", java.time.Duration.ofMillis(25))).thenReturn(mockResult); + job.setDefinition(new CpsFlowDefinition("" + + "node {\n" + + " cfnUpdateStackSetForAccounts(stackSet: 'foo', pollInterval: 25, accounts:['123'], regions: ['us-east-1'])" + + "}\n", true) + ); + jenkinsRule.assertBuildStatusSuccess(job.scheduleBuild2(0)); + + PowerMockito.verifyNew(CloudFormationStackSet.class, Mockito.atLeastOnce()) + .withArguments( + Mockito.any(AmazonCloudFormation.class), + Mockito.eq("foo"), + Mockito.any(TaskListener.class), + Mockito.eq(SleepStrategy.EXPONENTIAL_BACKOFF_STRATEGY) + ); + Mockito.verify(stackSet).create(Mockito.anyString(), Mockito.anyString(), Mockito.anyCollectionOf(Parameter.class), Mockito.anyCollectionOf(Tag.class), Mockito.isNull(String.class), Mockito.isNull(String.class)); + } + + @Test + public void createNonExistantStackWithCustomAdminArn() throws Exception { + WorkflowJob job = jenkinsRule.jenkins.createProject(WorkflowJob.class, "createNonExistantStackWithCustomAdminArn"); + Mockito.when(stackSet.exists()).thenReturn(false); + DescribeStackSetResult stackSetResult = Mockito.mock(DescribeStackSetResult.class); + Mockito.when(stackSet.waitForStackState(StackSetStatus.ACTIVE,java.time.Duration.ofMillis(25))).thenReturn(stackSetResult); + CreateStackInstancesResult createResult = Mockito.mock(CreateStackInstancesResult.class); + Mockito.when(createResult.getOperationId()).thenReturn("SomeId"); + StackSetOperation stackSetOps = Mockito.mock(StackSetOperation.class); + DescribeStackSetOperationResult mockResult = Mockito.mock(DescribeStackSetOperationResult.class); + Mockito.when(stackSet.createStackInstances(Mockito.anyCollectionOf(String.class), Mockito.anyCollectionOf(String.class))).thenReturn(new CreateStackInstancesResult().withOperationId("Mockito")); + Mockito.when(stackSet.waitForOperationToComplete("Mockito", java.time.Duration.ofMillis(25))).thenReturn(mockResult); + job.setDefinition(new CpsFlowDefinition("" + + "node {\n" + + " cfnUpdateStackSetForAccounts(stackSet: 'foo', administratorRoleArn: 'bar', executionRoleName: 'baz', pollInterval: 25, accounts:['123'], regions: ['us-east-1'])" + + "}\n", true) + ); + jenkinsRule.assertBuildStatusSuccess(job.scheduleBuild2(0)); + + PowerMockito.verifyNew(CloudFormationStackSet.class, Mockito.atLeastOnce()) + .withArguments( + Mockito.any(AmazonCloudFormation.class), + Mockito.eq("foo"), + Mockito.any(TaskListener.class), + Mockito.eq(SleepStrategy.EXPONENTIAL_BACKOFF_STRATEGY) + ); + Mockito.verify(stackSet).create(Mockito.anyString(), Mockito.anyString(), Mockito.anyCollectionOf(Parameter.class), Mockito.anyCollectionOf(Tag.class), Mockito.eq("bar"), Mockito.eq("baz")); + } + + @Test + public void updateExistantStack() throws Exception { + WorkflowJob job = jenkinsRule.jenkins.createProject(WorkflowJob.class, "updateExistantStack"); + Mockito.when(stackSet.exists()).thenReturn(true); + DescribeStackSetResult stackSetResult = Mockito.mock(DescribeStackSetResult.class); + Mockito.when(stackSet.waitForStackState(StackSetStatus.ACTIVE,java.time.Duration.ofMillis(25))).thenReturn(stackSetResult); + CreateStackInstancesResult createResult = Mockito.mock(CreateStackInstancesResult.class); + Mockito.when(createResult.getOperationId()).thenReturn("Mockito"); + StackSetOperation stackSetOps = Mockito.mock(StackSetOperation.class); + DescribeStackSetOperationResult mockResult = Mockito.mock(DescribeStackSetOperationResult.class); + Mockito.when(stackSet.createStackInstances(Mockito.anyCollectionOf(String.class), Mockito.anyCollectionOf(String.class))).thenReturn(new CreateStackInstancesResult().withOperationId("Mockito")); + Mockito.when(stackSet.waitForOperationToComplete("Mockito", java.time.Duration.ofMillis(25))).thenReturn(mockResult); + Mockito.when(stackSet.update(Mockito.anyString(), Mockito.anyString(), Mockito.any(UpdateStackSetRequest.class))) + .thenReturn(new UpdateStackSetResult() + .withOperationId("Mockito") + ); + + job.setDefinition(new CpsFlowDefinition("" + + "node {\n" + + " cfnUpdateStackSetForAccounts(stackSet: 'foo', pollInterval: 25, accounts:['123'], regions: ['us-east-1'], params: ['foo=bar'], paramsFile: 'params.json')" + + "}\n", true) + ); + try (PrintWriter writer = new PrintWriter(jenkinsRule.jenkins.getWorkspaceFor(job).child("params.json").write())) { + writer.println("[{\"ParameterKey\": \"foo1\", \"ParameterValue\": \"25\"}]"); + } + jenkinsRule.assertBuildStatusSuccess(job.scheduleBuild2(0)); + + PowerMockito.verifyNew(CloudFormationStackSet.class, Mockito.atLeastOnce()) + .withArguments( + Mockito.any(AmazonCloudFormation.class), + Mockito.eq("foo"), + Mockito.any(TaskListener.class), + Mockito.eq(SleepStrategy.EXPONENTIAL_BACKOFF_STRATEGY) + ); + ArgumentCaptor requestCapture = ArgumentCaptor.forClass(UpdateStackSetRequest.class); + Mockito.verify(stackSet).update(Mockito.anyString(), Mockito.anyString(), requestCapture.capture()); + Assertions.assertThat(requestCapture.getValue().getParameters()).containsExactlyInAnyOrder( + new Parameter() + .withParameterKey("foo") + .withParameterValue("bar"), + new Parameter() + .withParameterKey("foo1") + .withParameterValue("25") + ); + + Mockito.verify(stackSet).waitForOperationToComplete("Mockito", Duration.ofMillis(25)); + } + + @Test + public void updateExistingStackStackSetWithOperationPreferences() throws Exception { + WorkflowJob job = jenkinsRule.jenkins.createProject(WorkflowJob.class, "updateExistingStackSetWithOperationPreferences"); + Mockito.when(stackSet.exists()).thenReturn(true); + DescribeStackSetResult stackSetResult = Mockito.mock(DescribeStackSetResult.class); + Mockito.when(stackSet.waitForStackState(StackSetStatus.ACTIVE,java.time.Duration.ofMillis(25))).thenReturn(stackSetResult); + CreateStackInstancesResult createResult = Mockito.mock(CreateStackInstancesResult.class); + Mockito.when(createResult.getOperationId()).thenReturn("Mockito"); + StackSetOperation stackSetOps = Mockito.mock(StackSetOperation.class); + DescribeStackSetOperationResult mockResult = Mockito.mock(DescribeStackSetOperationResult.class); + Mockito.when(stackSet.createStackInstances(Mockito.anyCollectionOf(String.class), Mockito.anyCollectionOf(String.class))).thenReturn(new CreateStackInstancesResult().withOperationId("Mockito")); + Mockito.when(stackSet.waitForOperationToComplete("Mockito", java.time.Duration.ofMillis(25))).thenReturn(mockResult); + Mockito.when(stackSet.update(Mockito.anyString(), Mockito.anyString(), Mockito.any(UpdateStackSetRequest.class))) + .thenReturn(new UpdateStackSetResult() + .withOperationId("Mockito") + ); + String operationId = "Mockito"; + Mockito.when(stackSet.update(Mockito.anyString(), Mockito.anyString(), Mockito.any(UpdateStackSetRequest.class))) + .thenReturn(new UpdateStackSetResult() + .withOperationId(operationId) + ); + job.setDefinition(new CpsFlowDefinition("" + + "node {\n" + + " cfnUpdateStackSetForAccounts(stackSet: 'foo', pollInterval: 25, accounts:['123'], regions: ['us-east-1'], operationPreferences: [failureToleranceCount: 5, regionOrder: ['us-west-2'], failureTolerancePercentage: 17, maxConcurrentCount: 18, maxConcurrentPercentage: 34])" + + "}\n", true) + ); + jenkinsRule.assertBuildStatusSuccess(job.scheduleBuild2(0)); + + PowerMockito.verifyNew(CloudFormationStackSet.class, Mockito.atLeastOnce()) + .withArguments( + Mockito.any(AmazonCloudFormation.class), + Mockito.eq("foo"), + Mockito.any(TaskListener.class), + Mockito.eq(SleepStrategy.EXPONENTIAL_BACKOFF_STRATEGY) + ); + ArgumentCaptor requestCapture = ArgumentCaptor.forClass(UpdateStackSetRequest.class); + Mockito.verify(stackSet).update(Mockito.anyString(), Mockito.anyString(), requestCapture.capture()); + + Assertions.assertThat(requestCapture.getValue().getOperationPreferences()).isEqualTo(new StackSetOperationPreferences() + .withFailureToleranceCount(5) + .withRegionOrder("us-west-2") + .withFailureTolerancePercentage(17) + .withMaxConcurrentCount(18) + .withMaxConcurrentPercentage(34) + ); + + Mockito.verify(stackSet).waitForOperationToComplete(operationId, Duration.ofMillis(25)); + } + + @Test + public void updateExistingStackWithCustomAdminRole() throws Exception { + WorkflowJob job = jenkinsRule.jenkins.createProject(WorkflowJob.class, "updateExistingStackWithCustomAdminRole"); + Mockito.when(stackSet.exists()).thenReturn(true); + DescribeStackSetResult stackSetResult = Mockito.mock(DescribeStackSetResult.class); + Mockito.when(stackSet.waitForStackState(StackSetStatus.ACTIVE,java.time.Duration.ofMillis(25))).thenReturn(stackSetResult); + CreateStackInstancesResult createResult = Mockito.mock(CreateStackInstancesResult.class); + Mockito.when(createResult.getOperationId()).thenReturn("Mockito"); + StackSetOperation stackSetOps = Mockito.mock(StackSetOperation.class); + DescribeStackSetOperationResult mockResult = Mockito.mock(DescribeStackSetOperationResult.class); + Mockito.when(stackSet.createStackInstances(Mockito.anyCollectionOf(String.class), Mockito.anyCollectionOf(String.class))).thenReturn(new CreateStackInstancesResult().withOperationId("Mockito")); + Mockito.when(stackSet.waitForOperationToComplete("Mockito", java.time.Duration.ofMillis(25))).thenReturn(mockResult); + Mockito.when(stackSet.update(Mockito.anyString(), Mockito.anyString(), Mockito.any(UpdateStackSetRequest.class))) + .thenReturn(new UpdateStackSetResult() + .withOperationId("Mockito") + ); + String operationId = "Mockito"; + Mockito.when(stackSet.update(Mockito.anyString(), Mockito.anyString(), Mockito.any(UpdateStackSetRequest.class))) + .thenReturn(new UpdateStackSetResult() + .withOperationId(operationId) + ); + job.setDefinition(new CpsFlowDefinition("" + + "node {\n" + + " cfnUpdateStackSetForAccounts(stackSet: 'foo', administratorRoleArn: 'bar', executionRoleName: 'baz', pollInterval: 25, accounts:['123'], regions: ['us-east-1'])" + + "}\n", true) + ); + jenkinsRule.assertBuildStatusSuccess(job.scheduleBuild2(0)); + + PowerMockito.verifyNew(CloudFormationStackSet.class, Mockito.atLeastOnce()) + .withArguments( + Mockito.any(AmazonCloudFormation.class), + Mockito.eq("foo"), + Mockito.any(TaskListener.class), + Mockito.eq(SleepStrategy.EXPONENTIAL_BACKOFF_STRATEGY) + ); + Mockito.verify(stackSet).update(Mockito.anyString(), Mockito.anyString(), Mockito.any(UpdateStackSetRequest.class)); + } + + @Test + public void doNotCreateNonExistantStack() throws Exception { + WorkflowJob job = jenkinsRule.jenkins.createProject(WorkflowJob.class, "doNotCreateNonExistantStack"); + Mockito.when(stackSet.exists()).thenReturn(false); + DescribeStackSetResult stackSetResult = Mockito.mock(DescribeStackSetResult.class); + Mockito.when(stackSet.waitForStackState(StackSetStatus.ACTIVE,java.time.Duration.ofMillis(25))).thenReturn(stackSetResult); + CreateStackInstancesResult createResult = Mockito.mock(CreateStackInstancesResult.class); + Mockito.when(createResult.getOperationId()).thenReturn("Mockito"); + StackSetOperation stackSetOps = Mockito.mock(StackSetOperation.class); + DescribeStackSetOperationResult mockResult = Mockito.mock(DescribeStackSetOperationResult.class); + Mockito.when(stackSet.createStackInstances(Mockito.anyCollectionOf(String.class), Mockito.anyCollectionOf(String.class))).thenReturn(new CreateStackInstancesResult().withOperationId("Mockito")); + Mockito.when(stackSet.waitForOperationToComplete("Mockito", java.time.Duration.ofMillis(25))).thenReturn(mockResult); + Mockito.when(stackSet.update(Mockito.anyString(), Mockito.anyString(), Mockito.any(UpdateStackSetRequest.class))) + .thenReturn(new UpdateStackSetResult() + .withOperationId("Mockito") + ); + String operationId = "Mockito"; + Mockito.when(stackSet.update(Mockito.anyString(), Mockito.anyString(), Mockito.any(UpdateStackSetRequest.class))) + .thenReturn(new UpdateStackSetResult() + .withOperationId(operationId) + ); + job.setDefinition(new CpsFlowDefinition("" + + "node {\n" + + " cfnUpdateStackSetForAccounts(stackSet: 'foo', create: false, pollInterval: 25, accounts:['123'], regions: ['us-east-1'])" + + "}\n", true) + ); + jenkinsRule.assertBuildStatusSuccess(job.scheduleBuild2(0)); + + PowerMockito.verifyNew(CloudFormationStackSet.class, Mockito.atLeastOnce()) + .withArguments( + Mockito.any(AmazonCloudFormation.class), + Mockito.eq("foo"), + Mockito.any(TaskListener.class), + Mockito.eq(SleepStrategy.EXPONENTIAL_BACKOFF_STRATEGY) + ); + Mockito.verify(stackSet, Mockito.never()).create(Mockito.anyString(), Mockito.anyString(), Mockito.anyCollectionOf(Parameter.class), Mockito.anyCollectionOf(Tag.class), Mockito.isNull(String.class), Mockito.isNull(String.class)); + } + + @Test + public void updateWithRegionBatches() throws Exception { + WorkflowJob job = jenkinsRule.jenkins.createProject(WorkflowJob.class, "updateWithRegionBatches"); + Mockito.when(stackSet.exists()).thenReturn(true); + DescribeStackSetResult stackSetResult = Mockito.mock(DescribeStackSetResult.class); + Mockito.when(stackSet.waitForStackState(StackSetStatus.ACTIVE,java.time.Duration.ofMillis(25))).thenReturn(stackSetResult); + CreateStackInstancesResult createResult = Mockito.mock(CreateStackInstancesResult.class); + Mockito.when(createResult.getOperationId()).thenReturn("Mockito"); + StackSetOperation stackSetOps = Mockito.mock(StackSetOperation.class); + DescribeStackSetOperationResult mockResult = Mockito.mock(DescribeStackSetOperationResult.class); + Mockito.when(stackSet.createStackInstances(Mockito.anyCollectionOf(String.class), Mockito.anyCollectionOf(String.class))).thenReturn(new CreateStackInstancesResult().withOperationId("Mockito")); + Mockito.when(stackSet.waitForOperationToComplete("Mockito", java.time.Duration.ofMillis(25))).thenReturn(mockResult); + Mockito.when(stackSet.update(Mockito.anyString(), Mockito.anyString(), Mockito.any(UpdateStackSetRequest.class))) + .thenReturn(new UpdateStackSetResult() + .withOperationId("Mockito") + ); + String operationId = "Mockito"; + Mockito.when(stackSet.update(Mockito.anyString(), Mockito.anyString(), Mockito.any(UpdateStackSetRequest.class))) + .thenReturn(new UpdateStackSetResult() + .withOperationId(operationId) + ); + Mockito.when(stackSet.findStackSetInstances()).thenReturn(asList( + new StackInstanceSummary().withAccount("a1").withRegion("r1"), + new StackInstanceSummary().withAccount("a2").withRegion("r1"), + new StackInstanceSummary().withAccount("a2").withRegion("r2"), + new StackInstanceSummary().withAccount("a3").withRegion("r3") + )); + job.setDefinition(new CpsFlowDefinition("" + + "node {\n" + + " cfnUpdateStackSetForAccounts(stackSet: 'foo'," + + " pollInterval: 25, accounts:['a1','a2', 'a3'], regions: ['r1','r2','r3'], " + + " batchingOptions: [" + + " regions: true" + + " ]" + + " )" + + "}\n", true) + ); + jenkinsRule.assertBuildStatusSuccess(job.scheduleBuild2(0)); + + PowerMockito.verifyNew(CloudFormationStackSet.class, Mockito.atLeastOnce()) + .withArguments( + Mockito.any(AmazonCloudFormation.class), + Mockito.eq("foo"), + Mockito.any(TaskListener.class), + Mockito.eq(SleepStrategy.EXPONENTIAL_BACKOFF_STRATEGY) + ); + ArgumentCaptor requestCapture = ArgumentCaptor.forClass(UpdateStackSetRequest.class); + Mockito.verify(stackSet, Mockito.times(3)).update(Mockito.anyString(), Mockito.anyString(), requestCapture.capture()); + Map> capturedRegionAccounts = requestCapture.getAllValues() + .stream() + .flatMap(summary -> summary.getRegions() + .stream() + .flatMap(region -> summary.getAccounts().stream() + .map(accountId -> RegionAccountIdTuple.builder().accountId(accountId).region(region).build()) + )) + .collect(Collectors.groupingBy(RegionAccountIdTuple::getRegion, Collectors.mapping(RegionAccountIdTuple::getAccountId, Collectors.toList()))); + Assertions.assertThat(capturedRegionAccounts).containsAllEntriesOf(new HashMap>() { + { + put("r1", asList("a1", "a2")); + put("r2", singletonList("a2")); + put("r3", singletonList("a3")); + } + }); + + Mockito.verify(stackSet, Mockito.times(3)).waitForOperationToComplete(Mockito.any(), Mockito.any()); + } + + @Value + @Builder + private static class RegionAccountIdTuple { + String region, accountId; + } +} From 55d3493ad91e7b8247fa0f7eb683064b2c0086f4 Mon Sep 17 00:00:00 2001 From: pjnkumar Date: Tue, 10 Nov 2020 04:47:51 -0600 Subject: [PATCH 2/6] Minor updates --- .../stacksets/CFNUpdateStackSetForAccountsStep.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/CFNUpdateStackSetForAccountsStep.java b/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/CFNUpdateStackSetForAccountsStep.java index a8e7c212..00359fd4 100644 --- a/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/CFNUpdateStackSetForAccountsStep.java +++ b/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/CFNUpdateStackSetForAccountsStep.java @@ -182,7 +182,7 @@ public DescribeStackSetResult whenStackSetMissing(Collection paramete cfnStackSet.create(this.getStep().readTemplate(this), url, parameters, tags, this.getStep().getAdministratorRoleArn(), this.getStep().getExecutionRoleName()); DescribeStackSetResult describeResult = cfnStackSet.waitForStackState(StackSetStatus.ACTIVE, getStep().getPollConfiguration().getPollInterval()); - this.getListener().getLogger().format("Creation of Stackset - %s", describeResult.toString()); + this.getListener().getLogger().format("Creation of Stackset completed"); CreateStackInstancesResult stackInstanceCreate = cfnStackSet.createStackInstances(this.getStep().getAccounts(), this.getStep().getRegions()); String getOpsId = stackInstanceCreate.getOperationId(); java.time.Duration dur = getStep().getPollConfiguration().getPollInterval(); From 7e22f2dc4b7f55cfa92142c73aa90d66be971704 Mon Sep 17 00:00:00 2001 From: pjnkumar Date: Tue, 10 Nov 2020 12:44:01 -0600 Subject: [PATCH 3/6] Adding files missed to fix build errors --- .../stacksets/AbstractCFNCreateStackSetStep.java | 1 + .../cloudformation/stacksets/CloudFormationStackSet.java | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/AbstractCFNCreateStackSetStep.java b/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/AbstractCFNCreateStackSetStep.java index fa4bde92..d3eeaf6d 100644 --- a/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/AbstractCFNCreateStackSetStep.java +++ b/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/AbstractCFNCreateStackSetStep.java @@ -42,6 +42,7 @@ import javax.annotation.Nonnull; import java.io.IOException; import java.util.Collection; +import java.util.Objects; abstract class AbstractCFNCreateStackSetStep extends TemplateStepBase { diff --git a/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/CloudFormationStackSet.java b/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/CloudFormationStackSet.java index 7a6f6877..e145f17a 100644 --- a/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/CloudFormationStackSet.java +++ b/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/CloudFormationStackSet.java @@ -44,7 +44,8 @@ import com.amazonaws.services.cloudformation.model.UpdateStackSetRequest; import com.amazonaws.services.cloudformation.model.UpdateStackSetResult; import hudson.model.TaskListener; - +import com.amazonaws.services.cloudformation.model.CreateStackInstancesResult; +import com.amazonaws.services.cloudformation.model.CreateStackInstancesRequest; import java.time.Duration; import java.util.ArrayList; import java.util.Collection; @@ -99,7 +100,13 @@ public CreateStackSetResult create(String templateBody, String templateUrl, Coll this.listener.getLogger().println("Created Stack set stackSetId=" + result.getStackSetId()); return result; } + public CreateStackInstancesResult createStackInstances (Collection accounts, Collection regions) { + CreateStackInstancesRequest req = new CreateStackInstancesRequest().withStackSetName(this.stackSet).withAccounts(accounts).withRegions(regions); + CreateStackInstancesResult result = this.client.createStackInstances(req); + this.listener.getLogger().println("Created Stack set instances"); + return result; + } DescribeStackSetResult waitForStackState(StackSetStatus expectedStatus, Duration pollInterval) throws InterruptedException { DescribeStackSetResult result = describe(); this.listener.getLogger().println("stackSetId=" + result.getStackSet().getStackSetId() + " status=" + result.getStackSet().getStatus()); From 5d84296c313b68bdeda1a3b6342ee122407457c9 Mon Sep 17 00:00:00 2001 From: pjnkumar Date: Tue, 10 Nov 2020 13:11:48 -0600 Subject: [PATCH 4/6] Fixing unused variable style error --- .../stacksets/CFNUpdateStackSetForAccountsStep.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/CFNUpdateStackSetForAccountsStep.java b/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/CFNUpdateStackSetForAccountsStep.java index 00359fd4..4185f96f 100644 --- a/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/CFNUpdateStackSetForAccountsStep.java +++ b/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/CFNUpdateStackSetForAccountsStep.java @@ -181,7 +181,7 @@ public DescribeStackSetResult whenStackSetMissing(Collection paramete CloudFormationStackSet cfnStackSet = this.getCfnStackSet(); cfnStackSet.create(this.getStep().readTemplate(this), url, parameters, tags, this.getStep().getAdministratorRoleArn(), this.getStep().getExecutionRoleName()); - DescribeStackSetResult describeResult = cfnStackSet.waitForStackState(StackSetStatus.ACTIVE, getStep().getPollConfiguration().getPollInterval()); + cfnStackSet.waitForStackState(StackSetStatus.ACTIVE, getStep().getPollConfiguration().getPollInterval()); this.getListener().getLogger().format("Creation of Stackset completed"); CreateStackInstancesResult stackInstanceCreate = cfnStackSet.createStackInstances(this.getStep().getAccounts(), this.getStep().getRegions()); String getOpsId = stackInstanceCreate.getOperationId(); From 70639fb6d4560eee3544497cc81881dd40b929dc Mon Sep 17 00:00:00 2001 From: pjnkumar Date: Thu, 12 Nov 2020 22:05:25 -0600 Subject: [PATCH 5/6] UpdatedDocumentationAndFixedUnusedImport --- README.md | 26 +++++++++++++++++++ .../AbstractCFNCreateStackSetStep.java | 1 - 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index de6ea64d..1c840f23 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ This plugins adds Jenkins pipeline steps to interact with the AWS API. * [cfnCreateChangeSet](#cfncreatechangeset) * [cfnExecuteChangeSet](#cfnexecutechangeset) * [cfnUpdateStackSet](#cfnupdatestackset) +* [cfnUpdateStackSetForAccounts] (#cfnupdatestacksetforaccounts) * [cfnDeleteStackSet](#cfndeletestackset) * [snsPublish](#snspublish) * [deployAPI](#deployapi) @@ -562,6 +563,31 @@ To automatically batch via region (find all stack instances, group them by regio ```groovy cfnUpdateStackSet(stackSet:'myStackSet', url:'https://s3.amazonaws.com/my-templates-bucket/template.yaml', batchingOptions: [regions: true]) ``` +## cfnUpdateStackSetForAccounts + +Create a stack set with Accounts and Regions. Other than the mandatory accounts and regions parameters, the function is identical to cfnUpdateStackSet function above. Function will only create stack instances if the stackset does not exist. Otherwise, it will only try to update the specified account/region stackinstances if they already exist in the StackSet. Will monitor the resulting StackSet operation and will fail the build step if the operation does not complete successfully. + +To prevent running into rate limiting on the AWS API you can change the default polling interval of 1000 ms using the parameter `pollIntervall`. Using the value `0` disables event printing. + +```groovy + cfnUpdateStackSetForAccounts(stackSet:'myStackSet', url:'https://s3.amazonaws.com/my-templates-bucket/template.yaml', accounts:["012345678901","012345678902" ], regions:["us-east-1","us-east-2"]) +``` + +To set a custom administrator role ARN: +```groovy + cfnUpdateStackSet(stackSet:'myStackSet', url:'https://s3.amazonaws.com/my-templates-bucket/template.yaml', administratorRoleArn: 'mycustomarn', accounts:["012345678901","012345678902" ], regions:["us-east-1","us-east-2"]) +``` + +To set a operation preferences: +```groovy + cfnUpdateStackSet(stackSet:'myStackSet', url:'https://s3.amazonaws.com/my-templates-bucket/template.yaml', operationPreferences: [failureToleranceCount: 5], accounts:["012345678901","012345678902" ], regions:["us-east-1","us-east-2"]) +``` + +When the stack set gets really big, the recommendation from AWS is to batch the update requests. This option is *not* part of the AWS API, but is an implementation to facilitate updating a large stack set. +To automatically batch via region (find all stack instances, group them by region, and submit each region separately): ( +```groovy + cfnUpdateStackSet(stackSet:'myStackSet', url:'https://s3.amazonaws.com/my-templates-bucket/template.yaml', batchingOptions: [regions: true], accounts:["012345678901","012345678902" ], regions:["us-east-1","us-east-2"]) +``` ## cfnDeleteStackSet diff --git a/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/AbstractCFNCreateStackSetStep.java b/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/AbstractCFNCreateStackSetStep.java index d3eeaf6d..fa4bde92 100644 --- a/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/AbstractCFNCreateStackSetStep.java +++ b/src/main/java/de/taimos/pipeline/aws/cloudformation/stacksets/AbstractCFNCreateStackSetStep.java @@ -42,7 +42,6 @@ import javax.annotation.Nonnull; import java.io.IOException; import java.util.Collection; -import java.util.Objects; abstract class AbstractCFNCreateStackSetStep extends TemplateStepBase { From 90c5c122f5d00ec0f729fe3cdc1414a7c54dc4e9 Mon Sep 17 00:00:00 2001 From: pjnkumar <73760051+pjnkumar@users.noreply.github.com> Date: Sat, 14 Nov 2020 14:44:11 -0600 Subject: [PATCH 6/6] Updated read me to fix link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1c840f23..547d078a 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ This plugins adds Jenkins pipeline steps to interact with the AWS API. * [cfnCreateChangeSet](#cfncreatechangeset) * [cfnExecuteChangeSet](#cfnexecutechangeset) * [cfnUpdateStackSet](#cfnupdatestackset) -* [cfnUpdateStackSetForAccounts] (#cfnupdatestacksetforaccounts) +* [cfnUpdateStackSetForAccounts](#cfnupdatestacksetforaccounts) * [cfnDeleteStackSet](#cfndeletestackset) * [snsPublish](#snspublish) * [deployAPI](#deployapi)