fix(integ-runner): region is not passed to CDK App (#949)

When using the `toolkit-lib` deployment engine, the selected region was
not being passed to the CDK app.

In the old engine, we could set the `AWS_REGION` environment variable
and the CDK CLI would respect that, but the toolkit lib was passing that
environment variable directly to the CDK app which was not respecting
that. Fix that by threading the region through as an explicit parameter
and adding an override parameter to the `AwsCliCompatible` base
credentials: we don't necessarily want to read this only from the INI
files.

Additionally: add a validation to make sure that the CDK app respects
the region we are requesting. CDK integ tests should either not pass an
environment, or read `$CDK_DEFAULT_REGION` to determine the current
region. It was too easy to just override the region with a hardcoded
one, but that breaks the integ runner's attempts to isolate tests across
regions. Guard against that by validating.

Also in this PR: disable the update workflow. I've lost 2 hours
debugging why my deployment was failing with nonsenical values and the
answer was: the checked-in snapshot was nonsensical, and the update
workflow will first deploy the checked-in snapshot and only then deploy
the current snapshot. Good idea, bad execution: if existing snapshots
are not region-agnostic they are guaranteed to fail. We should probably
resynthesize them from the previous git commit instead. Whatever it is,
what we do right now doesn't work so I'm disabling it.

Fixes #

---
By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license

Author: rix0rrr

packages/@aws-cdk/integ-runner/lib/engines/toolkit-lib.ts

@@ -1,8 +1,9 @@
 import * as path from 'node:path';
 import type { DeployOptions, ICdk, ListOptions, SynthFastOptions, SynthOptions, WatchEvents } from '@aws-cdk/cdk-cli-wrapper';
 import type { DefaultCdkOptions, DestroyOptions } from '@aws-cdk/cloud-assembly-schema/lib/integ-tests';
-import type { DeploymentMethod, ICloudAssemblySource, IIoHost, IoMessage, IoRequest, NonInteractiveIoHostProps, StackSelector } from '@aws-cdk/toolkit-lib';
-import { ExpandStackSelection, MemoryContext, NonInteractiveIoHost, StackSelectionStrategy, Toolkit } from '@aws-cdk/toolkit-lib';
+import { UNKNOWN_REGION } from '@aws-cdk/cx-api';
+import type { DeploymentMethod, ICloudAssemblySource, IIoHost, IoMessage, IoRequest, IReadableCloudAssembly, NonInteractiveIoHostProps, StackSelector } from '@aws-cdk/toolkit-lib';
+import { BaseCredentials, ExpandStackSelection, MemoryContext, NonInteractiveIoHost, StackSelectionStrategy, Toolkit } from '@aws-cdk/toolkit-lib';
 import * as chalk from 'chalk';
 import * as fs from 'fs-extra';
 
@@ -27,6 +28,11 @@ export interface ToolkitLibEngineOptions {
    * @default false
    */
   readonly showOutput?: boolean;
+
+  /**
+   * The region the CDK app should synthesize itself for
+   */
+  readonly region: string;
 }
 
 /**
@@ -36,13 +42,23 @@ export class ToolkitLibRunnerEngine implements ICdk {
   private readonly toolkit: Toolkit;
   private readonly options: ToolkitLibEngineOptions;
   private readonly showOutput: boolean;
+  private readonly ioHost: IntegRunnerIoHost;
 
   public constructor(options: ToolkitLibEngineOptions) {
     this.options = options;
     this.showOutput = options.showOutput ?? false;
 
+    // We always create this for ourselves to emit warnings, but potentially
+    // don't pass it to the toolkit.
+    this.ioHost = new IntegRunnerIoHost();
+
     this.toolkit = new Toolkit({
-      ioHost: this.showOutput? new IntegRunnerIoHost() : new NoopIoHost(),
+      ioHost: this.showOutput ? this.ioHost : new NoopIoHost(),
+      sdkConfig: {
+        baseCredentials: BaseCredentials.awsCliCompatible({
+          defaultRegion: options.region,
+        }),
+      },
       // @TODO - these options are currently available on the action calls
       // but toolkit-lib needs them at the constructor level.
       // Need to decide what to do with them.
@@ -73,6 +89,7 @@ export class ToolkitLibRunnerEngine implements ICdk {
       stacks: this.stackSelector(options),
       validateStacks: options.validation,
     });
+    await this.validateRegion(lock);
     await lock.dispose();
   }
 
@@ -100,6 +117,7 @@ export class ToolkitLibRunnerEngine implements ICdk {
     try {
       // @TODO - use produce to mimic the current behavior more closely
       const lock = await cx.produce();
+      await this.validateRegion(lock);
       await lock.dispose();
       // We should fix this once we have stabilized toolkit-lib as engine.
       // What we really should do is this:
@@ -217,7 +235,6 @@ export class ToolkitLibRunnerEngine implements ICdk {
       workingDirectory: this.options.workingDirectory,
       outdir,
       lookups: options.lookups,
-      resolveDefaultEnvironment: false, // not part of the integ-runner contract
       contextStore: new MemoryContext(options.context),
       env: this.options.env,
       synthOptions: {
@@ -256,6 +273,53 @@ export class ToolkitLibRunnerEngine implements ICdk {
       method: options.deploymentMethod ?? 'change-set',
     };
   }
+
+  /**
+   * Check that the regions for the stacks in the CloudAssembly match the regions requested on the engine
+   *
+   * This prevents misconfiguration of the integ test app. People tend to put:
+   *
+   * ```ts
+   * new Stack(app, 'Stack', {
+   *   env: {
+   *     region: 'some-region-that-suits-me',
+   *   }
+   * });
+   * ```
+   *
+   * Into their integ tests, instead of:
+   *
+   * ```ts
+   * {
+   *   region: process.env.CDK_DEFAULT_REGION,
+   * }
+   * ```
+   *
+   * This catches that misconfiguration.
+   */
+  private async validateRegion(asm: IReadableCloudAssembly) {
+    for (const stack of asm.cloudAssembly.stacksRecursively) {
+      if (stack.environment.region !== this.options.region && stack.environment.region !== UNKNOWN_REGION) {
+        this.ioHost.notify({
+          action: 'deploy',
+          code: 'CDK_RUNNER_W0000',
+          time: new Date(),
+          level: 'warn',
+          message: `Stack ${stack.displayName} synthesizes for region ${stack.environment.region}, even though ${this.options.region} was requested. Please configure \`{ env: { region: process.env.CDK_DEFAULT_REGION, account: process.env.CDK_DEFAULT_ACCOUNT } }\`, or use no env at all. Do not hardcode a region or account.`,
+          data: {
+            stackName: stack.displayName,
+            stackRegion: stack.environment.region,
+            requestedRegion: this.options.region,
+          },
+        }).catch((e) => {
+          if (e) {
+            // eslint-disable-next-line no-console
+            console.error(e);
+          }
+        });
+      }
+    }
+  }
 }
 
 /**
@@ -269,9 +333,16 @@ class IntegRunnerIoHost extends NonInteractiveIoHost {
     });
   }
   public async notify(msg: IoMessage<unknown>): Promise<void> {
+    let color;
+    switch (msg.level) {
+      case 'error': color = chalk.red; break;
+      case 'warn': color = chalk.yellow; break;
+      default: color = chalk.gray;
+    }
+
     return super.notify({
       ...msg,
-      message: chalk.gray(msg.message),
+      message: color(msg.message),
     });
   }
 }

packages/@aws-cdk/integ-runner/lib/runner/engine.ts

@@ -18,9 +18,8 @@ export function makeEngine(options: IntegRunnerOptions): ICdk {
       return new ToolkitLibRunnerEngine({
         workingDirectory: options.test.directory,
         showOutput: options.showOutput,
-        env: {
-          ...options.env,
-        },
+        env: options.env,
+        region: options.region,
       });
     case 'cli-wrapper':
     default:
@@ -29,6 +28,8 @@ export function makeEngine(options: IntegRunnerOptions): ICdk {
         showOutput: options.showOutput,
         env: {
           ...options.env,
+          // The CDK CLI will interpret this and use it usefully
+          AWS_REGION: options.region,
         },
       });
   }

packages/@aws-cdk/integ-runner/lib/runner/runner-base.ts

@@ -26,6 +26,11 @@ export interface IntegRunnerOptions extends EngineOptions {
    */
   readonly test: IntegTest;
 
+  /**
+   * The region where the test should be deployed
+   */
+  readonly region: string;
+
   /**
    * The AWS profile to use when invoking the CDK CLI
    *

packages/@aws-cdk/integ-runner/lib/runner/snapshot-test-runner.ts

@@ -34,8 +34,11 @@ interface SnapshotAssembly {
  * the validation of the integration test snapshots
  */
 export class IntegSnapshotRunner extends IntegRunner {
-  constructor(options: IntegRunnerOptions) {
-    super(options);
+  constructor(options: Omit<IntegRunnerOptions, 'region'>) {
+    super({
+      ...options,
+      region: 'unused',
+    });
   }
 
   /**

packages/@aws-cdk/integ-runner/lib/workers/extract/extract_worker.ts

@@ -31,8 +31,8 @@ export async function integTestWorker(request: IntegTestBatchRequest): Promise<I
         engine: request.engine,
         test,
         profile: request.profile,
+        region: request.region,
         env: {
-          AWS_REGION: request.region,
           CDK_DOCKER: process.env.CDK_DOCKER ?? 'docker',
         },
         showOutput: verbosity >= 2,
@@ -99,8 +99,8 @@ export async function watchTestWorker(options: IntegWatchOptions): Promise<void>
     engine: options.engine,
     test,
     profile: options.profile,
+    region: options.region,
     env: {
-      AWS_REGION: options.region,
       CDK_DOCKER: process.env.CDK_DOCKER ?? 'docker',
     },
     showOutput: verbosity >= 2,

packages/@aws-cdk/integ-runner/test/engines/toolkit-lib-snapshot-path.test.ts

@@ -25,14 +25,15 @@ describe('ToolkitLibRunnerEngine - Snapshot Path Handling', () => {
 
     engine = new ToolkitLibRunnerEngine({
       workingDirectory: '/test/dir',
+      region: 'us-dummy-1',
     });
   });
 
   it('should use fromAssemblyDirectory when app is a path to existing snapshot directory', async () => {
     const snapshotPath = 'test.snapshot';
     const fullSnapshotPath = path.join('/test/dir', snapshotPath);
     const mockCx = { produce: jest.fn() };
-    const mockLock = { dispose: jest.fn() };
+    const mockLock = { dispose: jest.fn(), cloudAssembly: { stacksRecursively: [] } };
 
     // Mock fs to indicate the snapshot directory exists
     mockedFs.pathExistsSync.mockReturnValue(true);
@@ -55,7 +56,7 @@ describe('ToolkitLibRunnerEngine - Snapshot Path Handling', () => {
   it('should use fromCdkApp when app is not a path to existing directory', async () => {
     const appCommand = 'node bin/app.js';
     const mockCx = { produce: jest.fn() };
-    const mockLock = { dispose: jest.fn() };
+    const mockLock = { dispose: jest.fn(), cloudAssembly: { stacksRecursively: [] } };
 
     // Mock fs to indicate the path doesn't exist
     mockedFs.pathExistsSync.mockReturnValue(false);
@@ -76,7 +77,7 @@ describe('ToolkitLibRunnerEngine - Snapshot Path Handling', () => {
     const appPath = 'app.js';
     const fullAppPath = path.join('/test/dir', appPath);
     const mockCx = { produce: jest.fn() };
-    const mockLock = { dispose: jest.fn() };
+    const mockLock = { dispose: jest.fn(), cloudAssembly: { stacksRecursively: [] } };
 
     // Mock fs to indicate the path exists but is not a directory
     mockedFs.pathExistsSync.mockReturnValue(true);

packages/@aws-cdk/integ-runner/test/engines/toolkit-lib.test.ts

@@ -25,13 +25,14 @@ describe('ToolkitLibRunnerEngine', () => {
 
     engine = new ToolkitLibRunnerEngine({
       workingDirectory: '/test/dir',
+      region: 'us-dummy-1',
     });
   });
 
   describe('synth', () => {
     it('should call toolkit.synth with correct parameters', async () => {
       const mockCx = { produce: jest.fn() };
-      const mockLock = { dispose: jest.fn() };
+      const mockLock = { dispose: jest.fn(), cloudAssembly: { stacksRecursively: [] } };
       mockToolkit.fromCdkApp.mockResolvedValue(mockCx as any);
       mockToolkit.synth.mockResolvedValue(mockLock as any);
 
@@ -56,7 +57,7 @@ describe('ToolkitLibRunnerEngine', () => {
   describe('synthFast', () => {
     it('should use fromCdkApp and produce for fast synthesis', async () => {
       const mockCx = { produce: jest.fn() };
-      const mockLock = { dispose: jest.fn() };
+      const mockLock = { dispose: jest.fn(), cloudAssembly: { stacksRecursively: [] } };
       mockCx.produce.mockResolvedValue(mockLock);
       mockToolkit.fromCdkApp.mockResolvedValue(mockCx as any);
 
@@ -221,11 +222,12 @@ describe('ToolkitLibRunnerEngine', () => {
       const engineWithOutput = new ToolkitLibRunnerEngine({
         workingDirectory: '/test',
         showOutput: true,
+        region: 'us-dummy-1',
       });
 
-      expect(MockedToolkit).toHaveBeenCalledWith({
+      expect(MockedToolkit).toHaveBeenCalledWith(expect.objectContaining({
         ioHost: expect.any(Object),
-      });
+      }));
     });
 
     it('should throw error when no app is provided', async () => {

packages/@aws-cdk/toolkit-lib/lib/api/aws-auth/base-credentials.ts

@@ -92,10 +92,14 @@ export class BaseCredentials {
    */
   public static awsCliCompatible(options: AwsCliCompatibleOptions = {}): IBaseCredentialsProvider {
     return new class implements IBaseCredentialsProvider {
-      public sdkBaseConfig(ioHost: IActionAwareIoHost, clientConfig: SdkBaseClientConfig) {
+      public async sdkBaseConfig(ioHost: IActionAwareIoHost, clientConfig: SdkBaseClientConfig) {
         const ioHelper = IoHelper.fromActionAwareIoHost(ioHost);
         const awsCli = new AwsCliCompatible(ioHelper, clientConfig.requestHandler ?? {}, new IoHostSdkLogger(ioHelper));
-        return awsCli.baseConfig(options.profile);
+
+        const ret = await awsCli.baseConfig(options.profile);
+        return options.defaultRegion
+          ? { ...ret, defaultRegion: options.defaultRegion }
+          : ret;
       }
 
       public toString() {
@@ -140,6 +144,18 @@ export interface AwsCliCompatibleOptions {
    * @default - Use environment variable if set.
    */
   readonly profile?: string;
+
+  /**
+   * Use a different default region than the one in the profile
+   *
+   * If not supplied the environment variable AWS_REGION will be used, or
+   * whatever region is set in the indicated profile in `~/.aws/config`.
+   * If no region is set in the profile the region in `[default]` will
+   * be used.
+   *
+   * @default - Use region from `~/.aws/config`.
+   */
+  readonly defaultRegion?: string;
 }
 
 export interface CustomBaseCredentialsOption {