ES-2979 - project config resolution

lib/aws-sdk-sync.ts

@@ -2,16 +2,20 @@
 import { DynamoDBClient, DynamoDBClientConfig, PutItemOutput } from "@aws-sdk/client-dynamodb";
 import { S3ClientConfig, ListBucketsOutput } from "@aws-sdk/client-s3";
 import { GetSecretValueRequest, GetSecretValueResponse, SecretsManagerClientConfig } from "@aws-sdk/client-secrets-manager";
+import { SSMClientConfig, GetParameterRequest, GetParameterResult } from "@aws-sdk/client-ssm";
+import { STSClientConfig, GetCallerIdentityRequest, GetCallerIdentityResponse } from "@aws-sdk/client-sts";
+import { CloudFormationClientConfig, ListExportsInput, ListExportsOutput } from "@aws-sdk/client-cloudformation";
 import { spawnSync } from "child_process";
 
 // Build a function to call on a different process
 // use `module.require` to keep webpack from overriding the function.
 // This isn't run within the bundle
 // It is stringified and run in a different process
-export function invoke(req: any, service: string, method: string, config: any, params: any) {
+export function invoke(req: any, service: string, method: string, config: any, params: any, packageName?: string) {
 	let hasLogged = false;
 	try {
-		let serviceLib = req("@aws-sdk/client-" + service.replace(/[A-Z]+/g, (a) => "-" + a.toLowerCase()).replace(/^-/, ""));
+		let pkg = packageName || ("@aws-sdk/client-" + service.replace(/[A-Z]+/g, (a) => "-" + a.toLowerCase()).replace(/^-/, ""));
+		let serviceLib = req(pkg);
 		new serviceLib[service](config)[method](params, (err: any, data: any) => {
 			if (!hasLogged) {
 				hasLogged = true;
@@ -31,8 +35,8 @@ export function invoke(req: any, service: string, method: string, config: any, p
 	}
 }
 
-function run(service: string, method: string, config: any, params: any) {
-	let fn = `(${invoke.toString()})(require,"${service}", "${method}", ${JSON.stringify(config)}, ${JSON.stringify(params)})`;
+function run(service: string, method: string, config: any, params: any, packageName?: string) {
+	let fn = `(${invoke.toString()})(require,"${service}", "${method}", ${JSON.stringify(config)}, ${JSON.stringify(params)}, ${packageName ? JSON.stringify(packageName) : "undefined"})`;
 
 	// Spawn node with the function to run `node -e (()=>{})`
 	// Using `RESPONSE::{}::RESPONSE` to denote the response in the output
@@ -59,9 +63,10 @@ function run(service: string, method: string, config: any, params: any) {
 }
 
 export class Service<T> {
+	protected packageName?: string;
 	constructor(private options?: T) { }
 	protected invoke(method: string, params?: any): any {
-		return run(this.constructor.name, method, this.options, params);
+		return run(this.constructor.name, method, this.options, params, this.packageName);
 	}
 }
 
@@ -84,9 +89,31 @@ export class DynamoDB extends Service<DynamoDBClientConfig> {
 	}
 }
 
+export class SSM extends Service<SSMClientConfig> {
+	getParameter(params: GetParameterRequest): GetParameterResult {
+		return this.invoke("getParameter", params);
+	}
+}
+
+export class STS extends Service<STSClientConfig> {
+	getCallerIdentity(params?: GetCallerIdentityRequest): GetCallerIdentityResponse {
+		return this.invoke("getCallerIdentity", params || {});
+	}
+}
+
+export class CloudFormation extends Service<CloudFormationClientConfig> {
+	packageName = "@aws-sdk/client-cloudformation";
+	listExports(params?: ListExportsInput): ListExportsOutput {
+		return this.invoke("listExports", params || {});
+	}
+}
+
 export default {
 	Service,
 	SecretsManager,
 	S3,
-	DynamoDB
+	DynamoDB,
+	SSM,
+	STS,
+	CloudFormation
 };

lib/configuration-builder.ts

@@ -69,6 +69,10 @@ export class ConfigurationBuilder<T> {
 
 		options.stage = options.stage || process.env.STAGE || process.env.ENVIRONMENT || process.env.LEO_ENVIRONMENT;
 		options.region = options.region || process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || "us-east-1";
+		// ${Stage} resolves to capitalized value (e.g. "Prod"), ${stage} to lowercase (e.g. "prod")
+		if (options.stage) {
+			(options as any).Stage = (options as any).Stage || options.stage.charAt(0).toUpperCase() + options.stage.slice(1).toLowerCase();
+		}
 		let g = (global as any);
 		if (g.rstreams_project_config_cache == null) {
 			g.rstreams_project_config_cache = {};
@@ -87,6 +91,15 @@ export class ConfigurationBuilder<T> {
 				this.data = (process as any).rsf_config;
 			} else if (g.rsf_config) {
 				this.data = g.rsf_config;
+			} else if (process.env.AWS_LAMBDA_FUNCTION_NAME) {
+				throw new Error(
+					`RSF_CONFIG is not set for Lambda "${process.env.AWS_LAMBDA_FUNCTION_NAME}". ` +
+					`Config auto-discovery is only supported for local development. ` +
+					`Ensure RSF_CONFIG is set via the deployment configuration.`
+				);
+			} else {
+				// Auto-discover config definition from the project (local dev only)
+				this.data = discoverConfigDef();
 			}
 		}
 
@@ -101,12 +114,14 @@ export class ConfigurationBuilder<T> {
 				this.data = JSON.parse(this.data);
 			} else {
 				// config is the key to a secret
+				let secretId = resolveKeywords(this.data as string, options);
+				assertKeywordsResolved(secretId);
 
 				logger.time("get-rsf-config");
 				this.data = JSON.parse(new awsSdkSync.SecretsManager({
 					region: options.region
 				}).getSecretValue({
-					SecretId: this.data
+					SecretId: secretId
 				}).SecretString.replace(/"{/g, '{').replace(/}"/g, "}"));
 				logger.timeEnd("get-rsf-config");
 			}
@@ -194,6 +209,7 @@ export class ConfigurationBuilder<T> {
 		};
 		if (opts.stageEnvVar != null && process.env[opts.stageEnvVar] != null) {
 			opts.stage = process.env[opts.stageEnvVar];
+			(opts as any).Stage = opts.stage.charAt(0).toUpperCase() + opts.stage.slice(1).toLowerCase();
 		}
 
 		if (opts.regionEnvVar != null && process.env[opts.regionEnvVar] != null) {
@@ -227,17 +243,82 @@ export class ConfigurationBuilder<T> {
 	}
 
 	static Resolvers: Record<string, (ref: ResourceReference, cache: any) => any> = {
-		// ssm: (ref: ResourceReference) => {
-		// 	return process.env[`RS_ssm::${resolveKeywords(ref.key, ref.options)}`];
-		// },
-		// cf: (ref: ResourceReference) => {
-		// 	return process.env[`RS_cf::${resolveKeywords(ref.key, ref.options)}`];
-		// },
-		// stack: (ref: ResourceReference) => {
-		// 	return process.env[`RS_stack::${resolveKeywords(ref.key, ref.options)}`];
-		// },
+		ssm: (ref: ResourceReference, cache: any) => {
+			let resolvedKey = resolveKeywords(ref.key, ref.options);
+			assertKeywordsResolved(resolvedKey);
+			let envValue = process.env[`RS_ssm::${resolvedKey}`];
+			if (envValue != null) {
+				return envValue;
+			}
+			let cacheKey = `ssm::${resolvedKey}`;
+			if (cache[cacheKey] != null) {
+				return cache[cacheKey];
+			}
+			logger.log(`SSM GetParameter Key: ${resolvedKey}, Region: ${ref.options?.region}`);
+			logger.time("ssm-get");
+			let result = new awsSdkSync.SSM({
+				region: ref.options?.region
+			}).getParameter({
+				Name: resolvedKey,
+				WithDecryption: true
+			});
+			logger.timeEnd("ssm-get");
+			let value = result.Parameter?.Value;
+			cache[cacheKey] = value;
+			return value;
+		},
+		cf: (ref: ResourceReference, cache: any) => {
+			let resolvedKey = resolveKeywords(ref.key, ref.options);
+			assertKeywordsResolved(resolvedKey);
+			let envValue = process.env[`RS_cf::${resolvedKey}`];
+			if (envValue != null) {
+				return envValue;
+			}
+			let cacheKey = `cf::${resolvedKey}`;
+			if (cache[cacheKey] != null) {
+				return cache[cacheKey];
+			}
+			logger.log(`CloudFormation ListExports Key: ${resolvedKey}, Region: ${ref.options?.region}`);
+			logger.time("cf-get");
+			let cfClient = new awsSdkSync.CloudFormation({ region: ref.options?.region });
+			let nextToken: string | undefined;
+			do {
+				let result = cfClient.listExports(nextToken ? { NextToken: nextToken } : {});
+				for (let exp of result.Exports || []) {
+					cache[`cf::${exp.Name}`] = exp.Value;
+				}
+				nextToken = result.NextToken;
+			} while (nextToken && cache[cacheKey] == null);
+			logger.timeEnd("cf-get");
+			return cache[cacheKey];
+		},
+		stack: (ref: ResourceReference, cache: any) => {
+			let resolvedKey = resolveKeywords(ref.key, ref.options);
+			let envValue = process.env[`RS_stack::${resolvedKey}`];
+			if (envValue != null) {
+				return envValue;
+			}
+			// Handle pseudo-parameters like ${AWS::AccountId}, ${AWS::Region}
+			if (resolvedKey === "AWS::AccountId" || resolvedKey === "${AWS::AccountId}") {
+				if (cache["stack::AWS::AccountId"] != null) {
+					return cache["stack::AWS::AccountId"];
+				}
+				logger.time("sts-get");
+				let identity = new awsSdkSync.STS({
+					region: ref.options?.region
+				}).getCallerIdentity();
+				logger.timeEnd("sts-get");
+				cache["stack::AWS::AccountId"] = identity.Account;
+				return identity.Account;
+			}
+			if (resolvedKey === "AWS::Region" || resolvedKey === "${AWS::Region}") {
+				return ref.options?.region || process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || "us-east-1";
+			}
+			return undefined;
+		},
 		secret: (ref: ResourceReference, cache: any) => {
 			let resolvedKey = resolveKeywords(ref.key, ref.options);
+			assertKeywordsResolved(resolvedKey);
 			if (process.env[`RS_secret::${resolvedKey}`] && !process.env[`RS_secret::${resolvedKey}`].match(/^(true|false)$/)) {
 				return process.env[`RS_secret::${resolvedKey}`];
 			}
@@ -250,11 +331,18 @@ export class ConfigurationBuilder<T> {
 				logger.log(`SecretsManager  Key: ${key}, Region:${ref.options?.region}`);
 
 				logger.time("secret-get");
-				let value = JSON.parse(new awsSdkSync.SecretsManager({
+				let raw = new awsSdkSync.SecretsManager({
 					region: ref.options?.region
 				}).getSecretValue({
 					SecretId: resolveKeywords(key, ref.options)
-				}).SecretString);
+				}).SecretString;
+				let value: any;
+				try {
+					value = JSON.parse(raw);
+				} catch (e) {
+					// Secret is a plain string, not JSON
+					value = raw;
+				}
 				logger.timeEnd("secret-get");
 				cache[cacheKey] = value;
 				cachedValue = value;
@@ -278,6 +366,18 @@ export function resolveKeywords(template: string, data: any) {
 	}).replace(/[_-]{2,}/g, "");
 	return name;
 }
+
+export function assertKeywordsResolved(resolved: string) {
+	let unresolved = resolved.match(/\${(.*?)}/g);
+	if (unresolved) {
+		let fields = unresolved.map(m => m.replace(/^\${|}$/g, ""));
+		throw new Error(
+			`Unresolved template variable(s) in config name "${resolved}": ${fields.join(", ")}. ` +
+			`Set one of STAGE, ENVIRONMENT, or LEO_ENVIRONMENT env vars ` +
+			`(e.g. STAGE=prod) or pass { stage: "prod" } to .build().`
+		);
+	}
+}
 export function getDataSafe(data = {}, path = "") {
 	const pathArray = path.split(".").filter(a => a !== "");
 	if (pathArray.length === 0) {
@@ -287,6 +387,22 @@ export function getDataSafe(data = {}, path = "") {
 	return pathArray.reduce((parent, field) => parent[field] || {}, data)[lastField];
 }
 
+function discoverConfigDef(): ConfigurationData | undefined {
+	// Walk up from cwd looking for project-config.def.json
+	let dir = process.cwd();
+	let prev = "";
+	while (dir !== prev) {
+		let defFile = path.resolve(dir, "project-config.def.json");
+		if (fs.existsSync(defFile)) {
+			logger.log(`Auto-discovered config definition: ${defFile}`);
+			return JSON.parse(fs.readFileSync(defFile, "utf-8"));
+		}
+		prev = dir;
+		dir = path.dirname(dir);
+	}
+	return undefined;
+}
+
 const numberRegex = /^\d+(?:\.\d*)?$/;
 const boolRegex = /^(?:false|true)$/i;
 const nullRegex = /^null$/;