CA-18 RDS Postgres Initial Tests (#24)

* CI-159 Update Makefile

* adding first test

* adding first test

* simple test

* adding helper function for connectivity test

* CA-18 adding initial tests

* standard for test_examples folder name

* make lint, terraform/docs and fmt

* go mod tidy

* fix comments

* big refactor - test_structure + testCases

* updating versions

* fixes module name

* CA-71 - update tests/ to instantiate examples/

* using expectApplyError

* 1st version of cloudwatch logs integration

* adding upgrade logs and updating local.tfvars

* adding parameter log_statement=all

* creates param group log parameters from module variables

* fix: adding require lib

* adding comments

* adding test README

* smallfix

* update to README

* removing local.tfvars

* reverting change on output

* Updated documentation

* Changed indivual outputs to module output.

Co-authored-by: Rosemary Wang <66625360+rosemarywang@users.noreply.github.com>
Co-authored-by: Hugo Barros <hugo.barros@caylent.com>

Author: 3 people

.gitignore

@@ -1,6 +1,7 @@
 # Local .terraform directories
 **/.terraform/*
 
+*.terraform.lock.hcl
 # .tfstate files
 *.tfstate
 *.tfstate.*

examples/minimal/README.md

@@ -13,7 +13,6 @@ No provider.
 |------|-------------|------|---------|:--------:|
 | ingress\_cidr\_blocks | CIDR blocks to attach to security groups for ingress | `list(string)` | n/a | yes |
 | name\_prefix | A string to prepend to names of resources created by this example | `any` | n/a | yes |
-| security\_group\_ids | List of security group IDs to allow ingress from (i.e. Spark cluster SG IDs, Tamr VM SG ID) | `list(string)` | n/a | yes |
 | subnet\_ids | List of at least 2 subnets in different AZs for DB subnet group | `list(string)` | n/a | yes |
 | vpc\_id | VPC ID of network. | `string` | n/a | yes |
 | egress\_cidr\_blocks | CIDR blocks to attach to security groups for egress | `list(string)` | <pre>[<br>  "0.0.0.0/0"<br>]</pre> | no |
@@ -23,7 +22,6 @@ No provider.
 
 | Name | Description |
 |------|-------------|
-| ingress\_ports | List of ingress ports |
 | rds | n/a |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/minimal/main.tf

@@ -2,14 +2,14 @@ module "rds_postgres" {
   # source               = "git::https://github.com/Datatamer/terraform-aws-rds-postgres.git?ref=3.0.0"
   source = "../.."
 
-  identifier_prefix    = "example-rds-pg-"
+  identifier_prefix    = "${var.name_prefix}-example-rds-pg-"
   postgres_name        = "example0"
-  parameter_group_name = "example-rds-postgres-pg"
+  parameter_group_name = "${var.name_prefix}-example-rds-postgres-pg"
   username             = "exampleUsername"
   password             = "examplePassword" #tfsec:ignore:GEN003
 
   vpc_id            = var.vpc_id
-  subnet_group_name = "example_subnet_group"
+  subnet_group_name = "${var.name_prefix}_example_subnet_group"
   # Network requirement: DB subnet group needs a subnet in at least two Availability Zones
   rds_subnet_ids     = var.subnet_ids
   security_group_ids = module.rds-postgres-sg.security_group_ids

examples/minimal/outputs.tf

@@ -1,8 +1,3 @@
-output "ingress_ports" {
-  value       = module.sg-ports
-  description = "List of ingress ports"
-}
-
 output "rds" {
   value = module.rds_postgres
 }

examples/minimal/variables.tf

@@ -8,11 +8,6 @@ variable "subnet_ids" {
   description = "List of at least 2 subnets in different AZs for DB subnet group"
 }
 
-variable "security_group_ids" {
-  description = "List of security group IDs to allow ingress from (i.e. Spark cluster SG IDs, Tamr VM SG ID)"
-  type        = list(string)
-}
-
 variable "name_prefix" {
   description = "A string to prepend to names of resources created by this example"
 }

test/README.md

@@ -0,0 +1,45 @@
+# Tests
+
+This folder contains automated tests for this Module. All of the tests are written in [Go](https://golang.org/).
+Most of these are "integration tests" that deploy real infrastructure using Terraform and verify that infrastructure works as expected using a helper library called [Terratest](https://github.com/gruntwork-io/terratest).
+
+
+
+## WARNING WARNING WARNING
+
+**Note #1**: Many of these tests create real resources in an AWS account and then try to clean those resources up at the end of a test run. That means these tests may cost you money to run! When adding tests, please be considerate of the resources you create and take extra care to clean everything up when you're done!
+
+**Note #2**: Never forcefully shut the tests down (e.g. by hitting `CTRL + C`) or the cleanup tasks won't run!
+
+**Note #3**: We need to set `-timeout 60m` on all tests not because they necessarily take that long, but because Go has a default test timeout of 10 minutes, after which it forcefully kills the tests with a `SIGQUIT`, preventing the cleanup tasks from running. Therefore, we set an overlying long timeout to make sure all tests have enough time to finish and clean up.
+
+
+
+## Running the tests
+
+### Prerequisites
+
+- Install the latest version of [Go](https://golang.org/).
+- Install [Terraform](https://www.terraform.io/downloads.html).
+- Configure your AWS credentials using one of the [options supported by the AWS SDK](http://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html). Usually, the easiest option is to set the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables.
+
+
+### Run all the tests
+
+```bash
+go test -v -timeout 60m
+```
+
+
+### Run a specific test
+
+To run a specific test called `TestFoo`:
+
+```bash
+go test -v -timeout 60m -run TestFoo
+```
+
+When using `t.Run("test_name",...)` inside a test function, you may run a specific test with:
+```bash
+go test -v -timeout 60m -run TestFoo/test_name
+```

test/create_db_test.go

@@ -0,0 +1,108 @@
+package test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/gruntwork-io/terratest/modules/aws"
+	"github.com/gruntwork-io/terratest/modules/random"
+	"github.com/gruntwork-io/terratest/modules/terraform"
+	test_structure "github.com/gruntwork-io/terratest/modules/test-structure"
+	"github.com/stretchr/testify/require"
+)
+
+// These const are declared according to what is found at "../../examples/minimal/main.tf"
+const (
+	expectedPw       = "examplePassword"
+	expectedUsername = "exampleUsername"
+	expectedDBName   = "example0"
+)
+
+// initTestCases initializes a list of RdsTestCase
+func initTestCases() []RdsTestCase {
+	return []RdsTestCase{
+		{
+			testName:         "minimal",
+			expectApplyError: false,
+			vars: map[string]interface{}{
+				"vpc_cidr":            "172.18.0.0/18",
+				"database_subnets":    []string{"172.18.0.0/24", "172.18.1.0/24"},
+				"egress_cidr_blocks":  []string{"0.0.0.0/0"},
+				"ingress_cidr_blocks": []string{"0.0.0.0/0"},
+				"name_prefix":         "",
+			},
+		},
+	}
+}
+
+// TestTerraformCreateRDS runs all test cases
+func TestTerraformCreateRDS(t *testing.T) {
+
+	testCases := initTestCases()
+
+	for _, testCase := range testCases {
+		testCase := testCase
+
+		t.Run(testCase.testName, func(t *testing.T) {
+			t.Parallel()
+
+			// These will create a tempTestFolder for each bucketTestCase.
+			tempTestFolder := test_structure.CopyTerraformFolderToTemp(t, "..", "test_examples/minimal")
+
+			// this stage will generate a random `awsRegion` and a `uniqueId` to be used in tests.
+			test_structure.RunTestStage(t, "pick_new_randoms", func() {
+				usRegions := []string{"us-east-1", "us-east-2", "us-west-1", "us-west-2"}
+				// This function will first check for the Env Var TERRATEST_REGION and return its value if != ""
+				awsRegion := aws.GetRandomStableRegion(t, usRegions, nil)
+
+				test_structure.SaveString(t, tempTestFolder, "region", awsRegion)
+				test_structure.SaveString(t, tempTestFolder, "unique_id", strings.ToLower(random.UniqueId()))
+			})
+
+			defer test_structure.RunTestStage(t, "teardown", func() {
+				teraformOptions := test_structure.LoadTerraformOptions(t, tempTestFolder)
+				terraform.Destroy(t, teraformOptions)
+			})
+
+			test_structure.RunTestStage(t, "setup_options", func() {
+				awsRegion := test_structure.LoadString(t, tempTestFolder, "region")
+				uniqueID := test_structure.LoadString(t, tempTestFolder, "unique_id")
+
+				testCase.vars["name_prefix"] = uniqueID
+
+				terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{
+					TerraformDir: tempTestFolder,
+					Vars:         testCase.vars,
+					EnvVars: map[string]string{
+						"AWS_REGION": awsRegion,
+					},
+				})
+
+				test_structure.SaveTerraformOptions(t, tempTestFolder, terraformOptions)
+			})
+
+			test_structure.RunTestStage(t, "create_rds", func() {
+				terraformOptions := test_structure.LoadTerraformOptions(t, tempTestFolder)
+				_, err := terraform.InitAndApplyE(t, terraformOptions)
+
+				if testCase.expectApplyError {
+					require.Error(t, err)
+					// If it failed as expected, we should skip the rest (validate function).
+					t.SkipNow()
+				}
+			})
+
+			test_structure.RunTestStage(t, "validate", func() {
+				awsRegion := test_structure.LoadString(t, tempTestFolder, "region")
+				terraformOptions := test_structure.LoadTerraformOptions(t, tempTestFolder)
+				validateModuleOutputs(t,
+					terraformOptions,
+					awsRegion,
+					int64(5432),
+					expectedUsername,
+					expectedDBName,
+				)
+			})
+		})
+	}
+}

test/go.mod

@@ -0,0 +1,9 @@
+module github.com/Datatamer/terraform-aws-rds-postgres
+
+go 1.16
+
+require (
+	github.com/aws/aws-sdk-go v1.40.40
+	github.com/gruntwork-io/terratest v0.37.8
+	github.com/stretchr/testify v1.7.0
+)