Merge pull request #20 from tomarv2/updates

moving to use terraform 1.0.1

Author: tomarv2

README.md

@@ -35,9 +35,9 @@
 
 ## Versions
 
-- Module tested for Terraform 0.14.
-- `databrickslabs/databricks` provider version [0.3.3](https://registry.terraform.io/providers/databrickslabs/databricks/latest)
-- AWS provider version [3.30](https://registry.terraform.io/providers/hashicorp/aws/latest).
+- Module tested for Terraform 1.0.1.
+- `databrickslabs/databricks` provider version [0.3.5](https://registry.terraform.io/providers/databrickslabs/databricks/latest)
+- AWS provider version [3.47](https://registry.terraform.io/providers/hashicorp/aws/latest).
 - `main` branch: Provider versions not pinned to keep up with Terraform releases.
 - `tags` releases: Tags are pinned with versions (use <a href="https://github.com/tomarv2/terraform-databricks-aws-workspace/tags" alt="GitHub tag">
         <img src="https://img.shields.io/github/v/tag/tomarv2/terraform-databricks-aws-workspace" /></a>).
@@ -57,7 +57,7 @@ terraform destroy -var='teamid=tryme' -var='prjid=project1'
 
 ### Option 2:
 
-#### Recommended method (store remote state in S3 using `prjid` and `teamid` to create directory structure):
+#### Recommended method (stores remote state in S3 using `prjid` and `teamid` to create directory structure):
 
 - Create python 3.6+ virtual environment
 ```
@@ -66,41 +66,55 @@ python3 -m venv <venv name>
 
 - Install package:
 ```
-pip install tfremote
+pip install tfremote --upgrade
 ```
 
 - Set below environment variables:
 ```
 export TF_AWS_BUCKET=<remote state bucket name>
-export TF_AWS_PROFILE=default
 export TF_AWS_BUCKET_REGION=us-west-2
+export TF_AWS_PROFILE=<profile from ~/.ws/credentials>
 ```
 
-- Updated `examples` directory with required values.
+or
+
+- Set below environment variables:
+```
+export TF_AWS_BUCKET=<remote state bucket name>
+export TF_AWS_BUCKET_REGION=us-west-2
+export AWS_ACCESS_KEY_ID=<aws_access_key_id>
+export AWS_SECRET_ACCESS_KEY=<aws_secret_access_key>
+```
+
+- Update [main.tf](examples/sample/main.tf) file with required values.
 
 - Run and verify the output before deploying:
 ```
-tf -cloud aws plan -var='teamid=foo' -var='prjid=bar'
+tf -c=aws plan -var='teamid=foo' -var='prjid=bar'
 ```
 
 - Run below to deploy:
 ```
-tf -cloud aws apply -var='teamid=foo' -var='prjid=bar'
+tf -c=aws apply -var='teamid=foo' -var='prjid=bar'
 ```
 
 - Run below to destroy:
 ```
-tf -cloud aws destroy -var='teamid=foo' -var='prjid=bar'
+tf -c=aws destroy -var='teamid=foo' -var='prjid=bar'
 ```
 
 **NOTE:**
 
 - Read more on [tfremote](https://github.com/tomarv2/tfremote)
 
+### Databricks workspace creation with new role
 ```
 module "databricks_workspace" {
   source = "git::git@github.com:tomarv2/terraform-databricks-aws-workspace.git"
 
+  # NOTE: One of the below is required:
+  # - 'profile_for_iam' - for IAM creation (if none is provided 'default' is used)
+  # - 'existing_role_name'
   profile_for_iam             = "iam-admin"
   aws_region                  = "us-east-2"
   databricks_account_username = "example@example.com"
@@ -113,6 +127,26 @@ module "databricks_workspace" {
 }
 ```
 
+### Databricks workspace creation with existing role
+```
+module "databricks_workspace" {
+  source = "git::git@github.com:tomarv2/terraform-databricks-aws-workspace.git"
+
+  # NOTE: One of the below is required:
+  # - 'profile_for_iam' - for IAM creation (if none is provided 'default' is used)
+  # - 'existing_role_name'
+  existing_role_arn          = "arn:aws:iam::123456789012:role/demo-role"
+  aws_region                  = "us-east-2"
+  databricks_account_username = "example@example.com"
+  databricks_account_password = "sample123!"
+  databricks_account_id       = "1234567-1234-1234-1234-1234567"
+  # -----------------------------------------
+  # Do not change the teamid, prjid once set.
+  teamid = var.teamid
+  prjid  = var.prjid
+}
+```
+
 Please refer to examples directory [link](examples) for references.
 
 ## Coming up:

examples/sample/main.tf

@@ -1,7 +1,11 @@
 module "databricks_workspace" {
-  source = "git::git@github.com:tomarv2/terraform-databricks-aws-workspace.git?ref=v0.0.5"
+  source = "git::git@github.com:tomarv2/terraform-databricks-aws-workspace.git?ref=v0.0.6"
 
+  # NOTE: One of the below is required:
+  # - 'profile_for_iam' - for IAM creation (if none is provided 'default' is used)
+  # - 'existing_role_name'
   profile_for_iam             = "iam-admin"
+  existing_role_name          = "arn:aws:iam::123456789012:role/demo-role"
   aws_region                  = "us-east-2"
   databricks_account_username = "example@example.com"
   databricks_account_password = "sample123!"

examples/sample/outputs.tf

@@ -21,11 +21,13 @@ output "databricks_mws_network_id" {
 output "storage_configuration_id" {
   description = "databricks mws storage id"
   value       = module.databricks_workspace.databricks_mws_storage_id
+  sensitive   = true
 }
 
 output "databricks_host" {
   description = "databricks workspace url"
   value       = module.databricks_workspace.workspace_url
+  sensitive   = true
 }
 
 output "databricks_credentials_id" {
@@ -43,9 +45,9 @@ output "pat_token" {
   description = "databricks pat"
   value       = module.databricks_workspace.pat_token
 }
+*/
 
 output "pat_token_duration" {
   description = "databricks pat"
   value       = module.databricks_workspace.pat_token_duration
 }
-*/

examples/sample/remote_backend.tf

@@ -3,14 +3,3 @@ data "databricks_aws_assume_role_policy" "this" {
 }
 
 data "databricks_aws_crossaccount_policy" "cross_account_iam_policy" {}
-
-
-data "databricks_aws_bucket_policy" "this" {
-  bucket = module.s3.s3_bucket_name
-}
-
-resource "aws_s3_bucket_policy" "root_bucket_policy" {
-  bucket     = module.s3.s3_bucket_id
-  policy     = data.databricks_aws_bucket_policy.this.json
-  depends_on = [databricks_mws_networks.this]
-}

examples/test/remote_backend.tf

@@ -9,28 +9,38 @@ module "vpc" {
 }
 
 module "iam_role" {
-  source = "git::git@github.com:tomarv2/terraform-aws-iam-role.git//modules/iam_role_external?ref=v0.0.3"
+  source = "git::git@github.com:tomarv2/terraform-aws-iam-role.git//modules/iam_role_external?ref=v0.0.4"
+
+  count = var.existing_role_name == null ? 1 : 0
 
-  profile_to_use     = local.profile_to_use
   assume_role_policy = data.databricks_aws_assume_role_policy.this.json
   external_id        = var.databricks_account_id
   # -----------------------------------------
   # Do not change the teamid, prjid once set.
   teamid = var.teamid
   prjid  = "${var.prjid}-${local.suffix}"
+
+  providers = {
+    aws = aws.iam-management
+  }
 }
 
 module "iam_policies" {
-  source = "git::git@github.com:tomarv2/terraform-aws-iam-policies.git?ref=v0.0.3"
+  source = "git::git@github.com:tomarv2/terraform-aws-iam-policies.git?ref=v0.0.4"
 
-  profile_to_use = local.profile_to_use
-  role_name      = module.iam_role.iam_role_name
-  policy         = data.databricks_aws_crossaccount_policy.cross_account_iam_policy.json
-  inline_policy  = true
+  count = var.existing_role_name == null ? 1 : 0
+
+  role_name     = join("", module.iam_role.*.iam_role_name)
+  policy        = data.databricks_aws_crossaccount_policy.cross_account_iam_policy.json
+  inline_policy = true
   # -----------------------------------------
   # Do not change the teamid, prjid once set.
   teamid = var.teamid
   prjid  = "${var.prjid}-${local.suffix}"
+
+  providers = {
+    aws = aws.iam-management
+  }
 }
 
 module "s3" {

iam.tf

@@ -10,9 +10,10 @@ resource "databricks_mws_networks" "this" {
 resource "databricks_mws_credentials" "this" {
   provider         = databricks.mws
   account_id       = var.databricks_account_id
-  role_arn         = module.iam_role.iam_role_arn
+  role_arn         = var.existing_role_name != null ? var.existing_role_name : join("", module.iam_role.*.iam_role_arn)
   credentials_name = "${var.teamid}-${var.prjid}-${local.suffix}"
-  depends_on       = [module.iam_role]
+
+  depends_on = [module.iam_role]
 }
 
 resource "databricks_mws_storage_configurations" "this" {

main.tf

@@ -5,12 +5,12 @@ output "vpc_id" {
 
 output "iam_role_arn" {
   description = "iam role arn"
-  value       = module.iam_role.iam_role_arn
+  value       = module.iam_role.*.iam_role_arn
 }
 
 output "inline_policy_id" {
   description = "inline policy id"
-  value       = module.iam_policies.inline_policy_id
+  value       = module.iam_policies.*.inline_policy_id
 }
 
 output "s3_bucket_name" {
@@ -76,11 +76,11 @@ output "workspace_url" {
 /*
 output "pat_token" {
   description = "databricks pat"
-  value       = databricks_token.pat.id
+  value       = databricks_token.pat.pat_token
 }
+*/
 
 output "pat_token_duration" {
   description = "databricks pat"
   value       = databricks_token.pat.lifetime_seconds
 }
-*/

mws.tf

@@ -0,0 +1,9 @@
+data "databricks_aws_bucket_policy" "this" {
+  bucket = module.s3.s3_bucket_name
+}
+
+resource "aws_s3_bucket_policy" "root_bucket_policy" {
+  bucket     = module.s3.s3_bucket_id
+  policy     = data.databricks_aws_bucket_policy.this.json
+  depends_on = [databricks_mws_networks.this]
+}