Skip to content

Commit 5049325

Browse files
jeffreyavenjeffreyaven
committed
updates
1 parent dcb2e40 commit 5049325

24 files changed

+1400
-139
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ testcreds/
1212
*.log
1313
venv/
1414
.venv/
15+
nohup.out
1516

1617
# Byte-compiled / optimized / DLL files
1718
__pycache__/
@@ -87,4 +88,4 @@ docs/_build/
8788

8889
venv/
8990
.DS_Store
90-
myenv/
91+
myenv/

‎CHANGELOG.md‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
11
# Changelog
22

3+
## 1.8.5 (2025-06-30)
4+
5+
- Added support for resource scoped variables
6+
- Added developer credits in `info`
7+
38
## 1.8.3 (2025-02-08)
49

510
- Added walkthrough for databricks bootstrap on aws.
6-
- Bugfix for expport variables on dry run.
11+
- Bugfix for export variables on dry run.
712

813
## 1.8.2 (2025-01-16)
914

‎examples/databricks/all-purpose-cluster/stackql_manifest.yml‎

Lines changed: 333 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,339 @@ resources:
270270
Resource: ["*"]
271271
exports:
272272
- aws_iam_role_arn: databricks_compute_role_arn
273+
# ====================================================================================
274+
# UC Metastore with KMS Encryption
275+
# ====================================================================================
276+
- name: aws/kms/metastore_key
277+
props:
278+
- name: description
279+
value: "KMS key for {{ stack_name }}-{{ stack_env }} Unity Catalog metastore encryption"
280+
- name: key_policy
281+
value:
282+
Version: "2012-10-17"
283+
Statement:
284+
- Sid: "Enable IAM User Permissions"
285+
Effect: "Allow"
286+
Principal:
287+
AWS: "arn:aws:iam::{{ aws_account }}:root"
288+
Action: "kms:*"
289+
Resource: "*"
290+
- Sid: "Allow Metastore Role to Use Key"
291+
Effect: "Allow"
292+
Principal:
293+
AWS: "arn:aws:iam::{{ aws_account }}:role/{{ stack_name }}-{{ stack_env }}-metastore-role"
294+
Action:
295+
- "kms:Decrypt"
296+
- "kms:Encrypt"
297+
- "kms:GenerateDataKey"
298+
- "kms:DescribeKey"
299+
- "kms:CreateGrant"
300+
- "kms:RetireGrant"
301+
Resource: "*"
302+
- Sid: "Allow Databricks to Use Key"
303+
Effect: "Allow"
304+
Principal:
305+
AWS: "arn:aws:iam::{{ databricks_aws_account_id }}:root"
306+
Action:
307+
- "kms:Decrypt"
308+
- "kms:GenerateDataKey"
309+
- "kms:DescribeKey"
310+
Resource: "*"
311+
Condition:
312+
StringEquals:
313+
"kms:ViaService": "s3.{{ region }}.amazonaws.com"
314+
- name: tags
315+
value:
316+
- Key: Purpose
317+
Value: "UC Metastore Encryption"
318+
merge:
319+
- global_tags
320+
exports:
321+
- key_id: metastore_kms_key_id
322+
- key_arn: metastore_kms_key_arn
323+
324+
- name: aws/kms/metastore_key_alias
325+
props:
326+
- name: alias_name
327+
value: "alias/{{ stack_name }}-{{ stack_env }}-metastore"
328+
- name: target_key_id
329+
value: "{{ metastore_kms_key_id }}"
330+
331+
- name: aws/s3/metastore_bucket
332+
props:
333+
- name: bucket_name
334+
value: "{{ stack_name }}-{{ stack_env }}-metastore"
335+
- name: ownership_controls
336+
value:
337+
Rules:
338+
- ObjectOwnership: "BucketOwnerPreferred"
339+
- name: bucket_encryption
340+
value:
341+
ServerSideEncryptionConfiguration:
342+
- BucketKeyEnabled: true
343+
ServerSideEncryptionByDefault:
344+
SSEAlgorithm: "aws:kms"
345+
KMSMasterKeyID: "{{ metastore_kms_key_arn }}"
346+
- name: public_access_block_configuration
347+
value:
348+
BlockPublicAcls: true
349+
IgnorePublicAcls: true
350+
BlockPublicPolicy: true
351+
RestrictPublicBuckets: true
352+
- name: versioning_configuration
353+
value:
354+
Status: "Enabled"
355+
- name: lifecycle_configuration
356+
value:
357+
Rules:
358+
- Id: "DeleteOldVersions"
359+
Status: "Enabled"
360+
NoncurrentVersionExpiration:
361+
NoncurrentDays: 30
362+
AbortIncompleteMultipartUpload:
363+
DaysAfterInitiation: 7
364+
- name: logging_configuration
365+
value:
366+
TargetBucket: "{{ stack_name }}-{{ stack_env }}-logs"
367+
TargetPrefix: "s3-access-logs/metastore/"
368+
- name: tags
369+
value:
370+
- Key: Purpose
371+
Value: "Unity Catalog Metastore"
372+
- Key: DataClassification
373+
Value: "Metadata"
374+
merge:
375+
- global_tags
376+
exports:
377+
- bucket_name: aws_s3_metastore_bucket_name
378+
- bucket_arn: aws_s3_metastore_bucket_arn
379+
380+
- name: aws/iam/metastore_access_role
381+
file: aws/iam/iam_role.iql
382+
props:
383+
- name: role_name
384+
value: "{{ stack_name }}-{{ stack_env }}-metastore-role"
385+
- name: assume_role_policy_document
386+
value:
387+
Version: "2012-10-17"
388+
Statement:
389+
- Effect: "Allow"
390+
Principal:
391+
AWS:
392+
- "arn:aws:iam::414351767826:role/unity-catalog-prod-UCMasterRole-14S5ZJVKOTYTL"
393+
Action: "sts:AssumeRole"
394+
Condition:
395+
StringEquals:
396+
sts:ExternalId: "0000" # Placeholder
397+
- name: description
398+
value: 'Unity Catalog metastore access role for ({{ stack_name }}-{{ stack_env }})'
399+
- name: path
400+
value: '/'
401+
- name: policies
402+
value:
403+
- PolicyName: "MetastoreS3Access"
404+
PolicyDocument:
405+
Version: "2012-10-17"
406+
Statement:
407+
- Sid: "S3MetastoreBucketAccess"
408+
Effect: "Allow"
409+
Action:
410+
- "s3:GetObject"
411+
- "s3:PutObject"
412+
- "s3:DeleteObject"
413+
- "s3:ListBucket"
414+
- "s3:GetBucketLocation"
415+
- "s3:GetLifecycleConfiguration"
416+
- "s3:PutLifecycleConfiguration"
417+
- "s3:ListBucketMultipartUploads"
418+
- "s3:ListMultipartUploadParts"
419+
- "s3:AbortMultipartUpload"
420+
Resource:
421+
- "{{ aws_s3_metastore_bucket_arn }}/*"
422+
- "{{ aws_s3_metastore_bucket_arn }}"
423+
- Sid: "AssumeRoleSelfTrust"
424+
Effect: "Allow"
425+
Action: ["sts:AssumeRole"]
426+
Resource: ["arn:aws:iam::{{ aws_account }}:role/{{ stack_name }}-{{ stack_env }}-metastore-role"]
427+
- PolicyName: "MetastoreKMSAccess"
428+
PolicyDocument:
429+
Version: "2012-10-17"
430+
Statement:
431+
- Sid: "KMSKeyAccess"
432+
Effect: "Allow"
433+
Action:
434+
- "kms:Decrypt"
435+
- "kms:Encrypt"
436+
- "kms:GenerateDataKey"
437+
- "kms:DescribeKey"
438+
- "kms:CreateGrant"
439+
- "kms:RetireGrant"
440+
Resource:
441+
- "{{ metastore_kms_key_arn }}"
442+
Condition:
443+
StringEquals:
444+
"kms:ViaService": "s3.{{ region }}.amazonaws.com"
445+
- PolicyName: "MetastoreFileEvents"
446+
PolicyDocument:
447+
Version: "2012-10-17"
448+
Statement:
449+
- Sid: "ManagedFileEventsSetupStatement"
450+
Effect: "Allow"
451+
Action:
452+
- "s3:GetBucketNotification"
453+
- "s3:PutBucketNotification"
454+
- "sns:ListSubscriptionsByTopic"
455+
- "sns:GetTopicAttributes"
456+
- "sns:SetTopicAttributes"
457+
- "sns:CreateTopic"
458+
- "sns:TagResource"
459+
- "sns:Publish"
460+
- "sns:Subscribe"
461+
- "sqs:CreateQueue"
462+
- "sqs:DeleteMessage"
463+
- "sqs:ReceiveMessage"
464+
- "sqs:SendMessage"
465+
- "sqs:GetQueueUrl"
466+
- "sqs:GetQueueAttributes"
467+
- "sqs:SetQueueAttributes"
468+
- "sqs:TagQueue"
469+
- "sqs:ChangeMessageVisibility"
470+
- "sqs:PurgeQueue"
471+
Resource:
472+
- "{{ aws_s3_metastore_bucket_arn }}"
473+
- "arn:aws:sqs:{{ region }}:{{ aws_account }}:csms-*"
474+
- "arn:aws:sns:{{ region }}:{{ aws_account }}:csms-*"
475+
- Sid: "ManagedFileEventsListStatement"
476+
Effect: "Allow"
477+
Action: ["sqs:ListQueues", "sqs:ListQueueTags", "sns:ListTopics"]
478+
Resource: "*"
479+
- Sid: "ManagedFileEventsTeardownStatement"
480+
Effect: "Allow"
481+
Action: ["sns:Unsubscribe", "sns:DeleteTopic", "sqs:DeleteQueue"]
482+
Resource:
483+
- "arn:aws:sqs:{{ region }}:{{ aws_account }}:csms-*"
484+
- "arn:aws:sns:{{ region }}:{{ aws_account }}:csms-*"
485+
- name: tags
486+
value:
487+
- Key: Purpose
488+
Value: "Unity Catalog Storage Credential"
489+
merge:
490+
- global_tags
491+
exports:
492+
- aws_iam_role_arn: metastore_access_role_arn
493+
494+
- name: databricks_account/metastore
495+
props:
496+
- name: name
497+
value: "{{ stack_name }}-{{ stack_env }}-metastore"
498+
- name: storage_root
499+
value: "s3://{{ aws_s3_metastore_bucket_name }}"
500+
- name: region
501+
value: "{{ region }}"
502+
exports:
503+
- metastore_id: databricks_metastore_id
504+
505+
- name: databricks_account/uc_storage_credentials
506+
props:
507+
- name: metastore_id
508+
value: "{{ databricks_metastore_id }}"
509+
- name: credential_info
510+
value:
511+
name: "{{ stack_name }}-{{ stack_env }}-storage-credential"
512+
comment: "Storage credential for {{ stack_name }} {{ stack_env }} metastore S3 access"
513+
read_only: false
514+
aws_iam_role:
515+
role_arn: "{{ metastore_access_role_arn }}"
516+
skip_validation: false
517+
exports:
518+
- credential_id: storage_credential_id
519+
- external_id: storage_credential_external_id
520+
521+
- name: aws/iam/update_metastore_role_trust_policy
522+
type: command
523+
props:
524+
- name: role_name
525+
value: "{{ stack_name }}-{{ stack_env }}-metastore-role"
526+
- name: assume_role_policy_document
527+
value:
528+
Version: "2012-10-17"
529+
Statement:
530+
- Effect: "Allow"
531+
Principal:
532+
AWS:
533+
- "arn:aws:iam::414351767826:role/unity-catalog-prod-UCMasterRole-14S5ZJVKOTYTL"
534+
- "arn:aws:iam::{{ aws_account }}:role/{{ stack_name }}-{{ stack_env }}-metastore-role"
535+
Action: "sts:AssumeRole"
536+
Condition:
537+
StringEquals:
538+
sts:ExternalId: "{{ storage_credential_external_id }}"
539+
540+
- name: databricks_account/validate_storage_credential
541+
type: command
542+
props:
543+
- name: credential_id
544+
value: "{{ storage_credential_id }}"
545+
- name: metastore_id
546+
value: "{{ databricks_metastore_id }}"
547+
548+
- name: databricks_account/external_location
549+
props:
550+
- name: metastore_id
551+
value: "{{ databricks_metastore_id }}"
552+
- name: name
553+
value: "{{ stack_name }}-{{ stack_env }}-metastore-location"
554+
- name: url
555+
value: "s3://{{ aws_s3_metastore_bucket_name }}/"
556+
- name: credential_name
557+
value: "{{ stack_name }}-{{ stack_env }}-storage-credential"
558+
- name: comment
559+
value: "External location for {{ stack_name }} {{ stack_env }} metastore root"
560+
exports:
561+
- external_location_id
562+
563+
- name: databricks_account/catalog
564+
props:
565+
- name: metastore_id
566+
value: "{{ databricks_metastore_id }}"
567+
- name: name
568+
value: "{{ stack_name }}_{{ stack_env }}"
569+
- name: comment
570+
value: "Main catalog for {{ stack_name }} {{ stack_env }} environment"
571+
- name: storage_root
572+
value: "s3://{{ aws_s3_metastore_bucket_name }}/catalogs/{{ stack_name }}_{{ stack_env }}"
573+
exports:
574+
- catalog_id
575+
576+
- name: databricks_account/metastore_assignment
577+
props:
578+
- name: workspace_id
579+
value: "{{ databricks_workspace_id }}"
580+
- name: metastore_id
581+
value: "{{ databricks_metastore_id }}"
582+
- name: default_catalog_name
583+
value: "{{ stack_name }}_{{ stack_env }}"
584+
585+
- name: databricks_account/catalog_workspace_binding
586+
props:
587+
- name: catalog_name
588+
value: "{{ stack_name }}_{{ stack_env }}"
589+
- name: workspace_id
590+
value: "{{ databricks_workspace_id }}"
591+
592+
- name: databricks_account/catalog_permissions
593+
props:
594+
- name: catalog_name
595+
value: "{{ stack_name }}_{{ stack_env }}"
596+
- name: principal
597+
value: "{{ databricks_group_id }}"
598+
- name: privileges
599+
value:
600+
- "USE_CATALOG"
601+
- "CREATE_SCHEMA"
602+
- "CREATE_TABLE"
603+
- "CREATE_FUNCTION"
604+
605+
273606
# ====================================================================================
274607
# AWS VPC Networking
275608
# ====================================================================================

0 commit comments

Comments
 (0)