added sql_escape filter

Author: jeffreyaven

examples/databricks/snowflake-interoperability/resources/databricks_workspace/catalog.iql

@@ -25,8 +25,3 @@ SELECT name as catalog_name
 FROM databricks_workspace.unitycatalog.catalogs
 WHERE name = '{{ name }}' AND
 deployment_name = '{{ databricks_deployment_name }}';
-
-/*+ delete */
-DELETE FROM databricks_workspace.unitycatalog.catalogs
-WHERE name = '{{ name }}' AND
-deployment_name = '{{ deployment_name }}';

examples/databricks/snowflake-interoperability/resources/databricks_workspace/schema.iql

@@ -25,8 +25,3 @@ FROM databricks_workspace.unitycatalog.schemas
 WHERE deployment_name = '{{ databricks_deployment_name }}'
 AND catalog_name = '{{ catalog_name }}'
 AND name = '{{ name }}';
-
-/*+ delete */
-DELETE FROM databricks_workspace.unitycatalog.schemas
-WHERE full_name = '{{ name }}' AND
-deployment_name = '{{ databricks_deployment_name }}';

examples/databricks/snowflake-interoperability/resources/snowflake/statement.iql

@@ -9,7 +9,7 @@ data__warehouse,
 endpoint
 )
 SELECT 
-'{{ statement }}',
+'{{ statement | sql_escape }}',
 {{ timeout }},
 '{{ database }}',
 '{{ schema }}',

examples/databricks/snowflake-interoperability/stackql_manifest.yml

@@ -195,21 +195,21 @@ resources:
         value: stackql
       - name: statement
         value: |
-          CREATE OR REPLACE CATALOG INTEGRATION unity_catalog_demo_int
+          CREATE CATALOG INTEGRATION IF NOT EXISTS unity_catalog_demo_int 
           CATALOG_SOURCE = ICEBERG_REST
           TABLE_FORMAT = ICEBERG
-          CATALOG_NAMESPACE = ''{{ schema_name }}''
+          CATALOG_NAMESPACE = '{{ schema_name }}'
           REST_CONFIG = (
-              CATALOG_URI = ''https://{{ databricks_deployment_name }}.cloud.databricks.com/api/2.1/unity-catalog/iceberg-rest''
-              WAREHOUSE  = ''{{ catalog_name }}''
+              CATALOG_URI = 'https://{{ databricks_deployment_name }}.cloud.databricks.com/api/2.1/unity-catalog/iceberg-rest'
+              WAREHOUSE  = '{{ catalog_name }}'
               ACCESS_DELEGATION_MODE = VENDED_CREDENTIALS
           )
           REST_AUTHENTICATION = (
           TYPE = OAUTH
-              OAUTH_TOKEN_URI = ''https://{{ databricks_deployment_name }}.cloud.databricks.com/oidc/v1/token''
-              OAUTH_CLIENT_ID = ''{{ service_principal_application_id }}''
-              OAUTH_CLIENT_SECRET = ''{{ secret }}''
-              OAUTH_ALLOWED_SCOPES = (''all-apis'', ''sql'')
+              OAUTH_TOKEN_URI = 'https://{{ databricks_deployment_name }}.cloud.databricks.com/oidc/v1/token'
+              OAUTH_CLIENT_ID = '{{ service_principal_application_id }}'
+              OAUTH_CLIENT_SECRET = '{{ secret }}'
+              OAUTH_ALLOWED_SCOPES = ('all-apis', 'sql')
           )
           ENABLED = TRUE
           REFRESH_INTERVAL_SECONDS = 30
@@ -231,8 +231,8 @@ resources:
       - name: statement
         value: |
           CREATE OR REPLACE ICEBERG TABLE retail_sales_bronze
-          CATALOG = ''unity_catalog_demo_int''
-          CATALOG_TABLE_NAME = ''retail_sales_bronze''
+          CATALOG = 'unity_catalog_demo_int'
+          CATALOG_TABLE_NAME = 'retail_sales_bronze'
           AUTO_REFRESH = TRUE
       - name: timeout
         value: 30
@@ -252,8 +252,8 @@ resources:
       - name: statement
         value: |
           CREATE OR REPLACE ICEBERG TABLE retail_sales_silver
-          CATALOG = ''unity_catalog_demo_int''
-          CATALOG_TABLE_NAME = ''retail_sales_silver''
+          CATALOG = 'unity_catalog_demo_int'
+          CATALOG_TABLE_NAME = 'retail_sales_silver'
           AUTO_REFRESH = TRUE
       - name: timeout
         value: 30
@@ -273,8 +273,8 @@ resources:
       - name: statement
         value: |
           CREATE OR REPLACE ICEBERG TABLE retail_sales_gold
-          CATALOG = ''unity_catalog_demo_int''
-          CATALOG_TABLE_NAME = ''retail_sales_gold''
+          CATALOG = 'unity_catalog_demo_int'
+          CATALOG_TABLE_NAME = 'retail_sales_gold'
           AUTO_REFRESH = TRUE
       - name: timeout
         value: 30

stackql_deploy/cmd/teardown.py

@@ -23,26 +23,27 @@ def collect_exports(self, show_queries, dry_run):
             full_context = get_full_context(self.env, self.global_context, resource, self.logger)
 
             # get resource queries
-            if type == 'query' and 'sql' in resource:
-                # inline SQL specified in the resource
-                test_queries = {}
-                exports_query = render_inline_template(self.env,
-                                                        resource["name"],
-                                                        resource["sql"],
-                                                        full_context,
-                                                        self.logger)
-                exports_retries = 1
-                exports_retry_delay = 0
-            else:
-                test_queries = get_queries(self.env,
-                                               self.stack_dir,
-                                               'resources',
-                                               resource,
-                                               full_context,
-                                               self.logger)
-                exports_query = test_queries.get('exports', {}).get('rendered')
-                exports_retries = test_queries.get('exports', {}).get('options', {}).get('retries', 1)
-                exports_retry_delay = test_queries.get('exports', {}).get('options', {}).get('retry_delay', 0)
+            if type != 'command':
+                if type == 'query' and 'sql' in resource:
+                    # inline SQL specified in the resource
+                    test_queries = {}
+                    exports_query = render_inline_template(self.env,
+                                                            resource["name"],
+                                                            resource["sql"],
+                                                            full_context,
+                                                            self.logger)
+                    exports_retries = 1
+                    exports_retry_delay = 0
+                else:
+                    test_queries = get_queries(self.env,
+                                                self.stack_dir,
+                                                'resources',
+                                                resource,
+                                                full_context,
+                                                self.logger)
+                    exports_query = test_queries.get('exports', {}).get('rendered')
+                    exports_retries = test_queries.get('exports', {}).get('options', {}).get('retries', 1)
+                    exports_retry_delay = test_queries.get('exports', {}).get('options', {}).get('retry_delay', 0)
 
             if exports_query:
                 self.process_exports(

stackql_deploy/lib/filters.py

@@ -109,6 +109,25 @@ def sql_list(input_data):
     quoted_items = [f"'{str(item)}'" for item in python_list]
     return f"({','.join(quoted_items)})"
 
+def sql_escape(value):
+    """
+    Escapes a string for use as a SQL string literal by doubling any single quotes.
+    This is useful for nested SQL statements where single quotes need to be escaped.
+    
+    Args:
+        value: The string to escape
+        
+    Returns:
+        The escaped string with single quotes doubled
+    """
+    if value is None:
+        return None
+        
+    if not isinstance(value, str):
+        value = str(value)
+        
+    return value.replace("'", "''")
+
 #
 # exported functions
 #
@@ -121,11 +140,12 @@ def setup_environment(stack_dir, logger):
         loader=FileSystemLoader(os.getcwd()),
         autoescape=False
     )
-    env.filters['merge_lists'] = merge_lists
+    env.filters['from_json'] = from_json
     env.filters['base64_encode'] = base64_encode
+    env.filters['merge_lists'] = merge_lists
     env.filters['generate_patch_document'] = generate_patch_document
-    env.filters['from_json'] = from_json
     env.filters['sql_list'] = sql_list
+    env.filters['sql_escape'] = sql_escape
     env.globals['uuid'] = lambda: str(uuid.uuid4())
     logger.debug("custom Jinja filters registered: %s", env.filters.keys())
     return env

stackql_deploy/lib/templating.py

@@ -38,7 +38,7 @@ def render_queries(res_name, env, queries, context, logger):
                         properties, ensure_ascii=False, separators=(',', ':')
                     ).replace('True', 'true').replace('False', 'false')
                     # Correctly format JSON to use double quotes and pass directly since template handles quoting
-                    json_str = json_str.replace("'", "\\'")  # escape single quotes if any within strings
+                    # json_str = json_str.replace("'", "\\'")  # escape single quotes if any within strings
                     temp_context[ctx_key] = json_str
                 # No need to alter non-JSON strings, assume the template handles them correctly
 
@@ -147,7 +147,7 @@ def render_inline_template(env, resource_name, template_string, full_context, lo
                     properties, ensure_ascii=False, separators=(',', ':')
                 ).replace('True', 'true').replace('False', 'false')
                 # Correctly format JSON to use double quotes and pass directly since template handles quoting
-                json_str = json_str.replace("'", "\\'")  # escape single quotes if any within strings
+                # json_str = json_str.replace("'", "\\'")  # escape single quotes if any within strings
                 temp_context[ctx_key] = json_str
 
         # Render the template

website/docs/resource-query-files.md

@@ -206,113 +206,17 @@ AND project = '{{ project }}'
 AND zone = '{{ zone }}'
 ```
 
-## Template filters
+## Template Filters
 
-### `from_json`
-`from_json` is a custom `stackql-deploy` filter which is used to convert a `json` string to a python dictionary or list.  The common use case is to take advantage of Jinja2 templating within `stackql-deploy` to dynamically generate SQL statements for infrastructure provisioning by converting JSON strings into Python dictionaries or lists, allowing for iteration and more flexible configuration management.
+StackQL Deploy leverages Jinja2 templating capabilities and extends them with custom filters for infrastructure provisioning. For a complete reference of all available filters, see the [__Template Filters__](template-filters) documentation.
 
-```sql  {2}
-/*+ create */
-{% for network_interface in network_interfaces | from_json %}
-INSERT INTO google.compute.instances 
- (
-  zone,
-  project,
-  data__name,
-  data__machineType,
-  data__canIpForward,
-  data__deletionProtection,
-  data__scheduling,
-  data__networkInterfaces,
-  data__disks,
-  data__serviceAccounts,
-  data__tags
- ) 
- SELECT
-'{{ default_zone }}',
-'{{ project }}',
-'{{ instance_name_prefix }}-{{ loop.index }}',
-'{{ machine_type }}',
-true,
-false,
-'{{ scheduling }}',
-'[ {{ network_interface | tojson }} ]',
-'{{ disks }}',
-'{{ service_accounts }}',
-'{{ tags }}';
-{% endfor %}
-```
-
-### `tojson`
-`tojson` is a built-in Jinja2 filter to convert a Python dictionary or list into a `json` string.  This may be required if you have used the `from_json` filter as shown here:
-
-```sql  {25}
-/*+ create */
-{% for network_interface in network_interfaces | from_json %}
-INSERT INTO google.compute.instances 
- (
-  zone,
-  project,
-  data__name,
-  data__machineType,
-  data__canIpForward,
-  data__deletionProtection,
-  data__scheduling,
-  data__networkInterfaces,
-  data__disks,
-  data__serviceAccounts,
-  data__tags
- ) 
- SELECT
-'{{ default_zone }}',
-'{{ project }}',
-'{{ instance_name_prefix }}-{{ loop.index }}',
-'{{ machine_type }}',
-true,
-false,
-'{{ scheduling }}',
-'[ {{ network_interface | tojson }} ]',
-'{{ disks }}',
-'{{ service_accounts }}',
-'{{ tags }}';
-{% endfor %}
-```
-
-### `generate_patch_document`
-
-`generate_patch_document` is a custom `stackql-deploy` filter which generates a patch document for the given resource according to https://datatracker.ietf.org/doc/html/rfc6902, this is designed for the AWS Cloud Control API, which requires a patch document to update resources.  An example of this filter used to update the `NotificationConfiguration` for an existing AWS bucket is shown here:
-
-```sql  {3-5}
-/*+ createorupdate */
-update aws.s3.buckets 
-set data__PatchDocument = string('{{ {
-    "NotificationConfiguration": transfer_notification_config
-    } | generate_patch_document }}') 
-WHERE 
-region = '{{ region }}' 
-AND data__Identifier = '{{ transfer_bucket_name }}';
-```
-
-### `base64_encode`
+Here are a few commonly used filters:
 
-`base64_encode` is a custom `stackql-deploy` filter used to generate a `base64` encoded value for a given input string, this is often specified as an input requirement for a free text field in a resource.  This example shows how to `base64` encode the `UserData` field for an AWS EC2 instance:
-
-```sql {13}
-/*+ create */
-INSERT INTO aws.ec2.instances (
- ImageId,
- InstanceType,
- SubnetId,
- UserData,
- region
-)
-SELECT 
- '{{ ami_id }}',
- '{{ instance_type }}',
- '{{ instance_subnet_id }}',
- '{{ user_data | base64_encode }}',
- '{{ region }}';
-```
+- `from_json` - Converts JSON strings to Python objects for iteration and manipulation
+- `tojson` - Converts Python objects back to JSON strings
+- `sql_escape` - Properly escapes SQL string literals for nested SQL statements
+- `generate_patch_document` - Creates RFC6902-compliant patch documents for AWS resources
+- `base64_encode` - Encodes strings as base64 for API fields requiring binary data
 
 ## Examples