Enable Databricks external location file events

README.md

@@ -146,7 +146,7 @@ The following requirements are needed by this module:
 
 - <a name="requirement_azurerm"></a> [azurerm](#requirement\_azurerm) (~> 4.0)
 
-- <a name="requirement_databricks"></a> [databricks](#requirement\_databricks) (~> 1.81)
+- <a name="requirement_databricks"></a> [databricks](#requirement\_databricks) (~> 1.82)
 
 - <a name="requirement_fabric"></a> [fabric](#requirement\_fabric) (~> 1.0)
 

databricks.tf

@@ -65,6 +65,7 @@ module "databricks_data_application" {
     data_factory_id           = try(module.data_application[each.key].data_factory_details.data_factory_id, {})
     data_factory_name         = try(module.data_application[each.key].data_factory_details.data_factory_name, {})
     data_factory_principal_id = try(module.data_application[each.key].data_factory_details.data_factory_principal_id, {})
+    data_factory_client_id    = try(module.data_application[each.key].data_factory_details.data_factory_client_id, {})
   }
   storage_container_ids = try(module.data_application[each.key].storage_container_ids, {})
   storage_queue_ids     = try(module.data_application[each.key].storage_queue_ids, {})

modules/dataapplication/outputs.tf

@@ -69,6 +69,14 @@ output "storage_queue_ids" {
     workspace = azurerm_storage_queue.storage_queue_workspace.id
   }
   sensitive = false
+
+  depends_on = [
+    azurerm_role_assignment.role_assignment_storage_queue_provider_queue_data_contributor_accessconnector,
+    azurerm_role_assignment.role_assignment_storage_queue_raw_queue_data_contributor_accessconnector,
+    azurerm_role_assignment.role_assignment_storage_queue_enriched_queue_data_contributor_accessconnector,
+    azurerm_role_assignment.role_assignment_storage_queue_curated_queue_data_contributor_accessconnector,
+    azurerm_role_assignment.role_assignment_storage_queue_workspace_queue_data_contributor_accessconnector,
+  ]
 }
 
 # User assigned identity outputs

modules/databrickscore/externallocation.tf

@@ -1,13 +1,14 @@
 resource "databricks_external_location" "external_location_engineering_default" {
   name = replace("${local.prefix}-engnrng-default", "-", "_")
 
-  comment         = "Default storage layer for default catalog '${local.prefix}'."
-  credential_name = databricks_storage_credential.storage_credential_engineering_default.name
-  fallback        = false
-  force_destroy   = true
-  force_update    = true
-  isolation_mode  = "ISOLATION_MODE_ISOLATED"
-  read_only       = false
-  skip_validation = false
-  url             = "abfss://${azurerm_storage_container.storage_container_engineering_default.name}@${reverse(split("/", azurerm_storage_container.storage_container_engineering_default.storage_account_id))[0]}.dfs.core.windows.net/"
+  comment            = "Default storage layer for default catalog '${local.prefix}'."
+  credential_name    = databricks_storage_credential.storage_credential_engineering_default.name
+  enable_file_events = false
+  fallback           = false
+  force_destroy      = true
+  force_update       = true
+  isolation_mode     = "ISOLATION_MODE_ISOLATED"
+  read_only          = false
+  skip_validation    = false
+  url                = "abfss://${azurerm_storage_container.storage_container_engineering_default.name}@${reverse(split("/", azurerm_storage_container.storage_container_engineering_default.storage_account_id))[0]}.dfs.core.windows.net/"
 }

modules/databrickscore/terraform.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     databricks = {
       source  = "databricks/databricks"
-      version = "~> 1.81"
+      version = "~> 1.82"
       configuration_aliases = [
         databricks.account
       ]

modules/databricksdataapplication/externallocation.tf

@@ -3,69 +3,99 @@ resource "databricks_external_location" "external_location_provider" {
 
   name = replace("${local.prefix}-${each.key}-pro", "-", "_")
 
-  comment         = "Default provider storage layer for '${var.app_name}-${each.key}' data application data provider."
-  credential_name = one(databricks_storage_credential.storage_credential[*].name)
-  fallback        = false
-  force_destroy   = true
-  force_update    = true
-  isolation_mode  = "ISOLATION_MODE_ISOLATED"
-  read_only       = false
-  skip_validation = false
-  url             = "abfss://${local.storage_container_provider[each.key].storage_container_name}@${local.storage_container_provider[each.key].storage_account_name}.dfs.core.windows.net/"
+  comment            = "Default provider storage layer for '${var.app_name}-${each.key}' data application data provider."
+  credential_name    = one(databricks_storage_credential.storage_credential[*].name)
+  enable_file_events = true
+  fallback           = false
+  force_destroy      = true
+  force_update       = true
+  isolation_mode     = "ISOLATION_MODE_ISOLATED"
+  read_only          = false
+  skip_validation    = false
+  url                = "abfss://${local.storage_container_provider[each.key].storage_container_name}@${local.storage_container_provider[each.key].storage_account_name}.dfs.core.windows.net/"
+  file_event_queue {
+    provided_aqs {
+      queue_url = var.storage_queue_ids.provider[each.key]
+    }
+  }
 }
 
 resource "databricks_external_location" "external_location_raw" {
   name = replace("${local.prefix}-raw", "-", "_")
 
-  comment         = "Default raw storage layer for '${var.app_name}' data application."
-  credential_name = one(databricks_storage_credential.storage_credential[*].name)
-  fallback        = false
-  force_destroy   = true
-  force_update    = true
-  isolation_mode  = "ISOLATION_MODE_ISOLATED"
-  read_only       = false
-  skip_validation = false
-  url             = "abfss://${local.storage_container_raw.storage_container_name}@${local.storage_container_raw.storage_account_name}.dfs.core.windows.net/"
+  comment            = "Default raw storage layer for '${var.app_name}' data application."
+  credential_name    = one(databricks_storage_credential.storage_credential[*].name)
+  enable_file_events = true
+  fallback           = false
+  force_destroy      = true
+  force_update       = true
+  isolation_mode     = "ISOLATION_MODE_ISOLATED"
+  read_only          = false
+  skip_validation    = false
+  url                = "abfss://${local.storage_container_raw.storage_container_name}@${local.storage_container_raw.storage_account_name}.dfs.core.windows.net/"
+  file_event_queue {
+    provided_aqs {
+      queue_url = var.storage_queue_ids.raw
+    }
+  }
 }
 
 resource "databricks_external_location" "external_location_enriched" {
   name = replace("${local.prefix}-enr", "-", "_")
 
-  comment         = "Default enriched storage layer for '${var.app_name}' data application."
-  credential_name = one(databricks_storage_credential.storage_credential[*].name)
-  fallback        = false
-  force_destroy   = true
-  force_update    = true
-  isolation_mode  = "ISOLATION_MODE_ISOLATED"
-  read_only       = false
-  skip_validation = false
-  url             = "abfss://${local.storage_container_enriched.storage_container_name}@${local.storage_container_enriched.storage_account_name}.dfs.core.windows.net/"
+  comment            = "Default enriched storage layer for '${var.app_name}' data application."
+  credential_name    = one(databricks_storage_credential.storage_credential[*].name)
+  enable_file_events = true
+  fallback           = false
+  force_destroy      = true
+  force_update       = true
+  isolation_mode     = "ISOLATION_MODE_ISOLATED"
+  read_only          = false
+  skip_validation    = false
+  url                = "abfss://${local.storage_container_enriched.storage_container_name}@${local.storage_container_enriched.storage_account_name}.dfs.core.windows.net/"
+  file_event_queue {
+    provided_aqs {
+      queue_url = var.storage_queue_ids.enriched
+    }
+  }
 }
 
 resource "databricks_external_location" "external_location_curated" {
   name = replace("${local.prefix}-cur", "-", "_")
 
-  comment         = "Default curated storage layer for '${var.app_name}' data application."
-  credential_name = one(databricks_storage_credential.storage_credential[*].name)
-  fallback        = false
-  force_destroy   = true
-  force_update    = true
-  isolation_mode  = "ISOLATION_MODE_ISOLATED"
-  read_only       = false
-  skip_validation = false
-  url             = "abfss://${local.storage_container_curated.storage_container_name}@${local.storage_container_curated.storage_account_name}.dfs.core.windows.net/"
+  comment            = "Default curated storage layer for '${var.app_name}' data application."
+  credential_name    = one(databricks_storage_credential.storage_credential[*].name)
+  enable_file_events = true
+  fallback           = false
+  force_destroy      = true
+  force_update       = true
+  isolation_mode     = "ISOLATION_MODE_ISOLATED"
+  read_only          = false
+  skip_validation    = false
+  url                = "abfss://${local.storage_container_curated.storage_container_name}@${local.storage_container_curated.storage_account_name}.dfs.core.windows.net/"
+  file_event_queue {
+    provided_aqs {
+      queue_url = var.storage_queue_ids.curated
+    }
+  }
 }
 
 resource "databricks_external_location" "external_location_workspace" {
   name = replace("${local.prefix}-wks", "-", "_")
 
-  comment         = "Default workspace storage layer for '${var.app_name}' data application."
-  credential_name = one(databricks_storage_credential.storage_credential[*].name)
-  fallback        = false
-  force_destroy   = true
-  force_update    = true
-  isolation_mode  = "ISOLATION_MODE_ISOLATED"
-  read_only       = false
-  skip_validation = false
-  url             = "abfss://${local.storage_container_workspace.storage_container_name}@${local.storage_container_workspace.storage_account_name}.dfs.core.windows.net/"
+  comment            = "Default workspace storage layer for '${var.app_name}' data application."
+  credential_name    = one(databricks_storage_credential.storage_credential[*].name)
+  enable_file_events = true
+  fallback           = false
+  force_destroy      = true
+  force_update       = true
+  isolation_mode     = "ISOLATION_MODE_ISOLATED"
+  read_only          = false
+  skip_validation    = false
+  url                = "abfss://${local.storage_container_workspace.storage_container_name}@${local.storage_container_workspace.storage_account_name}.dfs.core.windows.net/"
+  file_event_queue {
+    provided_aqs {
+      queue_url = var.storage_queue_ids.workspace
+    }
+  }
 }

terraform.tf

@@ -20,7 +20,7 @@ terraform {
     }
     databricks = {
       source  = "databricks/databricks"
-      version = "~> 1.81"
+      version = "~> 1.82"
     }
     time = {
       source  = "hashicorp/time"