Defining unity catalog by terraform in azure

Question

How can I enable a unity catalog in Databricks azure from terraform when I have the below resources defined. Can someone help me? I came across this script but that doesnt work. I am missing something.

Here the parts I have defined till now in my terraform:

resource "azurerm_databricks_workspace" "databricks_cdf_audit_log" {
  name                = "databricks-cdf-audit-log"
  resource_group_name = data.azurerm_resource_group.cdf_audit_log_rg.name
  location            = var.databricks_location
  sku                 = "premium"

  tags = {
    Environment = var.environment
    service     = var.service
    team        = var.team
  }
}

resource "azurerm_databricks_access_connector" "databricks_it_access_connector" {
  name                = "databricks-cdf-connector"
  resource_group_name = data.azurerm_resource_group.dit_log_rg.name
  location            = var.databricks_location

  identity {
    type = "SystemAssigned"
  }

  tags = {
    Environment = var.environment
    service     = var.service
    team        = var.team
  }
}



resource "azurerm_storage_account" "storage_account" {
  name                     = "${var.storage_account_prefix}auditmeta"
  resource_group_name      = data.azurerm_resource_group.cdf_audit_log_rg.name
  location                 = var.databricks_location
  account_tier             = "Standard"
  account_replication_type = "LRS"

  tags = {
    Environment = var.environment
    service     = var.service
    team        = var.team
  }
}

resource "azurerm_storage_container" "storage_contain" {
  name                  = "databrdata"
  storage_account_name  = azurerm_storage_account.storage_account.name
  container_access_type = "private"
}
resource "azurerm_role_assignment" "databricks_admin" {
  scope                = azurerm_databricks_workspace.databrit_log.id
  role_definition_name = "Owner"
  principal_id         = data.azuread_group.access_audit_members.object_id
}

resource "azurerm_role_assignment" "databricks_connector_role_assignment" {
  scope                = azurerm_storage_account.storage_account.id
  role_definition_name = "Storage Blob Data Contributor"
  principal_id         = azurerm_databricks_access_connector.databricks_it_access_connector.identity[0].principal_id
}

Answer 1

Databricks Terraform provider documentation contains step-by-step guide for enabling Unity Catalog on Azure. You need to perform following steps:

1. Configure Azure objects - what you did already
2. Create a Unity Catalog metastore and link it to workspaces
3. Create Unity Catalog objects in the metastore (optional)

The second step involves using of the Databricks provider (excerpt from documentation, just follow the guide):

resource "databricks_metastore" "this" {
  name = "primary"
  storage_root = format("abfss://%s@%s.dfs.core.windows.net/",
    azurerm_storage_container.unity_catalog.name,
  azurerm_storage_account.unity_catalog.name)
  force_destroy = true
}

resource "databricks_metastore_data_access" "first" {
  metastore_id = databricks_metastore.this.id
  name         = "the-keys"
  azure_managed_identity {
    access_connector_id = azurerm_databricks_access_connector.unity.id
  }

  is_default = true
}

resource "databricks_metastore_assignment" "this" {
  workspace_id         = local.databricks_workspace_id
  metastore_id         = databricks_metastore.this.id
  default_catalog_name = "hive_metastore"
}

You can also use this example of Azure Databricks + Unity Catalog from the Databricks Terraform modules repo (announcement blog post).