From 2ad851ab3a87b7e2341ec992d9bf7e33da5e7524 Mon Sep 17 00:00:00 2001 From: Alex Ott Date: Mon, 17 Nov 2025 15:49:54 +0100 Subject: [PATCH] [Feature[ Add `role_arn` field to `databricks_mws_storage_configurations` resource to support UC by Default This allows sharing S3 buckets between root storage and Unity Catalog --- NEXT_CHANGELOG.md | 1 + docs/resources/mws_storage_configurations.md | 18 +++++ mws/mws.go | 1 + mws/resource_mws_storage_configurations.go | 21 ++++-- ...esource_mws_storage_configurations_test.go | 73 +++++++++++++++++++ 5 files changed, 107 insertions(+), 7 deletions(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 2328b3cc24..b822682196 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -8,6 +8,7 @@ * Add `databricks_users` data source ([#4028](https://github.com/databricks/terraform-provider-databricks/pull/4028)) * Improve `databricks_service_principals` data source ([#5164](https://github.com/databricks/terraform-provider-databricks/pull/5164)) +* Add `role_arn` field to `databricks_mws_storage_configurations` resource to support sharing S3 buckets between root storage and Unity Catalog ([#5222](https://github.com/databricks/terraform-provider-databricks/issues/5222)) ### Bug Fixes diff --git a/docs/resources/mws_storage_configurations.md b/docs/resources/mws_storage_configurations.md index a4a15c3b23..7889f19d45 100644 --- a/docs/resources/mws_storage_configurations.md +++ b/docs/resources/mws_storage_configurations.md @@ -38,6 +38,20 @@ resource "databricks_mws_storage_configurations" "this" { } ``` +### Example Usage with Role ARN + +When sharing an S3 bucket between root storage and a Unity Catalog metastore, you can specify a role ARN: + +```hcl +resource "databricks_mws_storage_configurations" "this" { + provider = databricks.mws + account_id = var.databricks_account_id + storage_configuration_name = "${var.prefix}-storage" + bucket_name = aws_s3_bucket.root_storage_bucket.bucket + role_arn = aws_iam_role.unity_catalog_role.arn +} +``` + ## Argument Reference The following arguments are required: @@ -46,6 +60,10 @@ The following arguments are required: * `account_id` - Account Id that could be found in the top right corner of [Accounts Console](https://accounts.cloud.databricks.com/) * `storage_configuration_name` - name under which this storage configuration is stored +The following arguments are optional: + +* `role_arn` - (Optional) The ARN of the IAM role that Databricks will assume to access the S3 bucket. This allows sharing an S3 bucket between root storage and the default catalog for a workspace. See the [Databricks API documentation](https://docs.databricks.com/api/account/storage/create) for more details. + ## Attribute Reference In addition to all arguments above, the following attributes are exported: diff --git a/mws/mws.go b/mws/mws.go index 3df789fbe7..94bb5ffd4e 100644 --- a/mws/mws.go +++ b/mws/mws.go @@ -45,6 +45,7 @@ type StorageConfiguration struct { StorageConfigurationID string `json:"storage_configuration_id,omitempty"` StorageConfigurationName string `json:"storage_configuration_name,omitempty"` RootBucketInfo *RootBucketInfo `json:"root_bucket_info,omitempty"` + RoleArn string `json:"role_arn,omitempty"` AccountID string `json:"account_id,omitempty"` CreationTime int64 `json:"creation_time,omitempty"` } diff --git a/mws/resource_mws_storage_configurations.go b/mws/resource_mws_storage_configurations.go index 2b979cf448..9a66ba2f30 100644 --- a/mws/resource_mws_storage_configurations.go +++ b/mws/resource_mws_storage_configurations.go @@ -21,15 +21,15 @@ type StorageConfigurationsAPI struct { } // Create creates a configuration for the root s3 bucket -func (a StorageConfigurationsAPI) Create(mwsAcctID, storageConfigurationName string, bucketName string) (StorageConfiguration, error) { +func (a StorageConfigurationsAPI) Create(mwsAcctID, storageConfigurationName string, bucketName string, roleArn string) (StorageConfiguration, error) { var mwsStorageConfigurations StorageConfiguration storageConfigurationAPIPath := fmt.Sprintf("/accounts/%s/storage-configurations", mwsAcctID) - err := a.client.Post(a.context, storageConfigurationAPIPath, StorageConfiguration{ + storageConfig := StorageConfiguration{ StorageConfigurationName: storageConfigurationName, - RootBucketInfo: &RootBucketInfo{ - BucketName: bucketName, - }, - }, &mwsStorageConfigurations) + RoleArn: roleArn, + RootBucketInfo: &RootBucketInfo{BucketName: bucketName}, + } + err := a.client.Post(a.context, storageConfigurationAPIPath, storageConfig, &mwsStorageConfigurations) return mwsStorageConfigurations, err } @@ -62,7 +62,8 @@ func ResourceMwsStorageConfigurations() common.Resource { name := d.Get("storage_configuration_name").(string) bucketName := d.Get("bucket_name").(string) accountID := d.Get("account_id").(string) - storageConfiguration, err := NewStorageConfigurationsAPI(ctx, c).Create(accountID, name, bucketName) + roleArn := d.Get("role_arn").(string) + storageConfiguration, err := NewStorageConfigurationsAPI(ctx, c).Create(accountID, name, bucketName, roleArn) if err != nil { return err } @@ -81,6 +82,7 @@ func ResourceMwsStorageConfigurations() common.Resource { } d.Set("storage_configuration_name", storageConifiguration.StorageConfigurationName) d.Set("bucket_name", storageConifiguration.RootBucketInfo.BucketName) + d.Set("role_arn", storageConifiguration.RoleArn) return d.Set("creation_time", storageConifiguration.CreationTime) }, Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { @@ -104,6 +106,11 @@ func ResourceMwsStorageConfigurations() common.Resource { Type: schema.TypeString, Required: true, }, + "role_arn": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, "creation_time": { Type: schema.TypeInt, Computed: true, diff --git a/mws/resource_mws_storage_configurations_test.go b/mws/resource_mws_storage_configurations_test.go index 740963a4e8..91b2825bb9 100644 --- a/mws/resource_mws_storage_configurations_test.go +++ b/mws/resource_mws_storage_configurations_test.go @@ -177,3 +177,76 @@ func TestResourceStorageConfigurationDelete_Error(t *testing.T) { qa.AssertErrorStartsWith(t, err, "Internal error happened") assert.Equal(t, "abc/scid", d.Id()) } + +func TestResourceStorageConfigurationCreateWithArn(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "POST", + Resource: "/api/2.0/accounts/abc/storage-configurations", + ExpectedRequest: StorageConfiguration{ + StorageConfigurationName: "Main Storage", + RootBucketInfo: &RootBucketInfo{ + BucketName: "bucket", + }, + RoleArn: "arn:aws:iam::123456789012:role/my-role", + }, + Response: StorageConfiguration{ + StorageConfigurationID: "scid", + }, + }, + { + Method: "GET", + Resource: "/api/2.0/accounts/abc/storage-configurations/scid", + Response: StorageConfiguration{ + StorageConfigurationID: "scid", + StorageConfigurationName: "Main Storage", + RootBucketInfo: &RootBucketInfo{ + BucketName: "bucket", + }, + RoleArn: "arn:aws:iam::123456789012:role/my-role", + }, + }, + }, + Resource: ResourceMwsStorageConfigurations(), + State: map[string]any{ + "account_id": "abc", + "bucket_name": "bucket", + "storage_configuration_name": "Main Storage", + "role_arn": "arn:aws:iam::123456789012:role/my-role", + }, + Create: true, + }.Apply(t) + assert.NoError(t, err) + assert.Equal(t, "abc/scid", d.Id()) + assert.Equal(t, "arn:aws:iam::123456789012:role/my-role", d.Get("role_arn")) +} + +func TestResourceStorageConfigurationReadWithArn(t *testing.T) { + d, err := qa.ResourceFixture{ + Fixtures: []qa.HTTPFixture{ + { + Method: "GET", + Resource: "/api/2.0/accounts/abc/storage-configurations/scid", + Response: StorageConfiguration{ + StorageConfigurationID: "scid", + StorageConfigurationName: "Main Storage", + RootBucketInfo: &RootBucketInfo{ + BucketName: "bucket", + }, + RoleArn: "arn:aws:iam::123456789012:role/my-role", + }, + }, + }, + Resource: ResourceMwsStorageConfigurations(), + Read: true, + ID: "abc/scid", + }.Apply(t) + assert.NoError(t, err) + assert.Equal(t, "abc/scid", d.Id(), "Id should not be empty") + assert.Equal(t, "bucket", d.Get("bucket_name")) + assert.Equal(t, "arn:aws:iam::123456789012:role/my-role", d.Get("role_arn")) + assert.Equal(t, 0, d.Get("creation_time")) + assert.Equal(t, "scid", d.Get("storage_configuration_id")) + assert.Equal(t, "Main Storage", d.Get("storage_configuration_name")) +}