[Feature] Add role_arn field to databricks_mws_storage_configurations resource to support UC by Default

NEXT_CHANGELOG.md

@@ -8,6 +8,7 @@
 
 * Add `databricks_users` data source ([#4028](https://github.com/databricks/terraform-provider-databricks/pull/4028))
 * Improve `databricks_service_principals` data source ([#5164](https://github.com/databricks/terraform-provider-databricks/pull/5164))
+* Add `role_arn` field to `databricks_mws_storage_configurations` resource to support sharing S3 buckets between root storage and Unity Catalog ([#5222](https://github.com/databricks/terraform-provider-databricks/issues/5222))
 
 ### Bug Fixes
 

docs/resources/mws_storage_configurations.md

@@ -38,6 +38,20 @@ resource "databricks_mws_storage_configurations" "this" {
 }
 ```
 
+### Example Usage with Role ARN
+
+When sharing an S3 bucket between root storage and a Unity Catalog metastore, you can specify a role ARN:
+
+```hcl
+resource "databricks_mws_storage_configurations" "this" {
+  provider                   = databricks.mws
+  account_id                 = var.databricks_account_id
+  storage_configuration_name = "${var.prefix}-storage"
+  bucket_name                = aws_s3_bucket.root_storage_bucket.bucket
+  role_arn                   = aws_iam_role.unity_catalog_role.arn
+}
+```
+
 ## Argument Reference
 
 The following arguments are required:
@@ -46,6 +60,10 @@ The following arguments are required:
 * `account_id` - Account Id that could be found in the top right corner of [Accounts Console](https://accounts.cloud.databricks.com/)
 * `storage_configuration_name` - name under which this storage configuration is stored
 
+The following arguments are optional:
+
+* `role_arn` - (Optional) The ARN of the IAM role that Databricks will assume to access the S3 bucket. This allows sharing an S3 bucket between root storage and the default catalog for a workspace. See the [Databricks API documentation](https://docs.databricks.com/api/account/storage/create) for more details.
+
 ## Attribute Reference
 
 In addition to all arguments above, the following attributes are exported:

mws/mws.go

@@ -45,6 +45,7 @@ type StorageConfiguration struct {
 	StorageConfigurationID   string          `json:"storage_configuration_id,omitempty"`
 	StorageConfigurationName string          `json:"storage_configuration_name,omitempty"`
 	RootBucketInfo           *RootBucketInfo `json:"root_bucket_info,omitempty"`
+	RoleArn                  string          `json:"role_arn,omitempty"`
 	AccountID                string          `json:"account_id,omitempty"`
 	CreationTime             int64           `json:"creation_time,omitempty"`
 }

mws/resource_mws_storage_configurations.go

@@ -21,15 +21,15 @@ type StorageConfigurationsAPI struct {
 }
 
 // Create creates a configuration for the root s3 bucket
-func (a StorageConfigurationsAPI) Create(mwsAcctID, storageConfigurationName string, bucketName string) (StorageConfiguration, error) {
+func (a StorageConfigurationsAPI) Create(mwsAcctID, storageConfigurationName string, bucketName string, roleArn string) (StorageConfiguration, error) {
 	var mwsStorageConfigurations StorageConfiguration
 	storageConfigurationAPIPath := fmt.Sprintf("/accounts/%s/storage-configurations", mwsAcctID)
-	err := a.client.Post(a.context, storageConfigurationAPIPath, StorageConfiguration{
+	storageConfig := StorageConfiguration{
 		StorageConfigurationName: storageConfigurationName,
-		RootBucketInfo: &RootBucketInfo{
-			BucketName: bucketName,
-		},
-	}, &mwsStorageConfigurations)
+		RoleArn:                  roleArn,
+		RootBucketInfo:           &RootBucketInfo{BucketName: bucketName},
+	}
+	err := a.client.Post(a.context, storageConfigurationAPIPath, storageConfig, &mwsStorageConfigurations)
 	return mwsStorageConfigurations, err
 }
 
@@ -62,7 +62,8 @@ func ResourceMwsStorageConfigurations() common.Resource {
 			name := d.Get("storage_configuration_name").(string)
 			bucketName := d.Get("bucket_name").(string)
 			accountID := d.Get("account_id").(string)
-			storageConfiguration, err := NewStorageConfigurationsAPI(ctx, c).Create(accountID, name, bucketName)
+			roleArn := d.Get("role_arn").(string)
+			storageConfiguration, err := NewStorageConfigurationsAPI(ctx, c).Create(accountID, name, bucketName, roleArn)
 			if err != nil {
 				return err
 			}
@@ -81,6 +82,7 @@ func ResourceMwsStorageConfigurations() common.Resource {
 			}
 			d.Set("storage_configuration_name", storageConifiguration.StorageConfigurationName)
 			d.Set("bucket_name", storageConifiguration.RootBucketInfo.BucketName)
+			d.Set("role_arn", storageConifiguration.RoleArn)
 			return d.Set("creation_time", storageConifiguration.CreationTime)
 		},
 		Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
@@ -104,6 +106,11 @@ func ResourceMwsStorageConfigurations() common.Resource {
 				Type:     schema.TypeString,
 				Required: true,
 			},
+			"role_arn": {
+				Type:     schema.TypeString,
+				Optional: true,
+				ForceNew: true,
+			},
 			"creation_time": {
 				Type:     schema.TypeInt,
 				Computed: true,

mws/resource_mws_storage_configurations_test.go

@@ -177,3 +177,76 @@ func TestResourceStorageConfigurationDelete_Error(t *testing.T) {
 	qa.AssertErrorStartsWith(t, err, "Internal error happened")
 	assert.Equal(t, "abc/scid", d.Id())
 }
+
+func TestResourceStorageConfigurationCreateWithArn(t *testing.T) {
+	d, err := qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "POST",
+				Resource: "/api/2.0/accounts/abc/storage-configurations",
+				ExpectedRequest: StorageConfiguration{
+					StorageConfigurationName: "Main Storage",
+					RootBucketInfo: &RootBucketInfo{
+						BucketName: "bucket",
+					},
+					RoleArn: "arn:aws:iam::123456789012:role/my-role",
+				},
+				Response: StorageConfiguration{
+					StorageConfigurationID: "scid",
+				},
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/accounts/abc/storage-configurations/scid",
+				Response: StorageConfiguration{
+					StorageConfigurationID:   "scid",
+					StorageConfigurationName: "Main Storage",
+					RootBucketInfo: &RootBucketInfo{
+						BucketName: "bucket",
+					},
+					RoleArn: "arn:aws:iam::123456789012:role/my-role",
+				},
+			},
+		},
+		Resource: ResourceMwsStorageConfigurations(),
+		State: map[string]any{
+			"account_id":                 "abc",
+			"bucket_name":                "bucket",
+			"storage_configuration_name": "Main Storage",
+			"role_arn":                   "arn:aws:iam::123456789012:role/my-role",
+		},
+		Create: true,
+	}.Apply(t)
+	assert.NoError(t, err)
+	assert.Equal(t, "abc/scid", d.Id())
+	assert.Equal(t, "arn:aws:iam::123456789012:role/my-role", d.Get("role_arn"))
+}
+
+func TestResourceStorageConfigurationReadWithArn(t *testing.T) {
+	d, err := qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/accounts/abc/storage-configurations/scid",
+				Response: StorageConfiguration{
+					StorageConfigurationID:   "scid",
+					StorageConfigurationName: "Main Storage",
+					RootBucketInfo: &RootBucketInfo{
+						BucketName: "bucket",
+					},
+					RoleArn: "arn:aws:iam::123456789012:role/my-role",
+				},
+			},
+		},
+		Resource: ResourceMwsStorageConfigurations(),
+		Read:     true,
+		ID:       "abc/scid",
+	}.Apply(t)
+	assert.NoError(t, err)
+	assert.Equal(t, "abc/scid", d.Id(), "Id should not be empty")
+	assert.Equal(t, "bucket", d.Get("bucket_name"))
+	assert.Equal(t, "arn:aws:iam::123456789012:role/my-role", d.Get("role_arn"))
+	assert.Equal(t, 0, d.Get("creation_time"))
+	assert.Equal(t, "scid", d.Get("storage_configuration_id"))
+	assert.Equal(t, "Main Storage", d.Get("storage_configuration_name"))
+}