Skip to content

Commit 48465d5

Browse files
tongyimingmikatong
tongyiming
and
mikatong
authored
feat(emr): [120069985] support multi_disks (#2919)
* emr support multi_disks * update --------- Co-authored-by: mikatong <mikatong@tencent.com>
1 parent 1f65ddc commit 48465d5

File tree

6 files changed

+400
-66
lines changed

6 files changed

+400
-66
lines changed

‎.changelog/2919.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
```release-note:enhancement
2+
resource/tencentcloud_emr_cluster: Support multi_disks.
3+
```

‎tencentcloud/services/emr/extension_emr.go‎

Lines changed: 169 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
package emr
22

33
import (
4+
"fmt"
5+
46
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
57
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
68
emr "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/emr/v20190103"
9+
"github.com/tencentcloudstack/terraform-provider-tencentcloud/tencentcloud/internal/helper"
710
)
811

912
const (
@@ -35,38 +38,123 @@ func buildResourceSpecSchema() *schema.Schema {
3538
MaxItems: 1,
3639
Elem: &schema.Resource{
3740
Schema: map[string]*schema.Schema{
38-
"spec": {Type: schema.TypeString, Optional: true},
39-
"storage_type": {Type: schema.TypeInt, Optional: true},
40-
"disk_type": {Type: schema.TypeString, Optional: true},
41-
"mem_size": {Type: schema.TypeInt, Optional: true},
42-
"cpu": {Type: schema.TypeInt, Optional: true},
43-
"disk_size": {Type: schema.TypeInt, Optional: true},
44-
"root_size": {Type: schema.TypeInt, Optional: true},
41+
"spec": {
42+
Type: schema.TypeString,
43+
Optional: true,
44+
ForceNew: true,
45+
Description: "Node specification description, such as CVM.SA2.",
46+
},
47+
"storage_type": {
48+
Type: schema.TypeInt,
49+
Optional: true,
50+
ForceNew: true,
51+
Description: "Storage type. Value range:\n" +
52+
" - 4: Represents cloud SSD;\n" +
53+
" - 5: Represents efficient cloud disk;\n" +
54+
" - 6: Represents enhanced SSD Cloud Block Storage;\n" +
55+
" - 11: Represents throughput Cloud Block Storage;\n" +
56+
" - 12: Represents extremely fast SSD Cloud Block Storage.",
57+
},
58+
"disk_type": {
59+
Type: schema.TypeString,
60+
Optional: true,
61+
ForceNew: true,
62+
Description: "disk types. Value range:\n" +
63+
" - CLOUD_SSD: Represents cloud SSD;\n" +
64+
" - CLOUD_PREMIUM: Represents efficient cloud disk;\n" +
65+
" - CLOUD_BASIC: Represents Cloud Block Storage.",
66+
},
67+
"mem_size": {
68+
Type: schema.TypeInt,
69+
Optional: true,
70+
ForceNew: true,
71+
Description: "Memory size in M.",
72+
},
73+
"cpu": {
74+
Type: schema.TypeInt,
75+
Optional: true,
76+
ForceNew: true,
77+
Description: "Number of CPU cores.",
78+
},
79+
"disk_size": {
80+
Type: schema.TypeInt,
81+
Optional: true,
82+
ForceNew: true,
83+
Description: "Data disk capacity.",
84+
},
85+
"root_size": {
86+
Type: schema.TypeInt,
87+
Optional: true,
88+
ForceNew: true,
89+
Description: "Root disk capacity.",
90+
},
91+
"multi_disks": {
92+
Type: schema.TypeSet,
93+
Optional: true,
94+
Computed: true,
95+
ForceNew: true,
96+
Description: "Cloud disk list. When the data disk is a cloud disk, use disk_type and disk_size parameters directly, and use multi_disks for excess parts.",
97+
Elem: &schema.Resource{
98+
Schema: map[string]*schema.Schema{
99+
"disk_type": {
100+
Type: schema.TypeString,
101+
Optional: true,
102+
ForceNew: true,
103+
Elem: &schema.Schema{
104+
Type: schema.TypeString,
105+
},
106+
Description: "Cloud disk type\n" +
107+
" - CLOUD_SSD: Represents cloud SSD;\n" +
108+
" - CLOUD_PREMIUM: Represents efficient cloud disk;\n" +
109+
" - CLOUD_HSSD: Represents enhanced SSD Cloud Block Storage.",
110+
},
111+
"volume": {
112+
Type: schema.TypeInt,
113+
Optional: true,
114+
ForceNew: true,
115+
Description: "Cloud disk size.",
116+
},
117+
"count": {
118+
Type: schema.TypeInt,
119+
Optional: true,
120+
ForceNew: true,
121+
Description: "Number of cloud disks of this type.",
122+
},
123+
},
124+
},
125+
Set: func(v interface{}) int {
126+
m := v.(map[string]interface{})
127+
return helper.HashString(fmt.Sprintf("%s-%d-%d", m["disk_type"].(string), m["volume"].(int), m["count"].(int)))
128+
129+
},
130+
},
45131
},
46132
},
133+
Description: "Resource details.",
47134
}
48135
}
49136

50-
func ParseMultiDisks(_multiDisks []map[string]interface{}) []*emr.MultiDisk {
51-
multiDisks := make([]*emr.MultiDisk, len(_multiDisks))
52-
for _, item := range _multiDisks {
137+
func ParseMultiDisks(_multiDisks []interface{}) []*emr.MultiDisk {
138+
multiDisks := make([]*emr.MultiDisk, 0, len(_multiDisks))
139+
for _, multiDisk := range _multiDisks {
140+
item := multiDisk.(map[string]interface{})
53141
var diskType string
54-
var volume int64
55-
var count int64
142+
var volume int
143+
var count int
56144
for subK, subV := range item {
57145
if subK == "disk_type" {
58146
diskType = subV.(string)
59147
} else if subK == "volume" {
60-
volume = subV.(int64)
148+
volume = subV.(int)
61149
} else if subK == "count" {
62-
count = subV.(int64)
150+
count = subV.(int)
63151
}
64152
}
65153
multiDisks = append(multiDisks,
66154
&emr.MultiDisk{
67-
DiskType: common.StringPtr(diskType),
68-
Volume: common.Int64Ptr(volume),
69-
Count: common.Int64Ptr(count),
155+
DiskType: helper.String(diskType),
156+
Volume: helper.IntInt64(volume),
157+
Count: helper.IntInt64(count),
70158
})
71159
}
72160

@@ -101,7 +189,7 @@ func ParseResource(_resource map[string]interface{}) *emr.Resource {
101189
} else if k == "root_size" {
102190
resultResource.RootSize = common.Int64Ptr((int64)(v.(int)))
103191
} else if k == "multi_disks" {
104-
multiDisks := v.([]map[string]interface{})
192+
multiDisks := v.(*schema.Set).List()
105193
resultResource.MultiDisks = ParseMultiDisks(multiDisks)
106194
} else if k == "tags" {
107195
tags := v.([]map[string]string)
@@ -116,3 +204,66 @@ func ParseResource(_resource map[string]interface{}) *emr.Resource {
116204
}
117205
return resultResource
118206
}
207+
208+
func validateMultiDisks(r map[string]interface{}) error {
209+
if _, ok := r["multi_disks"]; !ok {
210+
return nil
211+
}
212+
multiDiskList := r["multi_disks"].(*schema.Set).List()
213+
visited := make(map[string]struct{})
214+
215+
for _, multiDisk := range multiDiskList {
216+
multiDiskMap := multiDisk.(map[string]interface{})
217+
key := fmt.Sprintf("%s-%d", multiDiskMap["disk_type"].(string), multiDiskMap["volume"].(int))
218+
if _, ok := visited[key]; ok {
219+
return fmt.Errorf("Merge disks of the same specifications")
220+
} else {
221+
visited[key] = struct{}{}
222+
}
223+
}
224+
225+
return nil
226+
}
227+
228+
func fetchMultiDisks(v *emr.NodeHardwareInfo, r *emr.OutterResource) (multiDisks []interface{}) {
229+
var inputDataDiskTag string
230+
if r.DiskType != nil && r.DiskSize != nil {
231+
inputDataDiskTag = fmt.Sprintf("%s-%d", *r.DiskType, *r.DiskSize)
232+
}
233+
for _, item := range v.MCMultiDisk {
234+
outputDataDiskTag := ""
235+
multiDisk := make(map[string]interface{})
236+
if item.Type != nil {
237+
var diskType string
238+
if *item.Type == 4 {
239+
diskType = "CLOUD_SSD"
240+
}
241+
if *item.Type == 5 {
242+
diskType = "CLOUD_PREMIUM"
243+
}
244+
if *item.Type == 6 {
245+
diskType = "CLOUD_HSSD"
246+
}
247+
multiDisk["disk_type"] = diskType
248+
outputDataDiskTag = diskType
249+
}
250+
if item.Volume != nil {
251+
volume := int(*item.Volume / 1024 / 1024 / 1024)
252+
multiDisk["volume"] = volume
253+
outputDataDiskTag = fmt.Sprintf("%s-%d", outputDataDiskTag, volume)
254+
}
255+
var count int
256+
if item.Count != nil {
257+
count = int(*item.Count)
258+
if count > 0 && inputDataDiskTag == outputDataDiskTag {
259+
count -= 1
260+
}
261+
multiDisk["count"] = count
262+
}
263+
264+
if count != 0 {
265+
multiDisks = append(multiDisks, multiDisk)
266+
}
267+
}
268+
return
269+
}

‎tencentcloud/services/emr/resource_tc_emr_cluster.go‎

Lines changed: 52 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -40,18 +40,20 @@ func ResourceTencentCloudEmrCluster() *schema.Resource {
4040
Required: true,
4141
ForceNew: true,
4242
Description: "Product ID. Different products ID represents different EMR product versions. Value range:\n" +
43-
"- 16: represents EMR-V2.3.0\n" +
44-
"- 20: indicates EMR-V2.5.0\n" +
45-
"- 25: represents EMR-V3.1.0\n" +
46-
"- 27: represents KAFKA-V1.0.0\n" +
47-
"- 30: indicates EMR-V2.6.0\n" +
48-
"- 33: represents EMR-V3.2.1\n" +
49-
"- 34: stands for EMR-V3.3.0\n" +
50-
"- 36: represents STARROCKS-V1.0.0\n" +
51-
"- 37: indicates EMR-V3.4.0\n" +
52-
"- 38: represents EMR-V2.7.0\n" +
53-
"- 39: stands for STARROCKS-V1.1.0\n" +
54-
"- 41: represents DRUID-V1.1.0.",
43+
" - 16: represents EMR-V2.3.0\n" +
44+
" - 20: represents EMR-V2.5.0\n" +
45+
" - 25: represents EMR-V3.1.0\n" +
46+
" - 27: represents KAFKA-V1.0.0\n" +
47+
" - 30: represents EMR-V2.6.0\n" +
48+
" - 33: represents EMR-V3.2.1\n" +
49+
" - 34: represents EMR-V3.3.0\n" +
50+
" - 37: represents EMR-V3.4.0\n" +
51+
" - 38: represents EMR-V2.7.0\n" +
52+
" - 44: represents EMR-V3.5.0\n" +
53+
" - 50: represents KAFKA-V2.0.0\n" +
54+
" - 51: represents STARROCKS-V1.4.0\n" +
55+
" - 53: represents EMR-V3.6.0\n" +
56+
" - 54: represents STARROCKS-V2.0.0.",
5557
},
5658
"vpc_settings": {
5759
Type: schema.TypeMap,
@@ -534,6 +536,31 @@ func resourceTencentCloudEmrClusterRead(d *schema.ResourceData, meta interface{}
534536
}
535537

536538
_ = d.Set("instance_id", instanceId)
539+
clusterNodeMap := make(map[string]*emr.NodeHardwareInfo)
540+
err = resource.Retry(tccommon.ReadRetryTimeout, func() *resource.RetryError {
541+
result, err := emrService.DescribeClusterNodes(ctx, instanceId, "all", "all", 0, 10)
542+
543+
if err != nil {
544+
return resource.RetryableError(err)
545+
}
546+
547+
if len(result) > 0 {
548+
_ = d.Set("auto_renew", result[0].IsAutoRenew)
549+
for _, item := range result {
550+
node := item
551+
// 节点类型 0:common节点；1:master节点；2:core节点；3:task节点
552+
if node.Flag != nil {
553+
clusterNodeMap[strconv.FormatInt(*node.Flag, 10)] = node
554+
}
555+
}
556+
}
557+
558+
return nil
559+
})
560+
561+
if err != nil {
562+
return err
563+
}
537564
if instance != nil {
538565
_ = d.Set("product_id", instance.ProductId)
539566
_ = d.Set("vpc_settings", map[string]interface{}{
@@ -587,6 +614,9 @@ func resourceTencentCloudEmrClusterRead(d *schema.ResourceData, meta interface{}
587614
if masterResource.RootSize != nil {
588615
masterResourceSpec["root_size"] = *masterResource.RootSize
589616
}
617+
if v, ok := clusterNodeMap["1"]; ok {
618+
masterResourceSpec["multi_disks"] = fetchMultiDisks(v, masterResource)
619+
}
590620
resourceSpec["master_resource_spec"] = []interface{}{masterResourceSpec}
591621
}
592622

@@ -619,6 +649,10 @@ func resourceTencentCloudEmrClusterRead(d *schema.ResourceData, meta interface{}
619649
if coreResource.RootSize != nil {
620650
coreResourceSpec["root_size"] = *coreResource.RootSize
621651
}
652+
if v, ok := clusterNodeMap["2"]; ok {
653+
coreResourceSpec["multi_disks"] = fetchMultiDisks(v, coreResource)
654+
}
655+
622656
resourceSpec["core_resource_spec"] = []interface{}{coreResourceSpec}
623657
}
624658

@@ -651,6 +685,9 @@ func resourceTencentCloudEmrClusterRead(d *schema.ResourceData, meta interface{}
651685
if taskResource.RootSize != nil {
652686
taskResourceSpec["root_size"] = *taskResource.RootSize
653687
}
688+
if v, ok := clusterNodeMap["3"]; ok {
689+
taskResourceSpec["multi_disks"] = fetchMultiDisks(v, taskResource)
690+
}
654691
resourceSpec["task_resource_spec"] = []interface{}{taskResourceSpec}
655692
}
656693

@@ -683,6 +720,9 @@ func resourceTencentCloudEmrClusterRead(d *schema.ResourceData, meta interface{}
683720
if comResource.RootSize != nil {
684721
comResourceSpec["root_size"] = *comResource.RootSize
685722
}
723+
if v, ok := clusterNodeMap["0"]; ok {
724+
comResourceSpec["multi_disks"] = fetchMultiDisks(v, comResource)
725+
}
686726
resourceSpec["common_resource_spec"] = []interface{}{comResourceSpec}
687727
}
688728

@@ -713,22 +753,6 @@ func resourceTencentCloudEmrClusterRead(d *schema.ResourceData, meta interface{}
713753
return err
714754
}
715755
_ = d.Set("tags", tags)
716-
err = resource.Retry(tccommon.ReadRetryTimeout, func() *resource.RetryError {
717-
result, err := emrService.DescribeClusterNodes(ctx, instanceId, "all", "all", 0, 10)
718-
719-
if err != nil {
720-
return resource.RetryableError(err)
721-
}
722756

723-
if len(result) > 0 {
724-
_ = d.Set("auto_renew", result[0].IsAutoRenew)
725-
}
726-
727-
return nil
728-
})
729-
730-
if err != nil {
731-
return err
732-
}
733757
return nil
734758
}

0 commit comments

Comments
 (0)