Skip to content

Commit fd1abc5

Browse files
hyunsunhyunsun
authored
feat: add waiting for nodes phase (#194)
1 parent 5ae6633 commit fd1abc5

File tree

8 files changed

+115
-31
lines changed

8 files changed

+115
-31
lines changed

‎internal/rest/deletev2clustersname.go‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import (
99

1010
"k8s.io/apimachinery/pkg/api/errors"
1111
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
13+
"k8s.io/apimachinery/pkg/types"
1214

1315
"github.com/open-edge-platform/cluster-manager/v2/internal/core"
1416
"github.com/open-edge-platform/cluster-manager/v2/pkg/api"
@@ -27,12 +29,24 @@ func (s *Server) DeleteV2ClustersName(ctx context.Context, request api.DeleteV2C
2729
}
2830

2931
activeProjectID := request.Params.Activeprojectid.String()
30-
err := s.k8sclient.Resource(core.ClusterResourceSchema).Namespace(activeProjectID).Delete(ctx, name, v1.DeleteOptions{})
32+
err := s.unpauseClusterIfPaused(ctx, activeProjectID, name)
33+
if errors.IsNotFound(err) {
34+
message := fmt.Sprintf("cluster '%s' not found in namespace '%s'", name, activeProjectID)
35+
return api.DeleteV2ClustersName404JSONResponse{N404NotFoundJSONResponse: api.N404NotFoundJSONResponse{Message: &message}}, nil
36+
}
37+
if err != nil {
38+
slog.Error("failed to unpause cluster before deletion", "namespace", activeProjectID, "name", name, "error", err)
39+
return api.DeleteV2ClustersName500JSONResponse{
40+
N500InternalServerErrorJSONResponse: api.N500InternalServerErrorJSONResponse{
41+
Message: ptr("failed to unpause cluster before deletion"),
42+
},
43+
}, nil
44+
}
45+
err = s.k8sclient.Resource(core.ClusterResourceSchema).Namespace(activeProjectID).Delete(ctx, name, v1.DeleteOptions{})
3146
if errors.IsNotFound(err) {
3247
message := fmt.Sprintf("cluster '%s' not found in namespace '%s'", name, activeProjectID)
3348
return api.DeleteV2ClustersName404JSONResponse{N404NotFoundJSONResponse: api.N404NotFoundJSONResponse{Message: &message}}, nil
3449
}
35-
3650
if err != nil {
3751
slog.Error("failed to delete cluster", "namespace", activeProjectID, "name", name, "error", err)
3852
return api.DeleteV2ClustersName500JSONResponse{
@@ -45,3 +59,33 @@ func (s *Server) DeleteV2ClustersName(ctx context.Context, request api.DeleteV2C
4559
slog.Debug("cluster deleted", "namespace", activeProjectID, "name", name)
4660
return api.DeleteV2ClustersName204Response{}, nil
4761
}
62+
63+
func (s *Server) unpauseClusterIfPaused(ctx context.Context, namespace, name string) error {
64+
cli := s.k8sclient.Resource(core.ClusterResourceSchema).Namespace(namespace)
65+
clusterObj, err := cli.Get(ctx, name, v1.GetOptions{})
66+
if err != nil {
67+
return err
68+
}
69+
paused, found, err := unstructured.NestedBool(clusterObj.Object, "spec", "paused")
70+
if err != nil {
71+
return err
72+
}
73+
if !found || !paused {
74+
// not paused
75+
return nil
76+
}
77+
78+
err = unstructured.SetNestedField(clusterObj.Object, false, "spec", "paused")
79+
if err != nil {
80+
return err
81+
}
82+
83+
patchData := []byte(`{"spec":{"paused":false}}`)
84+
_, err = cli.Patch(ctx, name, types.MergePatchType, patchData, v1.PatchOptions{})
85+
if err != nil {
86+
return err
87+
}
88+
89+
slog.Info("cluster unpaused before deletion", "namespace", namespace, "name", name)
90+
return nil
91+
}

‎internal/rest/deletev2clustersname_test.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/stretchr/testify/require"
1616
"k8s.io/apimachinery/pkg/api/errors"
1717
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
18+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
1819
"k8s.io/apimachinery/pkg/runtime/schema"
1920

2021
"github.com/open-edge-platform/cluster-manager/v2/internal/core"
@@ -30,6 +31,7 @@ func TestDeleteV2ClustersName204(t *testing.T) {
3031

3132
// Mock the delete cluster to succeed
3233
resource := k8s.NewMockResourceInterface(t)
34+
resource.EXPECT().Get(mock.Anything, name, metav1.GetOptions{}).Return(&unstructured.Unstructured{}, nil)
3335
resource.EXPECT().Delete(mock.Anything, name, metav1.DeleteOptions{}).Return(nil)
3436
nsResource := k8s.NewMockNamespaceableResourceInterface(t)
3537
nsResource.EXPECT().Namespace(activeProjectID).Return(resource)
@@ -93,6 +95,7 @@ func TestDeleteV2ClustersName404(t *testing.T) {
9395

9496
// Mock the get cluster to succeed and delete cluster to fail
9597
resource := k8s.NewMockResourceInterface(t)
98+
resource.EXPECT().Get(mock.Anything, name, metav1.GetOptions{}).Return(&unstructured.Unstructured{}, nil)
9699
resource.EXPECT().Delete(mock.Anything, name, metav1.DeleteOptions{}).Return(errors.NewNotFound(schema.GroupResource{Group: "core", Resource: "clusters"}, name))
97100
nsResource := k8s.NewMockNamespaceableResourceInterface(t)
98101
nsResource.EXPECT().Namespace(activeProjectID).Return(resource)
@@ -129,6 +132,7 @@ func TestDeleteV2ClustersName500(t *testing.T) {
129132

130133
// Mock the get cluster to succeed and delete cluster to fail
131134
resource := k8s.NewMockResourceInterface(t)
135+
resource.EXPECT().Get(mock.Anything, name, metav1.GetOptions{}).Return(&unstructured.Unstructured{}, nil)
132136
resource.EXPECT().Delete(mock.Anything, name, metav1.DeleteOptions{}).Return(fmt.Errorf("delete error"))
133137
nsResource := k8s.NewMockNamespaceableResourceInterface(t)
134138
nsResource.EXPECT().Namespace(activeProjectID).Return(resource)
@@ -158,6 +162,7 @@ func TestDeleteV2ClustersName500(t *testing.T) {
158162

159163
func createDeleteV2ClustersNameStubServer(t *testing.T) *Server {
160164
resource := k8s.NewMockResourceInterface(t)
165+
resource.EXPECT().Get(mock.Anything, mock.Anything, metav1.GetOptions{}).Return(&unstructured.Unstructured{}, nil)
161166
resource.EXPECT().Delete(mock.Anything, mock.Anything, metav1.DeleteOptions{}).Return(nil).Maybe()
162167
nsResource := k8s.NewMockNamespaceableResourceInterface(t)
163168
nsResource.EXPECT().Namespace(mock.Anything).Return(resource).Maybe()

‎internal/rest/getv2clusters_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ func generateClusterInfo(name, version string, lifecycleIndicator api.StatusIndi
165165
LifecyclePhase: &api.GenericStatus{Indicator: statusIndicatorPtr(lifecycleIndicator), Message: ptr(lifecycleMessage), Timestamp: ptr(uint64(0))},
166166
ControlPlaneReady: &api.GenericStatus{Indicator: statusIndicatorPtr(lifecycleIndicator), Message: ptr("condition not found"), Timestamp: ptr(uint64(0))},
167167
InfrastructureReady: &api.GenericStatus{Indicator: statusIndicatorPtr(lifecycleIndicator), Message: ptr("condition not found"), Timestamp: ptr(uint64(0))},
168-
NodeHealth: &api.GenericStatus{Indicator: statusIndicatorPtr(api.STATUSINDICATIONIDLE), Message: ptr("nodes are healthy"), Timestamp: ptr(uint64(0))},
168+
NodeHealth: &api.GenericStatus{Indicator: statusIndicatorPtr(api.STATUSINDICATIONUNSPECIFIED), Message: ptr("condition not found"), Timestamp: ptr(uint64(0))},
169169
NodeQuantity: ptr(0),
170170
ProviderStatus: &api.GenericStatus{Indicator: statusIndicatorPtr(api.STATUSINDICATIONUNSPECIFIED), Message: ptr("condition not found"), Timestamp: ptr(uint64(0))},
171171
}

‎internal/rest/getv2clusterssummary_test.go‎

Lines changed: 32 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -83,14 +83,40 @@ var clusterStatusUnknown = capi.ClusterStatus{
8383

8484
func createMockServer(t *testing.T, clusters []capi.Cluster, projectID string, options ...bool) *rest.Server {
8585
unstructuredClusters := make([]unstructured.Unstructured, len(clusters))
86+
unstructuredMachines := make([]unstructured.Unstructured, len(clusters))
8687
for i, cluster := range clusters {
8788
unstructuredCluster, err := convert.ToUnstructured(cluster)
8889
require.NoError(t, err, "convertClusterToUnstructured() error = %v, want nil")
8990
unstructuredClusters[i] = *unstructuredCluster
91+
92+
machine := capi.Machine{
93+
ObjectMeta: metav1.ObjectMeta{
94+
Name: cluster.Name,
95+
Labels: map[string]string{
96+
"cluster.x-k8s.io/cluster-name": cluster.Name,
97+
},
98+
},
99+
Spec: capi.MachineSpec{ClusterName: cluster.Name},
100+
Status: capi.MachineStatus{
101+
Phase: "Running",
102+
Conditions: []capi.Condition{
103+
{Type: "HealthCheckSucceed", Status: "True"},
104+
{Type: "InfrastructureReady", Status: "True"},
105+
{Type: "NodeHealthy", Status: "True"},
106+
},
107+
},
108+
}
109+
110+
unstructuredMachine, err := convert.ToUnstructured(machine)
111+
require.NoError(t, err, "convertClusterToUnstructured() error = %v, want nil")
112+
unstructuredMachines[i] = *unstructuredMachine
90113
}
91114
unstructuredClusterList := &unstructured.UnstructuredList{
92115
Items: unstructuredClusters,
93116
}
117+
unstructuredMachineList := &unstructured.UnstructuredList{
118+
Items: unstructuredMachines,
119+
}
94120
// default is to set up k8s client and machineResource mocks
95121
setupK8sMocks := true
96122
mockMachineResource := true
@@ -101,31 +127,18 @@ func createMockServer(t *testing.T, clusters []capi.Cluster, projectID string, o
101127
var mockedk8sclient *k8s.MockInterface
102128
mockedk8sclient = k8s.NewMockInterface(t)
103129
if setupK8sMocks {
104-
machine := capi.Machine{
105-
Status: capi.MachineStatus{
106-
Phase: "Running",
107-
Conditions: []capi.Condition{
108-
{Type: "HealthCheckSucceed", Status: "True"},
109-
{Type: "InfrastructureReady", Status: "True"},
110-
{Type: "NodeHealthy", Status: "True"},
111-
},
112-
},
113-
}
114-
unstructuredMachine, err := convert.ToUnstructured(machine)
115-
require.NoError(t, err, "convertMachineToUnstructured() error = %v, want nil")
116130
resource := k8s.NewMockResourceInterface(t)
117131
resource.EXPECT().List(mock.Anything, metav1.ListOptions{}).Return(unstructuredClusterList, nil)
118132
nsResource := k8s.NewMockNamespaceableResourceInterface(t)
119133
nsResource.EXPECT().Namespace(projectID).Return(resource)
120134
mockedk8sclient = k8s.NewMockInterface(t)
121135
mockedk8sclient.EXPECT().Resource(core.ClusterResourceSchema).Return(nsResource)
122136
if mockMachineResource {
123-
for _, cluster := range clusters {
124-
resource.EXPECT().List(mock.Anything, metav1.ListOptions{
125-
LabelSelector: "cluster.x-k8s.io/cluster-name=" + cluster.Name,
126-
}).Return(&unstructured.UnstructuredList{Items: []unstructured.Unstructured{*unstructuredMachine}}, nil).Maybe()
127-
}
128-
mockedk8sclient.EXPECT().Resource(core.MachineResourceSchema).Return(nsResource).Maybe()
137+
machineResource := k8s.NewMockResourceInterface(t)
138+
machineResource.EXPECT().List(mock.Anything, metav1.ListOptions{}).Return(unstructuredMachineList, nil)
139+
nsMachineResource := k8s.NewMockNamespaceableResourceInterface(t)
140+
nsMachineResource.EXPECT().Namespace(projectID).Return(machineResource)
141+
mockedk8sclient.EXPECT().Resource(core.MachineResourceSchema).Return(nsMachineResource).Maybe()
129142
}
130143
}
131144
return rest.NewServer(mockedk8sclient)
@@ -142,7 +155,7 @@ func generateClusterWithStatus(name, version *string, status capi.ClusterStatus)
142155
}
143156
return capi.Cluster{
144157
ObjectMeta: metav1.ObjectMeta{Name: clusterName},
145-
Spec: capi.ClusterSpec{Topology: &capi.Topology{Version: clusterVersion}},
158+
Spec: capi.ClusterSpec{Paused: false, Topology: &capi.Topology{Version: clusterVersion}},
146159
Status: status,
147160
}
148161
}

‎internal/rest/postv2clusters.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,7 @@ func (s *Server) createCluster(ctx context.Context, cli *k8s.Client, namespace,
186186
},
187187
Variables: variables,
188188
},
189+
Paused: true,
189190
},
190191
}
191192

‎internal/rest/postv2clusters_test.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ func TestPostV2Clusters201(t *testing.T) {
5656
},
5757
},
5858
Spec: capi.ClusterSpec{
59+
Paused: true,
5960
ClusterNetwork: &capi.ClusterNetwork{
6061
Pods: &capi.NetworkRanges{
6162
CIDRBlocks: []string{"10.0.0.0/16"},
@@ -196,6 +197,7 @@ func TestPostV2Clusters201(t *testing.T) {
196197
},
197198
},
198199
Spec: capi.ClusterSpec{
200+
Paused: true,
199201
ClusterNetwork: &capi.ClusterNetwork{
200202
Pods: &capi.NetworkRanges{
201203
CIDRBlocks: []string{"10.0.0.0/16"},
@@ -360,6 +362,7 @@ func TestPostV2Clusters201K3sAirGap(t *testing.T) {
360362
},
361363
},
362364
Spec: capi.ClusterSpec{
365+
Paused: true,
363366
ClusterNetwork: &capi.ClusterNetwork{
364367
Pods: &capi.NetworkRanges{
365368
CIDRBlocks: []string{"10.0.0.0/16"},
@@ -851,6 +854,7 @@ func TestPostV2Clusters500(t *testing.T) {
851854
},
852855
},
853856
Spec: capi.ClusterSpec{
857+
Paused: true,
854858
ClusterNetwork: &capi.ClusterNetwork{
855859
Pods: &capi.NetworkRanges{
856860
CIDRBlocks: []string{"10.0.0.0/16"},

‎internal/rest/utils.go‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -162,13 +162,22 @@ func getClusterLifecyclePhase(cluster *capi.Cluster) (*api.GenericStatus, []erro
162162
Message: new(string),
163163
Timestamp: new(uint64),
164164
}
165+
165166
var errorReasons []error
166167
if len(cluster.Status.Conditions) == 0 {
167168
*status.Indicator = api.STATUSINDICATIONUNSPECIFIED
168169
*status.Message = "Condition not found"
169170
*status.Timestamp = 0
170171
return &status, errorReasons
171172
}
173+
174+
*status.Timestamp = uint64(cluster.Status.Conditions[0].LastTransitionTime.UTC().Unix())
175+
176+
if cluster.Spec.Paused {
177+
*status.Indicator = api.STATUSINDICATIONIDLE
178+
*status.Message = "waiting for nodes"
179+
return &status, errorReasons
180+
}
172181
// ClusterPhase is a string representation of a Cluster Phase.
173182
// It is a high-level indicator of the status of the Cluster
174183
// as it is provisioned, from the API user’s perspective.
@@ -212,7 +221,6 @@ func getClusterLifecyclePhase(cluster *capi.Cluster) (*api.GenericStatus, []erro
212221
return nil, errorReasons
213222
}
214223

215-
*status.Timestamp = uint64(cluster.Status.Conditions[0].LastTransitionTime.UTC().Unix())
216224
return &status, errorReasons
217225
}
218226

@@ -279,7 +287,17 @@ func getNodeHealth(cluster *capi.Cluster, machines []unstructured.Unstructured)
279287
}
280288
}
281289

282-
if inProgress {
290+
if len(cluster.Status.Conditions) > 0 {
291+
*status.Timestamp = uint64(cluster.Status.Conditions[0].LastTransitionTime.UTC().Unix())
292+
} else {
293+
*status.Timestamp = 0
294+
}
295+
296+
if totalMachines == 0 {
297+
*status.Indicator = api.STATUSINDICATIONUNSPECIFIED
298+
*status.Message = "condition not found"
299+
*status.Timestamp = 0
300+
} else if inProgress {
283301
*status.Indicator = api.STATUSINDICATIONINPROGRESS
284302
*status.Message = fmt.Sprintf("node(s) health unknown (%v/%v);%s", machinesRunning, totalMachines, messages)
285303
} else if allHealthy && machinesRunning == totalMachines {
@@ -290,12 +308,6 @@ func getNodeHealth(cluster *capi.Cluster, machines []unstructured.Unstructured)
290308
*status.Message = fmt.Sprintf("nodes are unhealthy (%v/%v);%s", machinesRunning, totalMachines, machineMessage)
291309
}
292310

293-
if len(cluster.Status.Conditions) > 0 {
294-
*status.Timestamp = uint64(cluster.Status.Conditions[0].LastTransitionTime.UTC().Unix())
295-
} else {
296-
*status.Timestamp = 0
297-
}
298-
299311
return status
300312
}
301313

‎test/service/service_suite_test.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,12 @@ var _ = Describe("Cluster create/delete flow", Ordered, func() {
165165
}
166166
return resp.StatusCode(), nil
167167
}, 30*time.Second, 3*time.Second).Should(Equal(201))
168+
169+
// Unpause cluster to simulate cluster-agent behaviour
170+
patchData := []byte(`{"spec":{"paused":false}}`)
171+
cmd := exec.Command("kubectl", "-n", testTenantID.String(), "patch", "cl", clusterName, "--type=merge", "-p", string(patchData))
172+
err := cmd.Run()
173+
Expect(err).ToNot(HaveOccurred())
168174
})
169175

170176
It("Should return 200 and a list of clusters with one element on /v2/clusters", func() {
@@ -188,7 +194,6 @@ var _ = Describe("Cluster create/delete flow", Ordered, func() {
188194
fmt.Printf("unexpected number of clusters: %d\n", resp.JSON200.TotalElements)
189195
return false, fmt.Errorf("unexpected number of clusters: %d", resp.JSON200.TotalElements)
190196
}
191-
192197
if *(*resp.JSON200.Clusters)[0].NodeQuantity != 1 {
193198
return false, fmt.Errorf("unexpected number of nodes: %d", *(*resp.JSON200.Clusters)[0].NodeQuantity)
194199
}

0 commit comments

Comments
 (0)