With Fix, check to see if any test fail

Signed-off-by: Hanxi Zhang <hanxizh@amazon.com>

Author: hanxizh9910

tests/unit/cluster/test-flaky-migrate-slots.tcl

@@ -2,11 +2,12 @@ set base_conf [list cluster-enabled yes cluster-node-timeout 1000 cluster-databa
 
 test {Migrate the last slot away from a node using valkey-cli} {
     start_multiple_servers 4 [list overrides $base_conf] {
+
         # Create a cluster of 3 nodes
         exec src/valkey-cli --cluster-yes --cluster create \
-                        127.0.0.1:[srv 0 port] \
-                        127.0.0.1:[srv -1 port] \
-                        127.0.0.1:[srv -2 port]
+                           127.0.0.1:[srv 0 port] \
+                           127.0.0.1:[srv -1 port] \
+                           127.0.0.1:[srv -2 port]
 
         wait_for_condition 1000 50 {
             [CI 0 cluster_state] eq {ok} &&
@@ -16,11 +17,16 @@ test {Migrate the last slot away from a node using valkey-cli} {
             fail "Cluster doesn't stabilize"
         }
 
+        # Insert some data
+        assert_equal OK [exec src/valkey-cli -c -p [srv 0 port] SET foo bar]
+        set slot [exec src/valkey-cli -c -p [srv 0 port] CLUSTER KEYSLOT foo]
+
         # Add new node to the cluster
         exec src/valkey-cli --cluster-yes --cluster add-node \
-                    127.0.0.1:[srv -3 port] \
-                    127.0.0.1:[srv 0 port]
+                     127.0.0.1:[srv -3 port] \
+                     127.0.0.1:[srv 0 port]
 
+        # First we wait for new node to be recognized by entire cluster
         wait_for_cluster_size 4
 
         wait_for_condition 1000 50 {
@@ -32,52 +38,75 @@ test {Migrate the last slot away from a node using valkey-cli} {
             fail "Cluster doesn't stabilize"
         }
 
+        # Ensure all node IDs are propagated
+        wait_for_condition 1000 50 {
+            [CI 0 cluster_known_nodes] == 4 &&
+            [CI 1 cluster_known_nodes] == 4 &&
+            [CI 2 cluster_known_nodes] == 4 &&
+            [CI 3 cluster_known_nodes] == 4
+        } else {
+            fail "Cluster gossip hasn't propagated all node IDs"
+        }
+
         set newnode_r [valkey_client -3]
         set newnode_id [$newnode_r CLUSTER MYID]
-        
-        # Find a node with slots and migrate one slot to the new node
-        set source_r [valkey_client 0]
-        set source_id [$source_r CLUSTER MYID]
-        set slots [$source_r CLUSTER SLOTS]
-        set slot [lindex [lindex $slots 0] 0]
-        
-        # Migrate the slot using valkey-cli
-        exec src/valkey-cli --cluster reshard 127.0.0.1:[srv 0 port] \
-            --cluster-from $source_id \
-            --cluster-to $newnode_id \
-            --cluster-slots 1 \
-            --cluster-yes
 
+        # Find out which node has the key "foo" by asking the new node for a
+        # redirect.
+        catch { $newnode_r get foo } e
+        assert_match "MOVED $slot *" $e
+        lassign [split [lindex $e 2] :] owner_host owner_port
+        set owner_r [valkey $owner_host $owner_port 0 $::tls]
+        set owner_id [$owner_r CLUSTER MYID]
+
+        # Move slot to new node using plain commands
+        assert_equal OK [$newnode_r CLUSTER SETSLOT $slot IMPORTING $owner_id]
+        assert_equal OK [$owner_r CLUSTER SETSLOT $slot MIGRATING $newnode_id]
+        assert_equal {foo} [$owner_r CLUSTER GETKEYSINSLOT $slot 10]
+        assert_equal OK [$owner_r MIGRATE 127.0.0.1 [srv -3 port] "" 0 5000 KEYS foo]
+        assert_equal OK [$newnode_r CLUSTER SETSLOT $slot NODE $newnode_id]
+        assert_equal OK [$owner_r CLUSTER SETSLOT $slot NODE $newnode_id]
+
+        # Using --cluster check make sure we won't get `Not all slots are covered by nodes`.
+        # Wait for the cluster to become stable make sure the cluster is up during MIGRATE.
         wait_for_condition 1000 50 {
+            [catch {exec src/valkey-cli --cluster check 127.0.0.1:[srv 0 port]}] == 0 &&
+            [catch {exec src/valkey-cli --cluster check 127.0.0.1:[srv -1 port]}] == 0 &&
+            [catch {exec src/valkey-cli --cluster check 127.0.0.1:[srv -2 port]}] == 0 &&
+            [catch {exec src/valkey-cli --cluster check 127.0.0.1:[srv -3 port]}] == 0 &&
             [CI 0 cluster_state] eq {ok} &&
             [CI 1 cluster_state] eq {ok} &&
             [CI 2 cluster_state] eq {ok} &&
             [CI 3 cluster_state] eq {ok}
         } else {
-            fail "Cluster doesn't stabilize after migration"
+            fail "Cluster doesn't stabilize"
         }
 
-        # Now migrate the last slot away from the new node using valkey-cli
+        # Move the only slot back to original node using valkey-cli
         exec src/valkey-cli --cluster reshard 127.0.0.1:[srv -3 port] \
             --cluster-from $newnode_id \
-            --cluster-to $source_id \
+            --cluster-to $owner_id \
             --cluster-slots 1 \
             --cluster-yes
 
+        # The empty node will become a replica of the new owner before the
+        # `MOVED` check, so let's wait for the cluster to become stable.
         wait_for_condition 1000 50 {
             [CI 0 cluster_state] eq {ok} &&
             [CI 1 cluster_state] eq {ok} &&
             [CI 2 cluster_state] eq {ok} &&
             [CI 3 cluster_state] eq {ok}
         } else {
-            fail "Cluster doesn't stabilize after last slot migration"
+            fail "Cluster doesn't stabilize"
         }
 
-        # Verify the node with no slots becomes a replica
-        wait_for_condition 1000 50 {
-            [string match "*slave*" [$newnode_r role]]
-        } else {
-            fail "Empty node didn't become a replica"
-        }
+        # Check that the key foo has been migrated back to the original owner.
+        catch { $newnode_r get foo } e
+        assert_equal "MOVED $slot $owner_host:$owner_port" $e
+
+        # Check that the now empty primary node doesn't turn itself into
+        # a replica of any other nodes
+        wait_for_cluster_propagation
+        assert_match *master* [$owner_r role]
     }
 }