Skip to content

Commit 7eba151

Browse files
nmvknmvk
committed
Increase test timeout
With many tests, timeout needs to be increased to avoid random failures.
1 parent b5bb14d commit 7eba151

File tree

1 file changed

+32
-32
lines changed

1 file changed

+32
-32
lines changed

‎test/e2e/tests/test_replicationgroup.py‎

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -395,11 +395,11 @@ class TestReplicationGroup:
395395

396396
def test_rg_input_coverage(self, rg_input_coverage):
397397
(reference, _) = rg_input_coverage
398-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=40)
398+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
399399

400400
def test_rg_cmd_fromsnapshot(self, rg_cmd_fromsnapshot):
401401
(reference, _) = rg_cmd_fromsnapshot
402-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
402+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
403403

404404
# if primaryClusterID is a nonexistent node, the terminal condition should be set
405405
def test_rg_invalid_primary(self, rg_invalid_primary):
@@ -413,7 +413,7 @@ def test_rg_invalid_primary(self, rg_invalid_primary):
413413
@pytest.mark.blocked # TODO: remove when passing
414414
def test_rg_cme_uneven_shards(self, rg_cme_uneven_shards, rg_cme_uneven_shards_input):
415415
(reference, _) = rg_cme_uneven_shards
416-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
416+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
417417
ngid1 = rg_cme_uneven_shards_input['NGID1'][1:-1] # need to strip double quotes off node group ID
418418
ngid2 = rg_cme_uneven_shards_input['NGID2'][1:-1]
419419

@@ -447,7 +447,7 @@ def test_rg_cme_uneven_shards(self, rg_cme_uneven_shards, rg_cme_uneven_shards_i
447447
}
448448
_ = k8s.patch_custom_resource(reference, patch)
449449
sleep(DEFAULT_WAIT_SECS)
450-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
450+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
451451

452452
# assert new state
453453
resource = k8s.get_resource(reference)
@@ -463,7 +463,7 @@ def test_rg_cme_uneven_shards(self, rg_cme_uneven_shards, rg_cme_uneven_shards_i
463463
# increase and decrease replica count evenly across all shards in a CME RG
464464
def test_rg_cme_even_shards(self, rg_cme_even_shards, rg_cme_even_shards_input):
465465
(reference, _) = rg_cme_even_shards
466-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
466+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
467467
nng = int(rg_cme_even_shards_input['NUM_NODE_GROUPS'])
468468
rpng = int(rg_cme_even_shards_input['REPLICAS_PER_NODE_GROUP'])
469469

@@ -477,7 +477,7 @@ def test_rg_cme_even_shards(self, rg_cme_even_shards, rg_cme_even_shards_input):
477477
patch = {"spec": {"replicasPerNodeGroup": rpng}}
478478
_ = k8s.patch_custom_resource(reference, patch)
479479
sleep(DEFAULT_WAIT_SECS)
480-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
480+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
481481

482482
# assert replica count has increased
483483
resource = k8s.get_resource(reference)
@@ -489,7 +489,7 @@ def test_rg_cme_even_shards(self, rg_cme_even_shards, rg_cme_even_shards_input):
489489
patch = {"spec": {"replicasPerNodeGroup": rpng}}
490490
_ = k8s.patch_custom_resource(reference, patch)
491491
sleep(DEFAULT_WAIT_SECS)
492-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
492+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
493493

494494
# assert replica count has decreased
495495
resource = k8s.get_resource(reference)
@@ -499,7 +499,7 @@ def test_rg_cme_even_shards(self, rg_cme_even_shards, rg_cme_even_shards_input):
499499
# test update behavior of controller (engine version and replica count)
500500
def test_rg_upgrade_ev(self, rg_upgrade_ev_input, rg_upgrade_ev):
501501
(reference, _) = rg_upgrade_ev
502-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
502+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
503503

504504
# assert initial state
505505
cc = retrieve_cache_cluster(rg_upgrade_ev_input['RG_ID'])
@@ -511,7 +511,7 @@ def test_rg_upgrade_ev(self, rg_upgrade_ev_input, rg_upgrade_ev):
511511
patch = {"spec": {"engineVersion": desired_engine_version}}
512512
_ = k8s.patch_custom_resource(reference, patch)
513513
sleep(DEFAULT_WAIT_SECS)
514-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
514+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
515515

516516
# assert new state after upgrading engine version
517517
resource = k8s.get_resource(reference)
@@ -524,7 +524,7 @@ def test_rg_upgrade_ev(self, rg_upgrade_ev_input, rg_upgrade_ev):
524524
# test update of fields that can be changed quickly
525525
def test_rg_update_misc(self, rg_update_misc_input, rg_update_misc):
526526
(reference, _) = rg_update_misc
527-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
527+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
528528

529529
# desired initial state
530530
pmw = rg_update_misc_input['PMW']
@@ -549,7 +549,7 @@ def test_rg_update_misc(self, rg_update_misc_input, rg_update_misc):
549549
}
550550
_ = k8s.patch_custom_resource(reference, patch)
551551
sleep(DEFAULT_WAIT_SECS)
552-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=5)
552+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
553553

554554
# assert new state
555555
assert_misc_fields(reference, rg_update_misc_input['RG_ID'], pmw, description, srl, sw)
@@ -558,7 +558,7 @@ def test_rg_update_misc(self, rg_update_misc_input, rg_update_misc):
558558
@pytest.mark.blocked # TODO: remove when passing
559559
def test_rg_fault_tolerance(self, rg_fault_tolerance):
560560
(reference, _) = rg_fault_tolerance
561-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
561+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
562562

563563
# assert initial state
564564
resource = k8s.get_resource(reference)
@@ -581,7 +581,7 @@ def test_rg_fault_tolerance(self, rg_fault_tolerance):
581581
patch = {"spec": {"automaticFailoverEnabled": False, "multiAZEnabled": False}}
582582
_ = k8s.patch_custom_resource(reference, patch)
583583
sleep(DEFAULT_WAIT_SECS)
584-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=10)
584+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
585585

586586
# assert new state
587587
resource = k8s.get_resource(reference)
@@ -592,7 +592,7 @@ def test_rg_fault_tolerance(self, rg_fault_tolerance):
592592
patch = {"spec": {"primaryClusterID": node2, "automaticFailoverEnabled": True, "multiAZEnabled": True}}
593593
_ = k8s.patch_custom_resource(reference, patch)
594594
sleep(DEFAULT_WAIT_SECS)
595-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=15)
595+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
596596

597597
# assert roles
598598
resource = k8s.get_resource(reference)
@@ -614,7 +614,7 @@ def test_rg_fault_tolerance(self, rg_fault_tolerance):
614614
@pytest.mark.blocked # TODO: remove when passing
615615
def test_rg_associate_resources(self, rg_associate_resources_input, rg_associate_resources, bootstrap_resources):
616616
(reference, _) = rg_associate_resources
617-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
617+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
618618

619619
# associate resources, wait for RG to sync
620620
sg_list = [bootstrap_resources.SecurityGroup1, bootstrap_resources.SecurityGroup2]
@@ -623,7 +623,7 @@ def test_rg_associate_resources(self, rg_associate_resources_input, rg_associate
623623
patch = {"spec": {"securityGroupIDs": sg_list, "notificationTopicARN": sns_topic, "userGroupIDs": ug_list}}
624624
_ = k8s.patch_custom_resource(reference, patch)
625625
sleep(DEFAULT_WAIT_SECS)
626-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=10)
626+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
627627

628628
# assert new state
629629
assert_associated_resources(rg_associate_resources_input['RG_ID'], sg_list, sns_topic, ug_list)
@@ -635,15 +635,15 @@ def test_rg_associate_resources(self, rg_associate_resources_input, rg_associate
635635
patch = {"spec": {"securityGroupIDs": sg_list, "notificationTopicARN": sns_topic, "userGroupIDs": ug_list}}
636636
_ = k8s.patch_custom_resource(reference, patch)
637637
sleep(DEFAULT_WAIT_SECS)
638-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=10)
638+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
639639

640640
# assert new state
641641
assert_associated_resources(rg_associate_resources_input['RG_ID'], sg_list, sns_topic, ug_list)
642642

643643
def test_rg_update_cpg(self, rg_update_cpg_input, rg_update_cpg, bootstrap_resources):
644644
# wait for resource to sync and retrieve initial state
645645
(reference, _) = rg_update_cpg
646-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
646+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
647647

648648
# update, wait for resource to sync
649649
patch = {"spec": {"cacheParameterGroupName": bootstrap_resources.CPGName}}
@@ -658,7 +658,7 @@ def test_rg_update_cpg(self, rg_update_cpg_input, rg_update_cpg, bootstrap_resou
658658
@pytest.mark.blocked # TODO: remove when passing
659659
def test_rg_scale_vertically(self, rg_scale_vertically_input, rg_scale_vertically):
660660
(reference, _) = rg_scale_vertically
661-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
661+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
662662

663663
# assert initial state
664664
rg = retrieve_replication_group(rg_scale_vertically_input['RG_ID'])
@@ -669,7 +669,7 @@ def test_rg_scale_vertically(self, rg_scale_vertically_input, rg_scale_verticall
669669
patch = {"spec": {"cacheNodeType": cnt}}
670670
_ = k8s.patch_custom_resource(reference, patch)
671671
sleep(DEFAULT_WAIT_SECS)
672-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
672+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
673673

674674
# assert scale up complete
675675
rg = retrieve_replication_group(rg_scale_vertically_input['RG_ID'])
@@ -680,7 +680,7 @@ def test_rg_scale_vertically(self, rg_scale_vertically_input, rg_scale_verticall
680680
patch = {"spec": {"cacheNodeType": cnt}}
681681
_ = k8s.patch_custom_resource(reference, patch)
682682
sleep(DEFAULT_WAIT_SECS)
683-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
683+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
684684

685685
# assert scale down complete
686686
rg = retrieve_replication_group(rg_scale_vertically_input['RG_ID'])
@@ -689,7 +689,7 @@ def test_rg_scale_vertically(self, rg_scale_vertically_input, rg_scale_verticall
689689
@pytest.mark.blocked # TODO: remove when passing
690690
def test_rg_scale_horizontally(self, rg_scale_horizontally_input, rg_scale_horizontally):
691691
(reference, _) = rg_scale_horizontally
692-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
692+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
693693

694694
# assert initial state
695695
rg = retrieve_replication_group(rg_scale_horizontally_input['RG_ID'])
@@ -701,7 +701,7 @@ def test_rg_scale_horizontally(self, rg_scale_horizontally_input, rg_scale_horiz
701701
patch = {"spec": {"numNodeGroups": nng}}
702702
_ = k8s.patch_custom_resource(reference, patch)
703703
sleep(DEFAULT_WAIT_SECS)
704-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
704+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
705705

706706
# assert scale out complete
707707
rg = retrieve_replication_group(rg_scale_horizontally_input['RG_ID'])
@@ -712,7 +712,7 @@ def test_rg_scale_horizontally(self, rg_scale_horizontally_input, rg_scale_horiz
712712
patch = {"spec": {"numNodeGroups": nng}}
713713
_ = k8s.patch_custom_resource(reference, patch)
714714
sleep(DEFAULT_WAIT_SECS)
715-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
715+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
716716

717717
# assert scale in complete
718718
rg = retrieve_replication_group(rg_scale_horizontally_input['RG_ID'])
@@ -721,7 +721,7 @@ def test_rg_scale_horizontally(self, rg_scale_horizontally_input, rg_scale_horiz
721721
# add and modify log delivery configuration to replication group
722722
def test_rg_log_delivery(self, rg_log_delivery_input, rg_log_delivery, bootstrap_resources):
723723
(reference, _) = rg_log_delivery
724-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
724+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
725725

726726
# add log delivery config
727727
config = {
@@ -738,7 +738,7 @@ def test_rg_log_delivery(self, rg_log_delivery_input, rg_log_delivery, bootstrap
738738
patch = {"spec": {"logDeliveryConfigurations": [config]}}
739739
k8s.patch_custom_resource(reference, patch)
740740
sleep(DEFAULT_WAIT_SECS)
741-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=10)
741+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
742742

743743
# assert log delivery added
744744
assert_log_delivery_config(reference, config)
@@ -749,7 +749,7 @@ def test_rg_log_delivery(self, rg_log_delivery_input, rg_log_delivery, bootstrap
749749
patch = {"spec": {"logDeliveryConfigurations": [config]}}
750750
k8s.patch_custom_resource(reference, patch)
751751
sleep(DEFAULT_WAIT_SECS)
752-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=10)
752+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
753753

754754
# assert configuration modified
755755
assert_log_delivery_config(reference, config)
@@ -759,7 +759,7 @@ def test_rg_log_delivery(self, rg_log_delivery_input, rg_log_delivery, bootstrap
759759
patch = {"spec": {"logDeliveryConfigurations": [config]}}
760760
k8s.patch_custom_resource(reference, patch)
761761
sleep(DEFAULT_WAIT_SECS)
762-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=10)
762+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
763763

764764
# assert error message present
765765
resource = k8s.get_resource(reference)
@@ -771,23 +771,23 @@ def test_rg_log_delivery(self, rg_log_delivery_input, rg_log_delivery, bootstrap
771771
patch = {"spec": {"logDeliveryConfigurations": [config]}}
772772
k8s.patch_custom_resource(reference, patch)
773773
sleep(DEFAULT_WAIT_SECS)
774-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=10)
774+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
775775

776776
# assert log delivery disabled
777777
assert_log_delivery_config(reference, config)
778778

779779
def test_rg_auth_token(self, rg_auth_token, secrets):
780780
(reference, _) = rg_auth_token
781-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
781+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
782782

783783
patch = {"spec": {"authToken": {"name": secrets['NAME2'], "key": secrets['KEY2']}}}
784784
k8s.patch_custom_resource(reference, patch)
785785
sleep(DEFAULT_WAIT_SECS)
786-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
786+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
787787

788788
def test_rg_deletion(self, rg_deletion_input, rg_deletion, rg_deletion_waiter):
789789
(reference, _) = rg_deletion
790-
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=30)
790+
assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
791791

792792
# assertions after initial creation
793793
resource = k8s.get_resource(reference)

0 commit comments

Comments
 (0)