Increase wait times and parallel runs (#143)

Increase training job wait time from 15 mins to 30 mins
Increase processing job wait time from 15 mins to 30 mins
Increase parallel runs from 10 to 15

Author: akartsky

test/canary/scripts/run_test.sh

@@ -85,5 +85,5 @@ pushd $E2E_DIR
 
   # run tests
   echo "Run Tests"
-  pytest -n 10 --dist loadfile --log-cli-level INFO -m canary
+  pytest -n 15 --dist loadfile --log-cli-level INFO -m canary
 popd

test/e2e/__init__.py

@@ -282,7 +282,7 @@ def get_training_resource_status(reference: k8s.CustomResourceReference):
 def wait_resource_training_status(
     reference: k8s.CustomResourceReference,
     expected_status: str,
-    wait_periods: int = 30,
+    wait_periods: int = 60,
     period_length: int = 30,
 ):
     return wait_for_status(
@@ -297,7 +297,7 @@ def wait_resource_training_status(
 def wait_sagemaker_training_status(
     training_job_name,
     expected_status: str,
-    wait_periods: int = 30,
+    wait_periods: int = 60,
     period_length: int = 30,
 ):
     return wait_for_status(

test/e2e/tests/test_processingjob.py

@@ -91,7 +91,7 @@ def _wait_resource_processing_status(
         self,
         reference: k8s.CustomResourceReference,
         expected_status: str,
-        wait_periods: int = 30,
+        wait_periods: int = 60,
         period_length: int = 30,
     ):
         return wait_for_status(
@@ -106,7 +106,7 @@ def _wait_sagemaker_processing_status(
         self,
         processing_job_name,
         expected_status: str,
-        wait_periods: int = 30,
+        wait_periods: int = 60,
         period_length: int = 30,
     ):
         return wait_for_status(