Remove 404s from terminal codes (#84)

Fixes aws-controllers-k8s/community#1228

Description of changes:
404 error codes should not be included as terminal errors, as they prevent proper resource referencing. The better behaviour is for the controller to fall off exponentially while attempting to retry until the resource becomes available. This produces more noise in the controller logs, but properly updates the resources with the conditions until the resource can finally be created.

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

Author: RedbackThomson

apis/v1alpha1/ack-generate-metadata.yaml

@@ -1,13 +1,13 @@
 ack_generate_info:
-  build_date: "2022-03-02T19:01:08Z"
-  build_hash: ade2429bb444ab635916395ea5773d141ba135e1
-  go_version: go1.17.5
+  build_date: "2022-03-16T20:57:08Z"
+  build_hash: 7052f6808b237f97a2000e4c4f296b9ed2a0d7c0
+  go_version: go1.17.8
   version: v0.17.2
 api_directory_checksum: 246af92291668493e03954b4e4117c21f5b62790
 api_version: v1alpha1
 aws_sdk_go_version: v1.42.0
 generator_config_info:
-  file_checksum: 23f444fb86eaa5c1acb4d4f51430e606e7ee554a
+  file_checksum: 4fd6f317804ecb5192bfb88c569ae335d418248b
   original_file_name: generator.yaml
 last_modification:
   reason: API generation

apis/v1alpha1/generator.yaml

@@ -25,18 +25,14 @@ resources:
         - InvalidParameterValue
         - InvalidParameterCombination
         - InsufficientCacheClusterCapacity
-        - CacheSecurityGroupNotFound
-        - CacheSubnetGroupNotFoundFault
         - ClusterQuotaForCustomerExceeded
         - NodeQuotaForClusterExceeded
         - NodeQuotaForCustomerExceeded
         - InvalidVPCNetworkStateFault
         - TagQuotaPerResourceExceeded
         - NodeGroupsPerReplicationGroupQuotaExceeded
         - InvalidCacheSecurityGroupState
-        - CacheParameterGroupNotFound
         - InvalidKMSKeyFault
-        - CacheClusterNotFound
     fields:
       AllowedScaleUpModifications:
         is_read_only: true
@@ -94,8 +90,6 @@ resources:
         - InvalidParameterValue
         - InvalidParameterCombination
         - SnapshotAlreadyExistsFault
-        - CacheClusterNotFound
-        - ReplicationGroupNotFoundFault
         - SnapshotQuotaExceededFault
         - SnapshotFeatureNotSupportedFault
     fields:
@@ -133,14 +127,16 @@ resources:
       custom_method_name: customUpdateCacheParameterGroup
   User:
     exceptions:
+      errors:
+        404:
+          code: UserNotFound
       terminal_codes:
         - UserAlreadyExists
         - UserQuotaExceeded
         - DuplicateUserName
         - InvalidParameterValue
         - InvalidParameterCombination
         - InvalidUserState
-        - UserNotFound
         - DefaultUserAssociatedToUserGroup
     fields:
       LastRequestedAccessString:
@@ -178,7 +174,6 @@ resources:
         - DefaultUserRequired
         - UserGroupQuotaExceededFault
         - TagQuotaPerResourceExceeded
-        - UserNotFoundFault
     update_operation:
       custom_method_name: customUpdateUserGroup
 operations:

generator.yaml

@@ -25,18 +25,14 @@ resources:
         - InvalidParameterValue
         - InvalidParameterCombination
         - InsufficientCacheClusterCapacity
-        - CacheSecurityGroupNotFound
-        - CacheSubnetGroupNotFoundFault
         - ClusterQuotaForCustomerExceeded
         - NodeQuotaForClusterExceeded
         - NodeQuotaForCustomerExceeded
         - InvalidVPCNetworkStateFault
         - TagQuotaPerResourceExceeded
         - NodeGroupsPerReplicationGroupQuotaExceeded
         - InvalidCacheSecurityGroupState
-        - CacheParameterGroupNotFound
         - InvalidKMSKeyFault
-        - CacheClusterNotFound
     fields:
       AllowedScaleUpModifications:
         is_read_only: true
@@ -94,8 +90,6 @@ resources:
         - InvalidParameterValue
         - InvalidParameterCombination
         - SnapshotAlreadyExistsFault
-        - CacheClusterNotFound
-        - ReplicationGroupNotFoundFault
         - SnapshotQuotaExceededFault
         - SnapshotFeatureNotSupportedFault
     fields:
@@ -133,14 +127,16 @@ resources:
       custom_method_name: customUpdateCacheParameterGroup
   User:
     exceptions:
+      errors:
+        404:
+          code: UserNotFound
       terminal_codes:
         - UserAlreadyExists
         - UserQuotaExceeded
         - DuplicateUserName
         - InvalidParameterValue
         - InvalidParameterCombination
         - InvalidUserState
-        - UserNotFound
         - DefaultUserAssociatedToUserGroup
     fields:
       LastRequestedAccessString:
@@ -178,7 +174,6 @@ resources:
         - DefaultUserRequired
         - UserGroupQuotaExceededFault
         - TagQuotaPerResourceExceeded
-        - UserNotFoundFault
     update_operation:
       custom_method_name: customUpdateUserGroup
 operations:

go.local.mod

@@ -1,20 +1,72 @@
 module github.com/aws-controllers-k8s/elasticache-controller
 
-go 1.14
+go 1.17
 
 replace github.com/aws-controllers-k8s/runtime => ../runtime
 
 require (
 	github.com/aws-controllers-k8s/runtime v0.0.0-20210204203051-91a6bd53a48c
-	github.com/aws/aws-sdk-go v1.37.4
-	github.com/go-logr/logr v0.1.0
-	github.com/google/go-cmp v0.3.1
+	github.com/aws/aws-sdk-go v1.42.0
+	github.com/ghodss/yaml v1.0.0
+	github.com/go-logr/logr v1.2.0
+	github.com/google/go-cmp v0.5.5
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.5.1
-	go.uber.org/zap v1.10.0
-	k8s.io/api v0.18.2
-	k8s.io/apimachinery v0.18.6
-	k8s.io/client-go v0.18.2
-	sigs.k8s.io/controller-runtime v0.6.0
+	github.com/stretchr/testify v1.7.0
+	go.uber.org/zap v1.19.1
+	k8s.io/api v0.23.0
+	k8s.io/apimachinery v0.23.0
+	k8s.io/client-go v0.23.0
+	sigs.k8s.io/controller-runtime v0.11.0
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/go-logr/zapr v1.2.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/uuid v1.1.2 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/jaypipes/envutil v1.0.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_golang v1.11.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.28.0 // indirect
+	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/stretchr/objx v0.2.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	golang.org/x/net v0.0.0-20210825183410-e898025ed96a // indirect
+	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
+	golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8 // indirect
+	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	k8s.io/apiextensions-apiserver v0.23.0 // indirect
+	k8s.io/component-base v0.23.0 // indirect
+	k8s.io/klog/v2 v2.30.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
+	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
+	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.0 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )

pkg/resource/replication_group/sdk.go

@@ -24,7 +24,7 @@
 from e2e import service_marker, CRD_GROUP, CRD_VERSION, load_elasticache_resource
 from e2e.bootstrap_resources import get_bootstrap_resources
 from e2e.util import retrieve_cache_cluster, assert_even_shards_replica_count, retrieve_replication_group, \
-    assert_terminal_condition_set
+    assert_recoverable_condition_set
 
 RESOURCE_PLURAL = "replicationgroups"
 DEFAULT_WAIT_SECS = 30
@@ -401,13 +401,13 @@ def test_rg_cmd_fromsnapshot(self, rg_cmd_fromsnapshot):
         (reference, _) = rg_cmd_fromsnapshot
         assert k8s.wait_on_condition(reference, "ACK.ResourceSynced", "True", wait_periods=90)
 
-    # if primaryClusterID is a nonexistent node, the terminal condition should be set
+    # if primaryClusterID is a nonexistent node, the recoverable condition should be set
     def test_rg_invalid_primary(self, rg_invalid_primary):
         (reference, _) = rg_invalid_primary
         sleep(DEFAULT_WAIT_SECS)
 
         resource = k8s.get_resource(reference)
-        assert_terminal_condition_set(resource)
+        assert_recoverable_condition_set(resource)
 
     # increase and decrease replica counts per-shard in a CME RG
     @pytest.mark.blocked  # TODO: remove when passing

pkg/resource/snapshot/sdk.go

@@ -112,6 +112,15 @@ def assert_terminal_condition_set(resource):
     assert terminal is not None
     assert terminal['status'] == "True"
 
+# given the latest state of the resource, assert that the recoverable condition is set
+def assert_recoverable_condition_set(resource):
+    recoverable = None
+    for cond in resource['status']['conditions']:
+        if cond['type'] == "ACK.Recoverable":
+            recoverable = cond
+
+    assert recoverable is not None
+    assert recoverable['status'] == "True"
 
 # provide a basic nodeGroupConfiguration object of desired size
 def provide_node_group_configuration(size: int):