feat: implement templating for the PROXY_CACHE_USE_STALE env var

Fixes #437

This change modifies the startup scripts to support the optional
parameter `PROXY_CACHE_USE_STALE` which corresponds to the
NGINX directive with the same name. When this environment
variable is not set, a default is used.

Signed-off-by: Elijah Zupancic <e.zupancic@f5.com>
Co-Authored-By: Elijah Zupancic <e.zupancic@f5.com>

Author: mhbahmani

common/docker-entrypoint.d/00-check-for-required-env.sh

@@ -24,7 +24,7 @@ failed=0
 
 required=("S3_BUCKET_NAME" "S3_SERVER" "S3_SERVER_PORT" "S3_SERVER_PROTO"
 "S3_REGION" "S3_STYLE" "ALLOW_DIRECTORY_LIST" "AWS_SIGS_VERSION"
-"CORS_ENABLED")
+"CORS_ENABLED" "PROXY_CACHE_USE_STALE")
 
 # Require some form of authentication to be configured.
 
@@ -44,7 +44,7 @@ elif [[ -v AWS_SESSION_TOKEN ]]; then
 # b) Using Instance Metadata Service (IMDS) credentials, if IMDS is present at http://169.254.169.254.
 #    See https://docs.aws.amazon.com/sdkref/latest/guide/feature-imds-credentials.html.
 #    Example: We are running inside an EC2 instance.
-elif TOKEN=`curl -X PUT --silent --fail --connect-timeout 2 --max-time 2 "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"` && curl  -H "X-aws-ec2-metadata-token: $TOKEN" --output /dev/null --silent --head --fail --connect-timeout 2 --max-time 5 "http://169.254.169.254"; then 
+elif TOKEN=`curl -X PUT --silent --fail --connect-timeout 2 --max-time 2 "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"` && curl  -H "X-aws-ec2-metadata-token: $TOKEN" --output /dev/null --silent --head --fail --connect-timeout 2 --max-time 5 "http://169.254.169.254"; then
   echo "Running inside an EC2 instance, using IMDS for credentials"
 
 # c) Using assume role credentials. This is indicated by AWS_WEB_IDENTITY_TOKEN_FILE being set.
@@ -126,6 +126,10 @@ if [ -n "${HEADER_PREFIXES_TO_STRIP+x}" ]; then
   fi
 fi
 
+if [[ ! "${PROXY_CACHE_USE_STALE}" ]]; then
+  >&2 echo "PROXY_CACHE_USE_STALE must not be blank"
+  failed=1
+fi
 
 if [ $failed -gt 0 ]; then
   exit 1
@@ -144,6 +148,7 @@ echo "Directory Listing Path Prefix: ${DIRECTORY_LISTING_PATH_PREFIX}"
 echo "Provide Index Pages Enabled: ${PROVIDE_INDEX_PAGE}"
 echo "Append slash for directory enabled: ${APPEND_SLASH_FOR_POSSIBLE_DIRECTORY}"
 echo "Stripping the following headers from responses: x-amz-;${HEADER_PREFIXES_TO_STRIP}"
-echo "Allow the following headers from responses (these take precendence over the above): ${HEADER_PREFIXES_ALLOWED}"
+echo "Allow the following headers from responses (these take precedence over the above): ${HEADER_PREFIXES_ALLOWED}"
 echo "CORS Enabled: ${CORS_ENABLED}"
 echo "CORS Allow Private Network Access: ${CORS_ALLOW_PRIVATE_NETWORK_ACCESS}"
+echo "Proxy cache using stale setting: ${PROXY_CACHE_USE_STALE}"

common/docker-entrypoint.sh

@@ -71,7 +71,7 @@ fi
 # See documentation for this feature. We do not parse this as a boolean
 # since "true" and "false" are the required values of the header this populates
 if [ "${CORS_ALLOW_PRIVATE_NETWORK_ACCESS}" != "true" ] && [ "${CORS_ALLOW_PRIVATE_NETWORK_ACCESS}" != "false" ]; then
-  export CORS_ALLOW_PRIVATE_NETWORK_ACCESS=""  
+  export CORS_ALLOW_PRIVATE_NETWORK_ACCESS=""
 fi
 
 # This is the primary logic to determine the s3 host used for the
@@ -94,6 +94,10 @@ else
   export S3_HOST_HEADER="${S3_BUCKET_NAME}.${S3_SERVER}"
 fi
 
+# Use default proxy_cache_use_stale settings if the variable is not defined
+if [[ ! -v PROXY_CACHE_USE_STALE ]]; then
+  export PROXY_CACHE_USE_STALE="error timeout http_500 http_502 http_503 http_504"
+fi
 
 # Nothing is modified under this line
 

common/etc/nginx/nginx.conf

@@ -38,6 +38,7 @@ env PROXY_CACHE_VALID_OK;
 env PROXY_CACHE_SLICE_SIZE;
 env PROXY_CACHE_VALID_NOTFOUND;
 env PROXY_CACHE_VALID_FORBIDDEN;
+env PROXY_CACHE_USE_STALE;
 env HEADER_PREFIXES_TO_STRIP;
 env FOUR_O_FOUR_ON_EMPTY_BUCKET;
 env STRIP_LEADING_DIRECTORY_PATH;

common/etc/nginx/templates/default.conf.template

@@ -56,7 +56,7 @@ server {
     proxy_cache_revalidate on;
     proxy_cache_background_update on;
     proxy_cache_lock on;
-    proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504;
+    proxy_cache_use_stale ${PROXY_CACHE_USE_STALE};
     proxy_cache_key "$request_method$host$uri";
 
     # If you need to support proxying range request, refer to this article:

deployments/s3_express/settings.s3express.example

@@ -20,3 +20,4 @@ PROXY_CACHE_INACTIVE=60m
 PROXY_CACHE_VALID_OK=1h
 PROXY_CACHE_VALID_NOTFOUND=1m
 PROXY_CACHE_VALID_FORBIDDEN=30s
+PROXY_CACHE_USE_STALE='error timeout http_500 http_502 http_503 http_504'

docs/getting_started.md

@@ -20,4 +20,5 @@ PROXY_CACHE_INACTIVE=60m
 PROXY_CACHE_VALID_OK=1h
 PROXY_CACHE_VALID_NOTFOUND=1m
 PROXY_CACHE_VALID_FORBIDDEN=30s
+PROXY_CACHE_USE_STALE='error timeout http_500 http_502 http_503 http_504'
 STRIP_LEADING_DIRECTORY_PATH=/somepath

settings.example

@@ -96,6 +96,7 @@ echo "Slice of slice for byte range requests: ${PROXY_CACHE_SLICE_SIZE}"
 echo "Proxy Caching Time for Valid Response: ${PROXY_CACHE_VALID_OK}"
 echo "Proxy Caching Time for Not Found Response: ${PROXY_CACHE_VALID_NOTFOUND}"
 echo "Proxy Caching Time for Forbidden Response: ${PROXY_CACHE_VALID_FORBIDDEN}"
+echo "Proxy Cache Using Stale: ${PROXY_CACHE_USE_STALE}"
 echo "CORS Enabled: ${CORS_ENABLED}"
 echo "CORS Allow Private Network Access: ${CORS_ALLOW_PRIVATE_NETWORK_ACCESS}"
 
@@ -179,6 +180,8 @@ PROXY_CACHE_VALID_OK=${PROXY_CACHE_VALID_OK:-'1h'}
 PROXY_CACHE_VALID_NOTFOUND=${PROXY_CACHE_VALID_NOTFOUND:-'1m'}
 # Proxy caching time for response code 403
 PROXY_CACHE_VALID_FORBIDDEN=${PROXY_CACHE_VALID_FORBIDDEN:-'30s'}
+# Proxy cache using stale data when error occurs
+PROXY_CACHE_USE_STALE=${PROXY_CACHE_USE_STALE:-'error timeout http_500 http_502 http_503 http_504'}
 # Enables or disables CORS for the S3 Gateway (true=enabled, false=disabled)
 CORS_ENABLED=${CORS_ENABLED:-'false'}
 # Configure portion of URL to be removed (optional)