Support the latest versions of Logstash and the AWS SDK for Ruby. (#260)

Support the latest versions of Logstash and the AWS SDK for Ruby.

Adds the account_id configuration to the AWSIAMCredential struct to work with the latest versions of the AWS SDK for Ruby.

Test against the latest versions of Logstash during the integration testing to also test against these versions of the AWS SDK.

Use docker compose instead of docker-compose.

Resolves #258

Signed-off-by: David Venable <dlv@amazon.com>

Author: dlvenable

.github/workflows/CI.yml

@@ -11,7 +11,7 @@ jobs:
   Unit-test:
     strategy:
       matrix:
-        logstash: [ "7.16.3", "7.17.1", "8.3.2", "8.12.1" ]
+        logstash: [ "7.16.3", "7.17.24", "8.3.2", "8.12.1", "8.15.3" ]
     name: Unit Test logstash-output-opensearch
     runs-on: ubuntu-latest
     env:
@@ -33,8 +33,8 @@ jobs:
   Integration-Test-OpenSearch:
     strategy:
       matrix:
-        logstash: [ "7.16.3", "7.17.1", "8.3.2", "8.12.1" ]
-        opensearch: [ "1.3.4", "2.1.0", "2.12.0" ]
+        logstash: [ "7.16.3", "7.17.24", "8.3.2", "8.12.1", "8.15.3" ]
+        opensearch: [ "1.3.4", "2.1.0", "2.17.1" ]
         secure: [ true, false ]
 
     name: Integration Test logstash-output-opensearch against OpenSearch
@@ -58,7 +58,7 @@ jobs:
   Integration-Test-OpenDistro:
     strategy:
       matrix:
-        logstash: [ "7.16.3", "7.17.1", "8.12.1" ]
+        logstash: [ "7.16.3", "7.17.24", "8.3.2", "8.12.1", "8.15.3" ]
         opendistro: [ "1.13.3" ]
         secure: [ true, false ]
 

lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb

@@ -33,7 +33,8 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
     :profile,
     :instance_profile_credentials_retries,
     :instance_profile_credentials_timeout,
-    :region)
+    :region,
+    :account_id)
 
   class ManticoreAdapter
     attr_reader :manticore, :logger
@@ -78,10 +79,11 @@ def aws_iam_auth_initialization(options)
       instance_cred_retries = options[:auth_type]["instance_profile_credentials_retries"] || AWS_DEFAULT_PROFILE_CREDENTIAL_RETRY
       instance_cred_timeout = options[:auth_type]["instance_profile_credentials_timeout"] || AWS_DEFAULT_PROFILE_CREDENTIAL_TIMEOUT
       region = options[:auth_type]["region"] || AWS_DEFAULT_REGION
+      account_id = nil
       set_aws_region(region)
       set_service_name(options[:auth_type]["service_name"] || AWS_SERVICE)
 
-      credential_config = AWSIAMCredential.new(aws_access_key_id, aws_secret_access_key, session_token, profile, instance_cred_retries, instance_cred_timeout, region)
+      credential_config = AWSIAMCredential.new(aws_access_key_id, aws_secret_access_key, session_token, profile, instance_cred_retries, instance_cred_timeout, region, account_id)
       @credentials = Aws::CredentialProviderChain.new(credential_config).resolve
     end
 

scripts/opendistro/docker-run.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
-# This is intended to be run inside the docker container as the command of the docker-compose.
+# This is intended to be run inside the docker container as the command of the docker compose.
 set -ex
 
 cd scripts/opendistro
 
-docker-compose up --exit-code-from logstash
+docker compose up --exit-code-from logstash

scripts/opendistro/docker-setup.sh

@@ -9,5 +9,5 @@ if [ -f Gemfile.lock ]; then
 fi
 cd scripts/opendistro;
 
-docker-compose down
-docker-compose build
+docker compose down
+docker compose build

scripts/opensearch/docker-run.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
-# This is intended to be run inside the docker container as the command of the docker-compose.
+# This is intended to be run inside the docker container as the command of the docker compose.
 set -ex
 
 cd scripts/opensearch
 
-docker-compose up --exit-code-from logstash
+docker compose up --exit-code-from logstash

scripts/opensearch/docker-setup.sh

@@ -9,5 +9,5 @@ if [ -f Gemfile.lock ]; then
 fi
 cd scripts/opensearch;
 
-docker-compose down
-docker-compose build
+docker compose down
+docker compose build

scripts/run.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# This is intended to be run inside the docker container as the command of the docker-compose.
+# This is intended to be run inside the docker container as the command of the docker compose.
 
 env
 

scripts/unit-test/docker-run.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-# This is intended to be run inside the docker container as the command of the docker-compose.
+# This is intended to be run inside the docker container as the command of the docker compose.
 set -ex
 
 cd scripts/unit-test;
-docker-compose up --exit-code-from logstash logstash
+docker compose up --exit-code-from logstash logstash

scripts/unit-test/docker-setup.sh

@@ -9,5 +9,5 @@ if [ -f Gemfile.lock ]; then
 fi
 
 cd scripts/unit-test;
-docker-compose down
-docker-compose build logstash
+docker compose down
+docker compose build logstash