Merge branch 'develop'

Author: Chris Wiechmann

README.md

@@ -509,6 +509,20 @@ When Elasticsearch is finally started:
 ```
 Status YELLOW is expected when running Elasticsearch on a single node, as it can achieve the desired replicas. You may use Kibana Development tools or curl to get additional information.
 
+### vm.max_map_count is too low
+```
+ERROR: [1] bootstrap checks failed
+[1]: max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
+```
+Run the following to set it temporarly:
+`sudo sysctl -w vm.max_map_count=262144`
+Or the following:
+```
+1. sudo vi /etc/sysctl.conf
+2. add vm.max_map_count=262144
+3. sudo service sysctl restart
+```
+
 #### No results from Elasticsearch
 If you don't get any results from ElasticSearch for valid queries it might be a missing template configuration for the logstash-openlog index. Elastic-Search is doing by default a dynamic template mapping that is trying to figure out field types. However, for some of the fields the mapping must be overwritten. Therefore make sure so to include the sample mapping file: `configs/openlog_index_template.json` is used by your Logstash process. See `configs/logstash.conf` as an example.  
 The template mapping is pre-configured when using the docker-compose configuration. 

elasticsearch/docker-compose.es01.yml

@@ -3,13 +3,14 @@ services:
   # Elasticsearch node 1
   elasticsearch1:
     image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
+    # Custom entrypoint to configured ES based on given ELASTICSEARCH_HOSTS
+    entrypoint: /usr/share/elasticsearch/scripts/custom-entrypoint.sh
     container_name: elasticsearch1
     environment:
       - "ES_JAVA_OPTS=${ES_JAVA_OPTS}"
       - cluster.name=${ELASTICSEARCH_CLUSTERNAME}
       - node.name=elasticsearch1
-      - network.publish_host=${ELASTICSEARCH_HOST1}
-      - transport.port=${ELASTICSEARCH_HOST1_TRANSPORT}
+      - ELASTICSEARCH_HOSTS=${ELASTICSEARCH_HOSTS}
       - xpack.security.enabled=true
       - xpack.security.http.ssl.enabled=true
       - xpack.security.http.ssl.key=/usr/share/elasticsearch/${ELASTICSEARCH_KEY}
@@ -20,6 +21,10 @@ services:
       - xpack.security.transport.ssl.key=/usr/share/elasticsearch/${ELASTICSEARCH_KEY}
       - xpack.security.transport.ssl.certificate=/usr/share/elasticsearch/${ELASTICSEARCH_CRT}
       - xpack.security.transport.ssl.certificate_authorities=/usr/share/elasticsearch/${ELASTICSEARCH_CA}
+      # Activate the following parameters for advanced setup
+      #- network.publish_host=${ELASTICSEARCH_PUBLISH_HOST1}
+      #- http.port=${ELASTICSEARCH_HOST1_HTTP}
+      #- transport.port=${ELASTICSEARCH_HOST1_TRANSPORT}
     ulimits:
       memlock:
         soft: -1
@@ -38,11 +43,12 @@ services:
         target: /usr/share/elasticsearch/data
       - ${PWD}/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
       - ${PWD}/certificates:/usr/share/elasticsearch/config/certificates
+      - ${PWD}/elasticsearch/scripts:/usr/share/elasticsearch/scripts
     networks:
       - elastic
     ports:
-      - 9200:9200
-      - ${ELASTICSEARCH_HOST1_TRANSPORT}:${ELASTICSEARCH_HOST1_TRANSPORT}
+      - ${ELASTICSEARCH_HOST1_HTTP:-9200}:${ELASTICSEARCH_HOST1_HTTP:-9200}
+      - ${ELASTICSEARCH_HOST1_TRANSPORT:-9300}:${ELASTICSEARCH_HOST1_TRANSPORT:-9300}
     logging:
       options:
         max-file: "5"

elasticsearch/docker-compose.es01init.yml

@@ -1,6 +1,5 @@
 version: '3.7'
 services:
-  # Extend Elasticsearch node 1 configuration with second and third node
   elasticsearch1:
     environment:
-      - cluster.initial_master_nodes=elasticsearch1
+      - initCluster=true

elasticsearch/docker-compose.es02.yml

@@ -3,14 +3,14 @@ services:
   # Elasticsearch node 2
   elasticsearch2:
     image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
+    # Custom entrypoint to configured ES based on given ELASTICSEARCH_HOSTS
+    entrypoint: /usr/share/elasticsearch/scripts/custom-entrypoint.sh
     container_name: elasticsearch2
     environment:
       - "ES_JAVA_OPTS=${ES_JAVA_OPTS}"
       - cluster.name=${ELASTICSEARCH_CLUSTERNAME}
       - node.name=elasticsearch2
-      - network.publish_host=${ELASTICSEARCH_HOST2}
-      - transport.port=${ELASTICSEARCH_HOST2_TRANSPORT}
-      - discovery.seed_hosts=${ELASTICSEARCH_HOST1}:${ELASTICSEARCH_HOST1_TRANSPORT}
+      - ELASTICSEARCH_HOSTS=${ELASTICSEARCH_HOSTS}
       - xpack.security.enabled=true
       - xpack.security.http.ssl.enabled=true
       - xpack.security.http.ssl.key=/usr/share/elasticsearch/${ELASTICSEARCH_KEY}
@@ -21,6 +21,10 @@ services:
       - xpack.security.transport.ssl.key=/usr/share/elasticsearch/${ELASTICSEARCH_KEY}
       - xpack.security.transport.ssl.certificate=/usr/share/elasticsearch/${ELASTICSEARCH_CRT}
       - xpack.security.transport.ssl.certificate_authorities=/usr/share/elasticsearch/${ELASTICSEARCH_CA}
+      # Activate the following parameters for advanced setup
+      #- network.publish_host=${ELASTICSEARCH_PUBLISH_HOST2}
+      #- http.port=${ELASTICSEARCH_HOST2_HTTP}
+      #- transport.port=${ELASTICSEARCH_HOST2_TRANSPORT}
     ulimits:
       memlock:
         soft: -1
@@ -39,11 +43,12 @@ services:
         target: /usr/share/elasticsearch/data
       - ${PWD}/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
       - ${PWD}/certificates:/usr/share/elasticsearch/config/certificates
+      - ${PWD}/elasticsearch/scripts:/usr/share/elasticsearch/scripts
     networks:
       - elastic
     ports:
-      - 9201:9201
-      - ${ELASTICSEARCH_HOST2_TRANSPORT}:${ELASTICSEARCH_HOST2_TRANSPORT}
+      - ${ELASTICSEARCH_HOST1_HTTP:-9201}:${ELASTICSEARCH_HOST1_HTTP:-9201}
+      - ${ELASTICSEARCH_HOST1_TRANSPORT:-9301}:${ELASTICSEARCH_HOST1_TRANSPORT:-9301}
     logging:
       options:
         max-file: "5"

elasticsearch/docker-compose.es03.yml

@@ -3,14 +3,14 @@ services:
   # Elasticsearch node 3
   elasticsearch3:
     image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
+    # Custom entrypoint to configured ES based on given ELASTICSEARCH_HOSTS
+    entrypoint: /usr/share/elasticsearch/scripts/custom-entrypoint.sh
     container_name: elasticsearch3
     environment:
       - "ES_JAVA_OPTS=${ES_JAVA_OPTS}"
       - cluster.name=${ELASTICSEARCH_CLUSTERNAME}
       - node.name=elasticsearch3
-      - network.publish_host=${ELASTICSEARCH_HOST3}
-      - transport.port=${ELASTICSEARCH_HOST3_TRANSPORT}
-      - discovery.seed_hosts=${ELASTICSEARCH_HOST1}:${ELASTICSEARCH_HOST1_TRANSPORT}, ${ELASTICSEARCH_HOST2}:${ELASTICSEARCH_HOST2_TRANSPORT}
+      - ELASTICSEARCH_HOSTS=${ELASTICSEARCH_HOSTS}
       - xpack.security.enabled=true
       - xpack.security.http.ssl.enabled=true
       - xpack.security.http.ssl.key=/usr/share/elasticsearch/${ELASTICSEARCH_KEY}
@@ -21,6 +21,10 @@ services:
       - xpack.security.transport.ssl.key=/usr/share/elasticsearch/${ELASTICSEARCH_KEY}
       - xpack.security.transport.ssl.certificate=/usr/share/elasticsearch/${ELASTICSEARCH_CRT}
       - xpack.security.transport.ssl.certificate_authorities=/usr/share/elasticsearch/${ELASTICSEARCH_CA}
+      # Activate the following parameters for advanced setup
+      #- network.publish_host=${ELASTICSEARCH_PUBLISH_HOST3}
+      #- http.port=${ELASTICSEARCH_HOST3_HTTP}
+      #- transport.port=${ELASTICSEARCH_HOST3_TRANSPORT}
     ulimits:
       memlock:
         soft: -1
@@ -39,11 +43,12 @@ services:
         target: /usr/share/elasticsearch/data
       - ${PWD}/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
       - ${PWD}/certificates:/usr/share/elasticsearch/config/certificates
+      - ${PWD}/elasticsearch/scripts:/usr/share/elasticsearch/scripts
     networks:
       - elastic
     ports:
-      - 9202:9202
-      - ${ELASTICSEARCH_HOST3_TRANSPORT}:${ELASTICSEARCH_HOST3_TRANSPORT}
+      - ${ELASTICSEARCH_HOST1_HTTP:-9202}:${ELASTICSEARCH_HOST1_HTTP:-9202}
+      - ${ELASTICSEARCH_HOST1_TRANSPORT:-9302}:${ELASTICSEARCH_HOST1_TRANSPORT:-9302}
     logging:
       options:
         max-file: "5"

elasticsearch/scripts/custom-entrypoint.sh

@@ -0,0 +1,78 @@
+#!/bin/bash -e
+
+# This script is using the given ELASTICSEARCH_HOSTS to populate required parameters
+# to setup a Multi-Node Elasticsearch cluster.
+# If parameters (network.publish_host, http.port, transport.port) are already 
+# given externally they are used with precedence.
+
+# Get the number of the Elasticsearch node based on the node.name (e.g. elasticsearch2)
+nodeBasename=`echo node.name | awk 'match(ENVIRON[$0], /([a-zA-Z]*)([0-9]{1,})/, v) { print v[1] }'`
+nodeNumber=`echo node.name | awk 'match(ENVIRON[$0], /([a-zA-Z]*)([0-9]{1,})/, v) { print v[2] }'`
+
+count=1
+params=""
+seedHosts=""
+initialMasterNodes=""
+for host in ${ELASTICSEARCH_HOSTS//,/ }
+do
+    # Use all nodes as initial master node, when initializing the cluster
+    # It is assumed, that node-names are sequentially counted elasticsearch1, elasticsearch2, ...
+    if [ "${initCluster}" = "true" ]; then
+        if [ "${initialMasterNodes}" == "" ]; then
+            initialMasterNodes="-E cluster.initial_master_nodes=${nodeBasename}${count}"
+        else 
+            initialMasterNodes="${initialMasterNodes},${nodeBasename}${count}"
+        fi
+    fi
+    # Use all declared hosts as seed hosts if 
+    # Seeds hosts are not given externally and the standard transport ports are used
+    discoverySeedHosts=`echo discovery.seed_hosts | awk '{print ENVIRON[$0]}'`
+    discoveryTransportPort=`echo transport.port | awk '{print ENVIRON[$0]}'`
+    if [ -z "${discoverySeedHosts}" -a -z "${discoveryTransportPort}" ]
+    then
+        # Get the hostname and transport from the URL
+        discoverPublishHostname=`echo $host | awk 'match($0, /https?:\/\/(.*)\:(\d*)/, v) { print v[1] }'`
+        discoveryTransportPort=`echo $host | awk 'match($0,/https?:\/\/(.*)\:[0-9]{2}([0-9]{2})/, v) { print 93v[2] }'`
+        if [ "${seedHosts}" == "" ]; then
+            seedHosts="-E discovery.seed_hosts=${discoverPublishHostname}:${discoveryTransportPort}"
+        else 
+            seedHosts="${seedHosts},${discoverPublishHostname}:${discoveryTransportPort}"
+        fi
+    fi
+
+    if [ "${count}" == "${nodeNumber}" ]; then
+        echo "Setting up Elasticsearch node: ${nodeNumber}"
+        publishHost=`echo network.publish_host | awk '{print ENVIRON[$0]}'`
+        if [ -z "${publishHost}" ]; then
+            publishHost=`echo $host | awk 'match($0, /https?:\/\/(.*)\:(\d*)/, v) { print v[1] }'`
+            params="$params -E network.publish_host=${publishHost}"
+            echo "Set network.publish_host=${publishHost} based on given host: ${host}"
+        else
+            echo "network.publish_host=${publishHost} taken from envionment variable"
+        fi
+
+        httpPort=`echo http.port | awk '{print ENVIRON[$0]}'`
+        if [ -z "${httpPort}" ]
+        then
+            httpPort=`echo $host | awk 'match($0,/https?:\/\/(.*)\:([0-9]*)/, v) { print v[2] }'`
+            params="$params -E http.port=${httpPort}"
+            echo "Set http.port=${httpPort} based on given host: ${host}"
+        else
+            echo "http.port=${httpPort} taken from envionment variable"
+        fi
+
+        transportPort=`echo transport.port | awk '{print ENVIRON[$0]}'`
+        if [ -z "${transportPort}" ]
+        then
+            transportPort=`echo $host | awk 'match($0,/https?:\/\/(.*)\:[0-9]{2}([0-9]{2})/, v) { print 93v[2] }'`
+            params="$params -E transport.port=${transportPort}"
+            echo "Set transport.port=${transportPort} based on given host: ${host}"
+        else
+            echo "transport.port=${transportPort} taken from envionment variable"
+        fi
+    fi
+    count=`expr $count + 1`    
+done
+
+# Finally call the original Docker-Entrypoint
+/usr/local/bin/docker-entrypoint.sh elasticsearch ${params} ${seedHosts} ${initialMasterNodes}

env-sample

@@ -112,17 +112,25 @@ API_MANAGER_PASSWORD=changeme
 ################################################################################################
 #                            Configure Multi-Node ES-Cluster
 ################################################################################################
-# The following parameters are used by the elaticsearch/docker-compose.es<N>.yml files and only 
-# required if you are running Elasticsearch based on Docker-Compose. 
-# If you using an external Elasticsearch cluster, these parameters are not used.
-ELASTICSEARCH_HOST1=elasticsearch1
-ELASTICSEARCH_HOST2=elasticsearch2
-ELASTICSEARCH_HOST3=elasticsearch3
-
-# Ports are different just to be able to start multiple Elasticsearch containers on one host.
-ELASTICSEARCH_HOST1_TRANSPORT=9300
-ELASTICSEARCH_HOST2_TRANSPORT=9301
-ELASTICSEARCH_HOST3_TRANSPORT=9302
+# The following parameters are used to build and configure the Elasticsearch cluster. 
+# If you don't set this parameters the cluster is configured based on the configured 
+# ELASTICSEARCH_HOSTS.
+#
+# The publish host is used for internal cluster communication between each node. If must be 
+# set to a host that is reachable from other nodes. 
+# ELASTICSEARCH_PUBLISH_HOST1=elasticsearch1
+# ELASTICSEARCH_PUBLISH_HOST2=elasticsearch2
+# ELASTICSEARCH_PUBLISH_HOST3=elasticsearch3
+
+# The ES REST-API listen socket and used to map the port outside of the docker container.
+# ELASTICSEARCH_HOST1_HTTP=9200
+# ELASTICSEARCH_HOST2_HTTP=9201
+# ELASTICSEARCH_HOST3_HTTP=9202
+
+# The ES Transport protocol listen socket and used to map the port outside of the docker container.
+# ELASTICSEARCH_HOST1_TRANSPORT=9300
+# ELASTICSEARCH_HOST2_TRANSPORT=9301
+# ELASTICSEARCH_HOST3_TRANSPORT=9302
 
 ################################################################################################
 #                            Optional / Advanced parameters
@@ -220,7 +228,7 @@ ELASTICSEARCH_CLUSTERNAME=axway-apim-elasticsearch
 # ----------------------------------------------------------------------------------------------
 # Define the version of the Elastic-Stack to use. Based on this version Docker images are pulled
 # Used-By: Elasticsearch, Filebeat, Kibana, Logstash
-ELASTIC_VERSION=7.9.0
+ELASTIC_VERSION=7.9.2
 
 ################################################################################################
 #                         Certificates and Certificate authorities