Merge pull request #17 from Axway-API-Management-Plus/project-restructure

Major refactoring

Author: Chris Wiechmann

.env

@@ -1,6 +1,6 @@
 # Filebeat will mount that folder into the Filebeat Docker-Container to have access to the files
 APIGATEWAY_LOGS_FOLDER=/home/localuser/Axway-x.y.z/apigateway/logs/opentraffic
-APIGATEWAY_TRACES_FOLDER=/home/localuser/Axway-x.y.z/apigateway/groups/group-1/instance-1/trace
+APIGATEWAY_TRACES_FOLDER=/home/localuser/Axway-x.y.z/apigateway/groups/group-2/instance-1/trace
 
 # This variable is used by the API-Builder project to locate the Elasticsearch instance
 # Using the default docker-compose.yaml this setting is sufficient

.github/workflows/logstash.yml

@@ -1,10 +1,6 @@
 name: Logstash config test
 
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
+on: [push, pull_request]
 
 jobs:
   test-logstash:
@@ -26,4 +22,4 @@ jobs:
     - name: Run Logstash config tests
       run: |
         dir
-        ./logstash-filter-verifier
+        ./logstash-filter-verifier --diff-command="diff -y" --sockets ./logstash/test ./logstash/pipeline

README.md

@@ -13,8 +13,8 @@ Each API-Gateway instance is writing, [if configured](#enable-open-traffic-event
 
 Once the data is indexed by Elasticsearch it can be used by different clients. This process allows almost realtime monitoring of incoming requests. It takes around 5 seconds until a request is available in Elasticsearch.
 
-## Option 1 - Using the existing Traffic-Monitor
-One option is to use the existing API-Gateway Traffic-Monitor. That means, you use the same tooling as of today, but the underlying implementation of the Traffic-Monitor API is now pointing to Elasticsearch instead of the internal OPSDB hosted by each API-Gateway instance. This improves performance damatically, as Elasticsearch can scale across multiple machines if required and other dashboards can be created for instance with Kibana.  
+## Using the existing Traffic-Monitor
+Use the existing API-Gateway Traffic-Monitor. That means, you use the same tooling as of today, but the underlying implementation of the Traffic-Monitor API is now pointing to Elasticsearch instead of the internal OPSDB hosted by each API-Gateway instance. This improves performance damatically, as Elasticsearch can scale across multiple machines if required and other dashboards can be created for instance with Kibana.  
 The glue between Elasticsearch and the API-Gateway Traffic-Monitor is an [API-Builder project](./elk-traffic-monitor-api), that is exposing the same Traffic-Monitor API, but it is implemented using Elasticsearch instead of the OPSDB. The API-Builder is available as a ready to use Docker-Image and preconfigured in the docker-compose file.  
 Optionally you can import the API-Builder API into your API-Management system to apply additional security and by that secure access to your Elasticsearch instance.  
 
@@ -23,27 +23,6 @@ Finally, the Admin-Node-Manager has to be [configured](#configure-the-admin-node
 API-Builder exposing Traffic-Monitor API:  
 [![Traffic-Monitor API](https://github.com/Axway-API-Management-Plus/apigateway-openlogging-elk/workflows/Traffic-Monitor%20API/badge.svg)](https://github.com/Axway-API-Management-Plus/apigateway-openlogging-elk/actions)
 
-## Option 2 - Loginspector
-The Loginspector is a new separated user-interface with very basic set of functionalities. As part of the project the Loginspector is activated by default when using `docker-compose up -d`. If you don't wanna use it, it can be disabled by commenting out the following lines in the docker-compose.yml file:
-```yaml
-  nginx:
-    image: nginx:1.17.6
-    ports:
-      - 8888:90
-    volumes:
-      - ${PWD}/nginx/www:/usr/share/nginx/html
-      - ${PWD}/nginx/conf:/etc/nginx
-    depends_on:
-      - elasticsearch1
-    networks:
-      - elastic
-      - ingress
-```
-The Log-Inspector is accessible on the following URL: `http://hostname-to-your-docker-machine:8888/logspector.html`
-
-![Log-Inspector][img5]  
-
-
 ## Prerequisites
 For a simple deployment the prerequisites are very simple as all services can be started as a Docker-Container. In order to start all components in PoC-Like-Mode you just need:
 

configs/logstash.conf

@@ -2,7 +2,7 @@ version: '3.7'
 services:
   # The core component
   elasticsearch1:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.4.0
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.8.0
     container_name: elasticsearch1
     environment:
       - "ES_JAVA_OPTS=-Xms750M -Xmx750M"
@@ -19,7 +19,7 @@ services:
       - type: volume
         source: esdata1
         target: /usr/share/elasticsearch/data
-      - ${PWD}/configs/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
+      - ${PWD}/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
     networks:
       - elastic
       - ingress
@@ -29,7 +29,7 @@ services:
 
   # This is optional, but good to have to perform manual queries and create custom dashboards
   kibana:
-    image: docker.elastic.co/kibana/kibana:7.4.0
+    image: docker.elastic.co/kibana/kibana:7.8.0
     container_name: kibana
     environment:
       SERVER_NAME: localhost
@@ -52,15 +52,15 @@ services:
 
   # Supposed to run side-by-side with the API-Gateway to watch the Open-Traffic Event files and send event to Logstash
   filebeat:
-    image: docker.elastic.co/beats/filebeat:7.4.0
+    image: docker.elastic.co/beats/filebeat:7.8.0
     command: --strict.perms=false
     environment:
       - setup.kibana.host=kibana:5601
       - output.elasticsearch.hosts=["elasticsearch1:9200"]
     ports:
       - 9000:9000
     volumes:
-      - ${PWD}/configs/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml
+      - ${PWD}/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml
       - ${APIGATEWAY_LOGS_FOLDER}:/var/log/work
       - ${APIGATEWAY_TRACES_FOLDER}:/var/log/trace
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
@@ -70,7 +70,7 @@ services:
 
   # Is receiving events from Filebeat and does pre-processing
   logstash:
-    image: docker.elastic.co/logstash/logstash:7.4.0
+    image: docker.elastic.co/logstash/logstash:7.8.0
     links:
       - elasticsearch1
     environment:
@@ -79,9 +79,9 @@ services:
     ports:
       - 5044:5044
     volumes:
-      - ${PWD}/configs/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
-      - ${PWD}/configs/openlog_index_template.json:/usr/share/logstash/pipeline/openlog_index_template.json
-    command: logstash --path.config /usr/share/logstash/pipeline/logstash.conf --pipeline.batch.size 20 --pipeline.workers 1 --log.level info
+      - ${PWD}/logstash/pipeline/pipeline.conf:/usr/share/logstash/pipeline/pipeline.conf
+      - ${PWD}/logstash/config/traffic_details_index_template.json:/usr/share/logstash/config/traffic_details_index_template.json
+    command: logstash --path.config /usr/share/logstash/pipeline/pipeline.conf --pipeline.batch.size 20 --pipeline.workers 1 --log.level info
     depends_on:
       - elasticsearch1
     networks:
@@ -101,21 +101,6 @@ services:
     networks:
       - elastic
 
-  # This is the Logspector Web-Application
-  nginx:
-    image: nginx:1.17.6
-    ports:
-      - 8888:90
-    volumes:
-      - ${PWD}/nginx/www:/usr/share/nginx/html
-      - ${PWD}/nginx/conf:/etc/nginx
-    depends_on:
-      - elasticsearch1
-    networks:
-      - elastic
-      - ingress
-
-
 volumes:
   esdata1:
   esdata2: