Merge pull request #23 from Axway-API-Management-Plus/trace-messages

Chris Wiechmann · web-flow · commit 99e10df4b64f · 2020-08-25T09:52:08.000+02:00
Optimized ES-Index storing trace-messages
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -5,7 +5,9 @@ services:
     image: docker.elastic.co/elasticsearch/elasticsearch:7.8.1
     container_name: elasticsearch1
     environment:
-      - "ES_JAVA_OPTS=-Xms750M -Xmx750M"
+      # This should be 50% of your available memory, but not more than 31GB
+      # The remaining 50% of memory will be used by Apache Lucene
+      - ES_HEAP_SIZE: 1g
     ulimits:
       memlock:
         soft: -1
diff --git a/elk-traffic-monitor-api/test/_base.js b/elk-traffic-monitor-api/test/_base.js
@@ -66,12 +66,12 @@ function requestAsync(uri, options, cb) {
 	});
 }
 
-async function sendToElasticsearch(elasticConfig, index, dataset) {
+async function sendToElasticsearch(elasticConfig, index, template, dataset) {
 	console.log(`Creating connection to ElasticSearch cluster: ${elasticConfig.node}`)
 	const client = new Client({
 		node: elasticConfig.node
 	});
-	const mappingConfig = JSON.parse(fs.readFileSync('../logstash/config/traffic_details_index_template.json')).mappings;
+	const mappingConfig = JSON.parse(fs.readFileSync(`../logstash/config/${template}`)).mappings;
 	const createdIndexResponse = await client.indices.create({
 		index: index,
 		body: {
diff --git a/elk-traffic-monitor-api/test/asAdmin/test_circuitpath.js b/elk-traffic-monitor-api/test/asAdmin/test_circuitpath.js
@@ -9,7 +9,7 @@ describe('Traffic Monitor API', function () {
 	this.timeout(30000);
 	let server;
 	let auth;
-	const indexName = `circuitpath_test_${getRandomInt(9999)}`;
+	const indexName = `apigw-traffic-circuitpath_test_${getRandomInt(9999)}`;
 
 	beforeEach(() => {
 		// Simulate all responses in this test-file to be an admin, which will not lead to any result restriction
@@ -40,7 +40,7 @@ describe('Traffic Monitor API', function () {
 			server.started
 			.then(() => {
 				const entryset = require('../documents/basic/circuitpath_test_documents');
-				sendToElasticsearch(elasticConfig, indexName, entryset)
+				sendToElasticsearch(elasticConfig, indexName, 'traffic_details_index_template.json', entryset)
 				.then(() => {
 					resolve();
 				})
diff --git a/elk-traffic-monitor-api/test/asAdmin/test_getinfo_endpoint.js b/elk-traffic-monitor-api/test/asAdmin/test_getinfo_endpoint.js
@@ -9,7 +9,7 @@ describe('Endpoints', function () {
 	this.timeout(30000);
 	let server;
 	let auth;
-	const indexName = `getinfo_test_${getRandomInt(9999)}`;
+	const indexName = `apigw-traffic-getinfo_test_${getRandomInt(9999)}`;
 
 	beforeEach(() => {
 		// Simulate all responses in this test-file to be an admin, which will not lead to any result restriction
@@ -40,7 +40,7 @@ describe('Endpoints', function () {
 			server.started
 			.then(() => {
 				const entryset = require('../documents/basic/getinfo_test_documents');
-				sendToElasticsearch(elasticConfig, indexName, entryset)
+				sendToElasticsearch(elasticConfig, indexName, 'traffic_details_index_template.json', entryset)
 				.then(() => {
 					resolve();
 				})
diff --git a/elk-traffic-monitor-api/test/asAdmin/test_search_count_AsAdmin.js b/elk-traffic-monitor-api/test/asAdmin/test_search_count_AsAdmin.js
@@ -9,7 +9,7 @@ describe('Endpoints', function () {
 	this.timeout(30000);
 	let server;
 	let auth;
-	const indexName = `search_count_test_${getRandomInt(9999)}`;
+	const indexName = `apigw-traffic-search_count_test_${getRandomInt(9999)}`;
 
 	beforeEach(() => {
 		// Simulate all responses in this test-file to be an admin, which will not lead to any result restriction
@@ -40,7 +40,7 @@ describe('Endpoints', function () {
 			server.started
 			.then(() => {
 				const entryset = require('../documents/basic/search_count_documents');
-				sendToElasticsearch(elasticConfig, indexName, entryset)
+				sendToElasticsearch(elasticConfig, indexName, 'traffic_details_index_template.json', entryset)
 				.then(() => {
 					resolve();
 				})
diff --git a/elk-traffic-monitor-api/test/asAdmin/test_search_endpoint_AsAdmin.js b/elk-traffic-monitor-api/test/asAdmin/test_search_endpoint_AsAdmin.js
@@ -10,7 +10,7 @@ describe('Endpoints', function () {
 	this.timeout(30000);
 	let server;
 	let auth;
-	const indexName = `search_test_${getRandomInt(9999)}`;
+	const indexName = `apigw-traffic-search_test_${getRandomInt(9999)}`;
 
 	beforeEach(() => {
 		// Simulate all responses in this test-file to be an admin, which will not lead to any result restriction
@@ -41,7 +41,7 @@ describe('Endpoints', function () {
 			server.started
 			.then(() => {
 				const entryset = require('../documents/basic/search_test_documents');
-				sendToElasticsearch(elasticConfig, indexName, entryset)
+				sendToElasticsearch(elasticConfig, indexName, 'traffic_details_index_template.json', entryset)
 				.then(() => {
 					resolve();
 				})
diff --git a/elk-traffic-monitor-api/test/asAdmin/test_trace_endpoint.js b/elk-traffic-monitor-api/test/asAdmin/test_trace_endpoint.js
@@ -9,7 +9,7 @@ describe('Traffic Monitor API', function () {
 	this.timeout(30000);
 	let server;
 	let auth;
-	const indexName = `trace_test_${getRandomInt(9999)}`;
+	const indexName = `apigw-trace-test_${getRandomInt(9999)}`;
 
 	beforeEach(() => {
 		// Simulate all responses in this test-file to be an admin, which will not lead to any result restriction
@@ -40,7 +40,7 @@ describe('Traffic Monitor API', function () {
 			server.started
 			.then(() => {
 				const entryset = require('../documents/basic/trace_test_documents');
-				sendToElasticsearch(elasticConfig, indexName, entryset)
+				sendToElasticsearch(elasticConfig, indexName, 'trace_messages_index_template.json', entryset)
 				.then(() => {
 					resolve();
 				})
diff --git a/elk-traffic-monitor-api/test/restricted/test_circuitpath_restricted.js b/elk-traffic-monitor-api/test/restricted/test_circuitpath_restricted.js
@@ -9,7 +9,7 @@ describe('Traffic Monitor API', function () {
 	this.timeout(30000);
 	let server;
 	let auth;
-	const indexName = `circuitpath_test_${getRandomInt(9999)}`;
+	const indexName = `apigw-traffic-circuitpath_test_${getRandomInt(9999)}`;
 
 	/**
 	 * Start API Builder.
@@ -30,7 +30,7 @@ describe('Traffic Monitor API', function () {
 			server.started
 			.then(() => {
 				const entryset = require('../documents/basic/circuitpath_test_documents');
-				sendToElasticsearch(elasticConfig, indexName, entryset)
+				sendToElasticsearch(elasticConfig, indexName, 'traffic_details_index_template.json', entryset)
 				.then(() => {
 					resolve();
 				})
diff --git a/elk-traffic-monitor-api/test/restricted/test_getinfo_endpoint_restricted.js b/elk-traffic-monitor-api/test/restricted/test_getinfo_endpoint_restricted.js
@@ -9,7 +9,7 @@ describe('Endpoints', function () {
 	this.timeout(30000);
 	let server;
 	let auth;
-	const indexName = `getinfo_test_${getRandomInt(9999)}`;
+	const indexName = `apigw-traffic-getinfo_test_${getRandomInt(9999)}`;
 
 	/**
 	 * Start API Builder.
@@ -30,7 +30,7 @@ describe('Endpoints', function () {
 			server.started
 			.then(() => {
 				const entryset = require('../documents/basic/getinfo_test_documents');
-				sendToElasticsearch(elasticConfig, indexName, entryset)
+				sendToElasticsearch(elasticConfig, indexName, 'traffic_details_index_template.json', entryset)
 				.then(() => {
 					resolve();
 				})
diff --git a/elk-traffic-monitor-api/test/restricted/test_search_restricted.js b/elk-traffic-monitor-api/test/restricted/test_search_restricted.js
@@ -9,7 +9,7 @@ describe('Endpoints', function () {
 	this.timeout(30000);
 	let server;
 	let auth;
-	const indexName = `search_count_test_${getRandomInt(9999)}`;
+	const indexName = `apigw-traffic-search_count_test_${getRandomInt(9999)}`;
 
 	/**
 	 * Start API Builder.
@@ -30,7 +30,7 @@ describe('Endpoints', function () {
 			server.started
 			.then(() => {
 				const entryset = require('../documents/basic/search_count_documents');
-				sendToElasticsearch(elasticConfig, indexName, entryset)
+				sendToElasticsearch(elasticConfig, indexName, 'traffic_details_index_template.json', entryset)
 				.then(() => {
 					resolve();
 				})
diff --git a/logstash/config/trace_messages_index_template.json b/logstash/config/trace_messages_index_template.json
@@ -0,0 +1,27 @@
+{
+    "index_patterns": [
+        "apigw-trace-*"
+    ],
+    "settings": {
+        "number_of_shards": 5, 
+        "number_of_replicas": 2
+    },
+    "mappings": {
+        "dynamic": false,
+        "properties": {
+            "@timestamp": {
+                "type" : "date"
+            },
+            "correlationId": {
+                "type": "keyword"
+            },
+            "level": {
+                "type": "keyword"
+            },
+            "message": {
+                "type": "text", 
+                "norms": false
+            }
+        }
+    }
+}
diff --git a/logstash/pipeline/pipeline.conf b/logstash/pipeline/pipeline.conf
@@ -160,11 +160,15 @@ output {
     elasticsearch {
       hosts => "elasticsearch1:9200"
       index => "apigw-trace-messages-%{+YYYY.MM.dd}"
+      template => "${HOME}/config/trace_messages_index_template.json"
+      template_overwrite => true
     }
   } else if [logtype] == "trace" {
     elasticsearch {
       hosts => "elasticsearch1:9200"
-      index => "apigw-traffic-trace-%{+YYYY.MM.dd}"
+      index => "apigw-trace-traffic-%{+YYYY.MM.dd}"
+      template => "${HOME}/config/trace_messages_index_template.json"
+      template_overwrite => true
     }
   } else if [type] == 'errorEvent' {
     elasticsearch {
