Skip to content
This repository was archived by the owner on Dec 14, 2022. It is now read-only.

Commit 30a4d5c

Browse files
author
Chris Wiechmann
committed
Custom-Properties now mapped also for trafficDetails index
#115
1 parent e80fd9a commit 30a4d5c

File tree

6 files changed

+82
-43
lines changed

6 files changed

+82
-43
lines changed

‎apibuilder4elastic/custom_flow_nodes/api-builder-plugin-axway-api-management/src/customProperties.js‎

Lines changed: 29 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -19,51 +19,64 @@
1919
* does not define "next", the first defined output).
2020
*/
2121
async function mergeCustomProperties(params, options) {
22+
const { customProperties, desiredIndexTemplate, actualIndexTemplate, customPropertiesSettings } = params;
2223
const { logger } = options;
23-
if (!params.customProperties) {
24+
if (!customProperties) {
2425
throw new Error('Missing required parameter: customProperties');
2526
}
26-
if (!params.desiredIndexTemplate) {
27+
if (!desiredIndexTemplate) {
2728
throw new Error('Missing required parameter: desiredIndexTemplate');
2829
}
29-
if (!params.desiredIndexTemplate.mappings) {
30+
if (!desiredIndexTemplate.mappings) {
3031
throw new Error('Missing mappings properties in desiredIndexTemplate.mappings');
3132
}
32-
if (params.mergeCustomProperties == undefined) {
33-
params.mergeCustomProperties = false;
33+
if (customPropertiesSettings == undefined) {
34+
customPropertiesSettings = { merge: false };
3435
}
35-
if (!params.mergeCustomProperties) {
36+
if (!customPropertiesSettings.merge) {
3637
logger.info(`Custom properties are ignore for this template.`);
37-
return options.setOutput('noUpdate', params.desiredIndexTemplate);
38+
return options.setOutput('noUpdate', desiredIndexTemplate);
3839
}
3940
var updateRequired = false;
40-
for (var prop in params.customProperties) {
41-
if (Object.prototype.hasOwnProperty.call(params.customProperties, prop)) {
42-
var customPropertyConfig = params.customProperties[prop];
43-
if(addMapping(prop, customPropertyConfig.type, params.actualIndexTemplate, params.desiredIndexTemplate)) {
41+
for (var prop in customProperties) {
42+
if (Object.prototype.hasOwnProperty.call(customProperties, prop)) {
43+
var customPropertyConfig = customProperties[prop];
44+
if(addMapping(prop, customPropertyConfig.type, actualIndexTemplate, desiredIndexTemplate, customPropertiesSettings)) {
4445
if(!updateRequired) updateRequired = true;
4546
}
4647
}
4748
}
4849
if(!updateRequired) {
49-
return options.setOutput('noUpdate', params.desiredIndexTemplate);
50+
return options.setOutput('noUpdate', desiredIndexTemplate);
5051
} else {
51-
return params.desiredIndexTemplate;
52+
return desiredIndexTemplate;
5253
}
5354

54-
function addMapping(customPropertyName, type, actualTemplate, desiredTemplate) {
55+
function addMapping(customPropertyName, type, actualTemplate, desiredTemplate, customPropertiesSettings) {
5556
if(desiredTemplate == undefined) return false;
57+
debugger;
5658
if(actualTemplate != undefined && actualTemplate.mappings != undefined && actualTemplate.mappings.properties[`customProperties.${customPropertyName}`] != undefined) {
5759
options.logger.info(`Mapping for custom property: ${customPropertyName} already exists. No update required.`);
5860
// Take over the actual custom properties mapping!
5961
desiredTemplate.mappings.properties[`customProperties.${customPropertyName}`] = actualTemplate.mappings.properties[`customProperties.${customPropertyName}`];
6062
return false;
6163
} else {
6264
options.logger.info(`Update required for custom property: ${customPropertyName}.`);
65+
if(customPropertiesSettings.parent && !desiredTemplate.mappings.properties[customPropertiesSettings.parent]) {
66+
desiredTemplate.mappings.properties[customPropertiesSettings.parent] = {};
67+
}
6368
if(type == "custom") {
64-
desiredTemplate.mappings.properties[`customProperties.${customPropertyName}`] = { type: "text", norms: false};
69+
if(customPropertiesSettings.parent) {
70+
desiredTemplate.mappings.properties[customPropertiesSettings.parent][`customProperties.${customPropertyName}`] = { type: "text", norms: false, fields: { "keyword": { type: "keyword"} } };
71+
} else {
72+
desiredTemplate.mappings.properties[`customProperties.${customPropertyName}`] = { type: "text", norms: false, fields: { "keyword": { type: "keyword"} } };
73+
}
6574
} else {
66-
desiredTemplate.mappings.properties[`customProperties.${customPropertyName}`] = { type: "keyword"};
75+
if(customPropertiesSettings.parent) {
76+
desiredTemplate.mappings.properties[customPropertiesSettings.parent][`customProperties.${customPropertyName}`] = { type: "keyword"};
77+
} else {
78+
desiredTemplate.mappings.properties[`customProperties.${customPropertyName}`] = { type: "keyword"};
79+
}
6780
}
6881
return true;
6982
}

‎apibuilder4elastic/custom_flow_nodes/api-builder-plugin-axway-api-management/src/flow-nodes.yml‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -285,13 +285,12 @@ flow-nodes:
285285
initialType: object
286286
schema:
287287
type: object
288-
mergeCustomProperties:
289-
name: Merge custom properties
290-
description: Convience flag which basically turns off this flow now. This helps to simplify the flow.
288+
customPropertiesSettings:
289+
name: Custom properties settings
290+
description: "Controls if and how custom properties should be merged into the index template. See config file: elasticsearch_config/index_config.json"
291291
required: false
292-
initialType: string
293292
schema:
294-
type: string
293+
type: object
295294
outputs:
296295
next:
297296
name: Next

‎apibuilder4elastic/custom_flow_nodes/api-builder-plugin-axway-api-management/test/test-customProperties-MappingMerge.js‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ describe('Merge custom properties tests', () => {
4949
const { value, output } = await flowNode.mergeCustomProperties({
5050
customProperties: JSON.parse(fs.readFileSync('./test/testInput/customPropertiesConfig.json'), null),
5151
desiredIndexTemplate: desiredIndexTemplate,
52-
mergeCustomProperties: false
52+
customPropertiesSettings: { merge: false }
5353
});
5454

5555
expect(output).to.equal('noUpdate');
@@ -60,12 +60,12 @@ describe('Merge custom properties tests', () => {
6060
const { value, output } = await flowNode.mergeCustomProperties({
6161
customProperties: JSON.parse(fs.readFileSync('./test/testInput/customPropertiesConfig.json'), null),
6262
desiredIndexTemplate: JSON.parse(fs.readFileSync('./test/testInput/desiredIndexTemplate.json'), null),
63-
mergeCustomProperties: true
63+
customPropertiesSettings: { merge: true, parent: "" }
6464
});
6565

6666
expect(output).to.equal('next');
6767
expect(value.mappings.properties['customProperties.customProperty1']).to.be.an('Object');
68-
expect(value.mappings.properties['customProperties.customProperty1'].type).to.equal('text');
68+
expect(value.mappings.properties['customProperties.customProperty1']).to.deep.equal({type: "text", norms: false, fields: {keyword: { type: "keyword"}}});
6969
expect(value.mappings.properties['customProperties.customProperty2']).to.be.an('Object');
7070
expect(value.mappings.properties['customProperties.customProperty2'].type).to.equal('keyword');
7171
expect(value.mappings.properties['customProperties.customProperty3']).to.be.an('Object');
@@ -77,11 +77,11 @@ describe('Merge custom properties tests', () => {
7777
customProperties: JSON.parse(fs.readFileSync('./test/testInput/customPropertiesConfig.json'), null),
7878
desiredIndexTemplate: JSON.parse(fs.readFileSync('./test/testInput/desiredIndexTemplate.json'), null),
7979
actualIndexTemplate: JSON.parse(fs.readFileSync('./test/testInput/actualIndexTemplate.json'), null),
80-
mergeCustomProperties: true
80+
customPropertiesSettings: { merge: true, parent: "" }
8181
});
8282
expect(output).to.equal('next');
8383
expect(value.mappings.properties['customProperties.customProperty1']).to.be.an('Object');
84-
expect(value.mappings.properties['customProperties.customProperty1'].type).to.equal('text');
84+
expect(value.mappings.properties['customProperties.customProperty1']).to.deep.equal({type: "text", norms: false, fields: {keyword: { type: "keyword"}}});
8585
expect(value.mappings.properties['customProperties.customProperty2']).to.be.an('Object');
8686
expect(value.mappings.properties['customProperties.customProperty2'].type).to.equal('keyword');
8787
expect(value.mappings.properties['customProperties.customProperty3']).to.be.an('Object');
@@ -93,11 +93,11 @@ describe('Merge custom properties tests', () => {
9393
customProperties: JSON.parse(fs.readFileSync('./test/testInput/customPropertiesConfig.json'), null),
9494
desiredIndexTemplate: JSON.parse(fs.readFileSync('./test/testInput/desiredIndexTemplateWithCustomProps.json'), null),
9595
actualIndexTemplate: JSON.parse(fs.readFileSync('./test/testInput/actualIndexTemplateWithCustomProps.json'), null),
96-
mergeCustomProperties: true
96+
customPropertiesSettings: { merge: true, parent: "" }
9797
});
9898
expect(output).to.equal('noUpdate');
9999
expect(value.mappings.properties['customProperties.customProperty1']).to.be.an('Object');
100-
expect(value.mappings.properties['customProperties.customProperty1'].type).to.equal('text');
100+
expect(value.mappings.properties['customProperties.customProperty1']).to.deep.equal({type: "text" });
101101
expect(value.mappings.properties['customProperties.customProperty2']).to.be.an('Object');
102102
expect(value.mappings.properties['customProperties.customProperty2'].type).to.equal('keyword');
103103
expect(value.mappings.properties['customProperties.customProperty3']).to.be.an('Object');
@@ -109,15 +109,32 @@ describe('Merge custom properties tests', () => {
109109
customProperties: JSON.parse(fs.readFileSync('./test/testInput/customPropertiesConfig.json'), null),
110110
desiredIndexTemplate: JSON.parse(fs.readFileSync('./test/testInput/desiredIndexTemplate.json'), null),
111111
actualIndexTemplate: JSON.parse(fs.readFileSync('./test/testInput/actualIndexTemplateWithLessCustomProps.json'), null),
112-
mergeCustomProperties: true
112+
customPropertiesSettings: { merge: true, parent: "" }
113113
});
114114
expect(output).to.equal('next');
115115
expect(value.mappings.properties['customProperties.customProperty1']).to.be.an('Object');
116-
expect(value.mappings.properties['customProperties.customProperty1'].type).to.equal('text');
116+
expect(value.mappings.properties['customProperties.customProperty1']).to.deep.equal({type: "text", norms: false, fields: {keyword: { type: "keyword"}}});
117117
expect(value.mappings.properties['customProperties.customProperty2']).to.be.an('Object');
118118
expect(value.mappings.properties['customProperties.customProperty2'].type).to.equal('keyword');
119119
expect(value.mappings.properties['customProperties.customProperty3']).to.be.an('Object');
120120
expect(value.mappings.properties['customProperties.customProperty3'].type).to.equal('keyword');
121121
});
122+
123+
it('should merge into indexMappingTemplate withd a defined parent as custom properties are missing', async () => {
124+
const { value, output } = await flowNode.mergeCustomProperties({
125+
customProperties: JSON.parse(fs.readFileSync('./test/testInput/customPropertiesConfig.json'), null),
126+
desiredIndexTemplate: JSON.parse(fs.readFileSync('./test/testInput/desiredIndexTemplate.json'), null),
127+
actualIndexTemplate: JSON.parse(fs.readFileSync('./test/testInput/actualIndexTemplate.json'), null),
128+
customPropertiesSettings: { merge: true, parent: "transactionSummary" }
129+
});
130+
131+
expect(output).to.equal('next');
132+
expect(value.mappings.properties.transactionSummary['customProperties.customProperty1']).to.be.an('Object');
133+
expect(value.mappings.properties.transactionSummary['customProperties.customProperty1']).to.deep.equal({type: "text", norms: false, fields: {keyword: { type: "keyword"}}});
134+
expect(value.mappings.properties.transactionSummary['customProperties.customProperty2']).to.be.an('Object');
135+
expect(value.mappings.properties.transactionSummary['customProperties.customProperty2'].type).to.equal('keyword');
136+
expect(value.mappings.properties.transactionSummary['customProperties.customProperty3']).to.be.an('Object');
137+
expect(value.mappings.properties.transactionSummary['customProperties.customProperty3'].type).to.equal('keyword');
138+
});
122139
});
123140
});

‎apibuilder4elastic/elasticsearch_config/index_config.json‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
"template": {
66
"config": "elasticsearch_config/traffic-summary/index_template.json",
77
"name": "traffic-summary",
8-
"mergeCustomProperties": true
8+
"customProperties": {
9+
"merge": true
10+
}
911
},
1012
"ilm" : {
1113
"config": "elasticsearch_config/traffic-summary/ilm_policy.json",
@@ -18,7 +20,10 @@
1820
"template": {
1921
"config": "elasticsearch_config/traffic-details/index_template.json",
2022
"name": "traffic-details",
21-
"mergeCustomProperties": true
23+
"customProperties": {
24+
"merge": true,
25+
"parent": "transactionSummary"
26+
}
2227
},
2328
"ilm" : {
2429
"config": "elasticsearch_config/traffic-details/ilm_policy.json",
@@ -31,7 +36,9 @@
3136
"template": {
3237
"config": "elasticsearch_config/trace-messages/index_template.json",
3338
"name": "trace-messages",
34-
"mergeCustomProperties": false
39+
"customProperties": {
40+
"merge": false
41+
}
3542
},
3643
"ilm" : {
3744
"config": "elasticsearch_config/trace-messages/ilm_policy.json",
@@ -44,7 +51,9 @@
4451
"template": {
4552
"config": "elasticsearch_config/trace-messages/index_template.json",
4653
"name": "trace-messages",
47-
"mergeCustomProperties": false
54+
"customProperties": {
55+
"merge": false
56+
}
4857
},
4958
"ilm" : {
5059
"config": "elasticsearch_config/trace-messages/ilm_policy.json",
@@ -57,7 +66,9 @@
5766
"template": {
5867
"config": "elasticsearch_config/apigw-domainaudit/index_template.json",
5968
"name": "apigw-apigw-domainaudit",
60-
"mergeCustomProperties": false
69+
"customProperties": {
70+
"merge": false
71+
}
6172
},
6273
"ilm" : {
6374
"config": "elasticsearch_config/apigw-domainaudit/ilm_policy.json",
@@ -70,7 +81,9 @@
7081
"template": {
7182
"config": "elasticsearch_config/apigw-monitoring/index_template.json",
7283
"name": "apigw-monitoring",
73-
"mergeCustomProperties": false
84+
"customProperties": {
85+
"merge": false
86+
}
7487
},
7588
"ilm" : {
7689
"config": "elasticsearch_config/apigw-monitoring/ilm_policy.json",

‎apibuilder4elastic/flows/SetupElasticsearchIndex.json‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -129,11 +129,9 @@
129129
"metaDescription": "The actual Index-Template mapping loaded from Elasticsearch used to compare if new custom properties have been added. If all custom properties are already part of the mapping, the noUpdate exit is used."
130130
},
131131
{
132-
"name": "mergeCustomProperties",
132+
"name": "customPropertiesSettings",
133133
"type": "jsonpath",
134-
"value": "$.indexConfig.template.mergeCustomProperties",
135-
"metaName": "Merge custom properties",
136-
"metaDescription": "Convience flag which basically turns off this flow now. This helps to simplify the flow."
134+
"value": "$.indexConfig.template.customProperties"
137135
}
138136
],
139137
"outputs": {

‎apibuilder4elastic/triggers/timer.yaml‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -71,15 +71,14 @@ triggers:
7171
config: $.config
7272
env: $.env
7373
indexName: apigw-domainaudit
74-
7574
update-rollover-alias:
7675
enabled: true
7776
name: Update Index Rollover-Alias
7877
parameters:
79-
interval: '600000' # Run every 10 minutes
78+
interval: '600000'
8079
startImmediately: 'false'
8180
invoke:
8281
flow: apim-elk-setup-update-rollover-alias
8382
parameters:
8483
config: $.config
85-
env: $.env
84+
env: $.env

0 commit comments

Comments
 (0)