Skip to content

Commit 076d5e8

Browse files
author
Chris Wiechmann
committed
Create Transform method fixed/changed
1 parent 4d4bb49 commit 076d5e8

File tree

5 files changed

+21
-153
lines changed

5 files changed

+21
-153
lines changed

‎api-builder-plugin-fn-elasticsearch/CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
44
The format is based on [Keep a Changelog](http://keepachangelog.com/)
55
and this project adheres to [Semantic Versioning](http://semver.org/).
66

7+
## [1.0.22] 2021-08-27
8+
### Fixed
9+
- Create transform method fixed
10+
711
## [1.0.21] 2021-08-27
812
### Added
913
- Added Create Transform method

‎api-builder-plugin-fn-elasticsearch/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@axway-api-builder-ext/api-builder-plugin-fn-elasticsearch",
3-
"version": "1.0.21",
3+
"version": "1.0.22",
44
"description": "Integrate Elasticsearch into your API-Builder flow to combine search data for instance with other data available in your flow.",
55
"author": "Chris Wiechmann <cwiechmann@axway.com> (http://www.axway.com)",
66
"license": "Apache-2.0",

‎api-builder-plugin-fn-elasticsearch/src/actions/transform.js‎

Lines changed: 15 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,6 @@ async function putTransform(params, options) {
3535
delete params.startTransform;
3636
}
3737

38-
var replaceWhenChanged = false;
39-
if(params.replaceWhenChanged != undefined) {
40-
replaceWhenChanged = params.replaceWhenChanged;
41-
delete params.replaceWhenChanged;
42-
}
43-
4438
var deletePreviousTransform = false;
4539
if(params.deletePreviousTransform != undefined) {
4640
deletePreviousTransform = params.deletePreviousTransform;
@@ -52,63 +46,38 @@ async function putTransform(params, options) {
5246
idSuffix = "";
5347
}
5448
delete params.idSuffix;
55-
debugger;
49+
5650
var client = new ElasticsearchClient(elasticSearchConfig).client;
5751
try {
5852
var actualTransform;
5953
var actualTransformId
60-
// Get all active (RUNNING ONLY) transforms with the given Transform-ID
54+
// Get all active (RUNNING ONLY) transforms with the given primary Transform-ID
6155
const allTransforms = await client.transform.getTransformStats({ transformId: `${params.transformId}*` }, { ignore: [404], maxRetries: 3 });
6256
var runningTransforms = [];
6357
for (i = 0; i < allTransforms.body.transforms.length; i++) {
6458
const transform = allTransforms.body.transforms[i];
65-
// Check if the Transform-ID+Suffix already exists
59+
// Check if the Transform already exists, which means nothing to do
6660
if(transform.id==`${params.transformId}${idSuffix}`) {
67-
options.logger.info(`Transform found: ${params.transformId}${idSuffix} already exists with state: ${transform.state}`);
61+
options.logger.info(`Transform found: ${params.transformId}${idSuffix} already exists with state: ${transform.state}. To update this transform, please provide an idSuffix (e.g. v2)`);
6862
if(startTransform && transform.state != "started" && transform.state != "indexing") {
6963
options.logger.info(`Existing transform: ${params.transformId}${idSuffix} is not running, going to start it.`);
7064
await client.transform.startTransform( {transformId: transform.id}, { ignore: [404], maxRetries: 3 });
71-
return transform;
65+
}
66+
actualTransform = transform;
67+
} else {
68+
// Stop all other transforms
69+
if(transform.state == "started" || transform.state == "indexing") {
70+
await client.transform.stopTransform( {transformId: transform.id}, { ignore: [404], maxRetries: 3 });
7271
}
7372
}
74-
if(transform.state == "started" || transform.state == "indexing") {
75-
runningTransforms.push(transform);
76-
}
77-
}
78-
// If we have multiple transforms running, we cannot determine, which one should be used to compare the config with,
79-
// But it's important to stop them, as we should have only ONE running at the same time
80-
if(runningTransforms.length>1) {
81-
for (i = 0; i < runningTransforms.length; i++) {
82-
const runningTransform = runningTransforms[i];
83-
await client.transform.stopTransform( {transformId: runningTransform.id}, { ignore: [404], maxRetries: 3 });
84-
}
85-
} else if (runningTransforms.length==1) {
86-
// Otherwise we consider the currently running transform as to be the actual and load the configuration
87-
const response = await client.transform.getTransform( {transformId: runningTransforms[0].id}, { ignore: [404], maxRetries: 3 });
88-
actualTransform = response.body.transforms[0];
89-
actualTransformId = actualTransform.id;
9073
}
91-
92-
// Compare the configuration with the currently running (actual) transform
93-
if(actualTransform != undefined && replaceWhenChanged) {
94-
delete actualTransform.id;
95-
delete actualTransform.create_time;
96-
delete actualTransform.version;
97-
if(JSON.stringify(actualTransform) === JSON.stringify(params.body)) {
98-
return options.setOutput('noUpdate', `No update required as desired Transform with new ID: '${params.transformId}${idSuffix}' equals to existing transform with ID: '${actualTransformId}'.`);
99-
}
74+
if(actualTransform) return actualTransform;
75+
// Stop all running transforms, as we expect only one transform to run
76+
for (i = 0; i < runningTransforms.length; i++) {
77+
const runningTransform = runningTransforms[i];
78+
await client.transform.stopTransform( {transformId: runningTransform.id}, { ignore: [404], maxRetries: 3 });
10079
}
10180

102-
// If an existing transform exists, the transform should be stopped as we only want one transform running at a time (this is by design of how this action works, not ES)
103-
if(actualTransform != undefined) {
104-
if(actualTransformId == params.transformId && idSuffix == "") {
105-
throw new Error(`Cannot replace existing transform using the same Transform-ID: '${actualTransformId}'. Please provide an ID-Suffix.`);
106-
}
107-
options.logger.info(`Existing Transform found: ${actualTransformId}. Going to stop before creating new transform.`);
108-
var stopResult = await client.transform.stopTransform( {transformId: actualTransformId, force: true}, { ignore: [404], maxRetries: 3 });
109-
} else {
110-
options.logger.info(`No running Transform found with primary ID: ${params.transformId}. Creating new transform with ID: '${params.transformId}${idSuffix}'.`);
111-
}
11281
params.transformId = `${params.transformId}${idSuffix}`;
11382
try {
11483
var putTransformResult = await client.transform.putTransform( params, { ignore: [404], maxRetries: 3 });

‎api-builder-plugin-fn-elasticsearch/src/flow-nodes.yml‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -966,7 +966,7 @@ flow-nodes:
966966
type: string
967967
idSuffix:
968968
name: Transform-ID Suffix
969-
description: Mandatory if an existing transform is to be replaced. The Transform-ID suffix is used to generate the final Transform-ID, which is used to create the transform. This suffix is especially necessary if an existing transform is used to be replaced. Existing transforms with the primary transforms ID are searched for, stopped/deleted and a new transform with transform ID + suffix is created.
969+
description: Mandatory if an existing transform is to be replaced. The Transform-ID suffix is used to generate the final Transform-ID, which is used to create the transform. Existing transforms with the primary transforms ID are searched for, stopped/deleted and a new transform with transform ID + suffix is created.
970970
required: false
971971
schema:
972972
type: string
@@ -976,14 +976,6 @@ flow-nodes:
976976
required: true
977977
schema:
978978
type: object
979-
replaceWhenChanged:
980-
name: Recreate only when changed
981-
description: If set, a possibly existing transform is only replaced if the configuration is different. For this purpose, the entire transform configuration is compared. If no transform exists, this parameter is ignored and a new job is created.
982-
required: false
983-
initialType: boolean
984-
schema:
985-
default: false
986-
type: boolean
987979
deletePreviousTransform:
988980
name: Delete previous transform
989981
description: If an existing transform with the same primary ID has been found which should be replaced, this switch decides whether the old transform should be deleted. If the previous transform is running, it will be stopped automatically.

‎api-builder-plugin-fn-elasticsearch/test/transform/Transform-Tests.js‎

Lines changed: 0 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -38,20 +38,6 @@ describe('Transform tests', () => {
3838
expect(output).to.equal('error');
3939
});
4040

41-
it('should fail if the same transform-id is re-used without a suffix', async () => {
42-
const mockedGetTransformStats = setupElasticsearchMock(client, 'transform.getTransformStats', './test/mock/transform/getTransformStatsResponseOneStarted.json', false);
43-
const mockedGetTransform = setupElasticsearchMock(client, 'transform.getTransform', './test/mock/transform/getTransformResponseOneResult.json', false);
44-
45-
const inputParameter = { transformId: 'apigw-summary-transform-hourly', body: JSON.parse(fs.readFileSync('./test/mock/transform/putTransformRequestBody.json')) };
46-
const { value, output } = await flowNode.putTransform(inputParameter);
47-
48-
expect(value).to.be.instanceOf(Error)
49-
.and.to.have.property('message', 'Cannot replace existing transform using the same Transform-ID: \'apigw-summary-transform-hourly\'. Please provide an ID-Suffix.');
50-
expect(output).to.equal('error');
51-
expect(mockedGetTransformStats.callCount).to.equals(1);
52-
expect(mockedGetTransform.callCount).to.equals(1);
53-
});
54-
5541
it('should pass - with 2 running transforms (means no actual) - Created & Started new transform', async () => {
5642
const mockedGetTransformStats = setupElasticsearchMock(client, 'transform.getTransformStats', './test/mock/transform/getTransformStatsResponseTwoStarted.json', false);
5743
const mockedStopTransform = setupElasticsearchMock(client, 'transform.stopTransform', './test/mock/transform/stopTransformResponse.json', false);
@@ -76,7 +62,6 @@ describe('Transform tests', () => {
7662

7763
it('should pass - 1 running transform (is the actual transform) - Created & Started new transform - Old stopped', async () => {
7864
const mockedGetTransformStats = setupElasticsearchMock(client, 'transform.getTransformStats', './test/mock/transform/getTransformStatsResponseOneStarted.json', false);
79-
const mockedGetTransform = setupElasticsearchMock(client, 'transform.getTransform', './test/mock/transform/getTransformResponseOneResult.json', false);
8065
const mockedPutTransform = setupElasticsearchMock(client, 'transform.putTransform', './test/mock/transform/putTransformResponse.json', false);
8166
const mockedDeleteTransform = setupElasticsearchMock(client, 'transform.deleteTransform', './test/mock/transform/stopTransformResponse.json', false);
8267
const mockedStartTransform = setupElasticsearchMock(client, 'transform.startTransform', './test/mock/transform/startTransformResponse.json', false);
@@ -90,7 +75,6 @@ describe('Transform tests', () => {
9075

9176
expect(output).to.equal('next');
9277
expect(mockedGetTransformStats.callCount).to.equals(1); // should be called once to get all transforms
93-
expect(mockedGetTransform.callCount).to.equals(1); // used to retrieve the transform configuration
9478
expect(mockedPutTransform.callCount).to.equals(1); // a new transform should be created
9579
expect(mockedStartTransform.callCount).to.equals(1); // and started
9680
expect(mockedDeleteTransform.callCount).to.equals(0); // There is nothing to delete
@@ -117,86 +101,6 @@ describe('Transform tests', () => {
117101
expect(value.body.acknowledged).to.equal(true);
118102
});
119103

120-
it('should result in NoUpdate if the Transform config is unchanged', async () => {
121-
const mockedGetTransformStats = setupElasticsearchMock(client, 'transform.getTransformStats', './test/mock/transform/getTransformStatsResponseOneStarted.json', false);
122-
const mockedGetTransform = setupElasticsearchMock(client, 'transform.getTransform', './test/mock/transform/getTransformResponseOneResult.json', false);
123-
const mockedPutTransform = setupElasticsearchMock(client, 'transform.putTransform', null, false);
124-
125-
const inputParameter = {
126-
transformId: 'traffic-summary-hourly',
127-
replaceWhenChanged: true,
128-
body: JSON.parse(fs.readFileSync('./test/mock/transform/putTransformRequestBody.json')) };
129-
const { value, output } = await flowNode.putTransform(inputParameter);
130-
131-
expect(output).to.equal('noUpdate');
132-
expect(value).to.equal('No update required as desired Transform with new ID: \'traffic-summary-hourly\' equals to existing transform with ID: \'apigw-summary-transform-hourly\'.');
133-
expect(mockedGetTransformStats.callCount).to.equals(1);
134-
expect(mockedGetTransform.callCount).to.equals(1);
135-
expect(mockedPutTransform.callCount).to.equals(0);
136-
});
137-
138-
it('should update/recreate the transformation if configuration is changed', async () => {
139-
const mockedDeleteTransform = setupElasticsearchMock(client, 'transform.deleteTransform', './test/mock/transform/stopTransformResponse.json', false);
140-
const mockedPutTransform = setupElasticsearchMock(client, 'transform.putTransform', null, false);
141-
const mockedGetTransformStats = setupElasticsearchMock(client, 'transform.getTransformStats', './test/mock/transform/getTransformStatsResponseOneStarted.json', false);
142-
const mockedGetTransform = setupElasticsearchMock(client, 'transform.getTransform', './test/mock/transform/getTransformResponseConfigChanged.json', false);
143-
const mockedStartTransform = setupElasticsearchMock(client, 'transform.startTransform', './test/mock/transform/startTransformResponse.json', false);
144-
const mockedStopTransform = setupElasticsearchMock(client, 'transform.stopTransform', './test/mock/transform/stopTransformResponse.json', false);
145-
146-
var rollupJobConfig = JSON.parse(fs.readFileSync('./test/mock/transform/putTransformRequestBody.json'))
147-
const inputParameter = {
148-
transformId: 'traffic-summary-yearly',
149-
idSuffix: 'v1',
150-
replaceWhenChanged: true,
151-
body: rollupJobConfig };
152-
const { value, output } = await flowNode.putTransform(inputParameter);
153-
154-
expect(output).to.equal('next');
155-
expect(mockedGetTransformStats.callCount).to.equals(1);
156-
expect(mockedGetTransform.callCount).to.equals(1);
157-
expect(mockedPutTransform.callCount).to.equals(1);
158-
expect(mockedDeleteTransform.callCount).to.equals(0);
159-
expect(mockedStopTransform.callCount).to.equals(1);
160-
expect(mockedStartTransform.callCount).to.equals(1);
161-
// Additionally in this test make sure, the internal API-Params are removed and NOT send to ES
162-
inputParameter.transformId = `${inputParameter.transformId}-${inputParameter.idSuffix}`;
163-
delete inputParameter.replaceWhenChanged;
164-
delete inputParameter.idSuffix;
165-
expect(mockedPutTransform.lastCall.arg).to.deep.equals(inputParameter);
166-
});
167-
168-
it('should update/recreate the transform if configuration is changed and the previous job should be deleted', async () => {
169-
const mockedDeleteTransform = setupElasticsearchMock(client, 'transform.deleteTransform', './test/mock/transform/stopTransformResponse.json', false);
170-
const mockedPutTransform = setupElasticsearchMock(client, 'transform.putTransform', null, false);
171-
const mockedGetTransformStats = setupElasticsearchMock(client, 'transform.getTransformStats', './test/mock/transform/getTransformStatsResponseOneStarted.json', false);
172-
const mockedGetTransform = setupElasticsearchMock(client, 'transform.getTransform', './test/mock/transform/getTransformResponseConfigChanged.json', false);
173-
const mockedStartTransform = setupElasticsearchMock(client, 'transform.startTransform', './test/mock/transform/startTransformResponse.json', false);
174-
const mockedStopTransform = setupElasticsearchMock(client, 'transform.stopTransform', './test/mock/transform/stopTransformResponse.json', false);
175-
176-
var rollupJobConfig = JSON.parse(fs.readFileSync('./test/mock/transform/putTransformRequestBody.json'))
177-
const inputParameter = {
178-
transformId: 'traffic-summary-daily',
179-
idSuffix: 'v1',
180-
replaceWhenChanged: true,
181-
deletePreviousTransform: true,
182-
body: rollupJobConfig };
183-
const { value, output } = await flowNode.putTransform(inputParameter);
184-
185-
expect(output).to.equal('next');
186-
expect(mockedGetTransform.callCount).to.equals(1);
187-
expect(mockedGetTransformStats.callCount).to.equals(1);
188-
expect(mockedPutTransform.callCount).to.equals(1);
189-
expect(mockedDeleteTransform.callCount).to.equals(1);
190-
expect(mockedStopTransform.callCount).to.equals(1);
191-
expect(mockedStartTransform.callCount).to.equals(1);
192-
// Additionally in this test make sure, the internal API-Params are removed and NOT send to ES
193-
inputParameter.transformId = `${inputParameter.transformId}-${inputParameter.idSuffix}`;
194-
delete inputParameter.replaceWhenChanged;
195-
delete inputParameter.deletePreviousTransform;
196-
delete inputParameter.idSuffix;
197-
expect(mockedPutTransform.lastCall.arg).to.deep.equals(inputParameter);
198-
});
199-
200104
it('should create the transform not yet exists', async () => {
201105
const mockedGetTransformStats = setupElasticsearchMock(client, 'transform.getTransformStats', './test/mock/transform/getTransformStatsResponseZeroResult.json', false);
202106
const mockedPutTransform = setupElasticsearchMock(client, 'transform.putTransform', null, false);
@@ -206,7 +110,6 @@ describe('Transform tests', () => {
206110

207111
const inputParameter = {
208112
transformId: 'traffic-summary-rollup-job',
209-
replaceWhenChanged: true,
210113
body: JSON.parse(fs.readFileSync('./test/mock/transform/putTransformRequestBody.json')) };
211114
const { value, output } = await flowNode.putTransform(inputParameter);
212115

0 commit comments

Comments
 (0)