Added Index document method and search now has multiple exists

Author: Chris Wiechmann

api-builder-plugin-fn-elasticsearch/CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [2.0.0] 2021-09-14
+### Changed
+- Search method now provides missingIndex & noResult exists making it easier to control your flow
+- New method Index document
+
 ## [1.0.23] 2021-09-10
 ### Fixed
 - No longer trying to read ca.pem

api-builder-plugin-fn-elasticsearch/config/elasticsearch.default.js

@@ -18,10 +18,10 @@ module.exports = {
 				// https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/7.x/client-configuration.html
 				maxRetries: 5,
 				requestTimeout: 60000,
-				//ssl: {
+				ssl: {
 				//	ca: fs.readFileSync('./cacert.pem'),
-				//	rejectUnauthorized: false
-				//}
+					rejectUnauthorized: ("false" == process.env.ELASTIC_SSL_REJECT_UNAUTHORIZED) ? false : true
+				}
 			},
 			// The connection to Elasticsearch is validated on API-Builder startup by default
 			// It can be disabled by setting this to false.

api-builder-plugin-fn-elasticsearch/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@axway-api-builder-ext/api-builder-plugin-fn-elasticsearch",
-	"version": "1.0.23",
+	"version": "2.0.0",
 	"description": "Integrate Elasticsearch into your API-Builder flow to combine search data for instance with other data available in your flow.",
 	"author": "Chris Wiechmann <cwiechmann@axway.com> (http://www.axway.com)",
 	"license": "Apache-2.0",

api-builder-plugin-fn-elasticsearch/src/actions/indexDocument.js

@@ -0,0 +1,57 @@
+const { ElasticsearchClient } = require('./ElasticsearchClient');
+
+/**
+ * Action method.
+ * @param {object} req - The flow request context passed in at runtime.  The
+ *	 parameters are resolved as `req.params` and the available authorization
+ * credentials are passed in as `req.authorizations`.
+ * @param {object} outputs - A set of output callbacks.  Use it to signal an
+ *	 event and pass the output result back to the runtime.  Only use an
+ *	 output callback once and only after all asyncronous tasks complete.
+ * @param {object} options - The additional options provided from the flow
+ * 	 engine.
+ * @param {object} The logger from API Builder that can be used to log messages
+ * 	 to the console. See https://docs.axway.com/bundle/API_Builder_4x_allOS_en/page/logging.html
+ *
+ * @return {undefined}
+ */
+
+async function indexDocument(params, options) {
+	const { index, body, addTimestamp} = params;
+	const elasticSearchConfig = options.pluginConfig.elastic;
+	if (typeof elasticSearchConfig.node === 'undefined' && typeof elasticSearchConfig.nodes === 'undefined') {
+		options.logger.error('Elasticsearch configuration is invalid: nodes or node is missing.');
+		throw new Error('Elasticsearch configuration is invalid: nodes or node is missing.');
+	}
+
+	// Getting the client (which is a singleton)
+	var client = new ElasticsearchClient(elasticSearchConfig).client;
+
+	if(!index) {
+		throw new Error(`Missing required parameter: index`);
+	}
+	if(!body) {
+		throw new Error(`Missing required parameter: body`);
+	}
+	// If a timestamp field is given, add the timestamp to the body
+	if(addTimestamp) {
+		body[addTimestamp] = Date.now();
+		delete params.addTimestamp;
+	}
+	
+	var result;
+	try {
+		result = await client.index( params, { ignore: [404], maxRetries: 3 });
+	} catch (ex) {
+		if(ex instanceof Error) throw ex;
+		throw new Error(JSON.stringify(ex));
+	}
+	if(result.statusCode!=201) {
+		throw new Error(JSON.stringify(result));
+	}
+	return result;
+}
+
+module.exports = {
+	indexDocument
+};

api-builder-plugin-fn-elasticsearch/src/actions/search.js

@@ -78,6 +78,7 @@ async function search(params, options) {
 	addQueryParam("version");
 
 	options.logger.debug(`Using elastic search body: ${JSON.stringify(searchBody)}`);
+
 	var queryResult;
 	try {
 		queryResult = await client.search( searchBody, { ignore: [404], maxRetries: 3 });
@@ -86,6 +87,14 @@ async function search(params, options) {
 		throw new Error(JSON.stringify(ex));
 	}
 
+	if(queryResult.statusCode === 404 && queryResult.body.error.type == "index_not_found_exception") {
+		return options.setOutput('missingIndex', queryResult);
+	}
+
+	if(queryResult.body.hits.total.value === 0) {
+		return options.setOutput('noResult', queryResult);
+	}
+
 	return queryResult;
 
 	function addQueryParam(field) {

api-builder-plugin-fn-elasticsearch/src/flow-nodes.yml

@@ -360,21 +360,31 @@ flow-nodes:
             schema:
               default: false
               type: boolean
-        returns:
-          name: Next
-          description: Context variable will contain the Elasticsearch result
-          context: $.result
-          schema:
-            type: object
-        throws:
-          name: Error
-          description: An unexpected error was encountered.
-          context: $.error
-          schema:
-            type: object
-            properties:
-              message:
-                type: string
+        outputs:
+          next:
+            name: Next
+            description: Variable contains the Elasticsearch result
+            context: $.result
+            schema:
+              type: object
+          noResult:
+            name: No result
+            description: Exit used if the query result contains no hits.
+            context: $.result
+            schema:
+              type: object
+          missingIndex:
+            name: Missing index
+            description: Exit used if the index does not exists in Elasticsearch.
+            context: $.error
+            schema:
+              type: object
+          error:
+            name: Error
+            description: An unexpected error happened
+            context: $.error
+            schema:
+              type: object
 
       getTemplate: 
         name: Get Index Template
@@ -484,6 +494,148 @@ flow-nodes:
             schema:
               type: object
 
+      indexDocument: 
+        name: Index document
+        description: This method sends a document to Elasticsearch
+        parameters:
+          index:
+            name: Index-Name
+            description: The name of the index
+            required: true
+            schema:
+              type: string
+          body:
+            name: Body
+            description: "Body of the document to index"
+            required: true
+            schema:
+              type: object
+          id:
+            name: ID
+            description: Document ID. If not given Elasticsearch creates a unique document ID.
+            required: false
+            schema:
+              type: string
+          addTimestamp:
+            name: Add timestamp
+            description: If a field name (e.g. @timestamp) is given, the flow-node will add the current Unix epoch time to the body of the document.
+            required: false
+            initialType: string
+            schema:
+              type: string
+          waitForActiveShards:
+            name: Wait for active shards
+            description: Sets the number of shard copies that must be active before proceeding with the index operation. Defaults to 1, meaning the primary shard only. Set to all for all shard copies, otherwise set to any non-negative value less than or equal to the total number of copies for the shard (number of replicas + 1)
+            required: false
+            group: Advanced Options
+            schema:
+              type: string
+          opType:
+            name: Explicit operation type
+            description: Defaults to index for requests with an explicit document ID, and to `create`for requests without an explicit document ID
+            required: false
+            group: Advanced Options
+            initialType: string
+            schema:
+              type: string
+              enum:
+                - index
+                - create
+          refresh:
+            name: Refresh
+            description: If true then refresh the affected shards to make this operation visible to search, if wait_for then wait for a refresh to make this operation visible to search, if false (the default) then do nothing with refreshes.
+            required: false
+            group: Advanced Options
+            initialType: string
+            schema:
+              type: string
+              enum:
+                - true
+                - false
+                - wait_for
+          routing:
+            name: Routing
+            description: Specific routing value
+            required: false
+            group: Advanced Options
+            initialType: string
+            schema:
+              type: string
+          timeout:
+            name: Timeout
+            description: Explicit operation timeout. https://www.elastic.co/guide/en/elasticsearch/reference/7.x/common-options.html#time-units
+            required: false
+            group: Advanced Options
+            initialType: string
+            schema:
+              type: string
+          version:
+            name: Version
+            description: Explicit version number for concurrency control
+            required: false
+            group: Advanced Options
+            initialType: string
+            schema:
+              type: string
+          versionType:
+            name: Version Type
+            description: Specific version type
+            required: false
+            group: Advanced Options
+            initialType: string
+            schema:
+              type: string
+              enum:
+                - internal
+                - external
+                - external_gte
+          ifSeqNo:
+            name: Version Type
+            description: Only perform the index operation if the last operation that has changed the document has the specified sequence number
+            required: false
+            group: Advanced Options
+            schema:
+              type: number
+          ifPrimaryTerm:
+            name: If primary term
+            description: Only perform the index operation if the last operation that has changed the document has the specified primary term
+            required: false
+            group: Advanced Options
+            schema:
+              type: number
+          pipeline:
+            name: Pipeline
+            description: The pipeline id to preprocess incoming documents with
+            required: false
+            group: Advanced Options
+            initialType: string
+            schema:
+              type: string
+          requireAlias:
+            name: Require alias
+            description: When true, requires destination to be an alias. Default is false
+            required: false
+            group: Advanced Options
+            initialType: boolean
+            schema:
+              default: false
+              type: boolean
+        returns:
+          name: Next
+          description: The response returned from Elasticsearch for the indexed document
+          context: $.result
+          schema:
+            type: object
+        throws:
+          name: Error
+          description: An unexpected error was encountered.
+          context: $.error
+          schema:
+            type: object
+            properties:
+              message:
+                type: string
+
       getMapping: 
         name: Get Index Mapping
         description: Gets an index mapping from Elasticsearch
@@ -830,7 +982,7 @@ flow-nodes:
               type: string
           alias:
             name: Alias name
-            description: The alias for the index. If an index configuration is given, this alias name is merged into it.
+            description: The alias for the index. If an index configuration is given, this alias name is merged into it. Please note, that the index- and alias-name must be different.
             required: false
             initialType: string
             schema:

api-builder-plugin-fn-elasticsearch/src/index.js

@@ -7,6 +7,7 @@ const { getILMPolicy, putILMPolicy } = require('./actions/ilmPolicy');
 const { getRollupJobs, putRollupJob } = require('./actions/rollupJobs');
 const { putTransform } = require('./actions/transform');
 const { indicesRollover, indicesCreate, indicesExists } = require('./actions/indices');
+const { indexDocument } = require('./actions/indexDocument');
 const { ElasticsearchClient } = require('./actions/ElasticsearchClient');
 const { isDeveloperMode } = require('./utils');
 
@@ -53,7 +54,8 @@ async function getPlugin(pluginConfig, options) {
 		indicesRollover,
 		indicesCreate,
 		indicesExists,
-		putTransform
+		putTransform,
+		indexDocument
 	}, pluginConfig);
 	const plugin = sdk.getPlugin();
 	return plugin;

api-builder-plugin-fn-elasticsearch/test/basic/test-config.js

@@ -12,7 +12,7 @@ describe('Basic: flow-node elasticsearch', () => {
 	let plugin;
 	let flowNode;
 
-	var client = new ElasticsearchClient({nodes:'http://api-env:9200'}).client;
+	var client = new ElasticsearchClient({nodes:'https://api-env:9200', ssl: { rejectUnauthorized: false }}).client;
 	beforeEach(async () => {
 		plugin = await MockRuntime.loadPlugin(getPlugin, validConfig);
 		validConfig.validateConnection = false;

api-builder-plugin-fn-elasticsearch/test/config/basic-config.js

@@ -4,7 +4,7 @@ module.exports = {
 	pluginConfig: {
 		'@axway-api-builder-ext/api-builder-plugin-fn-elasticsearch': {
 			'elastic': {
-				node: 'http://api-env:9200', 
+				node: 'https://api-env:9200', 
 				auth: {
 					/* Use an API-Key
 					apiKey: process.env.ELASTIC_API_KEY