Merge pull request #98 from SumoLogic/hpal_evtgridtopic_fix

adding condition system topic

Author: himanshu219

.github/workflows/arm-template-test.yml

@@ -53,8 +53,8 @@ jobs:
       - uses: bridgecrewio/checkov-action@master
         with:
           file: ${{ matrix.dir }}/${{ matrix.file }}
-          skip_check: CKV_AZURE_16,CKV_AZURE_17,CKV_AZURE_35
+          skip_check: CKV_AZURE_16,CKV_AZURE_17,CKV_AZURE_35,CKV_AZURE_78
           quiet: false
           framework: arm
           output_format: cli
-          output_bc_ids: true
+          output_bc_ids: true

AppendBlobReader/src/appendblobproducer.js

@@ -318,11 +318,7 @@ function getDateDifferenceInMinutes(date_a, date_b) {
  */
 function getTasksForUnlockedFiles(context) {
 
-    var maxIngestionDelayPerFile = 5;
-    let MS_PER_MINUTE = 60*1000;
-    var dateVal = new Date(new Date() - maxIngestionDelayPerFile*MS_PER_MINUTE);
-    // fetching unlocked files which were not enqueued in last 5 minutes
-    var existingFileQuery = `done eq ${false} and blobType eq '${'AppendBlob'}' and offset ge ${0} and ( not (lastEnqueLockTime gt '') or lastEnqueLockTime le '${dateVal.toISOString()}')`
+    var existingFileQuery = `done eq ${false} and blobType eq '${'AppendBlob'}' and offset ge ${0}`
     return new Promise(function (resolve, reject) {
         return queryFiles(existingFileQuery, context).then(function (allentities) {
             var newFiletasks = [];

AppendBlobReader/src/appendblobreaderdeploy.json

@@ -444,7 +444,7 @@
             "properties": {
                 "name": "[parameters('serverfarms_BlobReaderPlan_name')]",
                 "maximumElasticWorkerCount": 1,
-                "perSiteScaling": false,
+                "perSiteScaling": true,
                 "targetWorkerCount": 0,
                 "targetWorkerSizeId": 0,
                 "reserved": false,
@@ -470,7 +470,7 @@
             "properties": {
                 "name": "[parameters('serverfarms_ConsumerPlan_name')]",
                 "maximumElasticWorkerCount": 1,
-                "perSiteScaling": false,
+                "perSiteScaling": true,
                 "targetWorkerCount": 0,
                 "targetWorkerSizeId": 0,
                 "reserved": false,
@@ -501,6 +501,7 @@
                     "http20Enabled": true,
                     "minTlsVersion": "1.2",
                     "scmMinTlsVersion": "1.2",
+                    "netFrameworkVersion": "v6.0",
                     "alwaysOn": true,
                     "cors": {
                         "allowedOrigins": [ "https://portal.azure.com" ]
@@ -595,6 +596,7 @@
                     "http20Enabled": true,
                     "minTlsVersion": "1.2",
                     "scmMinTlsVersion": "1.2",
+                    "netFrameworkVersion": "v6.0",
                     "alwaysOn": true,
                     "cors": {
                         "allowedOrigins": [ "https://portal.azure.com" ]
@@ -701,6 +703,7 @@
                     "http20Enabled": true,
                     "minTlsVersion": "1.2",
                     "scmMinTlsVersion": "1.2",
+                    "netFrameworkVersion": "v6.0",
                     "alwaysOn": true,
                     "cors": {
                         "allowedOrigins": [ "https://portal.azure.com" ]

AppendBlobReader/src/consumer.js

@@ -131,6 +131,7 @@ function downloadErrorHandler(err, serviceBusTask, context) {
 async function archiveIngestedFile(serviceBusTask, context) {
     try {
         // Delete entity from Azure Table Storage
+        // blobName should not be used directly since they might be truncated
         await azureTableClient.deleteEntity(serviceBusTask.partitionKey, serviceBusTask.rowKey);
         context.log(`Entity deleted, rowKey: ${serviceBusTask.rowKey}`);
     } catch (error) {

AppendBlobReader/src/producer.js

@@ -13,7 +13,10 @@ function getRowKey(metadata) {
     var arr = metadata.url.split('/').slice(3);
     var keyArr = [storageName];
     Array.prototype.push.apply(keyArr, arr);
-    return keyArr.join("-");
+    // key cannot be greater than 1KB or 1024 bytes;
+    var rowKey = keyArr.join("-");
+    return rowKey.substr(0,Math.min(1024, rowKey.length)).replace(/^-+|-+$/g, '');
+
 }
 
 function getBlobMetadata(message) {
@@ -301,8 +304,7 @@ module.exports = async function (context, eventHubMessages) {
                 context.done();
             }
 
-        }
-        else {
+        } else {
             context.log(`eventHubMessages might not pertain to appendblob or files with supported extensions, Exit now!`);
             context.done();
         }

AppendBlobReader/target/appendblob_producer_build/AppendBlobTaskProducer/index.js

@@ -318,11 +318,7 @@ function getDateDifferenceInMinutes(date_a, date_b) {
  */
 function getTasksForUnlockedFiles(context) {
 
-    var maxIngestionDelayPerFile = 5;
-    let MS_PER_MINUTE = 60*1000;
-    var dateVal = new Date(new Date() - maxIngestionDelayPerFile*MS_PER_MINUTE);
-    // fetching unlocked files which were not enqueued in last 5 minutes
-    var existingFileQuery = `done eq ${false} and blobType eq '${'AppendBlob'}' and offset ge ${0} and ( not (lastEnqueLockTime gt '') or lastEnqueLockTime le '${dateVal.toISOString()}')`
+    var existingFileQuery = `done eq ${false} and blobType eq '${'AppendBlob'}' and offset ge ${0}`
     return new Promise(function (resolve, reject) {
         return queryFiles(existingFileQuery, context).then(function (allentities) {
             var newFiletasks = [];

AppendBlobReader/target/appendblob_producer_build/host.json

@@ -21,7 +21,7 @@
             "default": "Warning",
             "Host.Aggregator": "Trace",
             "Host.Results": "Information",
-            "Function": "Trace"
+            "Function": "Information"
         }
     },
     "functionTimeout": "00:10:00",

AppendBlobReader/target/appendblob_producer_build/package-lock.json

@@ -131,6 +131,7 @@ function downloadErrorHandler(err, serviceBusTask, context) {
 async function archiveIngestedFile(serviceBusTask, context) {
     try {
         // Delete entity from Azure Table Storage
+        // blobName should not be used directly since they might be truncated
         await azureTableClient.deleteEntity(serviceBusTask.partitionKey, serviceBusTask.rowKey);
         context.log(`Entity deleted, rowKey: ${serviceBusTask.rowKey}`);
     } catch (error) {

AppendBlobReader/target/consumer_build/AppendBlobTaskConsumer/index.js

@@ -28,7 +28,7 @@
         "default": "Warning",
         "Host.Aggregator": "Trace",
         "Host.Results": "Information",
-        "Function": "Trace"
+        "Function": "Information"
       }
     }
 }