Skip to content

Commit 2b346a8

Browse files
himanshu219himanshu219
authored
Merge pull request #77 from SumoLogic/SUMO-235034-fix-arm-and-unit-test-for-block-blob-collection
ARM template and Unit test
2 parents 97d0f1e + fd060a2 commit 2b346a8

29 files changed

+616
-582
lines changed

‎.github/workflows/package-version-test.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: "Package version tests"
2+
on: [workflow_dispatch, pull_request]
3+
4+
jobs:
5+
package-version-test:
6+
runs-on: ubuntu-latest
7+
strategy:
8+
matrix:
9+
include: # Includes one more job
10+
- dir: 'BlockBlobReader/target/consumer_build'
11+
- dir: 'BlockBlobReader/target/dlqprocessor_build'
12+
- dir: 'BlockBlobReader/target/producer_build'
13+
steps:
14+
- name: Checkout code
15+
uses: actions/checkout@v4
16+
17+
- working-directory: ${{ matrix.dir }}
18+
run: |
19+
set +e
20+
npm outdated >> output.txt
21+
cat output.txt
22+
cat output.txt | grep -v "Package" | awk '$3!=$4' | wc -l | awk '{if ($1!=0) {print $1 " Deprecated packages found";} else {print "No Deprecated packages found";} }'

‎BlockBlobReader/requirements.txt‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,10 @@
1-
requests==2.20.0
2-
azure-storage-blob==1.1.0
3-
azure-mgmt-resource==1.2.2
4-
azure-mgmt-storage==1.0.0
5-
azure-storage==0.34.3
6-
azure-mgmt-eventgrid==1.0.0
7-
azure-mgmt-eventhub==1.2.0
8-
azure-cosmosdb-table==1.0.2
9-
azure-mgmt-authorization==0.50.0
10-
azure-graphrbac==0.51.0
11-
12-
13-
1+
requests==2.31.0
2+
azure-identity==1.15.0
3+
sumologic-sdk==0.1.15
4+
azure-mgmt-resource==23.0.1
5+
azure-mgmt-loganalytics==12.0.0
6+
azure-monitor-query==1.2.0
7+
azure-mgmt-storage==21.1.0
8+
azure-storage-blob==12.19.0
9+
azure-storage==0.36.0
10+
azure-cosmosdb-table==1.0.6

‎BlockBlobReader/src/blobreaderdeploy.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -118,8 +118,8 @@
118118
},
119119
"variables": {
120120

121-
"consumer_roleGuid": "[guid(parameters('sites_blobreaderconsumer_name'), uniqueString(deployment().name, resourceGroup().id))]",
122-
"dlq_roleGuid": "[guid(parameters('sites_DLQProcessor_name'), uniqueString(deployment().name, resourceGroup().id))]",
121+
"consumer_roleGuid": "[guid(parameters('sites_blobreaderconsumer_name'), uniqueString(resourceGroup().id))]",
122+
"dlq_roleGuid": "[guid(parameters('sites_DLQProcessor_name'), uniqueString(resourceGroup().id))]",
123123
"roleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1')]",
124124
"roleScope": "[concat('/subscriptions/', subscription().subscriptionId, '/resourceGroups/', parameters('StorageAccountResourceGroupName'), '/providers/Microsoft.Storage/storageAccounts/', parameters('StorageAccountName'))]",
125125
"DLQProcessor_resourceId": "[resourceId('Microsoft.Web/sites/', parameters('sites_DLQProcessor_name'))]",
@@ -259,6 +259,7 @@
259259
"location": "[parameters('location')]",
260260
"tags": {},
261261
"properties": {
262+
"minimumTlsVersion": "TLS1_2",
262263
"networkAcls": {
263264
"bypass": "AzureServices",
264265
"defaultAction": "Allow"

‎BlockBlobReader/src/blobreaderdeploywithPremiumPlan.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,8 @@
114114
},
115115
"variables": {
116116

117-
"consumer_roleGuid": "[guid(parameters('sites_blobreaderconsumer_name'), uniqueString(deployment().name, resourceGroup().id))]",
118-
"dlq_roleGuid": "[guid(parameters('sites_DLQProcessor_name'), uniqueString(deployment().name, resourceGroup().id))]",
117+
"consumer_roleGuid": "[guid(parameters('sites_blobreaderconsumer_name'), uniqueString(resourceGroup().id))]",
118+
"dlq_roleGuid": "[guid(parameters('sites_DLQProcessor_name'), uniqueString(resourceGroup().id))]",
119119
"roleDefinitionId": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1')]",
120120
"roleScope": "[concat('/subscriptions/', subscription().subscriptionId, '/resourceGroups/', parameters('StorageAccountResourceGroupName'), '/providers/Microsoft.Storage/storageAccounts/', parameters('StorageAccountName'))]",
121121
"DLQProcessor_resourceId": "[resourceId('Microsoft.Web/sites/', parameters('sites_DLQProcessor_name'))]",
@@ -255,6 +255,7 @@
255255
"location": "[parameters('location')]",
256256
"tags": {},
257257
"properties": {
258+
"minimumTlsVersion": "TLS1_2",
258259
"networkAcls": {
259260
"bypass": "AzureServices",
260261
"defaultAction": "Allow"

‎BlockBlobReader/src/consumer.js‎

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -234,8 +234,8 @@ function messageHandler(serviceBusTask, context, sumoClient) {
234234
}
235235
var msghandler = {"log": logHandler, "csv": csvHandler, "json": jsonHandler, "blob": blobHandler, "nsg": nsgLogsHandler};
236236
if (!(file_ext in msghandler)) {
237-
context.log("Error in messageHandler: Unknown file extension - " + file_ext + " for blob: " + serviceBusTask.blobName)
238-
context.done();
237+
context.log.error("Error in messageHandler: Unknown file extension - " + file_ext + " for blob: " + serviceBusTask.blobName);
238+
context.done();
239239
return;
240240
}
241241
if (file_ext === "json" & serviceBusTask.containerName === "insights-logs-networksecuritygroupflowevent") {
@@ -266,7 +266,7 @@ function messageHandler(serviceBusTask, context, sumoClient) {
266266
});
267267
sumoClient.flushAll();
268268
}).catch(function (err) {
269-
context.log("Error in creating json from csv " + err);
269+
context.log.error("Error in creating json from csv.");
270270
context.done(err);
271271
});
272272
} else {
@@ -279,10 +279,10 @@ function messageHandler(serviceBusTask, context, sumoClient) {
279279
});
280280
}).catch(function (err) {
281281
if(err.statusCode === 404) {
282-
context.log("Error in messageHandler: blob file doesn't exist %s %d %d", serviceBusTask.blobName, serviceBusTask.startByte, serviceBusTask.endByte);
282+
context.log.error("Error in messageHandler: blob file doesn't exist " + serviceBusTask.blobName + " " + serviceBusTask.startByte + " " +serviceBusTask.endByte);
283283
context.done()
284284
} else {
285-
context.log("Error in messageHandler: Failed to send blob %s %d %d", serviceBusTask.blobName, serviceBusTask.startByte, serviceBusTask.endByte);
285+
context.log.error("Error in messageHandler: Failed to send blob " + serviceBusTask.blobName + " " + serviceBusTask.startByte + " " +serviceBusTask.endByte);
286286
context.done(err);
287287
}
288288

@@ -308,18 +308,20 @@ function servicebushandler(context, serviceBusTask) {
308308
};
309309
setSourceCategory(serviceBusTask, options);
310310
function failureHandler(msgArray, ctx) {
311-
ctx.log("Failed to send to Sumo");
311+
ctx.log("ServiceBus Task: ", serviceBusTask)
312+
ctx.log.error("Failed to send to Sumo");
312313
if (sumoClient.messagesAttempted === sumoClient.messagesReceived) {
313314
ctx.done("TaskConsumer failedmessages: " + sumoClient.messagesFailed);
314315
}
315316
}
316317
function successHandler(ctx) {
317-
ctx.log('Successfully sent to Sumo', serviceBusTask);
318318
if (sumoClient.messagesAttempted === sumoClient.messagesReceived) {
319+
ctx.log("ServiceBus Task: ", serviceBusTask)
319320
if (sumoClient.messagesFailed > 0) {
321+
ctx.log.error('Failed to send few messages to Sumo')
320322
ctx.done("TaskConsumer failedmessages: " + sumoClient.messagesFailed);
321323
} else {
322-
ctx.log('Exiting now.');
324+
ctx.log('Successfully sent to Sumo, Exiting now.');
323325
ctx.done();
324326
}
325327
}
@@ -340,8 +342,8 @@ async function timetriggerhandler(context, timetrigger) {
340342
var sbClient = new ServiceBusClient(process.env.APPSETTING_TaskQueueConnectionString);
341343
var queueReceiver = sbClient.createReceiver(process.env.APPSETTING_TASKQUEUE_NAME,{ subQueueType: "deadLetter", receiveMode: "peekLock" });
342344
}catch(err){
343-
context.log(err);
344-
context.done("Failed to create service bus client and receiver");
345+
context.log.error("Failed to create service bus client and receiver");
346+
context.done(err);
345347
}
346348
try {
347349
var messages = await queueReceiver.receiveMessages(1, {
@@ -367,34 +369,34 @@ async function timetriggerhandler(context, timetrigger) {
367369
setSourceCategory(serviceBusTask, options);
368370
var sumoClient;
369371
async function failureHandler(msgArray, ctx) {
370-
ctx.log("Failed to send to Sumo");
372+
ctx.log.error("Failed to send to Sumo");
371373
if (sumoClient.messagesAttempted === sumoClient.messagesReceived) {
372374
await queueReceiver.close();
373375
await sbClient.close();
374376
ctx.done("TaskConsumer failedmessages: " + sumoClient.messagesFailed);
375377
}
376378
}
377379
async function successHandler(ctx) {
378-
ctx.log('Successfully sent to Sumo');
379380
if (sumoClient.messagesAttempted === sumoClient.messagesReceived) {
380381
//TODO: Test Scenario for combination of successful and failed requests
381382
if (sumoClient.messagesFailed > 0) {
382383
await queueReceiver.close();
383384
await sbClient.close();
385+
ctx.log.error('Failed to send few messages to Sumo')
384386
ctx.done("DLQTaskConsumer failedmessages: " + sumoClient.messagesFailed);
385387
} else {
386-
ctx.log('Sent ' + sumoClient.messagesAttempted + ' data to Sumo. Exit now.');
388+
ctx.log('Successfully sent to Sumo, Exiting now.');
387389
try{
388390
await queueReceiver.completeMessage(messages[0]);
389391
}catch(err){
390392
await queueReceiver.close();
391393
await sbClient.close();
392394
if (!err) {
393-
context.log("sent and deleted");
395+
ctx.log("sent and deleted");
394396
ctx.done();
395397
} else {
396-
context.log(err)
397-
ctx.done("Messages Sent but failed delete from DeadLetterQueue");
398+
ctx.log.verbose("Messages Sent but failed delete from DeadLetterQueue");
399+
ctx.done(err);
398400
}
399401
}
400402
}
@@ -406,10 +408,10 @@ async function timetriggerhandler(context, timetrigger) {
406408
await queueReceiver.close();
407409
await sbClient.close();
408410
if (typeof error === 'string' && new RegExp("\\b" + "No messages" + "\\b", "gi").test(error)) {
409-
context.log(error);
411+
context.log.error(error);
410412
context.done();
411413
} else {
412-
context.log("Error in reading messages from DLQ: ", error, typeof(error));
414+
context.log.error("Error in reading messages from DLQ");
413415
context.done(error);
414416
}
415417
}

‎BlockBlobReader/src/producer.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ async function createTasksForBlob(partitionKey, rowKey, sortedcontentlengths, co
140140
return Promise.resolve({status: "success",rowKey: rowKey, message: tasks.length + " Tasks added for rowKey: " + rowKey});
141141
}catch(err){
142142
if (err && err.details && err.details.odataError && err.details.odataError.code === "UpdateConditionNotSatisfied" && err.statusCode === 412) {
143-
context.log("Need to Retry: " + rowKey);
143+
context.log.verbose("Need to Retry: " + rowKey);
144144
}
145145
return Promise.reject({status: "failed",rowKey: rowKey, message: "Unable to Update offset for rowKey: " + rowKey + " Error: " + err, lastoffset : lastoffset, currentoffset: currentoffset});
146146
}
@@ -171,7 +171,7 @@ module.exports = async function (context, eventHubMessages) {
171171
for (let response of responseValues){
172172
processed += 1;
173173
if(response.status === "failed"){
174-
context.log("creating duplicate task since retry failed for rowkey: " + response.rowKey);
174+
context.log.verbose("creating duplicate task since retry failed for rowkey: " + response.rowKey);
175175
var duplicateTask = Object.assign({
176176
startByte: response.currentoffset + 1,
177177
endByte: response.lastoffset
@@ -184,11 +184,12 @@ module.exports = async function (context, eventHubMessages) {
184184
if (totalRows === processed) {
185185
context.log("Tasks Created: " + JSON.stringify(context.bindings.tasks) + " Blobpaths: " + JSON.stringify(allcontentlengths));
186186
if (errArr.length > 0) {
187-
context.log(errArr.join('\n'));
187+
context.log.error(errArr.join('\n'));
188188
}
189189
context.done();
190190
}
191191
} catch (error) {
192+
context.log.error(error)
192193
context.done(error);
193194
}
194195
};

‎BlockBlobReader/target/consumer_build/BlobTaskConsumer/index.js‎

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -234,8 +234,8 @@ function messageHandler(serviceBusTask, context, sumoClient) {
234234
}
235235
var msghandler = {"log": logHandler, "csv": csvHandler, "json": jsonHandler, "blob": blobHandler, "nsg": nsgLogsHandler};
236236
if (!(file_ext in msghandler)) {
237-
context.log("Error in messageHandler: Unknown file extension - " + file_ext + " for blob: " + serviceBusTask.blobName)
238-
context.done();
237+
context.log.error("Error in messageHandler: Unknown file extension - " + file_ext + " for blob: " + serviceBusTask.blobName);
238+
context.done();
239239
return;
240240
}
241241
if (file_ext === "json" & serviceBusTask.containerName === "insights-logs-networksecuritygroupflowevent") {
@@ -266,7 +266,7 @@ function messageHandler(serviceBusTask, context, sumoClient) {
266266
});
267267
sumoClient.flushAll();
268268
}).catch(function (err) {
269-
context.log("Error in creating json from csv " + err);
269+
context.log.error("Error in creating json from csv.");
270270
context.done(err);
271271
});
272272
} else {
@@ -279,10 +279,10 @@ function messageHandler(serviceBusTask, context, sumoClient) {
279279
});
280280
}).catch(function (err) {
281281
if(err.statusCode === 404) {
282-
context.log("Error in messageHandler: blob file doesn't exist %s %d %d", serviceBusTask.blobName, serviceBusTask.startByte, serviceBusTask.endByte);
282+
context.log.error("Error in messageHandler: blob file doesn't exist " + serviceBusTask.blobName + " " + serviceBusTask.startByte + " " +serviceBusTask.endByte);
283283
context.done()
284284
} else {
285-
context.log("Error in messageHandler: Failed to send blob %s %d %d", serviceBusTask.blobName, serviceBusTask.startByte, serviceBusTask.endByte);
285+
context.log.error("Error in messageHandler: Failed to send blob " + serviceBusTask.blobName + " " + serviceBusTask.startByte + " " +serviceBusTask.endByte);
286286
context.done(err);
287287
}
288288

@@ -308,18 +308,20 @@ function servicebushandler(context, serviceBusTask) {
308308
};
309309
setSourceCategory(serviceBusTask, options);
310310
function failureHandler(msgArray, ctx) {
311-
ctx.log("Failed to send to Sumo");
311+
ctx.log("ServiceBus Task: ", serviceBusTask)
312+
ctx.log.error("Failed to send to Sumo");
312313
if (sumoClient.messagesAttempted === sumoClient.messagesReceived) {
313314
ctx.done("TaskConsumer failedmessages: " + sumoClient.messagesFailed);
314315
}
315316
}
316317
function successHandler(ctx) {
317-
ctx.log('Successfully sent to Sumo', serviceBusTask);
318318
if (sumoClient.messagesAttempted === sumoClient.messagesReceived) {
319+
ctx.log("ServiceBus Task: ", serviceBusTask)
319320
if (sumoClient.messagesFailed > 0) {
321+
ctx.log.error('Failed to send few messages to Sumo')
320322
ctx.done("TaskConsumer failedmessages: " + sumoClient.messagesFailed);
321323
} else {
322-
ctx.log('Exiting now.');
324+
ctx.log('Successfully sent to Sumo, Exiting now.');
323325
ctx.done();
324326
}
325327
}
@@ -340,8 +342,8 @@ async function timetriggerhandler(context, timetrigger) {
340342
var sbClient = new ServiceBusClient(process.env.APPSETTING_TaskQueueConnectionString);
341343
var queueReceiver = sbClient.createReceiver(process.env.APPSETTING_TASKQUEUE_NAME,{ subQueueType: "deadLetter", receiveMode: "peekLock" });
342344
}catch(err){
343-
context.log(err);
344-
context.done("Failed to create service bus client and receiver");
345+
context.log.error("Failed to create service bus client and receiver");
346+
context.done(err);
345347
}
346348
try {
347349
var messages = await queueReceiver.receiveMessages(1, {
@@ -367,34 +369,34 @@ async function timetriggerhandler(context, timetrigger) {
367369
setSourceCategory(serviceBusTask, options);
368370
var sumoClient;
369371
async function failureHandler(msgArray, ctx) {
370-
ctx.log("Failed to send to Sumo");
372+
ctx.log.error("Failed to send to Sumo");
371373
if (sumoClient.messagesAttempted === sumoClient.messagesReceived) {
372374
await queueReceiver.close();
373375
await sbClient.close();
374376
ctx.done("TaskConsumer failedmessages: " + sumoClient.messagesFailed);
375377
}
376378
}
377379
async function successHandler(ctx) {
378-
ctx.log('Successfully sent to Sumo');
379380
if (sumoClient.messagesAttempted === sumoClient.messagesReceived) {
380381
//TODO: Test Scenario for combination of successful and failed requests
381382
if (sumoClient.messagesFailed > 0) {
382383
await queueReceiver.close();
383384
await sbClient.close();
385+
ctx.log.error('Failed to send few messages to Sumo')
384386
ctx.done("DLQTaskConsumer failedmessages: " + sumoClient.messagesFailed);
385387
} else {
386-
ctx.log('Sent ' + sumoClient.messagesAttempted + ' data to Sumo. Exit now.');
388+
ctx.log('Successfully sent to Sumo, Exiting now.');
387389
try{
388390
await queueReceiver.completeMessage(messages[0]);
389391
}catch(err){
390392
await queueReceiver.close();
391393
await sbClient.close();
392394
if (!err) {
393-
context.log("sent and deleted");
395+
ctx.log("sent and deleted");
394396
ctx.done();
395397
} else {
396-
context.log(err)
397-
ctx.done("Messages Sent but failed delete from DeadLetterQueue");
398+
ctx.log.verbose("Messages Sent but failed delete from DeadLetterQueue");
399+
ctx.done(err);
398400
}
399401
}
400402
}
@@ -406,10 +408,10 @@ async function timetriggerhandler(context, timetrigger) {
406408
await queueReceiver.close();
407409
await sbClient.close();
408410
if (typeof error === 'string' && new RegExp("\\b" + "No messages" + "\\b", "gi").test(error)) {
409-
context.log(error);
411+
context.log.error(error);
410412
context.done();
411413
} else {
412-
context.log("Error in reading messages from DLQ: ", error, typeof(error));
414+
context.log.error("Error in reading messages from DLQ");
413415
context.done(error);
414416
}
415417
}

0 commit comments

Comments
 (0)