Skip to content

Commit 7e8e25b

Browse files
himanshu219himanshu219
authored
Merge pull request #73 from SumoLogic/azure-metrics-upgrade-v2
Azure metrics collection upgrade to v2
2 parents a864c0f + 05ce87e commit 7e8e25b

File tree

24 files changed

+594
-689
lines changed

24 files changed

+594
-689
lines changed

‎.github/workflows/arm-template-test.yml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
name: "ARM template tests"
2+
3+
on: [pull_request]
4+
5+
jobs:
6+
best-practices-test:
7+
name: "ARM template tests using arm-ttk"
8+
runs-on: "ubuntu-latest"
9+
steps:
10+
- name: Checkout repo
11+
uses: actions/checkout@v4
12+
13+
- name: Test ARM Templates using arm-ttk
14+
id: filestest
15+
uses: aliencube/arm-ttk-actions@v0.3
16+
with:
17+
path: EventHubs/src/azuredeploy_metrics.json
18+
19+
security-test:
20+
name: "ARM template tests using checkov"
21+
runs-on: "ubuntu-latest"
22+
steps:
23+
- name: Checkout repo
24+
uses: actions/checkout@v4
25+
26+
- name: Test ARM Templates using checkov
27+
id: checkov
28+
uses: bridgecrewio/checkov-action@v12
29+
with:
30+
file: EventHubs/src/azuredeploy_metrics.json
31+
skip_check: CKV_AZURE_16,CKV_AZURE_17,CKV_AZURE_35
32+
quiet: false
33+
framework: arm
34+
output_format: cli
35+
output_bc_ids: true

‎BlockBlobReader/target/consumer_build/BlobTaskConsumer/sumoclient.js‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -125,11 +125,10 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
125125
var self = this;
126126
let curOptions = Object.assign({},this.options);
127127

128-
//this.context.log("Flush buffer for metaKey:"+metaKey);
128+
this.context.debug("Flush buffer for metaKey:"+metaKey);
129129

130130
function httpSend(messageArray,data) {
131131
return new Promise( (resolve,reject) => {
132-
//self.context.log("Inside HTTP Send");
133132
var req = https.request(curOptions, function (res) {
134133
var body = '';
135134
res.setEncoding('utf8');
@@ -155,7 +154,7 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
155154
});
156155
req.write(data);
157156
req.end();
158-
});
157+
});
159158
}
160159

161160
if (targetBuffer) {
@@ -176,10 +175,10 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
176175

177176
return zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
178177
if (!e) {
179-
self.context.log("gzip successful");
178+
self.context.debug("gzip successful");
180179
return sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,compressed_data])
181180
.then(()=> {
182-
self.context.log("Succesfully sent " + self.messagesSent + " messages to Sumo");
181+
self.context.debug("Successfully sent to Sumo after "+self.MaxAttempts);
183182
self.success_callback(self.context);}
184183
)
185184
.catch((err) => {
@@ -194,7 +193,7 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
194193
self.context.log("Failed to gzip: " + JSON.stringify(e) + ' messagesAttempted: ' + self.messagesAttempted + ' messagesReceived: ' + self.messagesReceived);
195194
self.failure_callback(msgArray,self.context);
196195
}
197-
});
196+
});
198197
} else {
199198
//self.context.log('Send raw data to Sumo');
200199
return sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,msgArray.join('\n')])
@@ -253,7 +252,7 @@ function FlushFailureHandler (messageArray,ctx) {
253252
};
254253

255254
/**
256-
* Default built-in callback function to handle successful sents. It simply logs a success message
255+
* Default built-in callback function to handle successful sent. It simply logs a success message
257256
* @param ctx is a context variable that supports a log method
258257
* @constructor
259258
*/

‎BlockBlobReader/target/consumer_build/BlobTaskConsumer/sumometricclient.js‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22
/**
33
* Created by duc on 6/30/17. This is a client for metric
44
*/
5-
var https = require('https');
6-
var zlib= require('zlib');
7-
var url = require('url');
5+
var https = require('node:https');
6+
var zlib = require('node:zlib');
7+
var url = require('node:url');
88

99
var sumoclient = require('./sumoclient');
1010
var bucket = require('./messagebucket');
@@ -119,7 +119,7 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {
119119
var self = this;
120120
let curOptions = Object.assign({},this.options);
121121

122-
this.context.log("Flush METRIC buffer for metaKey:"+metaKey);
122+
this.context.debug("Flush METRIC buffer for metaKey:"+metaKey);
123123

124124
function httpSend(messageArray,data) {
125125

@@ -170,11 +170,12 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {
170170
if (curOptions.compress_data) {
171171
curOptions.headers['Content-Encoding'] = 'gzip';
172172

173-
zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
173+
return zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
174174
if (!e) {
175+
self.context.debug("gzip successful");
175176
sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,compressed_data])
176177
.then(()=> {
177-
//self.context.log("Succesfully sent to Sumo after "+self.MaxAttempts);
178+
self.context.debug("Successfully sent to Sumo after "+self.MaxAttempts);
178179
self.success_callback(self.context);}
179180
)
180181
.catch((err) => {
@@ -184,20 +185,21 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {
184185
self.failure_callback(msgArray,self.context);
185186
});
186187
} else {
188+
self.messagesFailed += msgArray.length;
187189
self.messagesAttempted += msgArray.length;
188190
self.context.log("Failed to gzip: " + JSON.stringify(e) + ' messagesAttempted: ' + self.messagesAttempted + ' messagesReceived: ' + self.messagesReceived);
189191
self.failure_callback(msgArray,self.context);
190192
}
191193
});
192194
} else {
193195
//self.context.log('Send raw data to Sumo');
194-
sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,msgArray.join('\n')])
196+
return sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,msgArray.join('\n')])
195197
.then(()=> { self.success_callback(self.context);})
196-
.catch((err) => {
197-
self.messagesFailed += msgArray.length;
198-
self.messagesAttempted += msgArray.length;
199-
self.context.log("Failed to send after retries: " + self.MaxAttempts + " " + JSON.stringify(err) + ' messagesAttempted: ' + self.messagesAttempted + ' messagesReceived: ' + self.messagesReceived);
200-
self.failure_callback(msgArray,self.context);
198+
.catch((err) => {
199+
self.messagesFailed += msgArray.length;
200+
self.messagesAttempted += msgArray.length;
201+
self.context.log("Failed to send after retries: " + self.MaxAttempts + " " + JSON.stringify(err) + ' messagesAttempted: ' + self.messagesAttempted + ' messagesReceived: ' + self.messagesReceived);
202+
self.failure_callback(msgArray,self.context);
201203
});
202204
}
203205
}
@@ -215,7 +217,7 @@ function FlushFailureHandler (messageArray,ctx) {
215217
};
216218

217219
/**
218-
* Default built-in callback function to handle successful sents. It simply logs a success message
220+
* Default built-in callback function to handle successful sent. It simply logs a success message
219221
* @param ctx is a context variable that supports a log method
220222
* @constructor
221223
*/
@@ -227,5 +229,4 @@ module.exports = {
227229
SumoMetricClient:SumoMetricClient,
228230
FlushFailureHandler:FlushFailureHandler,
229231
DefaultSuccessHandler:DefaultSuccessHandler
230-
}
231-
232+
}

‎BlockBlobReader/target/dlqprocessor_build/DLQTaskConsumer/sumoclient.js‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -125,11 +125,10 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
125125
var self = this;
126126
let curOptions = Object.assign({},this.options);
127127

128-
//this.context.log("Flush buffer for metaKey:"+metaKey);
128+
this.context.debug("Flush buffer for metaKey:"+metaKey);
129129

130130
function httpSend(messageArray,data) {
131131
return new Promise( (resolve,reject) => {
132-
//self.context.log("Inside HTTP Send");
133132
var req = https.request(curOptions, function (res) {
134133
var body = '';
135134
res.setEncoding('utf8');
@@ -155,7 +154,7 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
155154
});
156155
req.write(data);
157156
req.end();
158-
});
157+
});
159158
}
160159

161160
if (targetBuffer) {
@@ -176,10 +175,10 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
176175

177176
return zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
178177
if (!e) {
179-
self.context.log("gzip successful");
178+
self.context.debug("gzip successful");
180179
return sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,compressed_data])
181180
.then(()=> {
182-
self.context.log("Succesfully sent " + self.messagesSent + " messages to Sumo");
181+
self.context.debug("Successfully sent to Sumo after "+self.MaxAttempts);
183182
self.success_callback(self.context);}
184183
)
185184
.catch((err) => {
@@ -194,7 +193,7 @@ SumoClient.prototype.flushBucketToSumo = function(metaKey) {
194193
self.context.log("Failed to gzip: " + JSON.stringify(e) + ' messagesAttempted: ' + self.messagesAttempted + ' messagesReceived: ' + self.messagesReceived);
195194
self.failure_callback(msgArray,self.context);
196195
}
197-
});
196+
});
198197
} else {
199198
//self.context.log('Send raw data to Sumo');
200199
return sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,msgArray.join('\n')])
@@ -253,7 +252,7 @@ function FlushFailureHandler (messageArray,ctx) {
253252
};
254253

255254
/**
256-
* Default built-in callback function to handle successful sents. It simply logs a success message
255+
* Default built-in callback function to handle successful sent. It simply logs a success message
257256
* @param ctx is a context variable that supports a log method
258257
* @constructor
259258
*/

‎BlockBlobReader/target/dlqprocessor_build/DLQTaskConsumer/sumometricclient.js‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22
/**
33
* Created by duc on 6/30/17. This is a client for metric
44
*/
5-
var https = require('https');
6-
var zlib= require('zlib');
7-
var url = require('url');
5+
var https = require('node:https');
6+
var zlib = require('node:zlib');
7+
var url = require('node:url');
88

99
var sumoclient = require('./sumoclient');
1010
var bucket = require('./messagebucket');
@@ -119,7 +119,7 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {
119119
var self = this;
120120
let curOptions = Object.assign({},this.options);
121121

122-
this.context.log("Flush METRIC buffer for metaKey:"+metaKey);
122+
this.context.debug("Flush METRIC buffer for metaKey:"+metaKey);
123123

124124
function httpSend(messageArray,data) {
125125

@@ -170,11 +170,12 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {
170170
if (curOptions.compress_data) {
171171
curOptions.headers['Content-Encoding'] = 'gzip';
172172

173-
zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
173+
return zlib.gzip(msgArray.join('\n'),function(e,compressed_data){
174174
if (!e) {
175+
self.context.debug("gzip successful");
175176
sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,compressed_data])
176177
.then(()=> {
177-
//self.context.log("Succesfully sent to Sumo after "+self.MaxAttempts);
178+
self.context.debug("Successfully sent to Sumo after "+self.MaxAttempts);
178179
self.success_callback(self.context);}
179180
)
180181
.catch((err) => {
@@ -184,20 +185,21 @@ SumoMetricClient.prototype.flushBucketToSumo = function(metaKey) {
184185
self.failure_callback(msgArray,self.context);
185186
});
186187
} else {
188+
self.messagesFailed += msgArray.length;
187189
self.messagesAttempted += msgArray.length;
188190
self.context.log("Failed to gzip: " + JSON.stringify(e) + ' messagesAttempted: ' + self.messagesAttempted + ' messagesReceived: ' + self.messagesReceived);
189191
self.failure_callback(msgArray,self.context);
190192
}
191193
});
192194
} else {
193195
//self.context.log('Send raw data to Sumo');
194-
sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,msgArray.join('\n')])
196+
return sumoutils.p_retryMax(httpSend,self.MaxAttempts,self.RetryInterval,[msgArray,msgArray.join('\n')])
195197
.then(()=> { self.success_callback(self.context);})
196-
.catch((err) => {
197-
self.messagesFailed += msgArray.length;
198-
self.messagesAttempted += msgArray.length;
199-
self.context.log("Failed to send after retries: " + self.MaxAttempts + " " + JSON.stringify(err) + ' messagesAttempted: ' + self.messagesAttempted + ' messagesReceived: ' + self.messagesReceived);
200-
self.failure_callback(msgArray,self.context);
198+
.catch((err) => {
199+
self.messagesFailed += msgArray.length;
200+
self.messagesAttempted += msgArray.length;
201+
self.context.log("Failed to send after retries: " + self.MaxAttempts + " " + JSON.stringify(err) + ' messagesAttempted: ' + self.messagesAttempted + ' messagesReceived: ' + self.messagesReceived);
202+
self.failure_callback(msgArray,self.context);
201203
});
202204
}
203205
}
@@ -215,7 +217,7 @@ function FlushFailureHandler (messageArray,ctx) {
215217
};
216218

217219
/**
218-
* Default built-in callback function to handle successful sents. It simply logs a success message
220+
* Default built-in callback function to handle successful sent. It simply logs a success message
219221
* @param ctx is a context variable that supports a log method
220222
* @constructor
221223
*/
@@ -227,5 +229,4 @@ module.exports = {
227229
SumoMetricClient:SumoMetricClient,
228230
FlushFailureHandler:FlushFailureHandler,
229231
DefaultSuccessHandler:DefaultSuccessHandler
230-
}
231-
232+
}

‎EventHubs/requirements.txt‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1-
requests==2.20.0
2-
azure-mgmt-resource==1.2.2
3-
azure-mgmt-eventhub==1.2.0
4-
azure-servicebus==0.21.1
5-
azure-cosmosdb-table==1.0.2
6-
azure-mgmt-storage==1.0.0
7-
1+
requests==2.31.0
2+
azure-monitor-query==1.2.0
3+
azure-identity==1.15.0
4+
azure-core==1.29.6
5+
azure-mgmt-resource==23.0.1
6+
azure-mgmt-eventhub==11.0.0
7+
azure-eventhub==5.11.5
8+
azure-mgmt-storage==21.1.0
9+
azure-mgmt-loganalytics==12.0.0
10+
sumologic-sdk==0.1.15

0 commit comments

Comments
 (0)