completed testing, added dotnet framework version

Author: himanshu219

AppendBlobReader/src/appendblobreaderdeploy.json

@@ -501,6 +501,7 @@
                     "http20Enabled": true,
                     "minTlsVersion": "1.2",
                     "scmMinTlsVersion": "1.2",
+                    "netFrameworkVersion": "v6.0",
                     "alwaysOn": true,
                     "cors": {
                         "allowedOrigins": [ "https://portal.azure.com" ]
@@ -595,6 +596,7 @@
                     "http20Enabled": true,
                     "minTlsVersion": "1.2",
                     "scmMinTlsVersion": "1.2",
+                    "netFrameworkVersion": "v6.0",
                     "alwaysOn": true,
                     "cors": {
                         "allowedOrigins": [ "https://portal.azure.com" ]
@@ -701,6 +703,7 @@
                     "http20Enabled": true,
                     "minTlsVersion": "1.2",
                     "scmMinTlsVersion": "1.2",
+                    "netFrameworkVersion": "v6.0",
                     "alwaysOn": true,
                     "cors": {
                         "allowedOrigins": [ "https://portal.azure.com" ]

AppendBlobReader/tests/test_appendblobreader.py

@@ -139,7 +139,7 @@ def test_03_func_logs(self):
         self.assertTrue(self.filter_logs(captured_output, 'message', message),
                         f"No '{message}' log line found in '{azurefunction}' function logs")
 
-        message = "Updated offset result"
+        message = "Successfully updated OffsetMap"
         self.assertTrue(self.filter_logs(captured_output, 'message', message),
                         f"No '{message}' log line found in '{azurefunction}' function logs")
 
@@ -188,8 +188,8 @@ def test_03_func_logs(self):
             expected_filename = self.test_filename
 
         # Verify addition of _sourceCategory, _sourceHost, _sourceName and also additional metadata
-        self.assertTrue(source_name == f"{expected_filename}", f"_sourceName {source_name} metadata is incorrect")
-        self.assertTrue(source_host == f"{self.test_storageaccount_name}/{self.test_container_name}", f"_sourceHost {source_host} metadata is incorrect")
+        self.assertTrue(source_name == expected_filename, f"_sourceName: {source_name} expected_filename: {expected_filename} metadata is incorrect")
+        self.assertTrue(source_host == f"{self.test_storageaccount_name}/{self.test_container_name}", f"_sourceHost {source_host} expected_sourcehost: {self.test_storageaccount_name}/{self.test_container_name} metadata is incorrect")
 
     # def test_05_upload_filename_with_utf16_chars_having_utf16_chars_in_deep_folder():
     #     # Verify with a filename with special characters

BlockBlobReader/src/blobreaderdeploywithPremiumPlan.json

@@ -364,6 +364,7 @@
                     "http20Enabled": true,
                     "minTlsVersion": "1.2",
                     "scmMinTlsVersion": "1.2",
+                    "netFrameworkVersion": "v6.0",
                     "appSettings": [
                         {
                             "name": "FUNCTIONS_EXTENSION_VERSION",
@@ -461,6 +462,7 @@
                     "http20Enabled": true,
                     "minTlsVersion": "1.2",
                     "scmMinTlsVersion": "1.2",
+                    "netFrameworkVersion": "v6.0",
                     "appSettings": [
                         {
                             "name": "FUNCTIONS_EXTENSION_VERSION",
@@ -556,6 +558,7 @@
                     "http20Enabled": true,
                     "minTlsVersion": "1.2",
                     "scmMinTlsVersion": "1.2",
+                    "netFrameworkVersion": "v6.0",
                     "appSettings": [
                         {
                             "name": "FUNCTIONS_EXTENSION_VERSION",

BlockBlobReader/target/consumer_build/host.json

@@ -27,7 +27,7 @@
       "default": "Warning",
       "Host.Aggregator": "Trace",
       "Host.Results": "Information",
-      "Function": "Trace"
+      "Function": "Information"
     }
   }
 }

BlockBlobReader/target/dlqprocessor_build/host.json

@@ -22,7 +22,7 @@
       "default": "Warning",
       "Host.Aggregator": "Trace",
       "Host.Results": "Information",
-      "Function": "Trace"
+      "Function": "Information"
     }
   }
 }

BlockBlobReader/target/producer_build/host.json

@@ -29,7 +29,7 @@
       "default": "Warning",
       "Host.Aggregator": "Trace",
       "Host.Results": "Information",
-      "Function": "Trace"
+      "Function": "Information"
     }
   }
 }

BlockBlobReader/tests/blob_fixtures.json

@@ -1 +1,56 @@
-{"records":[{"time":"2023-05-15T10:08:26.1964863Z","systemId":"61171fa9-d0e7-44e4-8133-d6f4035bc533","macAddress":"002248340BBC","category":"NetworkSecurityGroupFlowEvent","resourceId":"/SUBSCRIPTIONS/C088DC46-D692-42AD-A4B6-9A542D28AD2A/RESOURCEGROUPS/CON-1774/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/AADDS-NSG","operationName":"NetworkSecurityGroupFlowEvents","properties":{"Version":2,"flows":[{"rule":"DefaultRule_DenyAllInBound","flows":[{"mac":"002248340BBC","flowTuples":["1684145260,198.199.98.123,10.1.1.6,38095,7199,T,I,D,B,,,,","1684145262,162.142.125.180,10.1.1.6,10023,18061,T,I,D,B,,,,","1684145264,162.142.125.249,10.1.1.6,9303,33712,T,I,D,B,,,,","1684145267,146.59.233.75,10.1.1.6,33678,22,T,I,D,B,,,,","1684145275,167.94.145.28,10.1.1.6,39678,13337,T,I,D,B,,,,","1684145281,162.142.125.184,10.1.1.6,56860,9800,T,I,D,B,,,,","1684145290,79.124.62.78,10.1.1.6,53146,46367,T,I,D,B,,,,","1684145297,162.142.125.180,10.1.1.6,16879,10030,T,I,D,B,,,,","1684145297,193.106.192.189,10.1.1.6,35021,23,T,I,D,B,,,,","1684145300,94.102.61.42,10.1.1.6,46560,8426,T,I,D,B,,,,"]}]}]}}]}
+{
+    "records": [
+        {
+            "time": "2017-06-16T20:59:43.7340000Z",
+            "systemId": "abcdef01-2345-6789-0abc-def012345678",
+            "category": "NetworkSecurityGroupFlowEvent",
+            "resourceId": "/SUBSCRIPTIONS/00000000-0000-0000-0000-000000000000/RESOURCEGROUPS/MYRESOURCEGROUP/PROVIDERS/MICROSOFT.NETWORK/NETWORKSECURITYGROUPS/MYNSG",
+            "operationName": "NetworkSecurityGroupFlowEvents",
+            "properties": {
+                "Version": 1,
+                "flows": [
+                    {
+                        "rule": "DefaultRule_AllowInternetOutBound",
+                        "flows": [
+                            {
+                                "mac": "000D3A18077E",
+                                "flowTuples": [
+                                    "1497646722,10.0.0.4,168.62.32.14,44904,443,T,O,A",
+                                    "1497646722,10.0.0.4,52.240.48.24,45218,443,T,O,A",
+                                    "1497646725,10.0.0.4,168.62.32.14,44910,443,T,O,A",
+                                    "1497646725,10.0.0.4,52.240.48.24,45224,443,T,O,A",
+                                    "1497646728,10.0.0.4,168.62.32.14,44916,443,T,O,A",
+                                    "1497646728,10.0.0.4,52.240.48.24,45230,443,T,O,A",
+                                    "1497646732,10.0.0.4,168.62.32.14,44922,443,T,O,A",
+                                    "1497646732,10.0.0.4,52.240.48.24,45236,443,T,O,A"
+                                ]
+                            }
+                        ]
+                    },
+                    {
+                        "rule": "DefaultRule_DenyAllInBound",
+                        "flows": []
+                    },
+                    {
+                        "rule": "UserRule_ssh-rule",
+                        "flows": []
+                    },
+                    {
+                        "rule": "UserRule_web-rule",
+                        "flows": [
+                            {
+                                "mac": "000D3A18077E",
+                                "flowTuples": [
+                                    "1497646738,13.82.225.93,10.0.0.4,1180,80,T,I,A",
+                                    "1497646750,13.82.225.93,10.0.0.4,1184,80,T,I,A",
+                                    "1497646768,13.82.225.93,10.0.0.4,1181,80,T,I,A",
+                                    "1497646780,13.82.225.93,10.0.0.4,1336,80,T,I,A"
+                                ]
+                            }
+                        ]
+                    }
+                ]
+            }
+        }
+    ]
+}

BlockBlobReader/tests/test_blobreader.py

@@ -30,7 +30,9 @@ def setUpClass(cls):
         cls.test_storageaccount_name = "testsa%s" % (test_datetime_value)
         # Verify when Test Storage Account and template deployment are in different regions
         cls.test_storageAccountRegion = "Central US"
-        cls.test_container_name = "testcontainer-%s" % (datetime_value)
+        cls.log_type = os.environ.get("LOG_TYPE", "blob")
+
+        cls.test_container_name = "testcontainer-%s" % (datetime_value) if cls.log_type != "json" else "insights-logs-networksecuritygroupflowevent"
         cls.test_filename_excluded_by_filter = "blockblob_test_filename_excluded_by_filter.blob"
         cls.test_filename_unsupported_extension = "blockblob_test.xml"
         # https://learn.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata
@@ -87,10 +89,9 @@ def upload_file_of_unknown_extension(self):
                                                      data_block, [])
 
     def test_03_func_logs(self):
-        log_type = os.environ.get("LOG_TYPE", "log")
-        self.logger.info("inserting mock %s data in BlobStorage" % log_type)
-        if log_type in ("csv", "log",  "blob"):
-            self.insert_mock_logs_in_BlobStorage(log_type)
+        self.logger.info("inserting mock %s data in BlobStorage" % cls.log_type)
+        if cls.log_type in ("csv", "log",  "blob"):
+            self.insert_mock_logs_in_BlobStorage(cls.log_type)
         else:
             self.insert_mock_json_in_BlobStorage()
 
@@ -126,7 +127,7 @@ def test_03_func_logs(self):
                          f"Warning messages found in {azurefunction} azure function logs")
 
         self.logger.info("fetching mock data count from sumo")
-        log_type = os.environ.get("LOG_TYPE", "json")
+
         query = f'_sourceCategory="{self.source_category}" | count by _sourceName, _sourceHost'
         relative_time_in_minutes = 30
         expected_record_count = {
@@ -148,8 +149,8 @@ def test_03_func_logs(self):
         except Exception as err:
             self.logger.info(f"Error in fetching sumo query results {err}")
 
-        self.assertTrue(record_count == expected_record_count.get(log_type),
-                        f"block blob file's record count: {record_count} differs from expected count {expected_record_count.get(log_type)} in sumo '{self.source_category}'")
+        self.assertTrue(record_count == expected_record_count.get(cls.log_type),
+                        f"block blob file's record count: {record_count} differs from expected count {expected_record_count.get(cls.log_type)} in sumo '{self.source_category}'")
 
         # Verify Filter Prefix field
         self.assertTrue(record_excluded_by_filter_count == 0,
@@ -159,14 +160,15 @@ def test_03_func_logs(self):
                         f"block blob file's record count: {record_unsupported_extension_count}, logs with unsupported blob extension should not be ingested")
 
         # Verify with a very long append blob filename (1024 characters)
+        file_ext = f".{cls.log_type}"
         if len(self.test_filename) > 128:
-            expected_filename = self.test_filename[:60] + "..." + self.test_filename[-60:]
+            expected_filename = self.test_filename[:60] + "..." + self.test_filename[-(60-len(file_ext)):] + file_ext
         else:
             expected_filename = self.test_filename
 
         # Verify addition of _sourceCategory, _sourceHost, _sourceName and also additional metadata
-        self.assertTrue(source_name == f"{expected_filename}", f"_sourceName {source_name} metadata is incorrect")
-        self.assertTrue(source_host == f"{self.test_storageaccount_name}/{self.test_container_name}", f"_sourceHost {source_host} metadata is incorrect")
+        self.assertTrue(source_name == expected_filename, f"_sourceName: {source_name} expected_filename: {expected_filename} metadata is incorrect")
+        self.assertTrue(source_host == f"{self.test_storageaccount_name}/{self.test_container_name}", f"_sourceHost {source_host} expected_sourcehost: {self.test_storageaccount_name}/{self.test_container_name} metadata is incorrect")
 
     def get_random_name(self, length=32):
         return str(uuid.uuid4())

deletetestresourcegroups.py

@@ -0,0 +1,111 @@
+# from azure.common.credentials import ServicePrincipalCredentials
+from azure.identity import ClientSecretCredential
+import json
+import os
+from azure.mgmt.resource import ResourceManagementClient
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from sumologic import SumoLogic
+
+config = {
+    "AZURE_SUBSCRIPTION_ID": "",
+    "AZURE_CLIENT_ID": "",
+    "AZURE_CLIENT_SECRET": "",
+    "AZURE_TENANT_ID": "",
+    "AZURE_DEFAULT_REGION": "",
+    "SUMO_ACCESS_ID": "",
+    "SUMO_ACCESS_KEY": "",
+    "SUMO_DEPLOYMENT": "us1"
+}
+
+# config_file = os.path.expanduser("~/.azure/azure_credentials.json")
+# config = json.load(open(config_file))
+subscription_id = str(config['AZURE_SUBSCRIPTION_ID'])
+credentials = ClientSecretCredential(
+    client_id=config['AZURE_CLIENT_ID'],
+    client_secret=config['AZURE_CLIENT_SECRET'],
+    tenant_id=config['AZURE_TENANT_ID']
+)
+location = str(config['AZURE_DEFAULT_REGION'])
+
+print("creating credentials", subscription_id)
+
+resource_client = ResourceManagementClient(credentials, subscription_id)
+
+
+def delete_resource_group(resource_group_name):
+    print("Found %s " % resource_group_name)
+    resp = resource_client.resource_groups.begin_delete(resource_group_name)
+    resp.wait()
+    print('Deleted {}'.format(resource_group_name), resp.status())
+
+
+groups = resource_client.resource_groups.list()
+future_to_group = {}
+with ThreadPoolExecutor(max_workers=10) as executor:
+    for group in groups:
+        group_name = group.name
+        if (group_name.startswith("TBL") or group_name.startswith("TABR") or group_name.startswith("testsumosa")):
+            print(f"scheduling {group_name}")
+            future_to_group[executor.submit(delete_resource_group, group_name)] = group_name
+
+if future_to_group:
+    for future in as_completed(future_to_group):
+        group_name = future_to_group[future]
+        try:
+            future.result()
+            print(f"Task completed for {group_name}")
+        except Exception as exc:
+            print('%r generated an exception: %s' % (group_name, exc))
+else:
+    print("No resource group found")
+
+
+def api_endpoint(sumo_deployment):
+    if sumo_deployment == "us1":
+        return "https://api.sumologic.com/api"
+    elif sumo_deployment in ["ca", "au", "de", "eu", "jp", "us2", "fed", "in"]:
+        return "https://api.%s.sumologic.com/api" % sumo_deployment
+    else:
+        return 'https://%s-api.sumologic.net/api' % sumo_deployment
+
+sumologic_cli = SumoLogic(config["SUMO_ACCESS_ID"], config["SUMO_ACCESS_KEY"], api_endpoint(config["SUMO_DEPLOYMENT"]))
+
+
+def delete_collector(collector_id, collector_name):
+    print("Found %s " % collector_name)
+    resp = sumologic_cli.delete_collector({"collector": {"id": collector_id}})
+    print('Deleted {}'.format(collector_name), resp)
+
+
+future_to_group = {}
+offset = 0
+with ThreadPoolExecutor(max_workers=5) as executor:
+    while True:
+        collectors = sumologic_cli.collectors(limit=1000, offset=offset, filter_type="hosted")
+        if not collectors:
+            break
+        for collector in collectors:
+            collector_name = collector["name"]
+            collector_id = collector["id"]
+            if (collector_name.startswith("azure_appendblob_unittest") or collector_name.startswith("azure_blockblob_unittest")):
+                print(f"scheduling {collector_name}")
+                future_to_group[executor.submit(delete_collector, collector_id, collector_name)] = collector_name
+        offset += 100
+
+
+if future_to_group:
+    for future in as_completed(future_to_group):
+        collector_name = future_to_group[future]
+        try:
+            future.result()
+            print(f"Task completed for {collector_name}")
+        except Exception as exc:
+            print('%r generated an exception: %s' % (collector_name, exc))
+else:
+    print("No collector found")
+
+
+
+
+
+