fix: null point exception in event parser (#1488)

amliuyong · web-flow · commit 861dc5289810 · 2024-06-27T12:12:10.000Z
diff --git a/src/data-pipeline/etl-common/src/main/java/software/aws/solution/clickstream/common/BaseEventParser.java b/src/data-pipeline/etl-common/src/main/java/software/aws/solution/clickstream/common/BaseEventParser.java
@@ -27,6 +27,7 @@
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Optional;
 
 import static software.aws.solution.clickstream.common.Util.deCodeUri;
 import static software.aws.solution.clickstream.common.Util.decompress;
@@ -149,11 +150,12 @@ protected void setPageViewUrl(final ClickstreamEvent clickstreamEvent, final Str
             return;
         }
         clickstreamEvent.setPageViewPageUrl(deCodeUri(url));
-        UrlParseResult urlParseResult = Util.parseUrl(url);
+        Optional<UrlParseResult> urlParseResultOpt = Util.parseUrl(url);
 
-        if (urlParseResult == null) {
+        if (urlParseResultOpt.isEmpty()) {
             return;
         }
+        UrlParseResult urlParseResult = urlParseResultOpt.get();
 
         if (urlParseResult.getPath() != null) {
             clickstreamEvent.setPageViewPageUrlPath(urlParseResult.getPath());
diff --git a/src/data-pipeline/etl-common/src/main/java/software/aws/solution/clickstream/common/ClickstreamEventParser.java b/src/data-pipeline/etl-common/src/main/java/software/aws/solution/clickstream/common/ClickstreamEventParser.java
@@ -23,9 +23,13 @@
 import software.aws.solution.clickstream.common.ingest.*;
 import software.aws.solution.clickstream.common.model.*;
 
-import java.sql.*;
-import java.time.*;
-import java.util.*;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -398,8 +402,11 @@ private void setPageView(final Event ingestEvent, final ClickstreamEvent clickst
         clickstreamEvent.setPageViewLatestReferrerHost(ingestEvent.getAttributes().getLatestReferrerHost());
         if (ingestEvent.getAttributes().getLatestReferrer() != null
                 && clickstreamEvent.getPageViewLatestReferrerHost() == null) {
-            UrlParseResult latestReferrerUrlParseResult = parseUrl(ingestEvent.getAttributes().getLatestReferrer());
-            clickstreamEvent.setPageViewLatestReferrerHost(latestReferrerUrlParseResult.getHostName());
+
+            Optional<UrlParseResult> r = parseUrl(ingestEvent.getAttributes().getLatestReferrer());
+            if (r.isPresent()) {
+                clickstreamEvent.setPageViewLatestReferrerHost(r.get().getHostName());
+            }
         }
         clickstreamEvent.setPageViewEntrances(ingestEvent.getAttributes().getEntrances());
     }
diff --git a/src/data-pipeline/etl-common/src/main/java/software/aws/solution/clickstream/common/Util.java b/src/data-pipeline/etl-common/src/main/java/software/aws/solution/clickstream/common/Util.java
@@ -38,6 +38,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.zip.GZIPInputStream;
 
 @Slf4j
@@ -185,9 +186,9 @@ public static Map<String, List<String>> getUriParams(final URI uriObj) {
         return params;
     }
 
-    public static UrlParseResult parseUrl(final String url) {
+    public static Optional<UrlParseResult> parseUrl(final String url) {
         if (url == null || url.isEmpty()) {
-            return null;
+            return Optional.empty();
         }
         String schemaUrl = url;
         if (!url.substring(0, Math.min(url.length(), 15)).contains("://")) {
@@ -210,7 +211,7 @@ public static UrlParseResult parseUrl(final String url) {
             }
         }
         log.debug("parseUrl(): result host: {}",  result.getHostName());
-        return result;
+        return Optional.of(result);
     }
 
     private static UrlParseResult extractFromUrl(final String schemaUrl) throws URISyntaxException {
diff --git a/src/data-pipeline/etl-common/src/main/java/software/aws/solution/clickstream/common/enrich/RuleBasedTrafficSourceHelper.java b/src/data-pipeline/etl-common/src/main/java/software/aws/solution/clickstream/common/enrich/RuleBasedTrafficSourceHelper.java
@@ -33,6 +33,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import static software.aws.solution.clickstream.common.Util.getUriParams;
 import static software.aws.solution.clickstream.common.Util.objectToJsonString;
@@ -189,7 +190,10 @@ public CategoryTrafficSource parse(final String pageUrl, final String pageReferr
         String pageHostName = null;
         if (pageUrl != null && !pageUrl.isEmpty()) {
             trafficSourceUtm = getUtmSourceFromUrl(pageUrl);
-            pageHostName = parseUrl(pageUrl).getHostName();
+            Optional<UrlParseResult> r = parseUrl(pageUrl);
+            if (r.isPresent()) {
+                pageHostName = r.get().getHostName();
+            }
         }
         CategoryTrafficSource result = parse(trafficSourceUtm, pageHostName, pageReferrer, latestReferrer, latestReferrerHost);
         categoryTrafficSourceCache.put(cachedKey, result);
@@ -339,7 +343,10 @@ public CategoryTrafficSource parse(final TrafficSourceUtm trafficSourceUtmInput,
 
         String pageReferrerHost = null;
         if (pageReferrer != null && !pageReferrer.isEmpty()) {
-             pageReferrerHost = parseUrl(pageReferrer).getHostName();
+             Optional<UrlParseResult> r = parseUrl(pageReferrer);
+             if (r.isPresent()) {
+                 pageReferrerHost = r.get().getHostName();
+             }
         }
         boolean isInternalReferrer = pageHostName != null && pageHostName.equalsIgnoreCase(pageReferrerHost);
         boolean isInternalLatestReferrer = pageHostName != null && pageHostName.equalsIgnoreCase(latestReferrerHost);
diff --git a/src/data-pipeline/etl-common/src/main/java/software/aws/solution/clickstream/common/enrich/ts/rule/CategoryListEvaluator.java b/src/data-pipeline/etl-common/src/main/java/software/aws/solution/clickstream/common/enrich/ts/rule/CategoryListEvaluator.java
@@ -31,6 +31,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -103,10 +104,16 @@ public SourceCategoryAndTerms evaluate(final String theReferrerUrl) {
             return categoryAndTerms;
         }
 
-        UrlParseResult r = Util.parseUrl(theReferrerUrl);
-        String hostName = r.getHostName();
-        String path = r.getPath();
-        Map<String, List<String>> urlParams = r.getQueryParameters();
+        Optional<UrlParseResult> r = Util.parseUrl(theReferrerUrl);
+        String hostName = "";
+        String path = "";
+        Map<String, List<String>> urlParams = new HashMap<>();
+
+        if (r.isPresent()) {
+             hostName = r.get().getHostName();
+             path = r.get().getPath();
+             urlParams = r.get().getQueryParameters();
+        }
 
         List<String> candidateUrls = getCandidateUrls(theReferrerUrl, hostName, path);
 
diff --git a/src/data-pipeline/etl-common/src/test/java/software/aws/solution/clickstream/common/ClickstreamEventParserTest.java b/src/data-pipeline/etl-common/src/test/java/software/aws/solution/clickstream/common/ClickstreamEventParserTest.java
@@ -524,4 +524,18 @@ void test_DeviceOperatingSystem_set_as_platform_web() throws IOException {
         Assertions.assertNull(event.getDeviceOperatingSystem());
     }
 
+    @Test
+    void test_parse_line_with_empty_latest_referrer() throws IOException {
+        // ./gradlew clean test --info --tests software.aws.solution.clickstream.common.ClickstreamEventParserTest.test_parse_line_with_empty_latest_referrer
+        String line = resourceFileContent("/empty_latest_referrer.json");
+        log.info(line);
+        ClickstreamEventParser clickstreamEventParser = getClickstreamEventParser();
+        String projectId = "test_project_id";
+        String fileName = "empty_latest_referrer.json";
+
+        ParseRowResult rowResult = clickstreamEventParser.parseLineToDBRow(line, projectId, fileName);
+
+        Assertions.assertEquals(1, rowResult.getClickstreamEventList().size());
+        Assertions.assertEquals(1, rowResult.getClickstreamUserList().size());
+    }
 }
diff --git a/src/data-pipeline/etl-common/src/test/java/software/aws/solution/clickstream/common/util/UtilTest.java b/src/data-pipeline/etl-common/src/test/java/software/aws/solution/clickstream/common/util/UtilTest.java
@@ -169,13 +169,13 @@ void test_readTextFile() throws IOException {
     void  test_parseUrl() {
         // ./gradlew clean test --info --tests software.aws.solution.clickstream.common.util.UtilTest.test_parseUrl
         String url = "https://www.example.com/abc/test?param1=value1&param2=value2";
-        UrlParseResult result = Util.parseUrl(url);
+        UrlParseResult result = Util.parseUrl(url).get();
         assertEquals("www.example.com", result.getHostName());
         assertEquals("param1=value1&param2=value2", result.getQueryString());
         assertEquals("/abc/test", result.getPath());
 
-        assertNull(Util.parseUrl(null));
-        assertNull(Util.parseUrl(""));
+        assertTrue(Util.parseUrl(null).isEmpty());
+        assertTrue(Util.parseUrl("").isEmpty());
         assertNotNull(Util.parseUrl("https://www.example.com/"));
         assertNotNull(Util.parseUrl("https://www.example.com/?"));
     }
diff --git a/src/data-pipeline/etl-common/src/test/resources/empty_latest_referrer.json b/src/data-pipeline/etl-common/src/test/resources/empty_latest_referrer.json
@@ -0,0 +1,19 @@
+{
+  "appId": "shop",
+  "client_timestamp": "1719469182900",
+  "compression": null,
+  "data": "[{\"event_type\":\"_page_view\",\"event_id\":\"9029c8fd-ed51-4da0-a7ea-40aad4d3ad45\",\"device_id\":\"564ea695-00d1-4fdf-beb2-9c54682a4217\",\"unique_id\":\"c631fe47-bf74-4fcf-a3ff-0e5380250d10\",\"app_id\":\"shop\",\"timestamp\":1719466410498,\"host_name\":\"test.cloudfront.net\",\"locale\":\"en-US\",\"system_language\":\"en\",\"country_code\":\"US\",\"zone_offset\":28800000,\"make\":\"Gecko\",\"platform\":\"Web\",\"screen_height\":1440,\"screen_width\":2560,\"viewport_height\":1302,\"viewport_width\":1276,\"sdk_name\":\"aws-solution-clickstream-sdk\",\"sdk_version\":\"0.12.1\",\"user\":{\"_user_first_touch_timestamp\":{\"value\":1719466021378,\"set_timestamp\":1719466021378},\"_user_id\":{\"value\":\"58\",\"set_timestamp\":1719466046946}},\"attributes\":{\"_traffic_source_source\":\"amazon\",\"_traffic_source_medium\":\"cpc\",\"_traffic_source_campaign\":\"summer_promotion\",\"_traffic_source_campaign_id\":\"summer_promotion_01\",\"_traffic_source_term\":\"running_shoes\",\"_traffic_source_content\":\"banner_ad_1\",\"_traffic_source_clid\":\"amazon_ad_123\",\"_traffic_source_clid_platform\":\"amazon_ads\",\"_session_id\":\"80250d10-20240627-052701378\",\"_session_start_timestamp\":1719466021378,\"_session_duration\":389120,\"_session_number\":1,\"_page_title\":\"Retail Demo Store\",\"_page_url\":\"https://test.cloudfront.net/product/ecc45e4c-9249-4b06-9f99-aa068eebddf4?feature=home_product_recs\",\"_latest_referrer\":\"\",\"_page_referrer\":\"https://test.cloudfront.net/\",\"_page_referrer_title\":\"Retail Demo Store\",\"_entrances\":0,\"_previous_timestamp\":1719466409801,\"_engagement_time_msec\":697}}]",
+  "date": "2024-06-27T06:19:45+00:00",
+  "fakeIp": null,
+  "ingest_time": 1719469185000,
+  "ip": "72.21.198.67",
+  "method": "POST",
+  "path": "/collect",
+  "platform": "Web",
+  "rid": "0ae1d0c2989e6a7a5619097967d65a7e",
+  "server_ingest_time": 1719469185000,
+  "source_type": "http_server",
+  "timestamp": "2024-06-27T06:19:45.547102438Z",
+  "ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:126.0) Gecko/20100101 Firefox/126.0",
+  "uri": "/collect?platform=Web&appId=shop&event_bundle_sequence_id=43&upload_timestamp=1719469182900&hashCode=366095c8"
+} 
diff --git a/src/data-pipeline/spark-etl/src/test/java/software/aws/solution/clickstream/ClickstreamDataConverterV3Test.java b/src/data-pipeline/spark-etl/src/test/java/software/aws/solution/clickstream/ClickstreamDataConverterV3Test.java
@@ -50,6 +50,18 @@ public void test_convert_data_v3() throws IOException {
         Assertions.assertEquals(expectedJson, dataJson);
     }
 
+    @Test
+    public void test_empty_latest_referrer() throws IOException {
+        // ./gradlew clean test --info --tests software.aws.solution.clickstream.ClickstreamDataConverterV3Test.test_empty_latest_referrer
+
+        String filePath = "/empty_latest_referrer.json";
+        Dataset<Row> dataset = readJsonDataset(filePath);
+        dataset = addFileName(dataset);
+        System.setProperty(PROJECT_ID_PROP, "projectId1");
+        Dataset<Row> result = converter.transform(dataset);
+        Assertions.assertEquals(88, result.count());
+    }
+
     @Test
     public void test_convert_data_v3_invalid_data() throws IOException {
         // ./gradlew clean test --info --tests software.aws.solution.clickstream.ClickstreamDataConverterV3Test.test_convert_data_v3_invalid_data
diff --git a/src/data-pipeline/spark-etl/src/test/resources/empty_latest_referrer.json b/src/data-pipeline/spark-etl/src/test/resources/empty_latest_referrer.json

Original file line number	Diff line number	Diff line change
`@@ -38,6 +38,7 @@`
`38`	`38`	`import java.util.HashMap;`
`39`	`39`	`import java.util.List;`
`40`	`40`	`import java.util.Map;`
	`41`	`+import java.util.Optional;`
`41`	`42`	`import java.util.zip.GZIPInputStream;`
`42`	`43`
`43`	`44`	`@Slf4j`
`@@ -185,9 +186,9 @@ public static Map<String, List<String>> getUriParams(final URI uriObj) {`
`185`	`186`	`return params;`
`186`	`187`	`}`
`187`	`188`
`188`		`- public static UrlParseResult parseUrl(final String url) {`
	`189`	`+ public static Optional<UrlParseResult> parseUrl(final String url) {`
`189`	`190`	`if (url == null \|\| url.isEmpty()) {`
`190`		`- return null;`
	`191`	`+ return Optional.empty();`
`191`	`192`	`}`
`192`	`193`	`String schemaUrl = url;`
`193`	`194`	`if (!url.substring(0, Math.min(url.length(), 15)).contains("://")) {`
`@@ -210,7 +211,7 @@ public static UrlParseResult parseUrl(final String url) {`
`210`	`211`	`}`
`211`	`212`	`}`
`212`	`213`	`log.debug("parseUrl(): result host: {}", result.getHostName());`
`213`		`- return result;`
	`214`	`+ return Optional.of(result);`
`214`	`215`	`}`
`215`	`216`
`216`	`217`	`private static UrlParseResult extractFromUrl(final String schemaUrl) throws URISyntaxException {`