Implemented service worker to intercept network traces, added tracking simple page navigation (#2743)

* Added sent page load metrics to backend logs
* Added service worker, network traces and tracking simple page navigation
* Clean sensitive data from traces parameters
* Added feature flag for client telemetry
* added clean up worker if feature flag is disabled
* extended user event logging for buttons clicks
* improved logging messages format
* set console logger by default and improved user interaction logging

Author: ygrik

.github/workflows/mainCI.yml

@@ -18,10 +18,10 @@ jobs:
       security-events: write
 
     steps:
-      - name: Use Node.js 16.x
+      - name: Use Node.js 20.x
         uses: actions/checkout@v2
         with:
-          node-version: 16.x
+          node-version: 20.x
 
       - name: Install
         run: npm install

src/apim.runtime.module.ts

@@ -88,10 +88,12 @@ import { TagService } from "./services/tagService";
 import { TenantService } from "./services/tenantService";
 import { UsersService } from "./services/usersService";
 import { TraceClick } from "./bindingHandlers/traceClick";
+import { ClientLogger } from "./logging/clientLogger";
 
 export class ApimRuntimeModule implements IInjectorModule {
     public register(injector: IInjector): void {
         injector.bindSingleton("logger", ConsoleLogger);
+        // injector.bindSingleton("logger", ClientLogger);
         injector.bindSingleton("traceClick", TraceClick);
         injector.bindToCollection("autostart", UnhandledErrorHandler);
         injector.bindToCollection("autostart", BalloonBindingHandler);

src/components/users/signin-social/ko/runtime/signin-aad-b2c.html

@@ -1,4 +1,4 @@
-<button id="signinB2C" class="button" data-bind="click: signIn, css: classNames, traceClick">
+<button id="signinB2C" class="button" data-action="B2C Sign in" data-bind="click: signIn, css: classNames, traceClick">
     <i class="icon-emb icon-svg-aad"></i>
     <span data-bind="text: label"></span>
 </button>

src/components/users/signin-social/ko/runtime/signin-aad.html

@@ -1,4 +1,4 @@
-<button id="signinAad" class="button" data-bind="click: signIn, css: classNames, traceClick">
+<button id="signinAad" class="button" data-action="AAD Sign in" data-bind="click: signIn, css: classNames, traceClick">
     <i class="icon-emb icon-svg-aad"></i>
     <span data-bind="text: label"></span>
 </button>

src/components/users/signup/ko/runtime/signup.html

@@ -34,16 +34,16 @@
 
         <!-- ko if: requireHipCaptcha -->
         <hip-captcha params="{ captchaData: captchaData, onInitComplete: onCaptchaCreated }"></hip-captcha>
-        <!-- /ko -->       
-        
+        <!-- /ko -->
+
         <!-- ko if: termsEnabled && termsOfUse -->
         <terms-of-use params="{ isConsentRequired: isConsentRequired, consented: consented, termsOfUse: termsOfUse }"></terms-of-use>
         <!-- /ko -->
 
         <div class="form-group">
-            
+
             <!-- ko ifnot: working -->
-            <button type="button" id="signup" class="button button-primary" data-bind="click: signup">
+            <button type="button" id="signup" class="button button-primary" data-action="Sign up" data-bind="click: signup">
                 Sign up
             </button>
             <!-- /ko -->

src/constants.ts

@@ -344,4 +344,15 @@ export const smallMobileBreakpoint = 400;
 /**
  * Key of the default admin user
  */
-export const integrationUserId = '/users/integration';
+export const integrationUserId = '/users/integration';
+
+/**
+  * This is used to store the unique user in local storage and identify the user session in client telemetry.
+ */
+export const USER_SESSION = "userSessionId";
+export const USER_ID = "userId";
+export const USER_ACTION = "data-action";
+
+// Feature flags
+export const FEATURE_FLAGS = "featureFlags";
+export const FEATURE_CLIENT_TELEMETRY = "clientTelemetry";

src/logging/clientLogger.ts

@@ -5,6 +5,7 @@ import { ISettingsProvider } from "@paperbits/common/configuration";
 import { ClientEvent } from "../models/logging/clientEvent";
 import { v4 as uuidv4 } from "uuid";
 import * as Constants from "../constants";
+import { Utils } from "../utils";
 
 export enum eventTypes {
     error = "Error",
@@ -33,6 +34,7 @@ export class ClientLogger implements Logger {
 
     public async trackEvent(eventName: string, properties?: Bag<string>): Promise<void> {
         const devPortalEvent = new ClientEvent();
+        this.addUserDataToEventData(properties);
 
         devPortalEvent.eventType = eventName;
         devPortalEvent.message = properties?.message;
@@ -43,6 +45,7 @@ export class ClientLogger implements Logger {
 
     public async trackError(error: Error, properties?: Bag<string>): Promise<void> {
         const devPortalEvent = new ClientEvent();
+        this.addUserDataToEventData(properties);
 
         devPortalEvent.eventType = eventTypes.error;
         devPortalEvent.message = error?.message;
@@ -54,6 +57,7 @@ export class ClientLogger implements Logger {
 
     public async trackView(viewName: string, properties?: Bag<string>): Promise<void> {
         const devPortalEvent = new ClientEvent();
+        this.addUserDataToEventData(properties);
 
         devPortalEvent.eventType = viewName;
         devPortalEvent.message = properties?.message;
@@ -70,6 +74,13 @@ export class ClientLogger implements Logger {
         // Not implemented
     }
 
+    private addUserDataToEventData(eventData?: Bag<string>) {
+        const userData = Utils.getUserData();
+        eventData = eventData || {};
+        eventData[Constants.USER_ID] = userData.userId;
+        eventData[Constants.USER_SESSION] = userData.sessionId;
+    }
+
     private async traceEvent(clientEvent: ClientEvent) {
         const datetime = new Date();
 

src/startup.runtime.ts

@@ -6,6 +6,10 @@ import { staticDataEnvironment } from "./../environmentConstants";
 import { define } from "mime";
 import { TraceClick } from "./bindingHandlers/traceClick";
 import { Logger } from "@paperbits/common/logging";
+import { TelemetryConfigurator } from "./telemetry/telemetryConfigurator";
+import { Utils } from "./utils";
+import { ISettingsProvider } from "@paperbits/common/configuration/ISettingsProvider";
+import { FEATURE_CLIENT_TELEMETRY, FEATURE_FLAGS } from "./constants";
 
 define({ "application/x-zip-compressed": ["zip"] }, true);
 
@@ -25,13 +29,46 @@ document.addEventListener("DOMContentLoaded", () => {
     traceClick.setupBinding();
 });
 
+initFeatures();
+
 window.onbeforeunload = () => {
     if (!location.pathname.startsWith("/signin-sso") &&
         !location.pathname.startsWith("/signup") &&
         !location.pathname.startsWith("/signin")) {
         const rest = location.href.split(location.pathname)[1];
         const returnUrl = location.pathname + rest;
         sessionStorage.setItem("returnUrl", returnUrl);
-        document.cookie = `returnUrl=${returnUrl}`; // for delegation
+        Utils.setCookie("returnUrl", returnUrl); // for delegation
+    }
+};
+
+function initFeatures() {
+    const logger = injector.resolve<Logger>("logger");
+    const settingsProvider = injector.resolve<ISettingsProvider>("settingsProvider");
+    checkIsFeatureEnabled(FEATURE_CLIENT_TELEMETRY, settingsProvider, logger)
+        .then((isEnabled) => {
+            logger.trackEvent("FeatureFlag", { feature: FEATURE_CLIENT_TELEMETRY, enabled: isEnabled.toString(), message: `Feature flag '${FEATURE_CLIENT_TELEMETRY}' - enabled` });
+            let telemetryConfigurator = new TelemetryConfigurator(injector);
+            if (isEnabled) {
+                telemetryConfigurator.configure();
+            } else {
+                telemetryConfigurator.cleanUp();
+            }
+        });
+}
+
+async function checkIsFeatureEnabled(featureFlagName: string, settingsProvider: ISettingsProvider, logger: Logger): Promise<boolean> {
+    try {
+        const settingsObject = await settingsProvider.getSetting(FEATURE_FLAGS);
+
+        const featureFlags = new Map(Object.entries(settingsObject ?? {}));
+        if (!featureFlags || !featureFlags.has(featureFlagName)) {
+            return false;
+        }
+
+        return featureFlags.get(featureFlagName) == true;
+    } catch (error) {
+        logger?.trackEvent("FeatureFlag", { message: "Feature flag check failed", data: error.message });
+        return false;
     }
-};
+}

src/telemetry/serviceWorker.ts

@@ -0,0 +1,67 @@
+import { Bag } from "@paperbits/common/bag";
+declare const clients: any;
+
+const allowedList = ["state", "session_state"];
+
+function sendMessageToClients(message: Bag<string>): void {
+    clients.matchAll().then((items: any[]) => {
+        if (items.length > 0) {
+            items.forEach(client => client.postMessage(message));
+        }
+    });
+}
+
+addEventListener("fetch", (event: FetchEvent) => {
+    const request = event.request;
+
+    event.respondWith(
+        (async () => {
+            const response = await fetch(request);
+
+            if (request.url.endsWith("/trace")) {
+                return response;
+            }
+
+            const cleanedUrl = request.url.indexOf("#code=") > -1 ? cleanUpUrlParams(request) : request.url;
+
+            const telemetryData = {
+                url: cleanedUrl,
+                method: request.method.toUpperCase(),
+                status: response.status.toString(),
+                responseHeaders: ""
+            };
+
+            const headers: { [key: string]: string } = {};
+            response.headers.forEach((value, key) => {
+                if (key.toLocaleLowerCase() === "authorization") {
+                    return;
+                }
+                headers[key] = value;
+            });
+            telemetryData.responseHeaders = JSON.stringify(headers);
+
+            sendMessageToClients(telemetryData);
+
+            return response;
+        })()
+    );
+});
+
+console.log("Telemetry worker started.");
+
+function cleanUpUrlParams(request: Request): string {
+    const url = new URL(request.url);
+    const hash = url.hash.substring(1); // Remove the leading '#'
+    const params = new URLSearchParams(hash);
+
+    // Remove all parameters except those in the allowedList
+    for (const key of params.keys()) {
+        if (!allowedList.includes(key)) {
+            // Replace the 'code' parameter value
+            params.set(key, "xxxxxxxxxx");
+        }
+    }
+
+    url.hash = params.toString();
+    return url.toString();
+}