[MERGE #5830 @thomasmo] Expose BGParse through new JSRT APIs

Merge pull request #5830 from thomasmo:bgparse_core

This change introduces new experimental JSRT APIs to expose the functionality of BGParseManager:
- JsQueueBackgroundParse_Experimental
- JsDiscardBackgroundParse_Experimental
- JsExecuteBackgroundParse_Experimental
This gives ChakraCore a programmatic means to move script parsing off of the UI thread.
Changes are also made in ch.exe to exercise these APIs for testing. Additional changes are made elsewhere for proper cleanup during shutdown.

This feature is still off by default.  To test in ch.exe, the following flags are required:
    "-bgparse -executewithbgparse -parserstatecache"

Author: Thomas Moore (CHAKRA)

bin/ChakraCore/ChakraCore.def

@@ -70,3 +70,7 @@ JsSerializeParserState
 JsRunScriptWithParserState
 JsGetPromiseState
 JsGetPromiseResult
+
+JsQueueBackgroundParse_Experimental
+JsDiscardBackgroundParse_Experimental
+JsExecuteBackgroundParse_Experimental

bin/ChakraCore/ChakraCoreDllFunc.cpp

@@ -153,6 +153,8 @@ EXTERN_C BOOL WINAPI DllMain(HINSTANCE hmod, DWORD dwReason, PVOID pvReserved)
 #if defined(CHECK_MEMORY_LEAK) || defined(LEAK_REPORT)
         else
         {
+            ThreadBoundThreadContextManager::DestroyAllContexts();
+            DetachProcess();
             ThreadContext::ReportAndCheckLeaksOnProcessDetach();
         }
 #endif

bin/ch/ChakraRtInterface.cpp

@@ -160,6 +160,10 @@ bool ChakraRTInterface::LoadChakraDll(ArgInfo* argInfo, HINSTANCE *outLibrary)
     m_jsApiHooks.pfJsrtSerializeParserState = (JsAPIHooks::JsrtSerializeParserState)GetChakraCoreSymbol(library, "JsSerializeParserState");
     m_jsApiHooks.pfJsrtRunScriptWithParserState = (JsAPIHooks::JsrtRunScriptWithParserState)GetChakraCoreSymbol(library, "JsRunScriptWithParserState");
 
+    m_jsApiHooks.pfJsrtQueueBackgroundParse_Experimental = (JsAPIHooks::JsrtQueueBackgroundParse_Experimental)GetChakraCoreSymbol(library, "JsQueueBackgroundParse_Experimental");
+    m_jsApiHooks.pfJsrtDiscardBackgroundParse_Experimental = (JsAPIHooks::JsrtDiscardBackgroundParse_Experimental)GetChakraCoreSymbol(library, "JsDiscardBackgroundParse_Experimental");
+    m_jsApiHooks.pfJsrtExecuteBackgroundParse_Experimental = (JsAPIHooks::JsrtExecuteBackgroundParse_Experimental)GetChakraCoreSymbol(library, "JsExecuteBackgroundParse_Experimental");
+
     m_jsApiHooks.pfJsrtTTDCreateRecordRuntime = (JsAPIHooks::JsrtTTDCreateRecordRuntimePtr)GetChakraCoreSymbol(library, "JsTTDCreateRecordRuntime");
     m_jsApiHooks.pfJsrtTTDCreateReplayRuntime = (JsAPIHooks::JsrtTTDCreateReplayRuntimePtr)GetChakraCoreSymbol(library, "JsTTDCreateReplayRuntime");
     m_jsApiHooks.pfJsrtTTDCreateContext = (JsAPIHooks::JsrtTTDCreateContextPtr)GetChakraCoreSymbol(library, "JsTTDCreateContext");

bin/ch/ChakraRtInterface.h

@@ -92,6 +92,10 @@ struct JsAPIHooks
 
     typedef JsErrorCode(WINAPI *JsrtSerializeParserState)(JsValueRef script, JsValueRef *buffer, JsParseScriptAttributes parseAttributes);
     typedef JsErrorCode(WINAPI *JsrtRunScriptWithParserState)(JsValueRef script, JsSourceContext sourceContext, JsValueRef sourceUrl, JsParseScriptAttributes parseAttributes, JsValueRef parserState, JsValueRef *result);
+    
+    typedef JsErrorCode(WINAPI *JsrtQueueBackgroundParse_Experimental)(JsScriptContents* contents, DWORD* dwBgParseCookie);
+    typedef JsErrorCode(WINAPI *JsrtDiscardBackgroundParse_Experimental)(DWORD dwBgParseCookie, void* buffer, bool* callerOwnsBuffer);
+    typedef JsErrorCode(WINAPI *JsrtExecuteBackgroundParse_Experimental)(DWORD dwBgParseCookie, JsValueRef script, JsSourceContext sourceContext, WCHAR *url, JsParseScriptAttributes parseAttributes, JsValueRef parserState, JsValueRef *result);
 
     typedef JsErrorCode(WINAPI *JsrtTTDCreateRecordRuntimePtr)(JsRuntimeAttributes attributes, bool enableDebugging, size_t snapInterval, size_t snapHistoryLength, TTDOpenResourceStreamCallback openResourceStream, JsTTDWriteBytesToStreamCallback writeBytesToStream, JsTTDFlushAndCloseStreamCallback flushAndCloseStream, JsThreadServiceCallback threadService, JsRuntimeHandle *runtime);
     typedef JsErrorCode(WINAPI *JsrtTTDCreateReplayRuntimePtr)(JsRuntimeAttributes attributes, const char* infoUri, size_t infoUriCount, bool enableDebugging, TTDOpenResourceStreamCallback openResourceStream, JsTTDReadBytesFromStreamCallback readBytesFromStream, JsTTDFlushAndCloseStreamCallback flushAndCloseStream, JsThreadServiceCallback threadService, JsRuntimeHandle *runtime);
@@ -195,6 +199,10 @@ struct JsAPIHooks
     JsrtSerializeParserState pfJsrtSerializeParserState;
     JsrtRunScriptWithParserState pfJsrtRunScriptWithParserState;
 
+    JsrtQueueBackgroundParse_Experimental pfJsrtQueueBackgroundParse_Experimental;
+    JsrtDiscardBackgroundParse_Experimental pfJsrtDiscardBackgroundParse_Experimental;
+    JsrtExecuteBackgroundParse_Experimental pfJsrtExecuteBackgroundParse_Experimental;
+
     JsrtTTDCreateRecordRuntimePtr pfJsrtTTDCreateRecordRuntime;
     JsrtTTDCreateReplayRuntimePtr pfJsrtTTDCreateReplayRuntime;
     JsrtTTDCreateContextPtr pfJsrtTTDCreateContext;
@@ -430,6 +438,10 @@ class ChakraRTInterface
 
     static JsErrorCode WINAPI JsSerializeParserState(JsValueRef script, JsValueRef *buffer, JsParseScriptAttributes parseAttributes) { return HOOK_JS_API(SerializeParserState(script, buffer, parseAttributes)); }
     static JsErrorCode WINAPI JsRunScriptWithParserState(JsValueRef script, JsSourceContext sourceContext, JsValueRef sourceUrl, JsParseScriptAttributes parseAttributes, JsValueRef parserState, JsValueRef * result) { return HOOK_JS_API(RunScriptWithParserState(script, sourceContext, sourceUrl, parseAttributes, parserState, result)); }
+
+    static JsErrorCode WINAPI JsQueueBackgroundParse_Experimental(JsScriptContents* contents, DWORD* dwBgParseCookie) { return HOOK_JS_API(QueueBackgroundParse_Experimental)(contents, dwBgParseCookie);  }
+    static JsErrorCode WINAPI JsDiscardBackgroundParse_Experimental(DWORD dwBgParseCookie, void* buffer, bool* callerOwnsBuffer) { return HOOK_JS_API(DiscardBackgroundParse_Experimental(dwBgParseCookie, buffer, callerOwnsBuffer)); }
+    static JsErrorCode WINAPI JsExecuteBackgroundParse_Experimental(DWORD dwBgParseCookie, JsValueRef script, JsSourceContext sourceContext, WCHAR *url, JsParseScriptAttributes parseAttributes, JsValueRef parserState, JsValueRef *result) { return HOOK_JS_API(ExecuteBackgroundParse_Experimental(dwBgParseCookie, script, sourceContext, url, parseAttributes, parserState, result)); }
 };
 
 class AutoRestoreContext

bin/ch/HostConfigFlagsList.h

@@ -20,6 +20,6 @@ FLAG(bool, Test262,                         "load Test262 harness", false)
 FLAG(bool, Module,                          "load the script as a module", false)
 FLAG(bool, TrackRejectedPromises,           "Enable tracking of unhandled promise rejections", false)
 FLAG(BSTR, CustomConfigFile,                "Custom config file to be used to pass in additional flags to Chakra", NULL)
-FLAG(bool, ExecuteWithBgParse,              "[No-op] Load script with bgparse (note: requires bgparse to be on as well)", false)
+FLAG(bool, ExecuteWithBgParse,              "Load script with bgparse (note: requires bgparse and parserstatecache be on as well)", false)
 #undef FLAG
 #endif

bin/ch/WScriptJsrt.h

@@ -89,6 +89,8 @@ class WScriptJsrt
             return _u("FatalError");
         case (JsErrorCode::JsErrorInExceptionState) :
             return _u("ErrorInExceptionState");
+        case (JsErrorCode::JsErrorBadSerializedScript):
+            return _u("ErrorBadSerializedScript ");
         default:
             AssertMsg(false, "Unexpected JsErrorCode");
             return nullptr;

bin/ch/ch.cpp

@@ -39,6 +39,8 @@ UINT32 snapHistoryLength = MAXUINT32;
 LPCWSTR connectionUuidString = NULL;
 UINT32 startEventCount = 1;
 
+HRESULT RunBgParseSync(LPCSTR fileContents, UINT lengthBytes, const char* fileName);
+
 extern "C"
 HRESULT __stdcall OnChakraCoreLoadedEntry(TestHooks& testHooks)
 {
@@ -456,14 +458,20 @@ HRESULT RunScript(const char* fileName, LPCSTR fileContents, size_t fileLength,
         {
             runScript = WScriptJsrt::ModuleEntryPoint(fileName, fileContents, fullPath);
         }
+        else if (HostConfigFlags::flags.ExecuteWithBgParse && !HostConfigFlags::flags.DebugLaunch)
+        {
+            unsigned int lengthBytes = (unsigned int) fileLength;
+            runScript = (JsErrorCode)RunBgParseSync(fileContents, lengthBytes, fileName);
+        }
         else // bufferValue == nullptr && parserStateCache == nullptr
         {
             JsValueRef scriptSource;
             IfJsErrorFailLog(ChakraRTInterface::JsCreateExternalArrayBuffer((void*)fileContents,
                 (unsigned int)fileLength,
                 fileContentsFinalizeCallback, (void*)fileContents, &scriptSource));
+
 #if ENABLE_TTD
-            if(doTTRecord)
+            if (doTTRecord)
             {
                 JsPropertyIdRef ttProperty = nullptr;
                 JsValueRef ttString = nullptr;
@@ -755,6 +763,47 @@ HRESULT CreateAndRunSerializedScript(const char* fileName, LPCSTR fileContents,
     return hr;
 }
 
+// Use the asynchronous BGParse JSRT APIs in a synchronous call
+HRESULT RunBgParseSync(LPCSTR fileContents, UINT lengthBytes, const char* fileName)
+{
+    JsValueRef scriptSource;
+    JsErrorCode e = (ChakraRTInterface::JsCreateExternalArrayBuffer((void*)fileContents,
+        (unsigned int)lengthBytes,
+        nullptr, (void*)fileContents, &scriptSource));
+
+    // What's the preferred way of doing this?
+    WCHAR fileNameWide[MAX_PATH] = { 0 };
+    size_t fileNameLength = strlen(fileName);
+    for (size_t i = 0; i < fileNameLength; i++)
+    {
+        fileNameWide[i] = fileName[i];
+    }
+
+    JsScriptContents scriptContents = { 0 };
+    scriptContents.container = (LPVOID)fileContents;
+    scriptContents.containerType = JsScriptContainerType::HeapAllocatedBuffer;
+    scriptContents.encodingType = JsScriptEncodingType::Utf8;
+    scriptContents.contentLengthInBytes = lengthBytes;
+    scriptContents.fullPath = fileNameWide;
+
+    DWORD cookie = 0;
+    e = ChakraRTInterface::JsQueueBackgroundParse_Experimental(&scriptContents, &cookie);
+    Assert(e == JsErrorCode::JsNoError);
+
+    JsValueRef bgResult = nullptr;
+    e = ChakraRTInterface::JsExecuteBackgroundParse_Experimental(
+        cookie,
+        scriptSource,
+        WScriptJsrt::GetNextSourceContext(),
+        (WCHAR*)scriptContents.fullPath,
+        JsParseScriptAttributes::JsParseScriptAttributeNone,
+        nullptr,//_In_ JsValueRef parserState,
+        &bgResult
+    );
+
+    return e;
+}
+
 HRESULT ExecuteTest(const char* fileName)
 {
     HRESULT hr = S_OK;

lib/Jsrt/ChakraCommon.h

@@ -625,6 +625,27 @@ typedef unsigned short uint16_t;
         JsDataView = 12,
     } JsValueType;
 
+    typedef enum _JsScriptEncodingType
+    {
+        Utf8,
+        Utf16
+    } JsScriptEncodingType;
+
+    typedef enum _JsScriptContainerType
+    {
+        HeapAllocatedBuffer
+    } JsScriptContainerType;
+
+    typedef struct _JsScriptContents
+    {
+        void * container;
+        JsScriptEncodingType encodingType;
+        JsScriptContainerType containerType;
+        JsSourceContext sourceContext;
+        size_t contentLengthInBytes;
+        WCHAR * fullPath;
+    } JsScriptContents;
+
     /// <summary>
     ///     User implemented callback routine for memory allocation events
     /// </summary>
@@ -2413,6 +2434,60 @@ typedef unsigned short uint16_t;
             _In_opt_ JsPromiseContinuationCallback promiseContinuationCallback,
             _In_opt_ void *callbackState);
 
+    /// <summary>
+    ///     Note: Experimental API
+    ///     Starts a request for background script parsing on another thread
+    /// </summary>
+    /// <param name="contents">ScriptContents struct with data needed to start parsing</param>
+    /// <param name="dwBgParseCookie">Identifier for subsequent BGParse operations</param>
+    /// <returns>
+    ///     The code <c>JsNoError</c> if the operation succeeded, a failure code otherwise.
+    /// </returns>
+    CHAKRA_API
+        JsQueueBackgroundParse_Experimental(
+            _In_ JsScriptContents* contents,
+            _Out_ DWORD* dwBgParseCookie);
+
+    /// <summary>
+    ///     Note: Experimental API
+    ///     Appropriately frees resources associated with a previously queued background parse
+    /// </summary>
+    /// <param name="dwBgParseCookie">Identifier for BGParse operation</param>
+    /// <param name="buffer">Pointer to script source buffer, used for validation</param>
+    /// <param name="callerOwnsBuffer">When <c>true</c>, caller is responsible for freeing buffer</param>
+    /// <returns>
+    ///     The code <c>JsNoError</c> if the operation succeeded, a failure code otherwise.
+    /// </returns>
+    CHAKRA_API
+        JsDiscardBackgroundParse_Experimental(
+            _In_ DWORD dwBgParseCookie,
+            _In_ void* buffer,
+            _Out_ bool* callerOwnsBuffer);
+
+    /// <summary>
+    ///     Note: Experimental API
+    ///     Executes the background parsed script
+    /// </summary>
+    /// <param name="dwBgParseCookie">Identifier for subsequent BGParse operations</param>
+    /// <param name="script">Pointer to script source</param>
+    /// <param name="sourceContext">JsSourceContext identifier</param>
+    /// <param name="url">Path to the parsed script</param>
+    /// <param name="parseAttributes"></param>
+    /// <param name="parserState">[May not be needed]</param>
+    /// <param name="result">Result of script execution</param>
+    /// <returns>
+    ///     The code <c>JsNoError</c> if the operation succeeded, a failure code otherwise.
+    /// </returns>
+    CHAKRA_API
+        JsExecuteBackgroundParse_Experimental(
+            _In_ DWORD dwBgParseCookie,
+            _In_ JsValueRef script,
+            _In_ JsSourceContext sourceContext,
+            _In_ WCHAR *url,
+            _In_ JsParseScriptAttributes parseAttributes,
+            _In_ JsValueRef parserState,
+            _Out_ JsValueRef *result);
+
 #ifdef _WIN32
 #include "ChakraCommonWindows.h"
 #endif // _WIN32