Prepare for more flow rr system languages (originally by Nedy); adapt to simpler rr-support record

.github/workflows/codetracer.yml

@@ -704,7 +704,7 @@ jobs:
           git config init.defaultBranch main
 
           git remote set-url origin https://x-access-token:${{ secrets.CODETRACER_PUSH_GITHUB_TOKEN }}@github.com/metacraft-labs/codetracer
-          
+
           nix develop .#devShells.x86_64-linux.default --command ./ci/deploy/docs.sh
 
   push-tag:

src/common/config.nim

@@ -9,9 +9,6 @@ type
   RRBackendConfig* = object
     enabled*:           bool
     path*:              string
-    ctPaths*:           string
-    debugInfoToolPath*: string
-    ctRRWorkerExe*:     string
 
   FlowConfigObjWrapper* = object
     enabled*:                             bool
@@ -63,9 +60,6 @@ type
     rrBackend* {.defaultVal: RRBackendConfig(
       enabled: false,
       path: "",
-      ctPaths: "",
-      debugInfoToolPath: "",
-      ctRRWorkerExe: "",
     ).}:                                                  RRBackendConfig
     skipInstall:                                          bool
 

src/common/ct_logging.nim

@@ -17,16 +17,20 @@ let LOG_LEVEL* = parseEnum[CtLogLevel](env.get("CODETRACER_LOG_LEVEL", codetrace
 
 template debugPrint*(args: varargs[untyped]) =
   if LOG_LEVEL <= CtLogLevel.Debug:
-    echo args
+    let i = instantiationInfo(0)
+    echo "INFO [ct](", i.filename & ":" & $i.line, "):", args
 
 template infoPrint*(args: varargs[untyped]) =
   if LOG_LEVEL <= CtLogLevel.Info:
-    echo args
+    let i = instantiationInfo(0)
+    echo "INFO [ct](", i.filename & ":" & $i.line, "):", args
 
 template warnPrint*(args: varargs[untyped]) =
   if LOG_LEVEL <= CtLogLevel.Warn:
-    echo "[warn]: ", args
+    let i = instantiationInfo(0)
+    echo "WARN [ct](", i.filename & ":" & $i.line, "):", args
 
 template errorPrint*(args: varargs[untyped]) =
   if LOG_LEVEL <= CtLogLevel.Error:
-    echo "[error]: ", args
+    let i = instantiationInfo(0)
+    echo "ERROR [ct](", i.filename & ":" & $i.line, "):", args

src/config/default_config.yaml

@@ -114,9 +114,6 @@ bindings:
 rrBackend:
   enabled: false
   path: ""
-  ctPaths: ""
-  debugInfoToolPath: ""
-  ctRRWorkerExe: ""
 
 skipInstall: true
 

src/ct/cli/build.nim

@@ -28,20 +28,20 @@ proc setupEnv*(configPath: string): StringTableRef =
 proc build*(programPath: string, outputPath: string) =
   let ctConfig = loadConfig(folder=getCurrentDir(), inTest=false)
   if ctConfig.rrBackend.enabled:
-    let configPath = ctConfig.rrBackend.ctPaths
+    # TODO: is it still required for ct-rr-support?
+    # let configPath = ctConfig.rrBackend.ctPaths
 
     try:
-      var env = setupEnv(configPath)
+      # var env = setupEnv(configPath)
       let args = @["build", programPath, outputPath]
       let p = startProcess(
         ctConfig.rrBackend.path,
         args = args,
-        env = env,
+        # env = env,
         options = {poParentStreams, poStdErrToStdOut})
       quit(waitForExit(p))
     except:
-      errorPrint "ct helper error: ", getCurrentExceptionMsg()
-      errorPrint "  ct paths config path: ", configPath
+      errorPrint "ct build plugin error: ", getCurrentExceptionMsg()
       quit(1)
   else:
     errorPrint "This functionality requires a codetracer-rr-backend installation"

src/ct/db_backend_record.nim

@@ -52,6 +52,31 @@ proc recordSymbols(sourceDir: string, outputFolder: string, lang: Lang) =
     echo "WARNING: Can't extract symbols. Some functionality may not work correctly!"
     echo ""
 
+proc recordWithRR(
+    ctRRSupportExe: string,
+    program: string, args: seq[string],
+    traceFolder: string,
+    traceId: int): Trace =
+
+  createDir(traceFolder)
+  let traceMetadataPath = traceFolder / "trace_metadata.json"
+  let traceDbMetadataPath = traceFolder / "trace_db_metadata.json"
+  let process = startProcess(
+    ctRRSupportExe,
+    args = @[
+      "record", "-o", traceFolder, program
+    ].concat(args),
+    options = {poEchoCmd, poParentStreams}
+  )
+  let code = waitForExit(process)
+  if code != 0:
+    echo fmt"error: ct-rr-support returned exit code ", code
+    quit(code)
+
+  # record pid and lang in trace_db_metadata.json
+  result = importTrace(traceFolder, traceId, NO_PID, LangUnknown, DB_SELF_CONTAINED_DEFAULT, traceKind="rr")
+
+
 # rr patches for ruby/other vm-s: not supported now, instead
 # in db backend support only direct traces
 
@@ -155,7 +180,7 @@ proc recordDb(
     echo fmt"error: recorder exited with {exitCode} for {lang}"
     quit(1)
 
-  result = importDbTrace(traceMetadataPath, traceId, recordPid, lang, DB_SELF_CONTAINED_DEFAULT)
+  result = importTrace(traceFolder, traceId, recordPid, lang, DB_SELF_CONTAINED_DEFAULT, traceKind="db")
 
 
 # record a program run
@@ -164,6 +189,7 @@ proc record(
     langArg: Lang, backend: string, stylusTrace: string,
     test = false, basic = false,
     traceIDRecord: int = -1, customPath: string = "", outputFolderArg: string = "",
+    traceKind: string = "db", rrSupportPath: string = "",
     pythonInterpreter: string = "", pythonActivationPath: string = "", pythonWithDiff: bool = false): Trace =
   var traceID: int
   if traceIDRecord == -1:
@@ -201,12 +227,15 @@ proc record(
   let lang = detectLang(executable, langArg)
   # echo "in db ", lang, " ", executable
   if lang == LangUnknown:
-    errorMessage fmt"error: lang unknown: probably an unsupported type of project/extension, or folder/path doesn't exist?"
-    quit(1)
+    if traceKind == "db":
+      errorMessage fmt"error: lang unknown: probably an unsupported type of project/extension, or folder/path doesn't exist?"
+      quit(1)
   elif not lang.isDbBased:
     # TODO integrate with rr/gdb backend
-    errorMessage fmt"error: {lang} not supported currently!"
-    quit(1)
+    if traceKind == "db":
+      errorMessage fmt"error: {lang} not supported currently with db: maybe you need a rr trace for it?"
+      quit(1)
+
   let (executableDir, executableFile, executableExt) = executable.splitFile
   discard executableDir
   discard executableExt
@@ -281,6 +310,15 @@ proc record(
         stylusTrace,
         traceId,
         pythonActivationPath = activationPathResolved)
+    elif traceKind == "rr":
+      echo "rr"
+      echo rrSupportPath
+      return recordWithRR(
+        rrSupportPath,
+        executable,
+        args,
+        outputFolder,
+        traceId)
     else:
       echo fmt"ERROR: unsupported lang {lang}"
       quit(1)
@@ -364,6 +402,7 @@ proc main*(): Trace =
   #   [-e/--export <export-zip>] [-c/--cleanup-output-folder]
   #   [-t/--stylus-trace <trace-path>]
   #   [-a/--address <address>] [--socket <socket-path>]
+  #   [--trace-kind db/rr] [--rr-support-path <rr-support-path>]
   #   <program> [<args>]
   let args = os.commandLineParams()
   if args.len == 0:
@@ -385,6 +424,10 @@ proc main*(): Trace =
   var socketPath = ""
   var isExportedWithArg = false
   var pythonInterpreter = ""
+  var traceKind = "db" # by default
+  var rrSupportPath = ""
+
+  echo args
 
   # for i, arg in args:
   var i = 0
@@ -444,6 +487,18 @@ proc main*(): Trace =
         return
       socketPath = args[i + 1]
       i += 2
+    elif arg == "--trace-kind":
+      if args.len() < i + 2:
+        displayHelp()
+        return
+      traceKind = args[i + 1]
+      i += 2
+    elif arg == "--rr-support-path":
+      if args.len() < i + 2:
+        displayHelp()
+        return
+      rrSupportPath = args[i + 1]
+      i += 2
     else:
       if program == "":
         program = arg
@@ -532,6 +587,7 @@ proc main*(): Trace =
     var trace = record(
       program, recordArgs, "", lang, backend, stylusTrace,
       traceIDRecord=traceID, outputFolderArg=outputFolder,
+      traceKind=traceKind, rrSupportPath=rrSupportPath,
       pythonInterpreter=pythonInterpreter)
     traceId = trace.id
     outputFolder = trace.outputFolder

src/ct/online_sharing/download.nim

@@ -38,7 +38,7 @@ proc downloadTrace*(url: string): int =
   let lang = detectLang(pathValue.extractFilename, LangUnknown, isWasm)
   let recordPid = NO_PID # for now not processing the pid , but it can be 
   # accessed from trace metadata file if we need it in the future
-  discard importDbTrace(traceMetadataPath, traceId, recordPid, lang, DB_SELF_CONTAINED_DEFAULT, url)
+  discard importTrace(unzippedLocation, traceId, recordPid, lang, DB_SELF_CONTAINED_DEFAULT, url)
   return traceId
 
 proc downloadTraceCommand*(traceDownloadUrl: string) =

src/ct/trace/record.nim

@@ -173,18 +173,18 @@ else:
     let combined = if diagnostics.len > 0: diagnostics else: lines.join("\n")
     return (recorderError, version, combined)
 
-proc recordInternal(exe: string, args: seq[string], withDiff: string, upload: bool, configPath: string): Trace =
-  let env = if configPath.len > 0:
-      setupEnv(configPath)
-    else:
-      var env = newStringTable(modeStyleInsensitive)
-      for name, value in envPairs():
-        env[name] = value
-      env
+proc recordInternal(exe: string, args: seq[string], withDiff: string, upload: bool): Trace =
+  # let env = if configPath.len > 0:
+  #     setupEnv(configPath)
+  #   else:
+  #     var env = newStringTable(modeStyleInsensitive)
+  #     for name, value in envPairs():
+  #       env[name] = value
+  #     env
   let p = startProcess(
     exe,
     args = args,
-    env = env,
+    # env = env,
     options = {poStdErrToStdOut, poUsePath})
 
   let (lines, exCode) = p.readLines
@@ -298,11 +298,11 @@ proc record*(lang: string,
     putEnv("CODETRACER_WRAPPER_PID", $getCurrentProcessId())
 
   if detectedLang in @[LangRubyDb, LangNoir, LangRustWasm, LangCppWasm, LangSmall, LangPythonDb]:
-    return recordInternal(dbBackendRecordExe, pargs, withDiff, upload, "")
+    return recordInternal(dbBackendRecordExe, pargs.concat(@["--trace-kind", "db"]), withDiff, upload)
   else:
     let ctConfig = loadConfig(folder=getCurrentDir(), inTest=false)
     if ctConfig.rrBackend.enabled:
-      let configPath = ctConfig.rrBackend.ctPaths
-      return recordInternal(ctConfig.rrBackend.path, concat(@["record"], pargs), withDiff, upload, configPath)
+      # let configPath = ctConfig.rrBackend.ctPaths
+      return recordInternal(dbBackendRecordExe, pargs.concat(@["--trace-kind", "rr", "--rr-support-path", ctConfig.rrBackend.path]), withDiff, upload)
     else:
       echo "This functionality requires a codetracer-rr-backend installation"

src/ct/trace/replay.nim

@@ -139,7 +139,7 @@ proc replay*(
     trace = findTraceForArgs(patternArg, traceIdArg, traceFolderArg)
 
     if trace.isNil and traceFolderArg.isSome:
-      trace = importDbTrace(traceFolderArg.get() / "trace_metadata.json", NO_TRACE_ID, NO_PID, LangUnknown)
+      trace = importTrace(traceFolderArg.get(), NO_TRACE_ID, NO_PID, LangUnknown)
     if trace.isNil:
       echo "ERROR: can't find or import trace"
       quit(1)