From 34af36f5deb41cbc6e395d7cc438d6f4b9fb0fe6 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Fri, 14 Nov 2025 10:23:55 +0000 Subject: [PATCH 1/9] Adds stew and results --- .gitmodules | 6 ++++++ config.nims | 3 +++ vendor/results | 1 + vendor/stew | 1 + 4 files changed, 11 insertions(+) create mode 160000 vendor/results create mode 160000 vendor/stew diff --git a/.gitmodules b/.gitmodules index c5b29ce6b..a403c8907 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,9 @@ [submodule "vendor/chronos"] path = vendor/chronos url = https://github.com/status-im/nim-chronos.git +[submodule "vendor/results"] + path = vendor/results + url = https://github.com/arnetheduck/nim-results.git +[submodule "vendor/stew"] + path = vendor/stew + url = https://github.com/status-im/nim-stew.git diff --git a/config.nims b/config.nims index 773f7d9f6..7768532b1 100644 --- a/config.nims +++ b/config.nims @@ -6,5 +6,8 @@ switch("define", "ssl") switch("path", "vendor" / "zippy" / "src") switch("path", "vendor" / "sat" / "src") switch("path", "vendor" / "checksums" / "src") +switch("path", "vendor" / "chronos") +switch("path", "vendor" / "results") +switch("path", "vendor" / "stew") switch("define", "zippyNoSimd") diff --git a/vendor/results b/vendor/results new file mode 160000 index 000000000..df8113dda --- /dev/null +++ b/vendor/results @@ -0,0 +1 @@ +Subproject commit df8113dda4c2d74d460a8fa98252b0b771bf1f27 diff --git a/vendor/stew b/vendor/stew new file mode 160000 index 000000000..b66168735 --- /dev/null +++ b/vendor/stew @@ -0,0 +1 @@ +Subproject commit b66168735d6f3841c5239c3169d3fe5fe98b1257 From a2da799ea78e4313b70f8bbe2640399c1a522b91 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Fri, 14 Nov 2025 11:58:42 +0000 Subject: [PATCH 2/9] Builds pkgs binaries in parallel for vnext --- src/nimble.nim | 16 ++-- src/nimblepkg/asyncfileops.nim | 39 +++++++++ src/nimblepkg/cli.nim | 70 +++++++-------- src/nimblepkg/nimscriptexecutor.nim | 42 ++++----- src/nimblepkg/packageinfo.nim | 24 ++++-- src/nimblepkg/packageinfotypes.nim | 7 +- src/nimblepkg/vcstools.nim | 12 +-- src/nimblepkg/vnext.nim | 128 +++++++++++++++++++--------- 8 files changed, 221 insertions(+), 117 deletions(-) create mode 100644 src/nimblepkg/asyncfileops.nim diff --git a/src/nimble.nim b/src/nimble.nim index 18721bbf4..9373bbcf0 100644 --- a/src/nimble.nim +++ b/src/nimble.nim @@ -5,7 +5,7 @@ import os, tables, strtabs, json, browsers, algorithm, sets, uri, sugar, sequtil strformat import std/options as std_opt - +import chronos import strutils except toLower from unicode import toLower import sat/sat @@ -660,10 +660,14 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options, # Copy this package's files based on the preferences specified in PkgInfo. var filesInstalled: HashSet[string] iterInstallFiles(realDir, pkgInfo, options, - proc (file: string) = - createDir(changeRoot(realDir, pkgDestDir, file.splitFile.dir)) - let dest = changeRoot(realDir, pkgDestDir, file) - filesInstalled.incl copyFileD(file, dest) + proc (file: string) {.raises: [].} = + try: + createDir(changeRoot(realDir, pkgDestDir, file.splitFile.dir)) + let dest = changeRoot(realDir, pkgDestDir, file) + filesInstalled.incl copyFileD(file, dest) + except CatchableError: + #TODO print msg + discard ) # Copy the .nimble file. @@ -2568,7 +2572,7 @@ proc run(options: Options, nimBin: string) = # Use vnext buildPkg for develop mode packages let isInRootDir = options.startDir == pkgInfo.myPath.parentDir and options.satResult.rootPackage.basicInfo.name == pkgInfo.basicInfo.name - buildPkg(nimBin, pkgInfo, isInRootDir, options) + waitFor buildPkg(nimBin, pkgInfo, isInRootDir, options) if options.getCompilationFlags.len > 0: displayWarning(ignoringCompilationFlagsMsg) diff --git a/src/nimblepkg/asyncfileops.nim b/src/nimblepkg/asyncfileops.nim new file mode 100644 index 000000000..24307c20c --- /dev/null +++ b/src/nimblepkg/asyncfileops.nim @@ -0,0 +1,39 @@ +# Async file operations using chronos async processes +# Similar to Node.js - uses external commands for file I/O + +import std/os +import chronos except Duration +import chronos/asyncproc + +export chronos except Duration +export asyncproc + +proc copyFileAsync*(source, dest: string): Future[void] {.async: (raises: [CatchableError, AsyncProcessError, AsyncProcessTimeoutError, CancelledError]).} = + ## Async file copy using chronos async processes + when defined(windows): + # Windows: use xcopy for better handling + let cmd = "xcopy /Y /Q " & quoteShell(source) & " " & quoteShell(dest) & "*" + else: + # Unix: use cp command with preserve permissions and recursive for dirs + let cmd = "cp -f -p -r " & quoteShell(source) & " " & quoteShell(dest) + + let exitCode = await execCommand(cmd) + if exitCode != 0: + raise newException(IOError, "Failed to copy file from " & source & " to " & dest & " (exit code: " & $exitCode & ")") + +proc copyDirAsync*(sourceDir, destDir: string): Future[void] {.async: (raises: [CatchableError, AsyncProcessError, AsyncProcessTimeoutError, CancelledError]).} = + ## Async directory copy using chronos async processes - copies entire directory tree + when defined(windows): + # Windows: use robocopy for robust directory copying + # /E = copy subdirs including empty, /NFL = no file list, /NDL = no dir list, /NJH = no job header, /NJS = no job summary, /NC = no class, /NS = no size, /NP = no progress + let cmd = "robocopy " & quoteShell(sourceDir) & " " & quoteShell(destDir) & " /E /NFL /NDL /NJH /NJS /NC /NS /NP" + let exitCode = await execCommand(cmd) + # robocopy exit codes: 0-7 are success (0=no files, 1=files copied, 2=extra files, etc.) + if exitCode > 7: + raise newException(IOError, "Failed to copy directory from " & sourceDir & " to " & destDir & " (exit code: " & $exitCode & ")") + else: + # Unix: use cp -r to copy entire directory recursively + let cmd = "cp -r -p " & quoteShell(sourceDir) & "/. " & quoteShell(destDir) + let exitCode = await execCommand(cmd) + if exitCode != 0: + raise newException(IOError, "Failed to copy directory from " & sourceDir & " to " & destDir & " (exit code: " & $exitCode & ")") diff --git a/src/nimblepkg/cli.nim b/src/nimblepkg/cli.nim index de67b921d..6cf7ab54c 100644 --- a/src/nimblepkg/cli.nim +++ b/src/nimblepkg/cli.nim @@ -76,22 +76,23 @@ proc displayInfoLine*(field, msg: string) = proc displayCategory(category: string, displayType: DisplayType, priority: Priority) = - if isSuppressed(displayType): - return + {.cast(gcsafe).}: + if isSuppressed(displayType): + return - # Calculate how much the `category` must be offset to align along a center - # line. - let offset = calculateCategoryOffset(category) - - # Display the category. - let text = "$1$2 " % [spaces(offset), category] - if globalCLI.showColor: - if priority != DebugPriority: - setForegroundColor(stdout, foregrounds[displayType]) - writeStyled(text, styles[priority]) - resetAttributes() - else: - stdout.write(text) + # Calculate how much the `category` must be offset to align along a center + # line. + let offset = calculateCategoryOffset(category) + + # Display the category. + let text = "$1$2 " % [spaces(offset), category] + if globalCLI.showColor: + if priority != DebugPriority: + setForegroundColor(stdout, foregrounds[displayType]) + writeStyled(text, styles[priority]) + resetAttributes() + else: + stdout.write(text) proc displayLine(category, line: string, displayType: DisplayType, @@ -106,27 +107,28 @@ proc displayLine(category, line: string, displayType: DisplayType, proc display*(category, msg: string, displayType = Message, priority = MediumPriority) = - # Multiple warnings containing the same messages should not be shown. - let warningPair = (category, msg) - if displayType == Warning: - if warningPair in globalCLI.warnings: - return - else: - globalCLI.warnings.incl(warningPair) + {.cast(gcsafe).}: + # Multiple warnings containing the same messages should not be shown. + let warningPair = (category, msg) + if displayType == Warning: + if warningPair in globalCLI.warnings: + return + else: + globalCLI.warnings.incl(warningPair) - # Suppress this message if its priority isn't high enough. - # TODO: Per-priority suppression counts? - if priority < globalCLI.level: - if priority != DebugPriority: - globalCLI.suppressionCount.inc - return + # Suppress this message if its priority isn't high enough. + # TODO: Per-priority suppression counts? + if priority < globalCLI.level: + if priority != DebugPriority: + globalCLI.suppressionCount.inc + return - # Display each line in the message. - var i = 0 - for line in msg.splitLines(): - if len(line) == 0: continue - displayLine(if i == 0: category else: "...", line, displayType, priority) - i.inc + # Display each line in the message. + var i = 0 + for line in msg.splitLines(): + if len(line) == 0: continue + displayLine(if i == 0: category else: "...", line, displayType, priority) + i.inc proc displayWarning*(message: string, priority = HighPriority) = display("Warning: ", message, Warning, priority) diff --git a/src/nimblepkg/nimscriptexecutor.nim b/src/nimblepkg/nimscriptexecutor.nim index 293d10187..4a5f3545a 100644 --- a/src/nimblepkg/nimscriptexecutor.nim +++ b/src/nimblepkg/nimscriptexecutor.nim @@ -5,30 +5,32 @@ import os, strutils, sets import packageparser, common, options, nimscriptwrapper, cli -proc execHook*(nimBin: string, options: Options, hookAction: ActionType, before: bool): bool = +proc execHook*(nimBin: string, options: Options, hookAction: ActionType, before: bool): bool {. raises: [].} = ## Returns whether to continue. result = true + {.cast(gcsafe).}: + # For certain commands hooks should not be evaluated. + if hookAction in noHookActions: + return - # For certain commands hooks should not be evaluated. - if hookAction in noHookActions: - return + var nimbleFile = "" + try: + nimbleFile = findNimbleFile(getCurrentDir(), true, options) - var nimbleFile = "" - try: - nimbleFile = findNimbleFile(getCurrentDir(), true, options) - except NimbleError: return true - # PackageInfos are cached so we can read them as many times as we want. - let pkgInfo = getPkgInfoFromFile(nimBin, nimbleFile, options) - let actionName = - if hookAction == actionCustom: options.action.command - else: ($hookAction)[6 .. ^1] - let hookExists = - if before: actionName.normalize in pkgInfo.preHooks - else: actionName.normalize in pkgInfo.postHooks - if pkgInfo.isNimScript and hookExists: - let res = execHook(nimBin, nimbleFile, actionName, before, options) - if res.success: - result = res.retVal + # PackageInfos are cached so we can read them as many times as we want. + let pkgInfo = getPkgInfoFromFile(nimBin, nimbleFile, options) + let actionName = + if hookAction == actionCustom: options.action.command + else: ($hookAction)[6 .. ^1] + let hookExists = + if before: actionName.normalize in pkgInfo.preHooks + else: actionName.normalize in pkgInfo.postHooks + if pkgInfo.isNimScript and hookExists: + let res = execHook(nimBin, nimbleFile, actionName, before, options) + if res.success: + result = res.retVal + except NimbleError: return true + except Exception: return false #TODO fix the propagation of Exception proc execCustom*(nimBin: string, nimbleFile: string, options: Options, execResult: var ExecutionResult[bool]): bool = diff --git a/src/nimblepkg/packageinfo.nim b/src/nimblepkg/packageinfo.nim index 7928eb0c2..e47f947d8 100644 --- a/src/nimblepkg/packageinfo.nim +++ b/src/nimblepkg/packageinfo.nim @@ -444,7 +444,7 @@ proc checkInstallDir(pkgInfo: PackageInfo, if thisDir == "nimcache": result = true proc iterFilesWithExt(dir: string, pkgInfo: PackageInfo, - action: proc (f: string)) = + action: proc (f: string): void {.raises: [].}) = ## Runs `action` for each filename of the files that have a whitelisted ## file extension. for kind, path in walkDir(dir): @@ -454,7 +454,7 @@ proc iterFilesWithExt(dir: string, pkgInfo: PackageInfo, if path.splitFile.ext.substr(1) in pkgInfo.installExt: action(path) -proc iterFilesInDir(dir: string, action: proc (f: string)) = +proc iterFilesInDir(dir: string, action: proc (f: string): void {.raises: [].}) = ## Runs `action` for each file in ``dir`` and any ## subdirectories that are in it. for kind, path in walkDir(dir): @@ -520,7 +520,7 @@ proc iterInstallFilesSimple*(realDir: string, pkgInfo: PackageInfo, action(file) proc iterInstallFiles*(realDir: string, pkgInfo: PackageInfo, - options: Options, action: proc (f: string)) = + options: Options, action: proc (f: string): void {.raises: [].}) = ## Runs `action` for each file within the ``realDir`` that should be ## installed. # Get the package root directory for skipDirs comparison @@ -670,9 +670,12 @@ proc needsRebuild*(pkgInfo: PackageInfo, bin: string, dir: string, options: Opti var rebuild = false iterFilesWithExt(dir, pkgInfo, proc (file: string) = - let srcTimestamp = getFileInfo(file).lastWriteTime - if binTimestamp < srcTimestamp: - rebuild = true + try: + let srcTimestamp = getFileInfo(file).lastWriteTime + if binTimestamp < srcTimestamp: + rebuild = true + except OSError: + discard ) return rebuild else: @@ -685,9 +688,12 @@ proc needsRebuild*(pkgInfo: PackageInfo, bin: string, dir: string, options: Opti var rebuild = false iterFilesWithExt(dir, pkgInfo, proc (file: string) = - let srcTimestamp = getFileInfo(file).lastWriteTime - if binTimestamp < srcTimestamp: - rebuild = true + try: + let srcTimestamp = getFileInfo(file).lastWriteTime + if binTimestamp < srcTimestamp: + rebuild = true + except OSError: + discard ) return rebuild diff --git a/src/nimblepkg/packageinfotypes.nim b/src/nimblepkg/packageinfotypes.nim index a73bc6c2d..aabcd9258 100644 --- a/src/nimblepkg/packageinfotypes.nim +++ b/src/nimblepkg/packageinfotypes.nim @@ -169,9 +169,10 @@ proc appendGloballyActiveFeatures*(pkgName: string, features: seq[string]) = proc getGloballyActiveFeatures*(): seq[string] = #returns features.{pkgName}.{feature} - for pkgName, features in globallyActiveFeatures: - for feature in features: - result.add(&"features.{pkgName}.{feature}") + {.cast(gcsafe).}: + for pkgName, features in globallyActiveFeatures: + for feature in features: + result.add(&"features.{pkgName}.{feature}") proc initSATResult*(pass: SATPass): SATResult = SATResult(pkgsToInstall: @[], solvedPkgs: @[], output: "", pkgs: initHashSet[PackageInfo](), diff --git a/src/nimblepkg/vcstools.nim b/src/nimblepkg/vcstools.nim index e2b921ed1..474f45bde 100644 --- a/src/nimblepkg/vcstools.nim +++ b/src/nimblepkg/vcstools.nim @@ -192,13 +192,13 @@ proc getVcsRevision*(dir: Path): Sha1Hash = ## - the external command fails. ## - the directory does not exist. ## - there is no vcsRevisions in the repository. + {.cast(gcsafe).}: + let vcsRevision = tryDoVcsCmd(dir, + gitCmd = "rev-parse HEAD", + hgCmd = "id -i --debug", + noVcsAction = $notSetSha1Hash) - let vcsRevision = tryDoVcsCmd(dir, - gitCmd = "rev-parse HEAD", - hgCmd = "id -i --debug", - noVcsAction = $notSetSha1Hash) - - return initSha1Hash(vcsRevision.strip(chars = Whitespace + {'+'})) + return initSha1Hash(vcsRevision.strip(chars = Whitespace + {'+'})) proc getVcsRevisions*(dir: Path): Sha1Hash = ## Returns current revision number if the directory `dir` is under version diff --git a/src/nimblepkg/vnext.nim b/src/nimblepkg/vnext.nim index 9034524c9..f4ac4653e 100644 --- a/src/nimblepkg/vnext.nim +++ b/src/nimblepkg/vnext.nim @@ -16,7 +16,7 @@ import std/[sequtils, sets, options, os, strutils, tables, strformat, algorithm] import nimblesat, packageinfotypes, options, version, declarativeparser, packageinfo, common, nimenv, lockfile, cli, downloadnim, packageparser, tools, nimscriptexecutor, packagemetadatafile, displaymessages, packageinstaller, reversedeps, developfile, urls - +import chronos, chronos/asyncproc when defined(windows): import std/strscans @@ -755,10 +755,11 @@ proc getPathsToBuildFor*(satResult: SATResult, pkgInfo: PackageInfo, recursive: result.incl(pkgInfo.expandPaths(nimBin, options)) proc getPathsAllPkgs*(options: Options): HashSet[string] = - let satResult = options.satResult - for pkg in satResult.pkgs: - for path in pkg.expandPaths(satResult.nimResolved.getNimBin(), options): - result.incl(path) + {.cast(gcsafe).}: + let satResult = options.satResult + for pkg in satResult.pkgs: + for path in pkg.expandPaths(satResult.nimResolved.getNimBin(), options): + result.incl(path) proc getNimBin(satResult: SATResult): string = #TODO change this so nim is passed as a parameter but we also need to change getPkgInfo so for the time being its also in options @@ -772,7 +773,7 @@ proc getNimBin(satResult: SATResult): string = raise newNimbleError[NimbleError]("No Nim found") proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], - args: seq[string], options: Options, nimBin: string) = + args: seq[string], options: Options, nimBin: string) {.async, raises: [].} = ## Builds a package as specified by ``pkgInfo``. # Handle pre-`build` hook. let @@ -822,6 +823,14 @@ proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], options.getCompilationBinary(pkgInfo).get("") else: "" + # Collect all binary build tasks to run in parallel + type BuildTask = object + bin: string + cmd: string + future: Future[CommandExResponse] + + var buildTasks: seq[BuildTask] + for bin, src in pkgInfo.bin: # Check if this is the only binary that we want to build. if binToBuild.len != 0 and binToBuild != bin: @@ -831,43 +840,45 @@ proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], let outputDir = pkgInfo.getOutputDir("") if dirExists(outputDir): if fileExists(outputDir / bin): - if not pkgInfo.needsRebuild(outputDir / bin, realDir, options): - display("Skipping", "$1/$2 (up-to-date)" % - [pkginfo.basicInfo.name, bin], priority = HighPriority) - binariesBuilt.inc() - continue + {.cast(gcsafe).}: + if not pkgInfo.needsRebuild(outputDir / bin, realDir, options): + display("Skipping", "$1/$2 (up-to-date)" % + [pkginfo.basicInfo.name, bin], priority = HighPriority) + binariesBuilt.inc() + continue else: createDir(outputDir) # Check if we can copy an existing binary from source directory when --noRebuild is used - if options.action.typ in {actionInstall, actionPath, actionUninstall, actionDevelop, actionUpgrade, actionLock, actionAdd} and + if options.action.typ in {actionInstall, actionPath, actionUninstall, actionDevelop, actionUpgrade, actionLock, actionAdd} and options.action.noRebuild: # When installing from a local directory, check for binary in the original directory - let sourceBinary = + let sourceBinary = if options.startDir != pkgDir: options.startDir / bin else: pkgDir / bin - + if fileExists(sourceBinary): # Check if the source binary is up-to-date - if not pkgInfo.needsRebuild(sourceBinary, realDir, options): - let targetBinary = outputDir / bin - display("Skipping", "$1/$2 (up-to-date)" % - [pkginfo.basicInfo.name, bin], priority = HighPriority) - copyFile(sourceBinary, targetBinary) - when not defined(windows): - # Preserve executable permissions - setFilePermissions(targetBinary, getFilePermissions(sourceBinary)) - binariesBuilt.inc() - continue + {.cast(gcsafe).}: + if not pkgInfo.needsRebuild(sourceBinary, realDir, options): + let targetBinary = outputDir / bin + display("Skipping", "$1/$2 (up-to-date)" % + [pkginfo.basicInfo.name, bin], priority = HighPriority) + copyFile(sourceBinary, targetBinary) + when not defined(windows): + # Preserve executable permissions + setFilePermissions(targetBinary, getFilePermissions(sourceBinary)) + binariesBuilt.inc() + continue let outputOpt = "-o:" & pkgInfo.getOutputDir(bin).quoteShell display("Building", "$1/$2 using $3 backend" % [pkginfo.basicInfo.name, bin, pkgInfo.backend], priority = HighPriority) # For installed packages, we need to handle srcDir correctly - let input = + let input = if pkgInfo.isInstalled and not pkgInfo.isLink and pkgInfo.srcDir != "": # For installed packages with srcDir, the source file is in srcDir realDir / pkgInfo.srcDir / src.changeFileExt("nim") @@ -878,13 +889,42 @@ proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], let cmd = "$# $# --colors:$# --noNimblePath $# $# $#" % [ options.satResult.getNimBin().quoteShell, pkgInfo.backend, if options.noColor: "off" else: "on", join(args, " "), outputOpt, input.quoteShell] - try: - # echo "***Executing cmd: ", cmd - doCmd(cmd) - binariesBuilt.inc() - except CatchableError as error: - raise buildFailed( - &"Build failed for the package: {pkgInfo.basicInfo.name}", details = error) + + display("Executing", cmd, priority = MediumPriority) + + # Start async build and collect future + var task: BuildTask + task.bin = bin + task.cmd = cmd + task.future = execCommandEx(cmd) + buildTasks.add(task) + + # Wait for all builds to complete in parallel + if buildTasks.len > 0: + let futures = buildTasks.mapIt(it.future) + discard await allFinished(futures) + + # Process results + for task in buildTasks: + try: + let response = task.future.read() + + # Display output + if response.stdOutput.len > 0: + display("Nim Output", response.stdOutput, priority = HighPriority) + if response.stdError.len > 0: + display("Nim Stderr", response.stdError, priority = HighPriority) + + # Check exit code + if response.status != QuitSuccess: + raise nimbleError( + "Execution failed with exit code $1\nCommand: $2" % + [$response.status, task.cmd]) + + binariesBuilt.inc() + except CatchableError as error: + raise buildFailed( + &"Build failed for package: {pkgInfo.basicInfo.name}, binary: {task.bin}", details = error) if binariesBuilt == 0: let binary = options.getCompilationBinary(pkgInfo).get("") @@ -960,9 +1000,11 @@ proc solutionToFullInfo*(satResult: SATResult, options: var Options) {.instrumen proc isRoot(pkgInfo: PackageInfo, satResult: SATResult): bool = pkgInfo.basicInfo.name == satResult.rootPackage.basicInfo.name and pkgInfo.basicInfo.version == satResult.rootPackage.basicInfo.version -proc buildPkg*(nimBin: string, pkgToBuild: PackageInfo, isRootInRootDir: bool, options: Options) {.instrument.} = +proc buildPkg*(nimBin: string, pkgToBuild: PackageInfo, isRootInRootDir: bool, options: Options) {.async.} = # let paths = getPathsToBuildFor(options.satResult, pkgToBuild, recursive = true, options) - let paths = getPathsAllPkgs(options) + let paths = try: getPathsAllPkgs(options) + except Exception: + initHashSet[string]() # echo "Paths ", paths # echo "Requires ", pkgToBuild.requires # echo "Package ", pkgToBuild.basicInfo.name @@ -975,12 +1017,15 @@ proc buildPkg*(nimBin: string, pkgToBuild: PackageInfo, isRootInRootDir: bool, o var pkgToBuild = pkgToBuild if isRootInRootDir: pkgToBuild.isInstalled = false - buildFromDir(pkgToBuild, paths, "-d:release" & flags, options, nimBin) + await buildFromDir(pkgToBuild, paths, "-d:release" & flags, options, nimBin) # For globally installed packages, always create symlinks # Only skip symlinks if we're building the root package in its own directory let shouldCreateSymlinks = not isRootInRootDir or options.action.typ == actionInstall if shouldCreateSymlinks: - createBinSymlink(pkgToBuild, options) + try: + createBinSymlink(pkgToBuild, options) + except Exception: + display("Error creating bin symlink", "Error creating bin symlink: " & getCurrentExceptionMsg(), Error, HighPriority) proc getVersionRangeFoPkgToInstall(satResult: SATResult, name: string, ver: Version): VersionRange = if satResult.rootPackage.basicInfo.name == name and satResult.rootPackage.basicInfo.version == ver: @@ -1125,12 +1170,14 @@ proc installPkgs*(satResult: var SATResult, options: var Options) {.instrument.} satResult.installedPkgs = installedPkgs.toSeq() for pkgInfo in satResult.installedPkgs: - # Run before-install hook now that package before the build step but after the package is copied over to the + # Run before-install hook now that package before the build step but after the package is copied over to the #install dir. let hookDir = pkgInfo.myPath.splitFile.dir if dirExists(hookDir): executeHook(nimBin, hookDir, options, actionInstall, before = true) + # Collect all build futures to run in parallel + var futPkgsToBuild = newSeq[Future[void]]() for pkgToBuild in pkgsToBuild: if pkgToBuild.bin.len == 0: if options.action.typ == actionBuild: @@ -1142,13 +1189,16 @@ proc installPkgs*(satResult: var SATResult, options: var Options) {.instrument.} # echo "Building package: ", pkgToBuild.basicInfo.name, " at ", pkgToBuild.myPath, " binaries: ", pkgToBuild.bin let isRoot = pkgToBuild.isRoot(options.satResult) and isInRootDir if isRoot and options.action.typ in rootBuildActions: - buildPkg(nimBin, pkgToBuild, isRoot, options) + futPkgsToBuild.add(buildPkg(nimBin, pkgToBuild, isRoot, options)) satResult.buildPkgs.add(pkgToBuild) elif not isRoot: #Build non root package for all actions that requires the package as a dependency - buildPkg(nimBin, pkgToBuild, isRoot, options) + futPkgsToBuild.add(buildPkg(nimBin, pkgToBuild, isRoot, options)) satResult.buildPkgs.add(pkgToBuild) + measureTime "Packages built in ", false: + waitFor allFutures(futPkgsToBuild) + for pkg in satResult.installedPkgs.mitems: satResult.pkgs.incl pkg From e23b89641fa274884376673249b310185ee0eea7 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Fri, 14 Nov 2025 12:03:35 +0000 Subject: [PATCH 3/9] Fix binaries ci --- src/nimblepkg/vnext.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nimblepkg/vnext.nim b/src/nimblepkg/vnext.nim index f4ac4653e..a4ca61916 100644 --- a/src/nimblepkg/vnext.nim +++ b/src/nimblepkg/vnext.nim @@ -773,7 +773,7 @@ proc getNimBin(satResult: SATResult): string = raise newNimbleError[NimbleError]("No Nim found") proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], - args: seq[string], options: Options, nimBin: string) {.async, raises: [].} = + args: seq[string], options: Options, nimBin: string): Future[void] {.async: (raises: [CatchableError, AsyncProcessError, AsyncProcessTimeoutError, CancelledError, Exception]).} = ## Builds a package as specified by ``pkgInfo``. # Handle pre-`build` hook. let @@ -1000,7 +1000,7 @@ proc solutionToFullInfo*(satResult: SATResult, options: var Options) {.instrumen proc isRoot(pkgInfo: PackageInfo, satResult: SATResult): bool = pkgInfo.basicInfo.name == satResult.rootPackage.basicInfo.name and pkgInfo.basicInfo.version == satResult.rootPackage.basicInfo.version -proc buildPkg*(nimBin: string, pkgToBuild: PackageInfo, isRootInRootDir: bool, options: Options) {.async.} = +proc buildPkg*(nimBin: string, pkgToBuild: PackageInfo, isRootInRootDir: bool, options: Options): Future[void] {.async: (raises: [CatchableError, AsyncProcessError, AsyncProcessTimeoutError, CancelledError, Exception]).} = # let paths = getPathsToBuildFor(options.satResult, pkgToBuild, recursive = true, options) let paths = try: getPathsAllPkgs(options) except Exception: From 86a0243bf2309a85522391215dae4a4b78a41ab6 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Mon, 17 Nov 2025 09:01:17 +0000 Subject: [PATCH 4/9] Attempt to fix ci --- src/nimble.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nimble.nim b/src/nimble.nim index 9373bbcf0..98473a0d9 100644 --- a/src/nimble.nim +++ b/src/nimble.nim @@ -665,8 +665,7 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options, createDir(changeRoot(realDir, pkgDestDir, file.splitFile.dir)) let dest = changeRoot(realDir, pkgDestDir, file) filesInstalled.incl copyFileD(file, dest) - except CatchableError: - #TODO print msg + except Exception: discard ) From 18e5886256544ae127d42f228a18984c480e7510 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Mon, 17 Nov 2025 12:06:36 +0000 Subject: [PATCH 5/9] Attempt to fix ci --- src/nimblepkg/vnext.nim | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/nimblepkg/vnext.nim b/src/nimblepkg/vnext.nim index a4ca61916..758a1c2bb 100644 --- a/src/nimblepkg/vnext.nim +++ b/src/nimblepkg/vnext.nim @@ -712,7 +712,8 @@ proc installFromDirDownloadInfo(nimBin: string,downloadDir: string, url: string, display("Warning:", "Skipped copy in project local deps mode", Warning) pkgInfo.isInstalled = true - displaySuccess(pkgInstalledMsg(pkgInfo.basicInfo.name), MediumPriority) + if pkgInfo.bin.len == 0: + displaySuccess(pkgInstalledMsg(pkgInfo.basicInfo.name), MediumPriority) pkgInfo proc activateSolvedPkgFeatures*(satResult: SATResult, options: Options) = @@ -1196,9 +1197,20 @@ proc installPkgs*(satResult: var SATResult, options: var Options) {.instrument.} futPkgsToBuild.add(buildPkg(nimBin, pkgToBuild, isRoot, options)) satResult.buildPkgs.add(pkgToBuild) - measureTime "Packages built in ", false: - waitFor allFutures(futPkgsToBuild) - + measureTime "Packages built in ", false: + waitFor allFutures(futPkgsToBuild) + + # Check each future for failures - allFutures doesn't propagate exceptions + for i, fut in futPkgsToBuild: + if fut.failed(): + # Re-raise the first build failure we encounter + raise fut.error() + + # Display success messages after builds complete successfully + # Only show for packages that were actually built (have binaries) + for pkgInfo in pkgsToBuild: + displaySuccess(pkgInstalledMsg(pkgInfo.basicInfo.name), MediumPriority) + for pkg in satResult.installedPkgs.mitems: satResult.pkgs.incl pkg From 857a02cd9541bdcd76f81cf6e76d478185b5ff3b Mon Sep 17 00:00:00 2001 From: jmgomez Date: Mon, 17 Nov 2025 16:23:15 +0000 Subject: [PATCH 6/9] fixes issue --- src/nimblepkg/vnext.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nimblepkg/vnext.nim b/src/nimblepkg/vnext.nim index 758a1c2bb..42102dd3e 100644 --- a/src/nimblepkg/vnext.nim +++ b/src/nimblepkg/vnext.nim @@ -774,7 +774,7 @@ proc getNimBin(satResult: SATResult): string = raise newNimbleError[NimbleError]("No Nim found") proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], - args: seq[string], options: Options, nimBin: string): Future[void] {.async: (raises: [CatchableError, AsyncProcessError, AsyncProcessTimeoutError, CancelledError, Exception]).} = + args: seq[string], options: Options, nimBin: string): Future[void] {.async: (raises: [CatchableError]).} = ## Builds a package as specified by ``pkgInfo``. # Handle pre-`build` hook. let @@ -1001,7 +1001,7 @@ proc solutionToFullInfo*(satResult: SATResult, options: var Options) {.instrumen proc isRoot(pkgInfo: PackageInfo, satResult: SATResult): bool = pkgInfo.basicInfo.name == satResult.rootPackage.basicInfo.name and pkgInfo.basicInfo.version == satResult.rootPackage.basicInfo.version -proc buildPkg*(nimBin: string, pkgToBuild: PackageInfo, isRootInRootDir: bool, options: Options): Future[void] {.async: (raises: [CatchableError, AsyncProcessError, AsyncProcessTimeoutError, CancelledError, Exception]).} = +proc buildPkg*(nimBin: string, pkgToBuild: PackageInfo, isRootInRootDir: bool, options: Options): Future[void] {.async: (raises: [CatchableError]).} = # let paths = getPathsToBuildFor(options.satResult, pkgToBuild, recursive = true, options) let paths = try: getPathsAllPkgs(options) except Exception: @@ -1026,7 +1026,7 @@ proc buildPkg*(nimBin: string, pkgToBuild: PackageInfo, isRootInRootDir: bool, o try: createBinSymlink(pkgToBuild, options) except Exception: - display("Error creating bin symlink", "Error creating bin symlink: " & getCurrentExceptionMsg(), Error, HighPriority) + displayError("Error creating bin symlink: " & getCurrentExceptionMsg()) proc getVersionRangeFoPkgToInstall(satResult: SATResult, name: string, ver: Version): VersionRange = if satResult.rootPackage.basicInfo.name == name and satResult.rootPackage.basicInfo.version == ver: From a59ee801d30873644a81cece4864ee4e10f7cdd4 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Tue, 18 Nov 2025 10:14:32 +0000 Subject: [PATCH 7/9] Refactors {.cast(gcsafe).} to threadvars when possible --- src/nimblepkg/cli.nim | 94 +++++++++--------- src/nimblepkg/declarativeparser.nim | 144 ++++++++++++++-------------- src/nimblepkg/nimscriptexecutor.nim | 43 ++++----- src/nimblepkg/packageinfo.nim | 4 +- src/nimblepkg/packageinfotypes.nim | 16 ++-- src/nimblepkg/vcstools.nim | 13 ++- src/nimblepkg/vnext.nim | 41 ++++---- 7 files changed, 180 insertions(+), 175 deletions(-) diff --git a/src/nimblepkg/cli.nim b/src/nimblepkg/cli.nim index 6cf7ab54c..121424121 100644 --- a/src/nimblepkg/cli.nim +++ b/src/nimblepkg/cli.nim @@ -48,7 +48,13 @@ proc newCLI(): CLI = showColor: true, ) -var globalCLI = newCLI() +var globalCLI {.threadvar.}: CLI + +proc getGlobalCLI(): CLI = + if globalCLI == nil: + globalCLI = newCLI() + return globalCLI + proc calculateCategoryOffset(category: string): int = assert category.len <= longestCategory @@ -57,14 +63,14 @@ proc calculateCategoryOffset(category: string): int = proc isSuppressed(displayType: DisplayType): bool = # Don't print any Warning, Message or Success messages when suppression of # warnings is enabled. That is, unless the user asked for --verbose output. - if globalCLI.suppressMessages and displayType >= Warning and - globalCLI.level == HighPriority: + if getGlobalCLI().suppressMessages and displayType >= Warning and + getGlobalCLI().level == HighPriority: return true proc displayFormatted*(displayType: DisplayType, msgs: varargs[string]) = ## for styling outputs lines using the DisplayTypes for msg in msgs: - if globalCLI.showColor: + if getGlobalCLI().showColor: stdout.styledWrite(foregrounds[displayType], msg) else: stdout.write(msg) @@ -76,23 +82,22 @@ proc displayInfoLine*(field, msg: string) = proc displayCategory(category: string, displayType: DisplayType, priority: Priority) = - {.cast(gcsafe).}: - if isSuppressed(displayType): - return + if isSuppressed(displayType): + return - # Calculate how much the `category` must be offset to align along a center - # line. - let offset = calculateCategoryOffset(category) - - # Display the category. - let text = "$1$2 " % [spaces(offset), category] - if globalCLI.showColor: - if priority != DebugPriority: - setForegroundColor(stdout, foregrounds[displayType]) - writeStyled(text, styles[priority]) - resetAttributes() - else: - stdout.write(text) + # Calculate how much the `category` must be offset to align along a center + # line. + let offset = calculateCategoryOffset(category) + + # Display the category. + let text = "$1$2 " % [spaces(offset), category] + if getGlobalCLI().showColor: + if priority != DebugPriority: + setForegroundColor(stdout, foregrounds[displayType]) + writeStyled(text, styles[priority]) + resetAttributes() + else: + stdout.write(text) proc displayLine(category, line: string, displayType: DisplayType, @@ -107,28 +112,27 @@ proc displayLine(category, line: string, displayType: DisplayType, proc display*(category, msg: string, displayType = Message, priority = MediumPriority) = - {.cast(gcsafe).}: - # Multiple warnings containing the same messages should not be shown. - let warningPair = (category, msg) - if displayType == Warning: - if warningPair in globalCLI.warnings: - return - else: - globalCLI.warnings.incl(warningPair) - - # Suppress this message if its priority isn't high enough. - # TODO: Per-priority suppression counts? - if priority < globalCLI.level: - if priority != DebugPriority: - globalCLI.suppressionCount.inc + # Multiple warnings containing the same messages should not be shown. + let warningPair = (category, msg) + if displayType == Warning: + if warningPair in getGlobalCLI().warnings: return + else: + getGlobalCLI().warnings.incl(warningPair) + + # Suppress this message if its priority isn't high enough. + # TODO: Per-priority suppression counts? + if priority < getGlobalCLI().level: + if priority != DebugPriority: + getGlobalCLI().suppressionCount.inc + return - # Display each line in the message. - var i = 0 - for line in msg.splitLines(): - if len(line) == 0: continue - displayLine(if i == 0: category else: "...", line, displayType, priority) - i.inc + # Display each line in the message. + var i = 0 + for line in msg.splitLines(): + if len(line) == 0: continue + displayLine(if i == 0: category else: "...", line, displayType, priority) + i.inc proc displayWarning*(message: string, priority = HighPriority) = display("Warning: ", message, Warning, priority) @@ -175,7 +179,7 @@ proc displayDebug*(msg: string) = proc displayTip*() = ## Called just before Nimble exits. Shows some tips for the user, for example ## the amount of messages that were suppressed and how to show them. - if globalCLI.suppressionCount > 0: + if getGlobalCLI().suppressionCount > 0: let msg = "$1 messages have been suppressed, use --verbose to show them." % $globalCLI.suppressionCount display("Tip:", msg, Warning, HighPriority) @@ -189,7 +193,7 @@ proc prompt*(forcePrompts: ForcePrompt, question: string): bool = display("Prompt:", question & " -> [forced no]", Warning, HighPriority) return false of dontForcePrompt: - if globalCLI.level > SilentPriority: + if getGlobalCLI().level > SilentPriority: display("Prompt:", question & " [y/N]", Warning, HighPriority) displayCategory("Answer:", Warning, HighPriority) let yn = stdin.readLine() @@ -323,10 +327,10 @@ proc promptList*(forcePrompts: ForcePrompt, question: string, args: openarray[st return promptListFallback(question, args) proc setVerbosity*(level: Priority) = - globalCLI.level = level + getGlobalCLI().level = level proc setShowColor*(val: bool) = - globalCLI.showColor = val + getGlobalCLI().showColor = val proc setSuppressMessages*(val: bool) = - globalCLI.suppressMessages = val + getGlobalCLI().suppressMessages = val diff --git a/src/nimblepkg/declarativeparser.nim b/src/nimblepkg/declarativeparser.nim index ab88ac0ca..53f444c16 100644 --- a/src/nimblepkg/declarativeparser.nim +++ b/src/nimblepkg/declarativeparser.nim @@ -210,24 +210,25 @@ proc getNimCompilationPath*(nimbleFile: string): string = let fileIdx = fileInfoIdx(conf, AbsoluteFile nimbleFile) var parser: Parser var includePath = "" - if setupParser(parser, fileIdx, newIdentCache(), conf): - let ast = parseAll(parser) - proc findIncludePath(n: PNode) = - case n.kind - of nkStmtList, nkStmtListExpr: - for child in n: - findIncludePath(child) - of nkIncludeStmt: - # Found an include statement - if n.len > 0 and n[0].kind in {nkStrLit..nkTripleStrLit}: - includePath = n[0].strVal - # echo "Found include: ", includePath - else: - for i in 0.. 0 and n[0].kind in {nkStrLit..nkTripleStrLit}: + includePath = n[0].strVal + # echo "Found include: ", includePath + else: + for i in 0.. 0: if includePath.contains("compilation.nim"): @@ -251,53 +252,53 @@ proc extractNimVersion*(nimbleFile: string): string = let compFileIdx = fileInfoIdx(conf, AbsoluteFile compilationPath) var parser: Parser - - if setupParser(parser, compFileIdx, newIdentCache(), conf): - let ast = parseAll(parser) - - # Process AST to find NimMajor, NimMinor, NimPatch definitions - proc processNode(n: PNode) = - case n.kind - of nkStmtList, nkStmtListExpr: - for child in n: - processNode(child) - of nkConstSection: - for child in n: - if child.kind == nkConstDef: - var identName = "" - case child[0].kind - of nkPostfix: - if child[0][1].kind == nkIdent: - identName = child[0][1].ident.s - of nkIdent: - identName = child[0].ident.s - of nkPragmaExpr: - # Handle pragma expression (like NimMajor* {.intdefine.}) - if child[0][0].kind == nkIdent: - identName = child[0][0].ident.s - elif child[0][0].kind == nkPostfix and child[0][0][1].kind == nkIdent: - identName = child[0][0][1].ident.s - else: discard - # echo "Unhandled node kind for const name: ", child[0].kind - # Extract value - if child.len > 2: - case child[2].kind - of nkIntLit: - let value = child[2].intVal.int - case identName - of "NimMajor": major = value - of "NimMinor": minor = value - of "NimPatch": patch = value - else: discard - else: - discard - else: - discard - - processNode(ast) - closeParser(parser) - # echo "Extracted version: ", major, ".", minor, ".", patch - return &"{major}.{minor}.{patch}" + {.cast(gcsafe).}: + if setupParser(parser, compFileIdx, newIdentCache(), conf): + let ast = parseAll(parser) + + # Process AST to find NimMajor, NimMinor, NimPatch definitions + proc processNode(n: PNode) = + case n.kind + of nkStmtList, nkStmtListExpr: + for child in n: + processNode(child) + of nkConstSection: + for child in n: + if child.kind == nkConstDef: + var identName = "" + case child[0].kind + of nkPostfix: + if child[0][1].kind == nkIdent: + identName = child[0][1].ident.s + of nkIdent: + identName = child[0].ident.s + of nkPragmaExpr: + # Handle pragma expression (like NimMajor* {.intdefine.}) + if child[0][0].kind == nkIdent: + identName = child[0][0].ident.s + elif child[0][0].kind == nkPostfix and child[0][0][1].kind == nkIdent: + identName = child[0][0][1].ident.s + else: discard + # echo "Unhandled node kind for const name: ", child[0].kind + # Extract value + if child.len > 2: + case child[2].kind + of nkIntLit: + let value = child[2].intVal.int + case identName + of "NimMajor": major = value + of "NimMinor": minor = value + of "NimPatch": patch = value + else: discard + else: + discard + else: + discard + + processNode(ast) + closeParser(parser) + # echo "Extracted version: ", major, ".", minor, ".", patch + return &"{major}.{minor}.{patch}" proc parseRequiresFile*(requiresFile: string): seq[string] = ## Parse a plain text requires file. @@ -340,12 +341,13 @@ proc extractRequiresInfo*(nimbleFile: string, options: Options): NimbleFileInfo localError(config, info, warnUser, msg) let fileIdx = fileInfoIdx(conf, AbsoluteFile nimbleFile) - var parser: Parser - if setupParser(parser, fileIdx, newIdentCache(), conf): - let ast = parseAll(parser) - extract(ast, conf, result, options) - closeParser(parser) - result.hasErrors = result.hasErrors or conf.errorCounter > 0 + {.cast(gcsafe).}: + var parser: Parser + if setupParser(parser, fileIdx, newIdentCache(), conf): + let ast = parseAll(parser) + extract(ast, conf, result, options) + closeParser(parser) + result.hasErrors = result.hasErrors or conf.errorCounter > 0 # Add requires from external requires file let nimbleDir = nimbleFile.splitFile.dir diff --git a/src/nimblepkg/nimscriptexecutor.nim b/src/nimblepkg/nimscriptexecutor.nim index 4a5f3545a..c42241d3f 100644 --- a/src/nimblepkg/nimscriptexecutor.nim +++ b/src/nimblepkg/nimscriptexecutor.nim @@ -5,32 +5,31 @@ import os, strutils, sets import packageparser, common, options, nimscriptwrapper, cli -proc execHook*(nimBin: string, options: Options, hookAction: ActionType, before: bool): bool {. raises: [].} = +proc execHook*(nimBin: string, options: Options, hookAction: ActionType, before: bool): bool {. raises: [], gcsafe.} = ## Returns whether to continue. result = true - {.cast(gcsafe).}: - # For certain commands hooks should not be evaluated. - if hookAction in noHookActions: - return + # For certain commands hooks should not be evaluated. + if hookAction in noHookActions: + return - var nimbleFile = "" - try: - nimbleFile = findNimbleFile(getCurrentDir(), true, options) + var nimbleFile = "" + try: + nimbleFile = findNimbleFile(getCurrentDir(), true, options) - # PackageInfos are cached so we can read them as many times as we want. - let pkgInfo = getPkgInfoFromFile(nimBin, nimbleFile, options) - let actionName = - if hookAction == actionCustom: options.action.command - else: ($hookAction)[6 .. ^1] - let hookExists = - if before: actionName.normalize in pkgInfo.preHooks - else: actionName.normalize in pkgInfo.postHooks - if pkgInfo.isNimScript and hookExists: - let res = execHook(nimBin, nimbleFile, actionName, before, options) - if res.success: - result = res.retVal - except NimbleError: return true - except Exception: return false #TODO fix the propagation of Exception + # PackageInfos are cached so we can read them as many times as we want. + let pkgInfo = getPkgInfoFromFile(nimBin, nimbleFile, options) + let actionName = + if hookAction == actionCustom: options.action.command + else: ($hookAction)[6 .. ^1] + let hookExists = + if before: actionName.normalize in pkgInfo.preHooks + else: actionName.normalize in pkgInfo.postHooks + if pkgInfo.isNimScript and hookExists: + let res = execHook(nimBin, nimbleFile, actionName, before, options) + if res.success: + result = res.retVal + except NimbleError: return true + except Exception: return false #TODO fix the propagation of Exception proc execCustom*(nimBin: string, nimbleFile: string, options: Options, execResult: var ExecutionResult[bool]): bool = diff --git a/src/nimblepkg/packageinfo.nim b/src/nimblepkg/packageinfo.nim index e47f947d8..6919dea54 100644 --- a/src/nimblepkg/packageinfo.nim +++ b/src/nimblepkg/packageinfo.nim @@ -444,7 +444,7 @@ proc checkInstallDir(pkgInfo: PackageInfo, if thisDir == "nimcache": result = true proc iterFilesWithExt(dir: string, pkgInfo: PackageInfo, - action: proc (f: string): void {.raises: [].}) = + action: proc (f: string): void {.raises: [], gcsafe.}) = ## Runs `action` for each filename of the files that have a whitelisted ## file extension. for kind, path in walkDir(dir): @@ -520,7 +520,7 @@ proc iterInstallFilesSimple*(realDir: string, pkgInfo: PackageInfo, action(file) proc iterInstallFiles*(realDir: string, pkgInfo: PackageInfo, - options: Options, action: proc (f: string): void {.raises: [].}) = + options: Options, action: proc (f: string): void {.raises: [], gcsafe.}) = ## Runs `action` for each file within the ``realDir`` that should be ## installed. # Get the package root directory for skipDirs comparison diff --git a/src/nimblepkg/packageinfotypes.nim b/src/nimblepkg/packageinfotypes.nim index aabcd9258..830174fc0 100644 --- a/src/nimblepkg/packageinfotypes.nim +++ b/src/nimblepkg/packageinfotypes.nim @@ -158,19 +158,23 @@ const noTask* = "" # Means that noTask is being ran. Use this as key for base de var satProccesedPackages*: Option[HashSet[PackageInfo]] #Package name -> features. When a package requires a feature, it is added to this table. #For instance, if a dependency of a package requires the feature "feature1", it will be added to this table although the root package may not explicitly require it. -var globallyActiveFeatures: Table[string, seq[string]] = initTable[string, seq[string]]() +var globallyActiveFeatures {.threadvar.}: TableRef[string, seq[string]] + +proc getGloballyActiveFeaturesTable(): TableRef[string, seq[string]] = + if globallyActiveFeatures == nil: + globallyActiveFeatures = newTable[string, seq[string]]() + return globallyActiveFeatures proc appendGloballyActiveFeatures*(pkgName: string, features: seq[string]) = - if pkgName notin globallyActiveFeatures: - globallyActiveFeatures[pkgName] = features + if pkgName notin getGloballyActiveFeaturesTable(): + getGloballyActiveFeaturesTable()[pkgName] = features else: for feature in features: - globallyActiveFeatures[pkgName].add(feature) + getGloballyActiveFeaturesTable()[pkgName].add(feature) proc getGloballyActiveFeatures*(): seq[string] = #returns features.{pkgName}.{feature} - {.cast(gcsafe).}: - for pkgName, features in globallyActiveFeatures: + for pkgName, features in getGloballyActiveFeaturesTable(): for feature in features: result.add(&"features.{pkgName}.{feature}") diff --git a/src/nimblepkg/vcstools.nim b/src/nimblepkg/vcstools.nim index 474f45bde..aa1bd4dfb 100644 --- a/src/nimblepkg/vcstools.nim +++ b/src/nimblepkg/vcstools.nim @@ -88,7 +88,7 @@ proc getVcsTypeAndSpecialDirPath*(dir: Path): VcsTypeAndSpecialDirPath = ## ## Raises a `NimbleError` in the case the directory `dir` does not exist. - var cache {.global.}: Table[Path, VcsTypeAndSpecialDirPath] + var cache {.global, threadvar.}: Table[Path, VcsTypeAndSpecialDirPath] if cache.hasKey(dir): return cache[dir] @@ -192,13 +192,12 @@ proc getVcsRevision*(dir: Path): Sha1Hash = ## - the external command fails. ## - the directory does not exist. ## - there is no vcsRevisions in the repository. - {.cast(gcsafe).}: - let vcsRevision = tryDoVcsCmd(dir, - gitCmd = "rev-parse HEAD", - hgCmd = "id -i --debug", - noVcsAction = $notSetSha1Hash) + let vcsRevision = tryDoVcsCmd(dir, + gitCmd = "rev-parse HEAD", + hgCmd = "id -i --debug", + noVcsAction = $notSetSha1Hash) - return initSha1Hash(vcsRevision.strip(chars = Whitespace + {'+'})) + return initSha1Hash(vcsRevision.strip(chars = Whitespace + {'+'})) proc getVcsRevisions*(dir: Path): Sha1Hash = ## Returns current revision number if the directory `dir` is under version diff --git a/src/nimblepkg/vnext.nim b/src/nimblepkg/vnext.nim index 42102dd3e..bf674906a 100644 --- a/src/nimblepkg/vnext.nim +++ b/src/nimblepkg/vnext.nim @@ -756,11 +756,10 @@ proc getPathsToBuildFor*(satResult: SATResult, pkgInfo: PackageInfo, recursive: result.incl(pkgInfo.expandPaths(nimBin, options)) proc getPathsAllPkgs*(options: Options): HashSet[string] = - {.cast(gcsafe).}: - let satResult = options.satResult - for pkg in satResult.pkgs: - for path in pkg.expandPaths(satResult.nimResolved.getNimBin(), options): - result.incl(path) + let satResult = options.satResult + for pkg in satResult.pkgs: + for path in pkg.expandPaths(satResult.nimResolved.getNimBin(), options): + result.incl(path) proc getNimBin(satResult: SATResult): string = #TODO change this so nim is passed as a parameter but we also need to change getPkgInfo so for the time being its also in options @@ -841,12 +840,11 @@ proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], let outputDir = pkgInfo.getOutputDir("") if dirExists(outputDir): if fileExists(outputDir / bin): - {.cast(gcsafe).}: - if not pkgInfo.needsRebuild(outputDir / bin, realDir, options): - display("Skipping", "$1/$2 (up-to-date)" % - [pkginfo.basicInfo.name, bin], priority = HighPriority) - binariesBuilt.inc() - continue + if not pkgInfo.needsRebuild(outputDir / bin, realDir, options): + display("Skipping", "$1/$2 (up-to-date)" % + [pkginfo.basicInfo.name, bin], priority = HighPriority) + binariesBuilt.inc() + continue else: createDir(outputDir) @@ -862,17 +860,16 @@ proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], if fileExists(sourceBinary): # Check if the source binary is up-to-date - {.cast(gcsafe).}: - if not pkgInfo.needsRebuild(sourceBinary, realDir, options): - let targetBinary = outputDir / bin - display("Skipping", "$1/$2 (up-to-date)" % - [pkginfo.basicInfo.name, bin], priority = HighPriority) - copyFile(sourceBinary, targetBinary) - when not defined(windows): - # Preserve executable permissions - setFilePermissions(targetBinary, getFilePermissions(sourceBinary)) - binariesBuilt.inc() - continue + if not pkgInfo.needsRebuild(sourceBinary, realDir, options): + let targetBinary = outputDir / bin + display("Skipping", "$1/$2 (up-to-date)" % + [pkginfo.basicInfo.name, bin], priority = HighPriority) + copyFile(sourceBinary, targetBinary) + when not defined(windows): + # Preserve executable permissions + setFilePermissions(targetBinary, getFilePermissions(sourceBinary)) + binariesBuilt.inc() + continue let outputOpt = "-o:" & pkgInfo.getOutputDir(bin).quoteShell display("Building", "$1/$2 using $3 backend" % From 25d42439f2bbb8edc21afc8f8390ebc60b83676d Mon Sep 17 00:00:00 2001 From: jmgomez Date: Tue, 18 Nov 2025 11:39:10 +0000 Subject: [PATCH 8/9] Fixes a random/race condition issue in the CI. Refactor in preparation for the job flag --- src/nimblepkg/packageinfo.nim | 18 +++-- src/nimblepkg/vnext.nim | 126 ++++++++++++++++++++-------------- 2 files changed, 90 insertions(+), 54 deletions(-) diff --git a/src/nimblepkg/packageinfo.nim b/src/nimblepkg/packageinfo.nim index 6919dea54..d705c4ebc 100644 --- a/src/nimblepkg/packageinfo.nim +++ b/src/nimblepkg/packageinfo.nim @@ -664,9 +664,14 @@ proc needsRebuild*(pkgInfo: PackageInfo, bin: string, dir: string, options: Opti if not options.isLegacy: if options.action.noRebuild: if not fileExists(bin): - return true - - let binTimestamp = getFileInfo(bin).lastWriteTime + return true + + let binTimestamp = + try: + getFileInfo(bin).lastWriteTime + except OSError: + # File disappeared or became inaccessible between fileExists check and here + return true var rebuild = false iterFilesWithExt(dir, pkgInfo, proc (file: string) = @@ -684,7 +689,12 @@ proc needsRebuild*(pkgInfo: PackageInfo, bin: string, dir: string, options: Opti if not options.action.noRebuild: return true - let binTimestamp = getFileInfo(bin).lastWriteTime + let binTimestamp = + try: + getFileInfo(bin).lastWriteTime + except OSError: + # File doesn't exist or became inaccessible + return true var rebuild = false iterFilesWithExt(dir, pkgInfo, proc (file: string) = diff --git a/src/nimblepkg/vnext.nim b/src/nimblepkg/vnext.nim index bf674906a..7192e2bad 100644 --- a/src/nimblepkg/vnext.nim +++ b/src/nimblepkg/vnext.nim @@ -772,8 +772,73 @@ proc getNimBin(satResult: SATResult): string = else: raise newNimbleError[NimbleError]("No Nim found") +type BuildTask = object + bin: string + cmd: string + future: Future[CommandExResponse] + +# Forward declaration +proc createBinSymlink(pkgInfo: PackageInfo, options: Options) + +proc processBuildTasks(buildTasks: seq[BuildTask], + pkgInfo: PackageInfo, nimBin: string, options: Options, + shouldCreateSymlinks: bool): Future[void] {.async: (raises: [CatchableError]).} = + ## Wait for all builds to complete in parallel, process their results, + ## and optionally create symlinks for the built binaries. + ## Returns the total number of binaries built. + + var binariesBuilt = 0 + + if buildTasks.len > 0: + let futures = buildTasks.mapIt(it.future) + discard await allFinished(futures) + + # Process results + for task in buildTasks: + try: + let response = task.future.read() + + # Display output + if response.stdOutput.len > 0: + display("Nim Output", response.stdOutput, priority = HighPriority) + if response.stdError.len > 0: + display("Nim Stderr", response.stdError, priority = HighPriority) + + # Check exit code + if response.status != QuitSuccess: + raise nimbleError( + "Execution failed with exit code $1\nCommand: $2" % + [$response.status, task.cmd]) + + binariesBuilt.inc() + except CatchableError as error: + raise buildFailed( + &"Build failed for package: {pkgInfo.basicInfo.name}, binary: {task.bin}", details = error) + + if binariesBuilt == 0: + let binary = options.getCompilationBinary(pkgInfo).get("") + if binary != "": + raise nimbleError(binaryNotDefinedInPkgMsg(binary, pkgInfo.basicInfo.name)) + + raise nimbleError( + "No binaries built, did you specify a valid binary name?" + ) + + # Handle post-`build` hook. + cd pkgInfo.myPath.parentDir(): # Make sure `execHook` executes the correct .nimble file. + discard execHook(nimBin, options, actionBuild, false) + + # Create symlinks if requested + if shouldCreateSymlinks: + try: + {.cast(gcsafe).}: + createBinSymlink(pkgInfo, options) + except Exception: + displayError("Error creating bin symlink: " & getCurrentExceptionMsg()) + + proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], - args: seq[string], options: Options, nimBin: string): Future[void] {.async: (raises: [CatchableError]).} = + args: seq[string], options: Options, nimBin: string): seq[BuildTask] = ## Builds a package as specified by ``pkgInfo``. # Handle pre-`build` hook. let @@ -824,10 +889,7 @@ proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], else: "" # Collect all binary build tasks to run in parallel - type BuildTask = object - bin: string - cmd: string - future: Future[CommandExResponse] + var buildTasks: seq[BuildTask] @@ -897,45 +959,8 @@ proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], task.future = execCommandEx(cmd) buildTasks.add(task) - # Wait for all builds to complete in parallel - if buildTasks.len > 0: - let futures = buildTasks.mapIt(it.future) - discard await allFinished(futures) - - # Process results - for task in buildTasks: - try: - let response = task.future.read() - - # Display output - if response.stdOutput.len > 0: - display("Nim Output", response.stdOutput, priority = HighPriority) - if response.stdError.len > 0: - display("Nim Stderr", response.stdError, priority = HighPriority) - - # Check exit code - if response.status != QuitSuccess: - raise nimbleError( - "Execution failed with exit code $1\nCommand: $2" % - [$response.status, task.cmd]) - - binariesBuilt.inc() - except CatchableError as error: - raise buildFailed( - &"Build failed for package: {pkgInfo.basicInfo.name}, binary: {task.bin}", details = error) - - if binariesBuilt == 0: - let binary = options.getCompilationBinary(pkgInfo).get("") - if binary != "": - raise nimbleError(binaryNotDefinedInPkgMsg(binary, pkgInfo.basicInfo.name)) - - raise nimbleError( - "No binaries built, did you specify a valid binary name?" - ) - - # Handle post-`build` hook. - cd pkgDir: # Make sure `execHook` executes the correct .nimble file. - discard execHook(nimBin, options, actionBuild, false) + return buildTasks + proc createBinSymlink(pkgInfo: PackageInfo, options: Options) = var binariesInstalled: HashSet[string] @@ -1015,15 +1040,16 @@ proc buildPkg*(nimBin: string, pkgToBuild: PackageInfo, isRootInRootDir: bool, o var pkgToBuild = pkgToBuild if isRootInRootDir: pkgToBuild.isInstalled = false - await buildFromDir(pkgToBuild, paths, "-d:release" & flags, options, nimBin) + + # Build and collect tasks + let buildTasks = buildFromDir(pkgToBuild, paths, "-d:release" & flags, options, nimBin) + # For globally installed packages, always create symlinks # Only skip symlinks if we're building the root package in its own directory let shouldCreateSymlinks = not isRootInRootDir or options.action.typ == actionInstall - if shouldCreateSymlinks: - try: - createBinSymlink(pkgToBuild, options) - except Exception: - displayError("Error creating bin symlink: " & getCurrentExceptionMsg()) + + # Process build tasks and create symlinks + await processBuildTasks(buildTasks, pkgToBuild, nimBin, options, shouldCreateSymlinks) proc getVersionRangeFoPkgToInstall(satResult: SATResult, name: string, ver: Version): VersionRange = if satResult.rootPackage.basicInfo.name == name and satResult.rootPackage.basicInfo.version == ver: From cc2595f12e2c2e14d4b89f456b017b2f3161f2ba Mon Sep 17 00:00:00 2001 From: jmgomez Date: Wed, 19 Nov 2025 10:23:45 +0000 Subject: [PATCH 9/9] Attempt to fix random race cond in the CI --- src/nimblepkg/vnext.nim | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/src/nimblepkg/vnext.nim b/src/nimblepkg/vnext.nim index 7192e2bad..1e2e5fa4b 100644 --- a/src/nimblepkg/vnext.nim +++ b/src/nimblepkg/vnext.nim @@ -972,33 +972,17 @@ proc createBinSymlink(pkgInfo: PackageInfo, options: Options) = # Set file permissions to +x for all binaries built, # and symlink them on *nix OS' to $nimbleDir/bin/ for bin, src in pkgInfo.bin: - let binDest = - # Issue #308 - if dirExists(pkgDestDir / bin): - bin & ".out" - elif dirExists(pkgDestDir / pkgInfo.binDir): - pkgInfo.binDir / bin - else: - bin - - # For develop mode packages, the binary is in the source directory, not installed directory - let symlinkDest = - if pkgInfo.isLink: - # Develop mode: binary is in the source directory - pkgInfo.getOutputDir(bin) - else: - # Installed package: binary is in the installed directory - pkgDestDir / binDest + let symlinkDest = pkgInfo.getOutputDir(bin) if not fileExists(symlinkDest): - raise nimbleError(&"Binary '{bin}' was not found at expected location: {symlinkDest}. BinDir is {binDir}. binDest is {binDest}. pkgDestDir is {pkgDestDir}. isLink is {pkgInfo.isLink}") + raise nimbleError(&"Binary '{bin}' was not found at expected location: {symlinkDest}. BinDir is {binDir}. isLink is {pkgInfo.isLink}") # if fileExists(symlinkDest) and not pkgInfo.isLink: # display("Warning:", ("Binary '$1' was already installed from source" & # " directory. Will be overwritten.") % bin, Warning, # MediumPriority) if not pkgInfo.isLink: - createDir((pkgDestDir / binDest).parentDir()) + createDir(symlinkDest.parentDir()) let symlinkFilename = options.getBinDir() / bin.extractFilename binariesInstalled.incl( setupBinSymlink(symlinkDest, symlinkFilename, options))