From 9dd966d3a875c7fa5336de811118975721d4980b Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 20 Nov 2025 10:15:29 +0000 Subject: [PATCH 01/29] Refactor display procs to be thread safe --- src/nimblepkg/cli.nim | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/nimblepkg/cli.nim b/src/nimblepkg/cli.nim index de67b921d..fe3545273 100644 --- a/src/nimblepkg/cli.nim +++ b/src/nimblepkg/cli.nim @@ -48,7 +48,12 @@ proc newCLI(): CLI = showColor: true, ) -var globalCLI = newCLI() +var globalCLI {.threadvar.}: CLI + +proc getGlobalCLI(): CLI = + if globalCLI == nil: + globalCLI = newCLI() + return globalCLI proc calculateCategoryOffset(category: string): int = assert category.len <= longestCategory @@ -57,14 +62,14 @@ proc calculateCategoryOffset(category: string): int = proc isSuppressed(displayType: DisplayType): bool = # Don't print any Warning, Message or Success messages when suppression of # warnings is enabled. That is, unless the user asked for --verbose output. - if globalCLI.suppressMessages and displayType >= Warning and - globalCLI.level == HighPriority: + if getGlobalCLI().suppressMessages and displayType >= Warning and + getGlobalCLI().level == HighPriority: return true proc displayFormatted*(displayType: DisplayType, msgs: varargs[string]) = ## for styling outputs lines using the DisplayTypes for msg in msgs: - if globalCLI.showColor: + if getGlobalCLI().showColor: stdout.styledWrite(foregrounds[displayType], msg) else: stdout.write(msg) @@ -85,7 +90,7 @@ proc displayCategory(category: string, displayType: DisplayType, # Display the category. let text = "$1$2 " % [spaces(offset), category] - if globalCLI.showColor: + if getGlobalCLI().showColor: if priority != DebugPriority: setForegroundColor(stdout, foregrounds[displayType]) writeStyled(text, styles[priority]) @@ -109,16 +114,16 @@ proc display*(category, msg: string, displayType = Message, # Multiple warnings containing the same messages should not be shown. let warningPair = (category, msg) if displayType == Warning: - if warningPair in globalCLI.warnings: + if warningPair in getGlobalCLI().warnings: return else: - globalCLI.warnings.incl(warningPair) + getGlobalCLI().warnings.incl(warningPair) # Suppress this message if its priority isn't high enough. # TODO: Per-priority suppression counts? - if priority < globalCLI.level: + if priority < getGlobalCLI().level: if priority != DebugPriority: - globalCLI.suppressionCount.inc + getGlobalCLI().suppressionCount.inc return # Display each line in the message. @@ -173,7 +178,7 @@ proc displayDebug*(msg: string) = proc displayTip*() = ## Called just before Nimble exits. Shows some tips for the user, for example ## the amount of messages that were suppressed and how to show them. - if globalCLI.suppressionCount > 0: + if getGlobalCLI().suppressionCount > 0: let msg = "$1 messages have been suppressed, use --verbose to show them." % $globalCLI.suppressionCount display("Tip:", msg, Warning, HighPriority) @@ -187,7 +192,7 @@ proc prompt*(forcePrompts: ForcePrompt, question: string): bool = display("Prompt:", question & " -> [forced no]", Warning, HighPriority) return false of dontForcePrompt: - if globalCLI.level > SilentPriority: + if getGlobalCLI().level > SilentPriority: display("Prompt:", question & " [y/N]", Warning, HighPriority) displayCategory("Answer:", Warning, HighPriority) let yn = stdin.readLine() @@ -321,10 +326,10 @@ proc promptList*(forcePrompts: ForcePrompt, question: string, args: openarray[st return promptListFallback(question, args) proc setVerbosity*(level: Priority) = - globalCLI.level = level + getGlobalCLI().level = level proc setShowColor*(val: bool) = - globalCLI.showColor = val + getGlobalCLI().showColor = val proc setSuppressMessages*(val: bool) = - globalCLI.suppressMessages = val + getGlobalCLI().suppressMessages = val From ab3f407405f0b1b8bf1101728114536563cc6254 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 20 Nov 2025 10:15:46 +0000 Subject: [PATCH 02/29] Green: doCmdExAsync executes command --- src/nimblepkg/tools.nim | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/nimblepkg/tools.nim b/src/nimblepkg/tools.nim index 5b2a155b8..c8de2f4c8 100644 --- a/src/nimblepkg/tools.nim +++ b/src/nimblepkg/tools.nim @@ -7,7 +7,7 @@ import osproc, pegs, strutils, os, uri, sets, json, parseutils, strformat, from net import SslCVerifyMode, newContext, SslContext -import version, cli, common, packageinfotypes, options, sha1hashes +import version, cli, common, packageinfotypes, options, sha1hashes, chronos, chronos/asyncproc from "$nim" / compiler/nimblecmd import getPathVersionChecksum proc extractBin(cmd: string): string = @@ -48,6 +48,14 @@ proc doCmd*(cmd: string) = proc doCmdEx*(cmd: string): ProcessOutput = displayDebug("Executing", cmd) + result = execCmdEx(cmd) + displayDebug("Output", result.output) + +proc doCmdExAsync*(cmd: string): Future[ProcessOutput] {.async.} = + displayDebug("Executing", cmd) + let res = await execCommandEx(cmd) + result = (res.stdOutput, res.status) + displayDebug("Output", result.output) let bin = extractBin(cmd) if findExe(bin) == "": raise nimbleError("'" & bin & "' not in PATH.") From 3e7817be55435c6f91d9addd4c24884fdb377371 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 20 Nov 2025 11:04:51 +0000 Subject: [PATCH 03/29] [OK] doCloneAsync clones a repo --- src/nimblepkg/cli.nim | 8 ++++---- src/nimblepkg/download.nim | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/src/nimblepkg/cli.nim b/src/nimblepkg/cli.nim index fe3545273..26b2fbe21 100644 --- a/src/nimblepkg/cli.nim +++ b/src/nimblepkg/cli.nim @@ -50,10 +50,10 @@ proc newCLI(): CLI = var globalCLI {.threadvar.}: CLI -proc getGlobalCLI(): CLI = - if globalCLI == nil: +proc getGlobalCLI*(): CLI = + if globalCLI.isNil: globalCLI = newCLI() - return globalCLI + result = globalCLI proc calculateCategoryOffset(category: string): int = assert category.len <= longestCategory @@ -180,7 +180,7 @@ proc displayTip*() = ## the amount of messages that were suppressed and how to show them. if getGlobalCLI().suppressionCount > 0: let msg = "$1 messages have been suppressed, use --verbose to show them." % - $globalCLI.suppressionCount + $getGlobalCLI().suppressionCount display("Tip:", msg, Warning, HighPriority) proc prompt*(forcePrompts: ForcePrompt, question: string): bool = diff --git a/src/nimblepkg/download.nim b/src/nimblepkg/download.nim index 3e26463fd..00742278f 100644 --- a/src/nimblepkg/download.nim +++ b/src/nimblepkg/download.nim @@ -2,7 +2,7 @@ # BSD License. Look at license.txt for more info. import parseutils, os, osproc, strutils, tables, uri, strformat, - httpclient, json, sequtils, urls + httpclient, json, sequtils, urls, chronos from algorithm import SortOrder, sorted @@ -19,6 +19,18 @@ proc updateSubmodules(dir: string) = discard tryDoCmdEx( &"git -C {dir.quoteShell} submodule update --init --recursive --depth 1") +proc tryDoCmdExAsync(cmd: string): Future[string] {.async.} = + ## Async version of tryDoCmdEx. Executes command and raises error if it fails. + let (output, exitCode) = await doCmdExAsync(cmd) + if exitCode != QuitSuccess: + raise nimbleError(tryDoCmdExErrorMessage(cmd, output, exitCode)) + return output + +proc updateSubmodulesAsync(dir: string): Future[void] {.async.} = + ## Async version of updateSubmodules. + discard await tryDoCmdExAsync( + &"git -C {dir.quoteShell} submodule update --init --recursive --depth 1") + proc doCheckout*(meth: DownloadMethod, downloadDir, branch: string, options: Options) = case meth of DownloadMethod.git: @@ -50,6 +62,26 @@ proc doClone(meth: DownloadMethod, url, downloadDir: string, branch = "", branchArg = if branch == "": "" else: &"-b {branch.quoteShell}" discard tryDoCmdEx(&"hg clone {tipArg} {branchArg} {url} {downloadDir.quoteShell}") +proc doCloneAsync*(meth: DownloadMethod, url, downloadDir: string, branch = "", + onlyTip = true, options: Options): Future[void] {.async.} = + ## Async version of doClone that uses doCmdExAsync for non-blocking execution. + case meth + of DownloadMethod.git: + let + submoduleFlag = if not options.ignoreSubmodules: " --recurse-submodules" else: "" + depthArg = if onlyTip: "--depth 1" else: "" + branchArg = if branch == "": "" else: &"-b {branch.quoteShell}" + discard await tryDoCmdExAsync( + "git clone --config core.autocrlf=false --config core.eol=lf " & + &"{submoduleFlag} {depthArg} {branchArg} {url} {downloadDir.quoteShell}") + if not options.ignoreSubmodules: + await downloadDir.updateSubmodulesAsync() + of DownloadMethod.hg: + let + tipArg = if onlyTip: "-r tip " else: "" + branchArg = if branch == "": "" else: &"-b {branch.quoteShell}" + discard await tryDoCmdExAsync(&"hg clone {tipArg} {branchArg} {url} {downloadDir.quoteShell}") + proc gitFetchTags*(repoDir: string, downloadMethod: DownloadMethod, options: Options) = case downloadMethod: of DownloadMethod.git: From 4ea1d96e55a91dda5cfaea11015ee96f519fb20e Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 20 Nov 2025 11:15:26 +0000 Subject: [PATCH 04/29] [OK] gitFetchTagsAsync fetches tags --- src/nimblepkg/download.nim | 10 ++++++++++ src/nimblepkg/tools.nim | 7 +++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/nimblepkg/download.nim b/src/nimblepkg/download.nim index 00742278f..bd14d71e5 100644 --- a/src/nimblepkg/download.nim +++ b/src/nimblepkg/download.nim @@ -91,6 +91,16 @@ proc gitFetchTags*(repoDir: string, downloadMethod: DownloadMethod, options: Opt # In Mercurial, pulling updates also fetches all remote tags tryDoCmdEx(&"hg --cwd {repoDir} pull") +proc gitFetchTagsAsync*(repoDir: string, downloadMethod: DownloadMethod, options: Options): Future[void] {.async.} = + ## Async version of gitFetchTags that uses doCmdExAsync for non-blocking execution. + case downloadMethod: + of DownloadMethod.git: + let submoduleFlag = if not options.ignoreSubmodules: " --recurse-submodules" else: "" + discard await tryDoCmdExAsync(&"git -C {repoDir} fetch --tags" & submoduleFlag) + of DownloadMethod.hg: + # In Mercurial, pulling updates also fetches all remote tags + discard await tryDoCmdExAsync(&"hg --cwd {repoDir} pull") + proc getTagsList*(dir: string, meth: DownloadMethod): seq[string] = var output: string cd dir: diff --git a/src/nimblepkg/tools.nim b/src/nimblepkg/tools.nim index c8de2f4c8..8c0901ed7 100644 --- a/src/nimblepkg/tools.nim +++ b/src/nimblepkg/tools.nim @@ -52,14 +52,13 @@ proc doCmdEx*(cmd: string): ProcessOutput = displayDebug("Output", result.output) proc doCmdExAsync*(cmd: string): Future[ProcessOutput] {.async.} = + let bin = extractBin(cmd) + if findExe(bin) == "": + raise nimbleError("'" & bin & "' not in PATH.") displayDebug("Executing", cmd) let res = await execCommandEx(cmd) result = (res.stdOutput, res.status) displayDebug("Output", result.output) - let bin = extractBin(cmd) - if findExe(bin) == "": - raise nimbleError("'" & bin & "' not in PATH.") - return execCmdEx(cmd) proc tryDoCmdExErrorMessage*(cmd, output: string, exitCode: int): string = &"Execution of '{cmd}' failed with an exit code {exitCode}.\n" & From cd0c204ef6fdc59166f31eac7f6bff3547f8d191 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 20 Nov 2025 11:23:30 +0000 Subject: [PATCH 05/29] [OK] getTagsListRemoteAsync queries remote tags --- src/nimblepkg/download.nim | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/nimblepkg/download.nim b/src/nimblepkg/download.nim index bd14d71e5..67989ee31 100644 --- a/src/nimblepkg/download.nim +++ b/src/nimblepkg/download.nim @@ -147,6 +147,27 @@ proc getTagsListRemote*(url: string, meth: DownloadMethod): seq[string] = # http://stackoverflow.com/questions/2039150/show-tags-for-remote-hg-repository raise nimbleError("Hg doesn't support remote tag querying.") +proc getTagsListRemoteAsync*(url: string, meth: DownloadMethod): Future[seq[string]] {.async.} = + ## Async version of getTagsListRemote that uses doCmdExAsync for non-blocking execution. + result = @[] + case meth + of DownloadMethod.git: + var (output, exitCode) = await doCmdExAsync(&"git ls-remote --tags {url}") + if exitCode != QuitSuccess: + raise nimbleError("Unable to query remote tags for " & url & + " . Git returned: " & output) + for i in output.splitLines(): + let refStart = i.find("refs/tags/") + # git outputs warnings, empty lines, etc + if refStart == -1: continue + let start = refStart+"refs/tags/".len + let tag = i[start .. i.len-1] + if not tag.endswith("^{}"): result.add(tag) + + of DownloadMethod.hg: + # http://stackoverflow.com/questions/2039150/show-tags-for-remote-hg-repository + raise nimbleError("Hg doesn't support remote tag querying.") + proc getVersionList*(tags: seq[string]): OrderedTable[Version, string] = ## Return an ordered table of Version -> git tag label. Ordering is ## in descending order with the most recent version first. From 281c4789a7b8880eecd5d73a56ff121d370a6ca2 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 20 Nov 2025 11:27:56 +0000 Subject: [PATCH 06/29] [OK] cloneSpecificRevisionAsync clones specific commit --- src/nimblepkg/download.nim | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/nimblepkg/download.nim b/src/nimblepkg/download.nim index 67989ee31..1a81ea598 100644 --- a/src/nimblepkg/download.nim +++ b/src/nimblepkg/download.nim @@ -231,6 +231,28 @@ proc cloneSpecificRevision(downloadMethod: DownloadMethod, of DownloadMethod.hg: discard tryDoCmdEx(&"hg clone {url} -r {($vcsRevision).quoteShell}") +proc cloneSpecificRevisionAsync*(downloadMethod: DownloadMethod, + url, downloadDir: string, + vcsRevision: Sha1Hash, options: Options): Future[void] {.async.} = + ## Async version of cloneSpecificRevision that uses doCmdExAsync for non-blocking execution. + assert vcsRevision != notSetSha1Hash + + display("Cloning", "revision: " & $vcsRevision, priority = MediumPriority) + case downloadMethod + of DownloadMethod.git: + let downloadDir = downloadDir.quoteShell + createDir(downloadDir) + discard await tryDoCmdExAsync(&"git -C {downloadDir.quoteShell} init") + discard await tryDoCmdExAsync(&"git -C {downloadDir.quoteShell} config core.autocrlf false") + discard await tryDoCmdExAsync(&"git -C {downloadDir.quoteShell} remote add origin {url}") + discard await tryDoCmdExAsync( + &"git -C {downloadDir.quoteShell} fetch --depth 1 origin {($vcsRevision).quoteShell}") + discard await tryDoCmdExAsync(&"git -C {downloadDir.quoteShell} reset --hard FETCH_HEAD") + if not options.ignoreSubmodules: + await downloadDir.updateSubmodulesAsync() + of DownloadMethod.hg: + discard await tryDoCmdExAsync(&"hg clone {url} -r {($vcsRevision).quoteShell}") + proc getTarExePath: string = ## Returns path to `tar` executable. var tarExePath {.global.}: string From 09aadb6e6856ce0a277d33cf03a280b295508ffb Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 20 Nov 2025 11:57:19 +0000 Subject: [PATCH 07/29] [OK] doDownloadTarballAsync downloads and extracts tarball --- src/nimblepkg/download.nim | 93 ++++++++++++++++++++++++++++++++++---- src/nimblepkg/tools.nim | 2 +- 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/src/nimblepkg/download.nim b/src/nimblepkg/download.nim index 1a81ea598..ebd623193 100644 --- a/src/nimblepkg/download.nim +++ b/src/nimblepkg/download.nim @@ -253,17 +253,38 @@ proc cloneSpecificRevisionAsync*(downloadMethod: DownloadMethod, of DownloadMethod.hg: discard await tryDoCmdExAsync(&"hg clone {url} -r {($vcsRevision).quoteShell}") -proc getTarExePath: string = +var tarExePathCache {.threadvar.}: string + +proc getTarExePath: string {.gcsafe.} = ## Returns path to `tar` executable. - var tarExePath {.global.}: string - once: - tarExePath = + if tarExePathCache == "": + tarExePathCache = when defined(Windows): - findExe("git").splitPath.head / "../usr/bin/tar.exe" + # On Windows, prefer Git's tar which supports --force-local + # Git for Windows includes tar at /usr/bin/tar.exe + let gitPath = findExe("git") + if gitPath != "": + # Navigate up from git.exe location to find Git root, then check usr/bin/tar.exe + var currentDir = gitPath.splitPath.head + var gitTar = "" + # Search up to 3 levels up for usr/bin/tar.exe + for i in 0..2: + let candidateTar = currentDir / "usr" / "bin" / "tar.exe" + if fileExists(candidateTar): + gitTar = candidateTar + break + currentDir = currentDir.parentDir + + if gitTar != "": + gitTar + else: + findExe("tar") + else: + findExe("tar") else: findExe("tar") - tarExePath = tarExePath.quoteShell - return tarExePath + tarExePathCache = tarExePathCache.quoteShell + return tarExePathCache proc hasTar: bool = ## Checks whether a `tar` external tool is available. @@ -375,7 +396,22 @@ proc getRevision(url, version: string): Sha1Hash = raise nimbleError(&"Cannot get revision for version \"{version}\" " & &"of package at \"{url}\".") -proc getTarCmdLine(downloadDir, filePath: string): string = +proc getRevisionAsync(url, version: string): Future[Sha1Hash] {.async.} = + ## Async version of getRevision that uses doCmdExAsync. + let output = await tryDoCmdExAsync(&"git ls-remote {url} {version}") + result = parseRevision(output) + if result == notSetSha1Hash: + if version.seemsLikeRevision: + try: + result = getFullRevisionFromGitHubApi(url, version) + except Exception: + raise nimbleError(&"Cannot get revision for version \"{version}\" " & + &"of package at \"{url}\".") + else: + raise nimbleError(&"Cannot get revision for version \"{version}\" " & + &"of package at \"{url}\".") + +proc getTarCmdLine(downloadDir, filePath: string): string {.gcsafe.} = ## Returns an OS specific command and arguments for extracting the downloaded ## tarball. when defined(Windows): @@ -436,6 +472,47 @@ proc doDownloadTarball(url, downloadDir, version: string, queryRevision: bool): filePath.removeFile return if queryRevision: getRevision(url, version) else: notSetSha1Hash +proc doDownloadTarballAsync*(url, downloadDir, version: string, queryRevision: bool): Future[Sha1Hash] {.async.} = + ## Async version of doDownloadTarball that uses doCmdExAsync for tar extraction. + ## Note: HTTP download is still synchronous, but tar extraction is async. + let downloadLink = getTarballDownloadLink(url, version) + display("Downloading", downloadLink) + let data = + try: + getUrlContent(downloadLink) + except Exception as e: + raise nimbleError("Failed to download tarball: " & e.msg) + display("Completed", "downloading " & downloadLink) + + let filePath = downloadDir / "tarball.tar.gz" + display("Saving", filePath) + downloadDir.createDir + writeFile(filePath, data) + display("Completed", "saving " & filePath) + + display("Unpacking", filePath) + let cmd = getTarCmdLine(downloadDir, filePath) + let (output, exitCode) = await doCmdExAsync(cmd) + if exitCode != QuitSuccess and not output.contains("Cannot create symlink to"): + raise nimbleError(tryDoCmdExErrorMessage(cmd, output, exitCode)) + display("Completed", "unpacking " & filePath) + + when defined(windows): + let listCmd = &"{getTarExePath()} -ztvf {filePath} --force-local" + let (cmdOutput, cmdExitCode) = await doCmdExAsync(listCmd) + if cmdExitCode != QuitSuccess: + raise nimbleError(tryDoCmdExErrorMessage(listCmd, cmdOutput, cmdExitCode)) + for line in cmdOutput.splitLines(): + if line.contains(" -> "): + let parts = line.split + let linkPath = parts[^1] + let linkNameParts = parts[^3].split('/') + let linkName = linkNameParts[1 .. ^1].foldl(a / b) + writeFile(downloadDir / linkName, linkPath) + + filePath.removeFile + return if queryRevision: await getRevisionAsync(url, version) else: notSetSha1Hash + {.warning[ProveInit]: off.} proc doDownload(url, downloadDir: string, verRange: VersionRange, downMethod: DownloadMethod, options: Options, diff --git a/src/nimblepkg/tools.nim b/src/nimblepkg/tools.nim index 8c0901ed7..a0305c62e 100644 --- a/src/nimblepkg/tools.nim +++ b/src/nimblepkg/tools.nim @@ -236,7 +236,7 @@ when defined(instrument): type CallRecord = object name: string - totalTime: Duration + totalTime: times.Duration callCount: int children: seq[int] # Indices of child records parent: int # Index of parent record (-1 for root) From 0cc76b71f0a025bb2cc15bb8ae065e17f6fcd2c7 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 20 Nov 2025 12:10:12 +0000 Subject: [PATCH 08/29] [OK] downloadPkgAsync downloads package --- src/nimblepkg/download.nim | 183 +++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) diff --git a/src/nimblepkg/download.nim b/src/nimblepkg/download.nim index ebd623193..eb0a5599f 100644 --- a/src/nimblepkg/download.nim +++ b/src/nimblepkg/download.nim @@ -615,6 +615,104 @@ proc doDownload(url, downloadDir: string, verRange: VersionRange, result.vcsRevision = downloadDir.getVcsRevision {.warning[ProveInit]: on.} +proc doDownloadAsync(url, downloadDir: string, verRange: VersionRange, + downMethod: DownloadMethod, options: Options, + vcsRevision: Sha1Hash): + Future[tuple[version: Version, vcsRevision: Sha1Hash]] {.async.} = + ## Async version of doDownload that uses async operations for clone, checkout, and tarball downloads. + template getLatestByTag(meth: untyped) {.dirty.} = + # Find latest version that fits our ``verRange``. + var latest = findLatest(verRange, versions) + ## Note: HEAD is not used when verRange.kind is verAny. This is + ## intended behaviour, the latest tagged version will be used in this case. + + # If no tagged versions satisfy our range latest.tag will be "". + # We still clone in that scenario because we want to try HEAD in that case. + # https://github.com/nim-lang/nimble/issues/22 + meth + if $latest.ver != "": + result.version = latest.ver + + result.vcsRevision = notSetSha1Hash + + removeDir(downloadDir) + if vcsRevision != notSetSha1Hash: + if downloadTarball(url, options): + discard await doDownloadTarballAsync(url, downloadDir, $vcsRevision, false) + else: + await cloneSpecificRevisionAsync(downMethod, url, downloadDir, vcsRevision, options) + result.vcsRevision = vcsRevision + elif verRange.kind == verSpecial: + # We want a specific commit/branch/tag here. + if verRange.spe == getHeadName(downMethod): + # Grab HEAD. + if downloadTarball(url, options): + result.vcsRevision = await doDownloadTarballAsync(url, downloadDir, "HEAD", true) + else: + await doCloneAsync(downMethod, url, downloadDir, + onlyTip = not options.forceFullClone, options = options) + else: + assert ($verRange.spe)[0] == '#', + "The special version must start with '#'." + let specialVersion = substr($verRange.spe, 1) + if downloadTarball(url, options): + result.vcsRevision = await doDownloadTarballAsync( + url, downloadDir, specialVersion, true) + else: + # Grab the full repo. + await doCloneAsync(downMethod, url, downloadDir, onlyTip = false, options = options) + # Then perform a checkout operation to get the specified branch/commit. + # `spe` starts with '#', trim it. + doCheckout(downMethod, downloadDir, specialVersion, options = options) + result.version = verRange.spe + else: + case downMethod + of DownloadMethod.git: + # For Git we have to query the repo remotely for its tags. This is + # necessary as cloning with a --depth of 1 removes all tag info. + result.version = getHeadName(downMethod) + let versions = (await getTagsListRemoteAsync(url, downMethod)).getVersionList() + if versions.len > 0: + getLatestByTag: + if downloadTarball(url, options): + let versionToDownload = + if latest.tag.len > 0: latest.tag else: "HEAD" + result.vcsRevision = await doDownloadTarballAsync( + url, downloadDir, versionToDownload, true) + else: + display("Cloning", "latest tagged version: " & latest.tag, + priority = MediumPriority) + await doCloneAsync(downMethod, url, downloadDir, latest.tag, + onlyTip = not options.forceFullClone, options = options) + else: + display("Warning:", &"The package {url} has no tagged releases, downloading HEAD instead.", Warning, + priority = HighPriority) + if downloadTarball(url, options): + result.vcsRevision = await doDownloadTarballAsync(url, downloadDir, "HEAD", true) + else: + # If no commits have been tagged on the repo we just clone HEAD. + await doCloneAsync(downMethod, url, downloadDir, onlyTip = not options.forceFullClone, options = options) # Grab HEAD. + of DownloadMethod.hg: + await doCloneAsync(downMethod, url, downloadDir, + onlyTip = not options.forceFullClone, options = options) + result.version = getHeadName(downMethod) + let versions = getTagsList(downloadDir, downMethod).getVersionList() + + if versions.len > 0: + getLatestByTag: + display("Switching", "to latest tagged version: " & latest.tag, + priority = MediumPriority) + doCheckout(downMethod, downloadDir, latest.tag, options = options) + else: + display("Warning:", &"The package {url} has no tagged releases, downloading HEAD instead.", Warning, + priority = HighPriority) + + if result.vcsRevision == notSetSha1Hash: + # In the case the package in not downloaded as tarball we must query its + # VCS revision from its download directory. + {.gcsafe.}: + result.vcsRevision = downloadDir.getVcsRevision + proc pkgDirHasNimble*(dir: string, options: Options): bool = try: discard findNimbleFile(dir, true, options) @@ -720,6 +818,91 @@ proc downloadPkg*(url: string, verRange: VersionRange, # moveDir(downloadDir, newDownloadDir) # result.dir = newDownloadDir / subdir +proc downloadPkgAsync*(url: string, verRange: VersionRange, + downMethod: DownloadMethod, + subdir: string, + options: Options, + downloadPath: string, + vcsRevision: Sha1Hash, + nimBin: string, + validateRange = true): Future[DownloadPkgResult] {.async.} = + ## Async version of downloadPkg that uses async operations for cloning and downloading. + ## Downloads the repository as specified by ``url`` and ``verRange`` using + ## the download method specified. + ## + ## If `downloadPath` isn't specified a location in /tmp/ will be used. + ## + ## Returns the directory where it was downloaded (subdir is appended) and + ## the concrete version which was downloaded. + ## + ## ``vcsRevision`` + ## If specified this parameter will cause specific VCS revision to be + ## checked out. + + let (downloadDir, pkgDir) = downloadPkgDir(url, verRange, subdir, options, vcsRevision, downloadPath) + result.dir = pkgDir + + #when using a persistent download dir we can skip the download if it's already done + if pkgDirHasNimble(result.dir, options): + return # already downloaded, skipping + + if options.offline: + raise nimbleError("Cannot download in offline mode.") + + let modUrl = modifyUrl(url, options.config.cloneUsingHttps) + + let downloadMethod = if downloadTarball(modUrl, options): + "http" else: $downMethod + + if subdir.len > 0: + display("Downloading", "$1 using $2 (subdir is '$3')" % + [modUrl, downloadMethod, subdir], + priority = HighPriority) + else: + display("Downloading", "$1 using $2" % [modUrl, downloadMethod], + priority = HighPriority) + + (result.version, result.vcsRevision) = await doDownloadAsync( + modUrl, downloadDir, verRange, downMethod, options, vcsRevision) + + var metaData = initPackageMetaData() + metaData.url = modUrl + metaData.vcsRevision = result.vcsRevision + saveMetaData(metaData, result.dir) + + var pkgInfo: PackageInfo + if validateRange and verRange.kind notin {verSpecial, verAny} or not options.isLegacy: + ## Makes sure that the downloaded package's version satisfies the requested + ## version range. + {.gcsafe.}: + try: + pkginfo = if options.satResult.pass == satNimSelection: #TODO later when in vnext we should just use this code path and fallback inside the toRequires if we can + getPkgInfoFromDirWithDeclarativeParser(result.dir, options, nimBin) + else: + getPkgInfo(result.dir, options, nimBin) + except Exception as e: + raise nimbleError("Failed to get package info: " & e.msg) + if pkginfo.basicInfo.version notin verRange: + raise nimbleError( + "Downloaded package's version does not satisfy requested version " & + "range: wanted $1 got $2." % + [$verRange, $pkginfo.basicInfo.version]) + + #TODO rework the pkgcache to handle this better + #ideally we should be able to know the version we are downloading upfront + #as for the constraints we need a way to invalidate the cache entry so it doesnt get outdated + # if options.isVNext: + # # Rename the download directory to use actual version if it's different from the version range + # # as constraints shouldnt be stored in the download cache but the actual package version + # # theorically this means that subsequent downloads of unconstraines packages will be re-download + # # but this shouldnt be an issue since when a package is installed we dont reach this point anymore + # let newDownloadDir = options.pkgCachePath / getDownloadDirName(url, pkginfo.basicInfo.version.toVersionRange(), notSetSha1Hash) + # if downloadDir != newDownloadDir: + # if dirExists(newDownloadDir): + # removeDir(newDownloadDir) + # moveDir(downloadDir, newDownloadDir) + # result.dir = newDownloadDir / subdir + proc echoPackageVersions*(pkg: Package) = let downMethod = pkg.downloadMethod case downMethod From c55ae6899c8de1a2e5934d2d356d0e20105510be Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 20 Nov 2025 12:20:48 +0000 Subject: [PATCH 09/29] Refactors declarative parser to be gcsafe --- src/nimblepkg/declarativeparser.nim | 149 ++++++++++++++-------------- src/nimblepkg/download.nim | 25 +++-- src/nimblepkg/vcstools.nim | 14 ++- 3 files changed, 98 insertions(+), 90 deletions(-) diff --git a/src/nimblepkg/declarativeparser.nim b/src/nimblepkg/declarativeparser.nim index ab88ac0ca..c1dbcb8e4 100644 --- a/src/nimblepkg/declarativeparser.nim +++ b/src/nimblepkg/declarativeparser.nim @@ -210,24 +210,25 @@ proc getNimCompilationPath*(nimbleFile: string): string = let fileIdx = fileInfoIdx(conf, AbsoluteFile nimbleFile) var parser: Parser var includePath = "" - if setupParser(parser, fileIdx, newIdentCache(), conf): - let ast = parseAll(parser) - proc findIncludePath(n: PNode) = - case n.kind - of nkStmtList, nkStmtListExpr: - for child in n: - findIncludePath(child) - of nkIncludeStmt: - # Found an include statement - if n.len > 0 and n[0].kind in {nkStrLit..nkTripleStrLit}: - includePath = n[0].strVal - # echo "Found include: ", includePath - else: - for i in 0.. 0 and n[0].kind in {nkStrLit..nkTripleStrLit}: + includePath = n[0].strVal + # echo "Found include: ", includePath + else: + for i in 0.. 0: if includePath.contains("compilation.nim"): @@ -251,51 +252,51 @@ proc extractNimVersion*(nimbleFile: string): string = let compFileIdx = fileInfoIdx(conf, AbsoluteFile compilationPath) var parser: Parser - - if setupParser(parser, compFileIdx, newIdentCache(), conf): - let ast = parseAll(parser) - - # Process AST to find NimMajor, NimMinor, NimPatch definitions - proc processNode(n: PNode) = - case n.kind - of nkStmtList, nkStmtListExpr: - for child in n: - processNode(child) - of nkConstSection: - for child in n: - if child.kind == nkConstDef: - var identName = "" - case child[0].kind - of nkPostfix: - if child[0][1].kind == nkIdent: - identName = child[0][1].ident.s - of nkIdent: - identName = child[0].ident.s - of nkPragmaExpr: - # Handle pragma expression (like NimMajor* {.intdefine.}) - if child[0][0].kind == nkIdent: - identName = child[0][0].ident.s - elif child[0][0].kind == nkPostfix and child[0][0][1].kind == nkIdent: - identName = child[0][0][1].ident.s - else: discard - # echo "Unhandled node kind for const name: ", child[0].kind - # Extract value - if child.len > 2: - case child[2].kind - of nkIntLit: - let value = child[2].intVal.int - case identName - of "NimMajor": major = value - of "NimMinor": minor = value - of "NimPatch": patch = value - else: discard - else: - discard - else: - discard - - processNode(ast) - closeParser(parser) + {.cast(gcSafe).}: + if setupParser(parser, compFileIdx, newIdentCache(), conf): + let ast = parseAll(parser) + + # Process AST to find NimMajor, NimMinor, NimPatch definitions + proc processNode(n: PNode) = + case n.kind + of nkStmtList, nkStmtListExpr: + for child in n: + processNode(child) + of nkConstSection: + for child in n: + if child.kind == nkConstDef: + var identName = "" + case child[0].kind + of nkPostfix: + if child[0][1].kind == nkIdent: + identName = child[0][1].ident.s + of nkIdent: + identName = child[0].ident.s + of nkPragmaExpr: + # Handle pragma expression (like NimMajor* {.intdefine.}) + if child[0][0].kind == nkIdent: + identName = child[0][0].ident.s + elif child[0][0].kind == nkPostfix and child[0][0][1].kind == nkIdent: + identName = child[0][0][1].ident.s + else: discard + # echo "Unhandled node kind for const name: ", child[0].kind + # Extract value + if child.len > 2: + case child[2].kind + of nkIntLit: + let value = child[2].intVal.int + case identName + of "NimMajor": major = value + of "NimMinor": minor = value + of "NimPatch": patch = value + else: discard + else: + discard + else: + discard + + processNode(ast) + closeParser(parser) # echo "Extracted version: ", major, ".", minor, ".", patch return &"{major}.{minor}.{patch}" @@ -340,12 +341,13 @@ proc extractRequiresInfo*(nimbleFile: string, options: Options): NimbleFileInfo localError(config, info, warnUser, msg) let fileIdx = fileInfoIdx(conf, AbsoluteFile nimbleFile) - var parser: Parser - if setupParser(parser, fileIdx, newIdentCache(), conf): - let ast = parseAll(parser) - extract(ast, conf, result, options) - closeParser(parser) - result.hasErrors = result.hasErrors or conf.errorCounter > 0 + {.cast(gcSafe).}: + var parser: Parser + if setupParser(parser, fileIdx, newIdentCache(), conf): + let ast = parseAll(parser) + extract(ast, conf, result, options) + closeParser(parser) + result.hasErrors = result.hasErrors or conf.errorCounter > 0 # Add requires from external requires file let nimbleDir = nimbleFile.splitFile.dir @@ -380,10 +382,11 @@ proc extractPluginInfo*(nimscriptFile: string, info: var PluginInfo) = conf.mainPackageNotes = {} let fileIdx = fileInfoIdx(conf, AbsoluteFile nimscriptFile) - var parser: Parser - if setupParser(parser, fileIdx, newIdentCache(), conf): - extractPlugin(nimscriptFile, parseAll(parser), conf, info) - closeParser(parser) + {.cast(gcSafe).}: + var parser: Parser + if setupParser(parser, fileIdx, newIdentCache(), conf): + extractPlugin(nimscriptFile, parseAll(parser), conf, info) + closeParser(parser) const Operators* = {'<', '>', '=', '&', '@', '!', '^'} diff --git a/src/nimblepkg/download.nim b/src/nimblepkg/download.nim index eb0a5599f..6e61b1646 100644 --- a/src/nimblepkg/download.nim +++ b/src/nimblepkg/download.nim @@ -874,19 +874,18 @@ proc downloadPkgAsync*(url: string, verRange: VersionRange, if validateRange and verRange.kind notin {verSpecial, verAny} or not options.isLegacy: ## Makes sure that the downloaded package's version satisfies the requested ## version range. - {.gcsafe.}: - try: - pkginfo = if options.satResult.pass == satNimSelection: #TODO later when in vnext we should just use this code path and fallback inside the toRequires if we can - getPkgInfoFromDirWithDeclarativeParser(result.dir, options, nimBin) - else: - getPkgInfo(result.dir, options, nimBin) - except Exception as e: - raise nimbleError("Failed to get package info: " & e.msg) - if pkginfo.basicInfo.version notin verRange: - raise nimbleError( - "Downloaded package's version does not satisfy requested version " & - "range: wanted $1 got $2." % - [$verRange, $pkginfo.basicInfo.version]) + try: + pkginfo = if options.satResult.pass == satNimSelection: #TODO later when in vnext we should just use this code path and fallback inside the toRequires if we can + getPkgInfoFromDirWithDeclarativeParser(result.dir, options, nimBin) + else: + getPkgInfo(result.dir, options, nimBin) + except Exception as e: + raise nimbleError("Failed to get package info: " & e.msg) + if pkginfo.basicInfo.version notin verRange: + raise nimbleError( + "Downloaded package's version does not satisfy requested version " & + "range: wanted $1 got $2." % + [$verRange, $pkginfo.basicInfo.version]) #TODO rework the pkgcache to handle this better #ideally we should be able to know the version we are downloading upfront diff --git a/src/nimblepkg/vcstools.nim b/src/nimblepkg/vcstools.nim index e2b921ed1..6bd82ff9c 100644 --- a/src/nimblepkg/vcstools.nim +++ b/src/nimblepkg/vcstools.nim @@ -76,6 +76,13 @@ proc hasVcsSubDir*(dir: Path): VcsType = else: result = vcsTypeNone +var vcsTypeAndSpecialDirPathCache {.threadvar.}: TableRef[Path, VcsTypeAndSpecialDirPath] + +proc getVcsTypeAndSpecialDirPathCache(): TableRef[Path, VcsTypeAndSpecialDirPath] = + if vcsTypeAndSpecialDirPathCache.isNil: + vcsTypeAndSpecialDirPathCache = newTable[Path, VcsTypeAndSpecialDirPath]() + return vcsTypeAndSpecialDirPathCache + proc getVcsTypeAndSpecialDirPath*(dir: Path): VcsTypeAndSpecialDirPath = ## By given directory `dir` gets the type of VCS under which is it by ## traversing the parent directories until some specific directory like @@ -88,9 +95,8 @@ proc getVcsTypeAndSpecialDirPath*(dir: Path): VcsTypeAndSpecialDirPath = ## ## Raises a `NimbleError` in the case the directory `dir` does not exist. - var cache {.global.}: Table[Path, VcsTypeAndSpecialDirPath] - if cache.hasKey(dir): - return cache[dir] + if getVcsTypeAndSpecialDirPathCache().hasKey(dir): + return getVcsTypeAndSpecialDirPathCache()[dir] if not dir.dirExists: raise nimbleError(dirDoesNotExistErrorMsg(dir)) @@ -111,7 +117,7 @@ proc getVcsTypeAndSpecialDirPath*(dir: Path): VcsTypeAndSpecialDirPath = dirIter = dirIter / vcsType.getVcsSpecialDir.Path result = (vcsType, dirIter) - cache[dir] = result + getVcsTypeAndSpecialDirPathCache()[dir] = result proc getVcsType*(dir: Path): VcsType = ## Returns VCS type of the given directory. From 7b72215cae7f57f23b0defba24364f856a2ff5f4 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 20 Nov 2025 12:40:06 +0000 Subject: [PATCH 10/29] [OK] getTagsListAsync lists tags from local repo --- src/nimblepkg/download.nim | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/nimblepkg/download.nim b/src/nimblepkg/download.nim index 6e61b1646..8fdd92eba 100644 --- a/src/nimblepkg/download.nim +++ b/src/nimblepkg/download.nim @@ -127,6 +127,33 @@ proc getTagsList*(dir: string, meth: DownloadMethod): seq[string] = else: result = @[] +proc getTagsListAsync*(dir: string, meth: DownloadMethod): Future[seq[string]] {.async.} = + ## Async version of getTagsList that uses doCmdExAsync for non-blocking execution. + var output: string + cd dir: + case meth + of DownloadMethod.git: + output = await tryDoCmdExAsync("git tag") + of DownloadMethod.hg: + output = await tryDoCmdExAsync("hg tags") + if output.len > 0: + case meth + of DownloadMethod.git: + result = @[] + for i in output.splitLines(): + if i == "": continue + result.add(i) + of DownloadMethod.hg: + result = @[] + for i in output.splitLines(): + if i == "": continue + var tag = "" + discard parseUntil(i, tag, ' ') + if tag != "tip": + result.add(tag) + else: + result = @[] + proc getTagsListRemote*(url: string, meth: DownloadMethod): seq[string] = result = @[] case meth From cc2d05ca2d27bbc05c4a651974f1c9c209b12576 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 20 Nov 2025 12:42:19 +0000 Subject: [PATCH 11/29] [OK] doCheckoutAsync --- src/nimblepkg/download.nim | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/nimblepkg/download.nim b/src/nimblepkg/download.nim index 8fdd92eba..ae5c939de 100644 --- a/src/nimblepkg/download.nim +++ b/src/nimblepkg/download.nim @@ -43,6 +43,19 @@ proc doCheckout*(meth: DownloadMethod, downloadDir, branch: string, options: Opt of DownloadMethod.hg: discard tryDoCmdEx(&"hg --cwd {downloadDir.quoteShell} checkout {branch.quoteShell}") +proc doCheckoutAsync*(meth: DownloadMethod, downloadDir, branch: string, options: Options): Future[void] {.async.} = + ## Async version of doCheckout that uses doCmdExAsync for non-blocking execution. + case meth + of DownloadMethod.git: + # Force is used here because local changes may appear straight after a clone + # has happened. Like in the case of git on Windows where it messes up the + # damn line endings. + discard await tryDoCmdExAsync(&"git -C {downloadDir.quoteShell} checkout --force {branch.quoteShell}") + if not options.ignoreSubmodules: + await downloadDir.updateSubmodulesAsync() + of DownloadMethod.hg: + discard await tryDoCmdExAsync(&"hg --cwd {downloadDir.quoteShell} checkout {branch.quoteShell}") + proc doClone(meth: DownloadMethod, url, downloadDir: string, branch = "", onlyTip = true, options: Options) = case meth From 7a47298996afa2927f05d4447cf8bca953dd0b81 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 20 Nov 2025 16:00:07 +0000 Subject: [PATCH 12/29] [OK] getPackageMinimalVersionsFromRepoAsync gets package versions --- src/nimblepkg/nimblesat.nim | 101 ++++++++++++++++++++++++++++++++++-- 1 file changed, 98 insertions(+), 3 deletions(-) diff --git a/src/nimblepkg/nimblesat.nim b/src/nimblepkg/nimblesat.nim index bc28eded0..9a7b4ec2c 100644 --- a/src/nimblepkg/nimblesat.nim +++ b/src/nimblepkg/nimblesat.nim @@ -1,8 +1,9 @@ -import sat/[sat, satvars] -import version, packageinfotypes, download, packageinfo, packageparser, options, +import sat/[sat, satvars] +import version, packageinfotypes, download, packageinfo, packageparser, options, sha1hashes, tools, downloadnim, cli, declarativeparser - + import std/[tables, sequtils, algorithm, sets, strutils, options, strformat, os, json, jsonutils] +import chronos type SatVarInfo* = object # attached information for a SAT variable @@ -704,6 +705,100 @@ proc getPackageMinimalVersionsFromRepo*(repoDir: string, pkg: PkgTuple, version: except CatchableError as e: displayWarning(&"Error cleaning up temporary directory {tempDir}: {e.msg}", LowPriority) +proc getPackageMinimalVersionsFromRepoAsync*(repoDir: string, pkg: PkgTuple, version: Version, downloadMethod: DownloadMethod, options: Options, nimBin: string): Future[seq[PackageMinimalInfo]] {.async.} = + ## Async version of getPackageMinimalVersionsFromRepo that uses async operations for VCS commands. + result = newSeq[PackageMinimalInfo]() + + let name = pkg[0] + try: + let taggedVersions = getTaggedVersions(repoDir, name, options) + if taggedVersions.isSome: + return taggedVersions.get.versions + except Exception: + discard # Continue with fetching from repo + + let tempDir = repoDir & "_versions" + try: + removeDir(tempDir) + copyDir(repoDir, tempDir) + var tags = initOrderedTable[Version, string]() + try: + await gitFetchTagsAsync(tempDir, downloadMethod, options) + tags = (await getTagsListAsync(tempDir, downloadMethod)).getVersionList() + except CatchableError as e: + displayWarning(&"Error fetching tags for {name}: {e.msg}", HighPriority) + + try: + {.gcsafe.}: + try: + if options.satResult.pass in {satNimSelection}: + # TODO test this code path + result.add getPkgInfoFromDirWithDeclarativeParser(repoDir, options, nimBin).getMinimalInfo(options) + else: + result.add getPkgInfo(repoDir, options, nimBin).getMinimalInfo(options) + except Exception as e: + raise newException(CatchableError, e.msg) + except CatchableError as e: + displayWarning(&"Error getting package info for {name}: {e.msg}", HighPriority) + + # Process tagged versions in the temporary copy + var checkedTags = 0 + for (ver, tag) in tags.pairs: + if options.maxTaggedVersions > 0 and checkedTags >= options.maxTaggedVersions: + break + inc checkedTags + + try: + let tagVersion = newVersion($ver) + + if not tagVersion.withinRange(pkg[1]): + displayInfo(&"Ignoring {name}:{tagVersion} because out of range {pkg[1]}", LowPriority) + continue + + await doCheckoutAsync(downloadMethod, tempDir, tag, options) + let nimbleFile = findNimbleFile(tempDir, true, options, warn = false) + {.gcsafe.}: + try: + if options.satResult.pass in {satNimSelection}: + result.addUnique getPkgInfoFromDirWithDeclarativeParser(tempDir, options, nimBin).getMinimalInfo(options) + elif options.useDeclarativeParser: + result.addUnique getMinimalInfo(nimbleFile, options, nimBin) + else: + let pkgInfo = getPkgInfoFromFile(nimBin, nimbleFile, options, useCache=false) + result.addUnique pkgInfo.getMinimalInfo(options) + except Exception as e: + raise newException(CatchableError, e.msg) + #here we copy the directory to its own folder so we have it cached for future usage + {.gcsafe.}: + try: + let downloadInfo = getPackageDownloadInfo((name, tagVersion.toVersionRange()), options) + if not dirExists(downloadInfo.downloadDir): + copyDir(tempDir, downloadInfo.downloadDir) + except Exception as e: + raise newException(CatchableError, e.msg) + + except CatchableError as e: + displayWarning( + &"Error reading tag {tag}: for package {name}. This may not be relevant as it could be an old version of the package. \n {e.msg}", + HighPriority) + if not (not options.isLegacy and options.satResult.pass == satNimSelection and options.satResult.declarativeParseFailed): + #Dont save tagged versions if we are in vNext and the declarative parser failed as this could cache the incorrect versions. + #its suboptimal in the sense that next packages after failure wont be saved in the first past but there is a guarantee that there is a second pass in the case + #the declarative parser fails so they will be saved then. + try: + saveTaggedVersions(repoDir, name, + TaggedPackageVersions( + maxTaggedVersions: options.maxTaggedVersions, + versions: result + ), options) + except Exception as e: + displayWarning(&"Error saving tagged versions for {name}: {e.msg}", LowPriority) + finally: + try: + removeDir(tempDir) + except CatchableError as e: + displayWarning(&"Error cleaning up temporary directory {tempDir}: {e.msg}", LowPriority) + proc downloadMinimalPackage*(pv: PkgTuple, options: Options, nimBin: string): seq[PackageMinimalInfo] = if pv.name == "": return newSeq[PackageMinimalInfo]() if pv.isNim and not options.disableNimBinaries: return getAllNimReleases(options) From 38fab27d1468ac6b89bebff1b681b079ea0fade5 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 20 Nov 2025 16:17:01 +0000 Subject: [PATCH 13/29] [OK] downloadMinimalPackageAsync downloads package with versions --- src/nimblepkg/nimblesat.nim | 80 ++++++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/src/nimblepkg/nimblesat.nim b/src/nimblepkg/nimblesat.nim index 9a7b4ec2c..c0f5a199c 100644 --- a/src/nimblepkg/nimblesat.nim +++ b/src/nimblepkg/nimblesat.nim @@ -807,15 +807,91 @@ proc downloadMinimalPackage*(pv: PkgTuple, options: Options, nimBin: string): se return if pv.ver.kind in [verSpecial, verEq]: #if special or equal, we dont retrieve more versions as we only need one. result = @[downloadPkInfoForPv(pv, options, false, nimBin).getMinimalInfo(options)] - else: + else: let (downloadRes, downloadMeth) = downloadPkgFromUrl(pv, options, false, nimBin) result = getPackageMinimalVersionsFromRepo(downloadRes.dir, pv, downloadRes.version, downloadMeth.get, options, nimBin) #Make sure the url is set for the package if pv.name.isUrl: - for r in result.mitems: + for r in result.mitems: if r.url == "": r.url = pv.name +proc downloadFromDownloadInfoAsync*(dlInfo: PackageDownloadInfo, options: Options, nimBin: string): Future[(DownloadPkgResult, Option[DownloadMethod])] {.async.} = + ## Async version of downloadFromDownloadInfo that uses async download operations. + if dlInfo.isFileUrl: + {.gcsafe.}: + try: + let pkgInfo = getPackageFromFileUrl(dlInfo.url, options, nimBin) + let downloadRes = (dir: pkgInfo.getNimbleFileDir(), version: pkgInfo.basicInfo.version, vcsRevision: notSetSha1Hash) + return (downloadRes, none(DownloadMethod)) + except Exception as e: + raise newException(CatchableError, e.msg) + else: + let downloadRes = await downloadPkgAsync(dlInfo.url, dlInfo.pv.ver, dlInfo.meth.get, dlInfo.subdir, options, + dlInfo.downloadDir, vcsRevision = notSetSha1Hash, nimBin = nimBin) + return (downloadRes, dlInfo.meth) + +proc downloadPkgFromUrlAsync*(pv: PkgTuple, options: Options, doPrompt = false, nimBin: string): Future[(DownloadPkgResult, Option[DownloadMethod])] {.async.} = + ## Async version of downloadPkgFromUrl that downloads from a package URL. + {.gcsafe.}: + try: + let dlInfo = getPackageDownloadInfo(pv, options, doPrompt) + return await downloadFromDownloadInfoAsync(dlInfo, options, nimBin) + except Exception as e: + raise newException(CatchableError, e.msg) + +proc downloadPkInfoForPvAsync*(pv: PkgTuple, options: Options, doPrompt = false, nimBin: string): Future[PackageInfo] {.async.} = + ## Async version of downloadPkInfoForPv that downloads and gets package info. + let downloadRes = await downloadPkgFromUrlAsync(pv, options, doPrompt, nimBin) + {.gcsafe.}: + try: + if options.satResult.pass in {satNimSelection}: + return getPkgInfoFromDirWithDeclarativeParser(downloadRes[0].dir, options, nimBin) + else: + return getPkgInfo(downloadRes[0].dir, options, nimBin, forValidation = false, onlyMinimalInfo = false) + except Exception as e: + raise newException(CatchableError, e.msg) + +proc downloadMinimalPackageAsync*(pv: PkgTuple, options: Options, nimBin: string): Future[seq[PackageMinimalInfo]] {.async.} = + ## Async version of downloadMinimalPackage that downloads package versions in parallel. + if pv.name == "": return newSeq[PackageMinimalInfo]() + + {.gcsafe.}: + try: + if pv.isNim and not options.disableNimBinaries: + return getAllNimReleases(options) + except Exception as e: + raise newException(CatchableError, e.msg) + + if pv.name.isFileURL: + {.gcsafe.}: + try: + result = @[getPackageFromFileUrl(pv.name, options, nimBin).getMinimalInfo(options)] + return + except Exception as e: + raise newException(CatchableError, e.msg) + + if pv.ver.kind in [verSpecial, verEq]: #if special or equal, we dont retrieve more versions as we only need one. + let pkgInfo = await downloadPkInfoForPvAsync(pv, options, false, nimBin) + {.gcsafe.}: + try: + result = @[pkgInfo.getMinimalInfo(options)] + except Exception as e: + raise newException(CatchableError, e.msg) + else: + let (downloadRes, downloadMeth) = await downloadPkgFromUrlAsync(pv, options, false, nimBin) + result = await getPackageMinimalVersionsFromRepoAsync(downloadRes.dir, pv, downloadRes.version, downloadMeth.get, options, nimBin) + + #Make sure the url is set for the package + {.gcsafe.}: + try: + if pv.name.isUrl: + for r in result.mitems: + if r.url == "": + r.url = pv.name + except Exception as e: + raise newException(CatchableError, e.msg) + proc fillPackageTableFromPreferred*(packages: var Table[string, PackageVersions], preferredPackages: seq[PackageMinimalInfo]) = for pkg in preferredPackages: if not hasVersion(packages, pkg.name, pkg.version): From f738e0a5defc628eb858da8e27f6817714769a2c Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 20 Nov 2025 16:26:07 +0000 Subject: [PATCH 14/29] Refactor a bunch of procs to be thread safe --- src/nimblepkg/download.nim | 5 +- src/nimblepkg/nimblesat.nim | 133 ++++++++++++++++------------------ src/nimblepkg/packageinfo.nim | 24 +++--- 3 files changed, 78 insertions(+), 84 deletions(-) diff --git a/src/nimblepkg/download.nim b/src/nimblepkg/download.nim index ae5c939de..94151ae94 100644 --- a/src/nimblepkg/download.nim +++ b/src/nimblepkg/download.nim @@ -295,7 +295,7 @@ proc cloneSpecificRevisionAsync*(downloadMethod: DownloadMethod, var tarExePathCache {.threadvar.}: string -proc getTarExePath: string {.gcsafe.} = +proc getTarExePath: string = ## Returns path to `tar` executable. if tarExePathCache == "": tarExePathCache = @@ -750,8 +750,7 @@ proc doDownloadAsync(url, downloadDir: string, verRange: VersionRange, if result.vcsRevision == notSetSha1Hash: # In the case the package in not downloaded as tarball we must query its # VCS revision from its download directory. - {.gcsafe.}: - result.vcsRevision = downloadDir.getVcsRevision + result.vcsRevision = downloadDir.getVcsRevision proc pkgDirHasNimble*(dir: string, options: Options): bool = try: diff --git a/src/nimblepkg/nimblesat.nim b/src/nimblepkg/nimblesat.nim index c0f5a199c..53b51cfd1 100644 --- a/src/nimblepkg/nimblesat.nim +++ b/src/nimblepkg/nimblesat.nim @@ -729,15 +729,14 @@ proc getPackageMinimalVersionsFromRepoAsync*(repoDir: string, pkg: PkgTuple, ver displayWarning(&"Error fetching tags for {name}: {e.msg}", HighPriority) try: - {.gcsafe.}: - try: - if options.satResult.pass in {satNimSelection}: - # TODO test this code path - result.add getPkgInfoFromDirWithDeclarativeParser(repoDir, options, nimBin).getMinimalInfo(options) - else: - result.add getPkgInfo(repoDir, options, nimBin).getMinimalInfo(options) - except Exception as e: - raise newException(CatchableError, e.msg) + try: + if options.satResult.pass in {satNimSelection}: + # TODO test this code path + result.add getPkgInfoFromDirWithDeclarativeParser(repoDir, options, nimBin).getMinimalInfo(options) + else: + result.add getPkgInfo(repoDir, options, nimBin).getMinimalInfo(options) + except Exception as e: + raise newException(CatchableError, e.msg) except CatchableError as e: displayWarning(&"Error getting package info for {name}: {e.msg}", HighPriority) @@ -757,25 +756,23 @@ proc getPackageMinimalVersionsFromRepoAsync*(repoDir: string, pkg: PkgTuple, ver await doCheckoutAsync(downloadMethod, tempDir, tag, options) let nimbleFile = findNimbleFile(tempDir, true, options, warn = false) - {.gcsafe.}: - try: - if options.satResult.pass in {satNimSelection}: - result.addUnique getPkgInfoFromDirWithDeclarativeParser(tempDir, options, nimBin).getMinimalInfo(options) - elif options.useDeclarativeParser: - result.addUnique getMinimalInfo(nimbleFile, options, nimBin) - else: - let pkgInfo = getPkgInfoFromFile(nimBin, nimbleFile, options, useCache=false) - result.addUnique pkgInfo.getMinimalInfo(options) - except Exception as e: - raise newException(CatchableError, e.msg) + try: + if options.satResult.pass in {satNimSelection}: + result.addUnique getPkgInfoFromDirWithDeclarativeParser(tempDir, options, nimBin).getMinimalInfo(options) + elif options.useDeclarativeParser: + result.addUnique getMinimalInfo(nimbleFile, options, nimBin) + else: + let pkgInfo = getPkgInfoFromFile(nimBin, nimbleFile, options, useCache=false) + result.addUnique pkgInfo.getMinimalInfo(options) + except Exception as e: + raise newException(CatchableError, e.msg) #here we copy the directory to its own folder so we have it cached for future usage - {.gcsafe.}: - try: - let downloadInfo = getPackageDownloadInfo((name, tagVersion.toVersionRange()), options) - if not dirExists(downloadInfo.downloadDir): - copyDir(tempDir, downloadInfo.downloadDir) - except Exception as e: - raise newException(CatchableError, e.msg) + try: + let downloadInfo = getPackageDownloadInfo((name, tagVersion.toVersionRange()), options) + if not dirExists(downloadInfo.downloadDir): + copyDir(tempDir, downloadInfo.downloadDir) + except Exception as e: + raise newException(CatchableError, e.msg) except CatchableError as e: displayWarning( @@ -819,13 +816,12 @@ proc downloadMinimalPackage*(pv: PkgTuple, options: Options, nimBin: string): se proc downloadFromDownloadInfoAsync*(dlInfo: PackageDownloadInfo, options: Options, nimBin: string): Future[(DownloadPkgResult, Option[DownloadMethod])] {.async.} = ## Async version of downloadFromDownloadInfo that uses async download operations. if dlInfo.isFileUrl: - {.gcsafe.}: - try: - let pkgInfo = getPackageFromFileUrl(dlInfo.url, options, nimBin) - let downloadRes = (dir: pkgInfo.getNimbleFileDir(), version: pkgInfo.basicInfo.version, vcsRevision: notSetSha1Hash) - return (downloadRes, none(DownloadMethod)) - except Exception as e: - raise newException(CatchableError, e.msg) + try: + let pkgInfo = getPackageFromFileUrl(dlInfo.url, options, nimBin) + let downloadRes = (dir: pkgInfo.getNimbleFileDir(), version: pkgInfo.basicInfo.version, vcsRevision: notSetSha1Hash) + return (downloadRes, none(DownloadMethod)) + except Exception as e: + raise newException(CatchableError, e.msg) else: let downloadRes = await downloadPkgAsync(dlInfo.url, dlInfo.pv.ver, dlInfo.meth.get, dlInfo.subdir, options, dlInfo.downloadDir, vcsRevision = notSetSha1Hash, nimBin = nimBin) @@ -833,64 +829,57 @@ proc downloadFromDownloadInfoAsync*(dlInfo: PackageDownloadInfo, options: Option proc downloadPkgFromUrlAsync*(pv: PkgTuple, options: Options, doPrompt = false, nimBin: string): Future[(DownloadPkgResult, Option[DownloadMethod])] {.async.} = ## Async version of downloadPkgFromUrl that downloads from a package URL. - {.gcsafe.}: - try: - let dlInfo = getPackageDownloadInfo(pv, options, doPrompt) - return await downloadFromDownloadInfoAsync(dlInfo, options, nimBin) - except Exception as e: - raise newException(CatchableError, e.msg) + try: + let dlInfo = getPackageDownloadInfo(pv, options, doPrompt) + return await downloadFromDownloadInfoAsync(dlInfo, options, nimBin) + except Exception as e: + raise newException(CatchableError, e.msg) proc downloadPkInfoForPvAsync*(pv: PkgTuple, options: Options, doPrompt = false, nimBin: string): Future[PackageInfo] {.async.} = ## Async version of downloadPkInfoForPv that downloads and gets package info. let downloadRes = await downloadPkgFromUrlAsync(pv, options, doPrompt, nimBin) - {.gcsafe.}: - try: - if options.satResult.pass in {satNimSelection}: - return getPkgInfoFromDirWithDeclarativeParser(downloadRes[0].dir, options, nimBin) - else: - return getPkgInfo(downloadRes[0].dir, options, nimBin, forValidation = false, onlyMinimalInfo = false) - except Exception as e: - raise newException(CatchableError, e.msg) + try: + if options.satResult.pass in {satNimSelection}: + return getPkgInfoFromDirWithDeclarativeParser(downloadRes[0].dir, options, nimBin) + else: + return getPkgInfo(downloadRes[0].dir, options, nimBin, forValidation = false, onlyMinimalInfo = false) + except Exception as e: + raise newException(CatchableError, e.msg) proc downloadMinimalPackageAsync*(pv: PkgTuple, options: Options, nimBin: string): Future[seq[PackageMinimalInfo]] {.async.} = ## Async version of downloadMinimalPackage that downloads package versions in parallel. if pv.name == "": return newSeq[PackageMinimalInfo]() + try: + if pv.isNim and not options.disableNimBinaries: + return getAllNimReleases(options) + except Exception as e: + raise newException(CatchableError, e.msg) - {.gcsafe.}: + if pv.name.isFileURL: try: - if pv.isNim and not options.disableNimBinaries: - return getAllNimReleases(options) + result = @[getPackageFromFileUrl(pv.name, options, nimBin).getMinimalInfo(options)] + return except Exception as e: raise newException(CatchableError, e.msg) - if pv.name.isFileURL: - {.gcsafe.}: - try: - result = @[getPackageFromFileUrl(pv.name, options, nimBin).getMinimalInfo(options)] - return - except Exception as e: - raise newException(CatchableError, e.msg) - if pv.ver.kind in [verSpecial, verEq]: #if special or equal, we dont retrieve more versions as we only need one. let pkgInfo = await downloadPkInfoForPvAsync(pv, options, false, nimBin) - {.gcsafe.}: - try: - result = @[pkgInfo.getMinimalInfo(options)] - except Exception as e: - raise newException(CatchableError, e.msg) + try: + result = @[pkgInfo.getMinimalInfo(options)] + except Exception as e: + raise newException(CatchableError, e.msg) else: let (downloadRes, downloadMeth) = await downloadPkgFromUrlAsync(pv, options, false, nimBin) result = await getPackageMinimalVersionsFromRepoAsync(downloadRes.dir, pv, downloadRes.version, downloadMeth.get, options, nimBin) #Make sure the url is set for the package - {.gcsafe.}: - try: - if pv.name.isUrl: - for r in result.mitems: - if r.url == "": - r.url = pv.name - except Exception as e: - raise newException(CatchableError, e.msg) + try: + if pv.name.isUrl: + for r in result.mitems: + if r.url == "": + r.url = pv.name + except Exception as e: + raise newException(CatchableError, e.msg) proc fillPackageTableFromPreferred*(packages: var Table[string, PackageVersions], preferredPackages: seq[PackageMinimalInfo]) = for pkg in preferredPackages: diff --git a/src/nimblepkg/packageinfo.nim b/src/nimblepkg/packageinfo.nim index 7928eb0c2..94f4974f7 100644 --- a/src/nimblepkg/packageinfo.nim +++ b/src/nimblepkg/packageinfo.nim @@ -191,11 +191,17 @@ proc fetchList*(list: PackageList, options: Options) = # Cache after first call var - gPackageJson: Table[string, JsonNode] + gPackageJson {.threadvar.}: TableRef[string, JsonNode] + +proc getGPackageJson(): TableRef[string, JsonNode] = + if gPackageJson.isNil: + gPackageJson = newTable[string, JsonNode]() + return gPackageJson + proc readPackageList(name: string, options: Options, ignorePackageCache = false): JsonNode = # If packages.json is not present ask the user if they want to download it. - if (not ignorePackageCache) and gPackageJson.hasKey(name): - return gPackageJson[name] + if (not ignorePackageCache) and getGPackageJson().hasKey(name): + return getGPackageJson()[name] if needsRefresh(options): if options.prompt("No local packages.json found, download it from " & @@ -205,16 +211,16 @@ proc readPackageList(name: string, options: Options, ignorePackageCache = false) else: # The user might not need a package list for now. So let's try # going further. - gPackageJson[name] = newJArray() - return gPackageJson[name] + getGPackageJson()[name] = newJArray() + return getGPackageJson()[name] let file = options.getNimbleDir() / "packages_" & name.toLowerAscii() & ".json" if file.fileExists: - gPackageJson[name] = parseFile(file) + getGPackageJson()[name] = parseFile(file) else: - gPackageJson[name] = newJArray() - return gPackageJson[name] + getGPackageJson()[name] = newJArray() + return getGPackageJson()[name] -proc getPackage*(pkg: string, options: Options, resPkg: var Package, ignorePackageCache = false): bool +proc getPackage*(pkg: string, options: Options, resPkg: var Package, ignorePackageCache = false): bool {.gcsafe.} proc resolveAlias(pkg: Package, options: Options): Package = result = pkg # Resolve alias. From fb951d130cffc93cc8c4457913049bbf62532b37 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 20 Nov 2025 16:43:24 +0000 Subject: [PATCH 15/29] [OK] getMinimalFromPreferredAsync returns preferred package --- src/nimblepkg/nimblesat.nim | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/nimblepkg/nimblesat.nim b/src/nimblepkg/nimblesat.nim index 53b51cfd1..79aa19f7e 100644 --- a/src/nimblepkg/nimblesat.nim +++ b/src/nimblepkg/nimblesat.nim @@ -46,6 +46,7 @@ type GetPackageMinimal* = proc (pv: PkgTuple, options: Options, nimBin: string): seq[PackageMinimalInfo] + GetPackageMinimalAsync* = proc (pv: PkgTuple, options: Options, nimBin: string): Future[seq[PackageMinimalInfo]] {.gcsafe.} TaggedPackageVersions = object maxTaggedVersions: int # Maximum number of tags. When number changes, we invalidate the cache @@ -898,6 +899,16 @@ proc getMinimalFromPreferred(pv: PkgTuple, getMinimalPackage: GetPackageMinimal return @[pp] getMinimalPackage(pv, options, nimBin) +proc getMinimalFromPreferredAsync*(pv: PkgTuple, getMinimalPackage: GetPackageMinimalAsync, preferredPackages: seq[PackageMinimalInfo], options: Options, nimBin: string): Future[seq[PackageMinimalInfo]] {.async.} = + ## Async version of getMinimalFromPreferred that uses async package fetching. + for pp in preferredPackages: + if (pp.name == pv.name or pp.url == pv.name) and pp.version.withinRange(pv.ver): + return @[pp] + try: + return await getMinimalPackage(pv, options, nimBin) + except Exception as e: + raise newException(CatchableError, e.msg) + proc hasSpecialVersion(versions: Table[string, PackageVersions], pkgName: string): bool = if pkgName in versions: for pkg in versions[pkgName].versions: From 5c4deb13576ef0d888d9cbd1e9a56ef41556999e Mon Sep 17 00:00:00 2001 From: jmgomez Date: Fri, 21 Nov 2025 11:23:59 +0000 Subject: [PATCH 16/29] uses collectAllVersionsAsync in solvePackages --- src/nimblepkg/nimblesat.nim | 191 ++++++++++++++++- tests/tasynctools.nim | 405 ++++++++++++++++++++++++++++++++++++ 2 files changed, 588 insertions(+), 8 deletions(-) create mode 100644 tests/tasynctools.nim diff --git a/src/nimblepkg/nimblesat.nim b/src/nimblepkg/nimblesat.nim index 79aa19f7e..697ac4740 100644 --- a/src/nimblepkg/nimblesat.nim +++ b/src/nimblepkg/nimblesat.nim @@ -847,8 +847,10 @@ proc downloadPkInfoForPvAsync*(pv: PkgTuple, options: Options, doPrompt = false, except Exception as e: raise newException(CatchableError, e.msg) -proc downloadMinimalPackageAsync*(pv: PkgTuple, options: Options, nimBin: string): Future[seq[PackageMinimalInfo]] {.async.} = - ## Async version of downloadMinimalPackage that downloads package versions in parallel. +var downloadCache {.threadvar.}: Table[string, Future[seq[PackageMinimalInfo]]] + +proc downloadMinimalPackageAsyncImpl(pv: PkgTuple, options: Options, nimBin: string): Future[seq[PackageMinimalInfo]] {.async.} = + ## Internal implementation of async download without caching. if pv.name == "": return newSeq[PackageMinimalInfo]() try: if pv.isNim and not options.disableNimBinaries: @@ -870,8 +872,11 @@ proc downloadMinimalPackageAsync*(pv: PkgTuple, options: Options, nimBin: string except Exception as e: raise newException(CatchableError, e.msg) else: - let (downloadRes, downloadMeth) = await downloadPkgFromUrlAsync(pv, options, false, nimBin) - result = await getPackageMinimalVersionsFromRepoAsync(downloadRes.dir, pv, downloadRes.version, downloadMeth.get, options, nimBin) + try: + let (downloadRes, downloadMeth) = await downloadPkgFromUrlAsync(pv, options, false, nimBin) + result = await getPackageMinimalVersionsFromRepoAsync(downloadRes.dir, pv, downloadRes.version, downloadMeth.get, options, nimBin) + except Exception as e: + raise newException(CatchableError, e.msg) #Make sure the url is set for the package try: @@ -882,6 +887,26 @@ proc downloadMinimalPackageAsync*(pv: PkgTuple, options: Options, nimBin: string except Exception as e: raise newException(CatchableError, e.msg) +proc downloadMinimalPackageAsync*(pv: PkgTuple, options: Options, nimBin: string): Future[seq[PackageMinimalInfo]] {.async.} = + ## Async version of downloadMinimalPackage with deduplication. + ## If multiple calls request the same package concurrently, they share the same download. + let cacheKey = pv.name & "@" & $pv.ver + + # Check if download is already in progress + if downloadCache.hasKey(cacheKey): + # Wait for the existing download to complete + return await downloadCache[cacheKey] + + # Start new download and cache the future + let downloadFuture = downloadMinimalPackageAsyncImpl(pv, options, nimBin) + downloadCache[cacheKey] = downloadFuture + + try: + result = await downloadFuture + finally: + # Remove from cache after completion (success or failure) + downloadCache.del(cacheKey) + proc fillPackageTableFromPreferred*(packages: var Table[string, PackageVersions], preferredPackages: seq[PackageMinimalInfo]) = for pkg in preferredPackages: if not hasVersion(packages, pkg.name, pkg.version): @@ -983,12 +1008,148 @@ proc processRequirements(versions: var Table[string, PackageVersions], pv: PkgTu # we need to avoid adding it to the package table as this will cause the solver to fail displayWarning(&"Error processing requirements for {pv.name}: {e.msg}", HighPriority) +proc processRequirementsAsync(pv: PkgTuple, visitedParam: HashSet[PkgTuple], getMinimalPackage: GetPackageMinimalAsync, preferredPackages: seq[PackageMinimalInfo] = newSeq[PackageMinimalInfo](), options: Options, nimBin: string): Future[Table[string, PackageVersions]] {.async.} = + ## Async version of processRequirements that returns computed versions instead of mutating shared state. + ## This allows for safe parallel execution since there's no shared mutable state. + ## Takes visited by value since we pass separate copies to each top-level dependency branch. + ## Processes all nested dependencies in parallel for maximum performance. + result = initTable[string, PackageVersions]() + + # Make a local mutable copy + var visited = visitedParam + + if pv in visited: + return + + visited.incl pv + + # For special versions, always process them even if we think we have the package + # This ensures the special version gets downloaded and added to the version table + try: + var pkgMins = await getMinimalFromPreferredAsync(pv, getMinimalPackage, preferredPackages, options, nimBin) + + # First, validate all requirements for all package versions before adding anything + var validPkgMins: seq[PackageMinimalInfo] = @[] + for pkgMin in pkgMins: + var allRequirementsValid = true + # Test if all requirements can be processed without errors + for req in pkgMin.requires: + try: + # Try to get minimal package info for the requirement to validate it exists + discard await getMinimalFromPreferredAsync(req, getMinimalPackage, preferredPackages, options, nimBin) + except CatchableError: + allRequirementsValid = false + displayWarning(&"Skipping package {pkgMin.name}@{pkgMin.version} due to invalid dependency: {req.name}", HighPriority) + break + + if allRequirementsValid: + validPkgMins.add pkgMin + + # Only add packages with valid requirements to the result table + for pkgMin in validPkgMins.mitems: + let pkgName = pkgMin.name.toLower + if pv.ver.kind == verSpecial: + # Keep both the commit hash and the actual semantic version + var specialVer = newVersion($pv.ver) + specialVer.speSemanticVersion = some($pkgMin.version) # Store the real version + pkgMin.version = specialVer + + # Special versions replace any existing versions + result[pkgName] = PackageVersions(pkgName: pkgName, versions: @[pkgMin]) + else: + # Add to result table + if not result.hasKey(pkgName): + result[pkgName] = PackageVersions(pkgName: pkgName, versions: @[pkgMin]) + else: + result[pkgName].versions.addUnique pkgMin + + # Process all requirements in parallel (full parallelization) + # Each branch gets its own copy of visited to avoid shared state issues + var reqFutures: seq[Future[Table[string, PackageVersions]]] = @[] + for req in pkgMin.requires: + reqFutures.add processRequirementsAsync(req, visited, getMinimalPackage, preferredPackages, options, nimBin) + + # Wait for all requirement processing to complete + if reqFutures.len > 0: + await allFutures(reqFutures) + + # Merge all requirement results + for reqFut in reqFutures: + let reqResult = reqFut.read() + for pkgName, pkgVersions in reqResult: + if not result.hasKey(pkgName): + result[pkgName] = pkgVersions + else: + for ver in pkgVersions.versions: + result[pkgName].versions.addUnique ver + + # Only add URL packages if we have valid versions + try: + if pv.name.isUrl and validPkgMins.len > 0: + result[pv.name] = PackageVersions(pkgName: pv.name, versions: validPkgMins) + except Exception as e: + raise newException(CatchableError, e.msg) + + except CatchableError as e: + # Some old packages may have invalid requirements (i.e repos that doesn't exist anymore) + # we need to avoid adding it to the package table as this will cause the solver to fail + displayWarning(&"Error processing requirements for {pv.name}: {e.msg}", HighPriority) proc collectAllVersions*(versions: var Table[string, PackageVersions], package: PackageMinimalInfo, options: Options, getMinimalPackage: GetPackageMinimal, preferredPackages: seq[PackageMinimalInfo] = newSeq[PackageMinimalInfo](), nimBin: string) {.instrument.} = var visited = initHashSet[PkgTuple]() for pv in package.requires: processRequirements(versions, pv, visited, getMinimalPackage, preferredPackages, options, nimBin) +proc mergeVersionTables(dest: var Table[string, PackageVersions], source: Table[string, PackageVersions]) = + ## Helper proc to merge version tables. Synchronous to avoid closure capture issues. + for pkgName, pkgVersions in source: + if not dest.hasKey(pkgName): + dest[pkgName] = pkgVersions + else: + # Merge versions, handling special versions + var hasSpecial = false + for ver in pkgVersions.versions: + if ver.version.isSpecial: + hasSpecial = true + # Special version replaces all + dest[pkgName] = PackageVersions(pkgName: pkgName, versions: @[ver]) + break + + if not hasSpecial: + # Check if existing has special version + var existingHasSpecial = false + for ver in dest[pkgName].versions: + if ver.version.isSpecial: + existingHasSpecial = true + break + + # Only add if existing doesn't have special version + if not existingHasSpecial: + for ver in pkgVersions.versions: + dest[pkgName].versions.addUnique ver + +proc collectAllVersionsAsync*(package: PackageMinimalInfo, options: Options, getMinimalPackage: GetPackageMinimalAsync, preferredPackages: seq[PackageMinimalInfo] = newSeq[PackageMinimalInfo](), nimBin: string): Future[Table[string, PackageVersions]] {.async.} = + ## Async version of collectAllVersions that processes top-level dependencies in parallel. + ## Uses return-based approach: each branch returns its computed versions, then we merge them. + ## This allows for safe parallel execution with no shared mutable state during processing. + ## Returns the merged version table instead of mutating a parameter. + + # Process all top-level requirements in parallel + # Each gets its own visited set to avoid race conditions + var futures: seq[Future[Table[string, PackageVersions]]] = @[] + for pv in package.requires: + var visitedCopy = initHashSet[PkgTuple]() + futures.add processRequirementsAsync(pv, visitedCopy, getMinimalPackage, preferredPackages, options, nimBin) + + # Wait for all to complete + await allFutures(futures) + + # Merge all results into a new table + result = initTable[string, PackageVersions]() + for fut in futures: + let resultTable = fut.read() + mergeVersionTables(result, resultTable) + proc topologicalSort*(solvedPkgs: seq[SolvedPackage]): seq[SolvedPackage] {.instrument.} = var inDegree = initTable[string, int]() var adjList = initTable[string, seq[string]]() @@ -1117,9 +1278,15 @@ proc postProcessSolvedPkgs*(solvedPkgs: var seq[SolvedPackage], options: Options proc solvePackages*(rootPkg: PackageInfo, pkgList: seq[PackageInfo], pkgsToInstall: var seq[(string, Version)], options: Options, output: var string, solvedPkgs: var seq[SolvedPackage], nimBin: string): HashSet[PackageInfo] {.instrument.} = var root: PackageMinimalInfo = rootPkg.getMinimalInfo(options) root.isRoot = true - var pkgVersionTable = initTable[string, PackageVersions]() - pkgVersionTable[root.name] = PackageVersions(pkgName: root.name, versions: @[root]) - collectAllVersions(pkgVersionTable, root, options, downloadMinimalPackage, pkgList.mapIt(it.getMinimalInfo(options)), nimBin) + var pkgVersionTable: Table[system.string, packageinfotypes.PackageVersions] + if options.isLegacy: + pkgVersionTable = initTable[string, PackageVersions]() + pkgVersionTable[root.name] = PackageVersions(pkgName: root.name, versions: @[root]) + collectAllVersions(pkgVersionTable, root, options, downloadMinimalPackage, pkgList.mapIt(it.getMinimalInfo(options)), nimBin) + else: + pkgVersionTable = waitFor collectAllVersionsAsync(root, options, downloadMinimalPackageAsync, pkgList.mapIt(it.getMinimalInfo(options)), nimBin) + pkgVersionTable[root.name] = PackageVersions(pkgName: root.name, versions: @[root]) + # if not options.isLegacy: pkgVersionTable.normalizeRequirements(options) options.satResult.pkgVersionTable = pkgVersionTable @@ -1170,7 +1337,15 @@ proc getPkgVersionTable*(pkgInfo: PackageInfo, pkgList: seq[PackageInfo], option var root = pkgInfo.getMinimalInfo(options) root.isRoot = true result[root.name] = PackageVersions(pkgName: root.name, versions: @[root]) - collectAllVersions(result, root, options, downloadMinimalPackage, pkgList.mapIt(it.getMinimalInfo(options)), nimBin) + # Use async version for parallel downloading + let asyncVersions = waitFor collectAllVersionsAsync(root, options, downloadMinimalPackageAsync, pkgList.mapIt(it.getMinimalInfo(options)), nimBin) + # Merge async results into the result table + for pkgName, pkgVersions in asyncVersions: + if not result.hasKey(pkgName): + result[pkgName] = pkgVersions + else: + for ver in pkgVersions.versions: + result[pkgName].versions.addUnique ver const maxPkgNameDisplayWidth = 40 # Cap package name width diff --git a/tests/tasynctools.nim b/tests/tasynctools.nim new file mode 100644 index 000000000..3412363a8 --- /dev/null +++ b/tests/tasynctools.nim @@ -0,0 +1,405 @@ +import unittest, chronos, strutils, os, tables +import nimblepkg/[tools, download, options, packageinfotypes, packageinfo, sha1hashes, version, nimblesat] + +suite "Async Tools": + test "doCmdExAsync executes command": + let (output, exitCode) = waitFor doCmdExAsync("echo hello") + check exitCode == 0 + check "hello" == output.strip + + test "doCloneAsync clones a repo": + let tmpDir = getTempDir() / "nimble_async_test" + let cloneDir = tmpDir / "clone" + + if dirExists(tmpDir): + removeDir(tmpDir) + createDir(tmpDir) + + let options = initOptions() + let repoUrl = "https://github.com/arnetheduck/nim-results" + waitFor doCloneAsync(DownloadMethod.git, repoUrl, cloneDir, options = options) + + check dirExists(cloneDir) + check fileExists(cloneDir / "README.md") + + removeDir(tmpDir) + + test "gitFetchTagsAsync fetches tags": + let tmpDir = getTempDir() / "nimble_async_test_tags" + let cloneDir = tmpDir / "clone" + + if dirExists(tmpDir): + removeDir(tmpDir) + createDir(tmpDir) + + # Clone a repo first (shallow clone without tags) + let options = initOptions() + let repoUrl = "https://github.com/arnetheduck/nim-results" + waitFor doCloneAsync(DownloadMethod.git, repoUrl, cloneDir, options = options) + + # Fetch tags asynchronously + waitFor gitFetchTagsAsync(cloneDir, DownloadMethod.git, options) + + # Verify tags were fetched by checking for a known stable release + let tags = getTagsList(cloneDir, DownloadMethod.git) + check tags.len > 0 + check "v0.4.0" in tags + + removeDir(tmpDir) + + test "getTagsListRemoteAsync queries remote tags": + let repoUrl = "https://github.com/arnetheduck/nim-results" + + # Query remote tags asynchronously + let tags = waitFor getTagsListRemoteAsync(repoUrl, DownloadMethod.git) + + # Verify we got tags and v0.4.0 exists + check tags.len > 0 + check "v0.4.0" in tags + + test "cloneSpecificRevisionAsync clones specific commit": + let tmpDir = getTempDir() / "nimble_async_test_revision" + let cloneDir = tmpDir / "clone" + + if dirExists(tmpDir): + removeDir(tmpDir) + createDir(tmpDir) + + # Clone a specific revision (v0.4.0 commit hash) + let options = initOptions() + let repoUrl = "https://github.com/arnetheduck/nim-results" + let revision = initSha1Hash("8a03fb2e00dccbf35807c869e0184933b0cffa37") + waitFor cloneSpecificRevisionAsync(DownloadMethod.git, repoUrl, cloneDir, revision, options) + + # Verify clone succeeded + check dirExists(cloneDir) + check fileExists(cloneDir / "README.md") + + removeDir(tmpDir) + + test "doDownloadTarballAsync downloads and extracts tarball": + # Skip on systems without tar + if findExe("tar") == "": + skip() + + let tmpDir = getTempDir() / "nimble_async_test_tarball" + let downloadDir = tmpDir / "download" + + if dirExists(tmpDir): + removeDir(tmpDir) + createDir(tmpDir) + + # Download nim-results v0.4.0 as tarball + let repoUrl = "https://github.com/arnetheduck/nim-results" + discard waitFor doDownloadTarballAsync(repoUrl, downloadDir, "v0.4.0", queryRevision = false) + + # Verify download succeeded + check dirExists(downloadDir) + check fileExists(downloadDir / "README.md") + + removeDir(tmpDir) + + test "getTagsListAsync lists tags from local repo": + let tmpDir = getTempDir() / "nimble_async_test_tagslist" + let cloneDir = tmpDir / "clone" + + if dirExists(tmpDir): + removeDir(tmpDir) + createDir(tmpDir) + + # Clone nim-results repo + let options = initOptions() + let repoUrl = "https://github.com/arnetheduck/nim-results" + waitFor doCloneAsync(DownloadMethod.git, repoUrl, cloneDir, + onlyTip = false, options = options) + + # Get tags list using async + let tags = waitFor getTagsListAsync(cloneDir, DownloadMethod.git) + + # Verify v0.4.0 tag exists + check tags.contains("v0.4.0") + check tags.len > 0 + + removeDir(tmpDir) + + test "doCheckoutAsync checks out branch/tag": + let tmpDir = getTempDir() / "nimble_async_test_checkout" + let cloneDir = tmpDir / "clone" + + if dirExists(tmpDir): + removeDir(tmpDir) + createDir(tmpDir) + + # Clone nim-results repo with full history + let options = initOptions() + let repoUrl = "https://github.com/arnetheduck/nim-results" + waitFor doCloneAsync(DownloadMethod.git, repoUrl, cloneDir, + onlyTip = false, options = options) + + # Checkout v0.4.0 tag using async + waitFor doCheckoutAsync(DownloadMethod.git, cloneDir, "v0.4.0", options) + + # Verify checkout succeeded by checking if we're on the right version + check dirExists(cloneDir) + check fileExists(cloneDir / "README.md") + + removeDir(tmpDir) + + test "downloadPkgAsync downloads package": + let tmpDir = getTempDir() / "nimble_async_test_pkg" + + if dirExists(tmpDir): + removeDir(tmpDir) + createDir(tmpDir) + + # Download nim-results using downloadPkgAsync + let options = initOptions() + let repoUrl = "https://github.com/arnetheduck/nim-results" + # Use #v0.4.0 to use verSpecial which skips validation + let verRange = parseVersionRange("#v0.4.0") + + # Find nim binary for validation + let nimBin = findExe("nim") + if nimBin == "": + skip() + + let result = waitFor downloadPkgAsync( + repoUrl, + verRange, + DownloadMethod.git, + subdir = "", + options, + downloadPath = tmpDir, + vcsRevision = notSetSha1Hash, + nimBin = nimBin, + validateRange = false + ) + + # Verify download succeeded + check dirExists(result.dir) + check fileExists(result.dir / "README.md") + # With verSpecial, version will be the special version string + check $result.version == "#v0.4.0" + + removeDir(tmpDir) + + test "getPackageMinimalVersionsFromRepoAsync gets package versions": + let tmpDir = getTempDir() / "nimble_async_test_minimalversions" + let repoDir = tmpDir / "repo" + + if dirExists(tmpDir): + removeDir(tmpDir) + createDir(tmpDir) + + # Clone nim-results repo with full history + let options = initOptions() + let repoUrl = "https://github.com/arnetheduck/nim-results" + waitFor doCloneAsync(DownloadMethod.git, repoUrl, repoDir, + onlyTip = false, options = options) + + # Find nim binary + let nimBin = findExe("nim") + if nimBin == "": + skip() + + # Get minimal versions for a version range + let pkg: PkgTuple = ("results", parseVersionRange(">= 0.4.0")) + let versions = waitFor getPackageMinimalVersionsFromRepoAsync( + repoDir, pkg, newVersion("0.4.0"), DownloadMethod.git, options, nimBin) + + # Verify we got some versions + check versions.len > 0 + # Verify we got v0.4.0 + var foundV040 = false + for v in versions: + if v.version == newVersion("0.4.0"): + foundV040 = true + break + check foundV040 + + removeDir(tmpDir) + + test "downloadMinimalPackageAsync downloads package with versions": + let tmpDir = getTempDir() / "nimble_async_test_minimalpackage" + + if dirExists(tmpDir): + removeDir(tmpDir) + createDir(tmpDir) + + # Find nim binary + let nimBin = findExe("nim") + if nimBin == "": + skip() + + # Create options with custom cache path + var options = initOptions() + options.pkgCachePath = tmpDir / "cache" + createDir(options.pkgCachePath) + + # Download nim-results package using downloadMinimalPackageAsync + # Use a version range to test the full flow of downloading and fetching versions + let pkg: PkgTuple = ("https://github.com/arnetheduck/nim-results", parseVersionRange(">= 0.4.0")) + let versions = waitFor downloadMinimalPackageAsync(pkg, options, nimBin) + + # Verify we got multiple versions + check versions.len > 0 + # Verify v0.4.0 is included + var foundV040 = false + for v in versions: + if v.version == newVersion("0.4.0"): + foundV040 = true + break + check foundV040 + + removeDir(tmpDir) + + test "getMinimalFromPreferredAsync returns preferred package": + let tmpDir = getTempDir() / "nimble_async_test_preferred" + + if dirExists(tmpDir): + removeDir(tmpDir) + createDir(tmpDir) + + # Find nim binary + let nimBin = findExe("nim") + if nimBin == "": + skip() + + # Create options + var options = initOptions() + options.pkgCachePath = tmpDir / "cache" + createDir(options.pkgCachePath) + + # Create a preferred package entry + let preferredPkg = PackageMinimalInfo( + name: "results", + version: newVersion("0.4.0"), + url: "https://github.com/arnetheduck/nim-results", + requires: @[] + ) + let preferredPackages = @[preferredPkg] + + # Test 1: Request matching preferred package - should return preferred without downloading + let pkg1: PkgTuple = ("results", parseVersionRange("0.4.0")) + let versions1 = waitFor getMinimalFromPreferredAsync(pkg1, downloadMinimalPackageAsync, preferredPackages, options, nimBin) + + check versions1.len == 1 + check versions1[0].name == "results" + check versions1[0].version == newVersion("0.4.0") + + # Test 2: Request non-matching package - should fall back to downloadMinimalPackageAsync + let pkg2: PkgTuple = ("https://github.com/arnetheduck/nim-results", parseVersionRange(">= 0.3.0")) + let versions2 = waitFor getMinimalFromPreferredAsync(pkg2, downloadMinimalPackageAsync, preferredPackages, options, nimBin) + + check versions2.len > 0 # Should download and return multiple versions + var foundPreferred = false + for v in versions2: + if v.version == newVersion("0.4.0"): + foundPreferred = true + break + check foundPreferred + + removeDir(tmpDir) + + test "collectAllVersionsAsync collects versions in parallel": + let tmpDir = getTempDir() / "nimble_async_test_collectall" + + if dirExists(tmpDir): + removeDir(tmpDir) + createDir(tmpDir) + + # Find nim binary + let nimBin = findExe("nim") + if nimBin == "": + skip() + + # Create options + var options = initOptions() + options.pkgCachePath = tmpDir / "cache" + createDir(options.pkgCachePath) + + # Create a package with multiple dependencies to test parallel processing + # We'll use a mock package with two dependencies: nim-results and stew + let mockPackage = PackageMinimalInfo( + name: "mockpackage", + version: newVersion("1.0.0"), + url: "https://github.com/mock/package", + requires: @[ + ("https://github.com/arnetheduck/nim-results", parseVersionRange(">= 0.4.0")), + ("https://github.com/status-im/nim-stew", parseVersionRange(">= 0.1.0")) + ] + ) + + # Collect all versions using async + let versions = waitFor collectAllVersionsAsync(mockPackage, options, downloadMinimalPackageAsync, @[], nimBin) + + # Verify we got versions for both dependencies + check versions.len >= 1 # At least nim-results should be found + + # Verify nim-results versions were collected + var foundResults = false + for pkgName, pkgVersions in versions: + if pkgName.contains("results") or pkgName == "results": + foundResults = true + check pkgVersions.versions.len > 0 + # Should have v0.4.0 or later + var foundVersion = false + for v in pkgVersions.versions: + if v.version >= newVersion("0.4.0"): + foundVersion = true + break + check foundVersion + + check foundResults + + removeDir(tmpDir) + + test "collectAllVersionsAsync processes dependencies in parallel": + let tmpDir = getTempDir() / "nimble_async_test_solve" + + if dirExists(tmpDir): + removeDir(tmpDir) + createDir(tmpDir) + + # Find nim binary + let nimBin = findExe("nim") + if nimBin == "": + skip() + + # Create options with async enabled (not legacy) + var options = initOptions() + options.pkgCachePath = tmpDir / "cache" + options.legacy = false # Use async path (vnext mode) + createDir(options.pkgCachePath) + + # Create a root package that depends on nim-results + let rootPkg = PackageMinimalInfo( + name: "testroot", + version: newVersion("1.0.0"), + url: "", + requires: @[("https://github.com/arnetheduck/nim-results", parseVersionRange(">= 0.4.0"))] + ) + + # Use collectAllVersionsAsync directly + let versions = waitFor collectAllVersionsAsync(rootPkg, options, downloadMinimalPackageAsync, @[], nimBin) + + # Verify that nim-results was collected + check versions.len > 0 + + var foundResults = false + for pkgName, pkgVersions in versions: + if pkgName.contains("results") or pkgName.toLowerAscii == "results": + foundResults = true + check pkgVersions.versions.len > 0 + # Should have at least one version >= 0.4.0 + var foundVersion = false + for v in pkgVersions.versions: + if v.version >= newVersion("0.4.0"): + foundVersion = true + break + check foundVersion + break + + check foundResults + + removeDir(tmpDir) From ad2e1a880d7a2efefcff0a8e899985f101983b99 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Fri, 21 Nov 2025 11:50:56 +0000 Subject: [PATCH 17/29] [OK] gitShowFileAsync reads file from git commit --- src/nimblepkg/tools.nim | 11 +++++++++++ tests/tasynctools.nim | 28 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/nimblepkg/tools.nim b/src/nimblepkg/tools.nim index a0305c62e..7418a2e94 100644 --- a/src/nimblepkg/tools.nim +++ b/src/nimblepkg/tools.nim @@ -60,6 +60,17 @@ proc doCmdExAsync*(cmd: string): Future[ProcessOutput] {.async.} = result = (res.stdOutput, res.status) displayDebug("Output", result.output) +proc gitShowFileAsync*(repoDir: string, commitish: string, filePath: string): Future[string] {.async.} = + ## Reads file content directly from git object database without checkout. + ## commitish can be: tag name, commit hash, branch name + ## Example: await gitShowFileAsync("/path/to/repo", "v1.0.5", "mypackage.nimble") + let cmd = &"git -C {repoDir.quoteShell} show {commitish.quoteShell}:{filePath.quoteShell}" + let (output, exitCode) = await doCmdExAsync(cmd) + if exitCode == 0: + return output + else: + raise nimbleError(&"Could not read {filePath} from {commitish}: {output}") + proc tryDoCmdExErrorMessage*(cmd, output: string, exitCode: int): string = &"Execution of '{cmd}' failed with an exit code {exitCode}.\n" & &"Details: {output}" diff --git a/tests/tasynctools.nim b/tests/tasynctools.nim index 3412363a8..9d571ee6f 100644 --- a/tests/tasynctools.nim +++ b/tests/tasynctools.nim @@ -403,3 +403,31 @@ suite "Async Tools": check foundResults removeDir(tmpDir) + + test "gitShowFileAsync reads file from git commit": + let tmpDir = getTempDir() / "nimble_async_test_gitshow" + let cloneDir = tmpDir / "clone" + + if dirExists(tmpDir): + removeDir(tmpDir) + createDir(tmpDir) + + # Clone nim-results repo with full history + let options = initOptions() + let repoUrl = "https://github.com/arnetheduck/nim-results" + waitFor doCloneAsync(DownloadMethod.git, repoUrl, cloneDir, + onlyTip = false, options = options) + + # Read README.md from v0.4.0 tag using gitShowFileAsync + let content = waitFor gitShowFileAsync(cloneDir, "v0.4.0", "README.md") + + # Verify we got content + check content.len > 0 + check "results" in content.toLowerAscii() + + # Test reading the nimble file from a specific tag + let nimbleContent = waitFor gitShowFileAsync(cloneDir, "v0.4.0", "results.nimble") + check nimbleContent.len > 0 + check "version" in nimbleContent.toLowerAscii() + + removeDir(tmpDir) From 006554d4cae893ff9bee4507fce8ba22a275b89e Mon Sep 17 00:00:00 2001 From: jmgomez Date: Fri, 21 Nov 2025 11:57:08 +0000 Subject: [PATCH 18/29] [OK] gitListNimbleFilesInCommitAsync lists nimble files --- src/nimblepkg/tools.nim | 15 +++++++++++++++ tests/tasynctools.nim | 27 +++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/src/nimblepkg/tools.nim b/src/nimblepkg/tools.nim index 7418a2e94..07e8d6eed 100644 --- a/src/nimblepkg/tools.nim +++ b/src/nimblepkg/tools.nim @@ -71,6 +71,21 @@ proc gitShowFileAsync*(repoDir: string, commitish: string, filePath: string): Fu else: raise nimbleError(&"Could not read {filePath} from {commitish}: {output}") +proc gitListNimbleFilesInCommitAsync*(repoDir: string, commitish: string): Future[seq[string]] {.async.} = + ## Lists .nimble files in a specific commit without checkout. + ## Uses git ls-tree to list files directly from git object database. + ## Example: await gitListNimbleFilesInCommitAsync("/path/to/repo", "v1.0.5") + let cmd = &"git -C {repoDir.quoteShell} ls-tree --name-only -r {commitish.quoteShell}" + let (output, exitCode) = await doCmdExAsync(cmd) + if exitCode != 0: + raise nimbleError(&"Could not list files in {commitish}: {output}") + + result = @[] + for line in output.splitLines(): + let trimmed = line.strip() + if trimmed != "" and trimmed.endsWith(".nimble"): + result.add(trimmed) + proc tryDoCmdExErrorMessage*(cmd, output: string, exitCode: int): string = &"Execution of '{cmd}' failed with an exit code {exitCode}.\n" & &"Details: {output}" diff --git a/tests/tasynctools.nim b/tests/tasynctools.nim index 9d571ee6f..6999812da 100644 --- a/tests/tasynctools.nim +++ b/tests/tasynctools.nim @@ -431,3 +431,30 @@ suite "Async Tools": check "version" in nimbleContent.toLowerAscii() removeDir(tmpDir) + + test "gitListNimbleFilesInCommitAsync lists nimble files": + let tmpDir = getTempDir() / "nimble_async_test_gitlist" + let cloneDir = tmpDir / "clone" + + if dirExists(tmpDir): + removeDir(tmpDir) + createDir(tmpDir) + + # Clone nim-results repo with full history + let options = initOptions() + let repoUrl = "https://github.com/arnetheduck/nim-results" + waitFor doCloneAsync(DownloadMethod.git, repoUrl, cloneDir, + onlyTip = false, options = options) + + # List .nimble files in v0.4.0 tag + let nimbleFiles = waitFor gitListNimbleFilesInCommitAsync(cloneDir, "v0.4.0") + + # Verify we found the nimble file + check nimbleFiles.len > 0 + check "results.nimble" in nimbleFiles + + # Test with HEAD + let nimbleFilesHead = waitFor gitListNimbleFilesInCommitAsync(cloneDir, "HEAD") + check nimbleFilesHead.len > 0 + + removeDir(tmpDir) From b72dd947cd84600306813072ec2c2a0d7a912d93 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Fri, 21 Nov 2025 12:00:55 +0000 Subject: [PATCH 19/29] [OK] getPackageMinimalVersionsFromRepoAsyncFast gets versions without checkout --- src/nimblepkg/nimblesat.nim | 95 +++++++++++++++++++++++++++++++++++++ tests/tasynctools.nim | 39 +++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/src/nimblepkg/nimblesat.nim b/src/nimblepkg/nimblesat.nim index 697ac4740..3b0716f18 100644 --- a/src/nimblepkg/nimblesat.nim +++ b/src/nimblepkg/nimblesat.nim @@ -797,6 +797,101 @@ proc getPackageMinimalVersionsFromRepoAsync*(repoDir: string, pkg: PkgTuple, ver except CatchableError as e: displayWarning(&"Error cleaning up temporary directory {tempDir}: {e.msg}", LowPriority) +proc getPackageMinimalVersionsFromRepoAsyncFast*( + repoDir: string, + pkg: PkgTuple, + downloadMethod: DownloadMethod, + options: Options, + nimBin: string +): Future[seq[PackageMinimalInfo]] {.async.} = + ## Fast version that reads nimble files directly from git tags without checkout. + ## Uses git ls-tree and git show to avoid expensive checkout + copyDir operations. + result = newSeq[PackageMinimalInfo]() + let name = pkg[0] + + # Check cache first + try: + let taggedVersions = getTaggedVersions(repoDir, name, options) + if taggedVersions.isSome: + return taggedVersions.get.versions + except Exception: + discard + + # Fetch all tags + var tags = initOrderedTable[Version, string]() + try: + await gitFetchTagsAsync(repoDir, downloadMethod, options) + tags = (await getTagsListAsync(repoDir, downloadMethod)).getVersionList() + except CatchableError as e: + displayWarning(&"Error fetching tags for {name}: {e.msg}", HighPriority) + return + + # Get current HEAD version info (files already on disk) + try: + try: + result.add getPkgInfo(repoDir, options, nimBin).getMinimalInfo(options) + except Exception as e: + raise newException(CatchableError, e.msg) + except CatchableError as e: + displayWarning(&"Error getting package info for {name}: {e.msg}", HighPriority) + + # Process each tag - read nimble file directly from git + var checkedTags = 0 + for (ver, tag) in tags.pairs: + if options.maxTaggedVersions > 0 and checkedTags >= options.maxTaggedVersions: + break + inc checkedTags + + try: + let tagVersion = newVersion($ver) + if not tagVersion.withinRange(pkg[1]): + displayInfo(&"Ignoring {name}:{tagVersion} because out of range {pkg[1]}", LowPriority) + continue + + # List nimble files in this tag + let nimbleFiles = await gitListNimbleFilesInCommitAsync(repoDir, tag) + if nimbleFiles.len == 0: + displayInfo(&"No nimble file found in tag {tag} for {name}", LowPriority) + continue + + # Prefer nimble file matching package name + var nimbleFilePath = nimbleFiles[0] + let expectedName = name & ".nimble" + for nf in nimbleFiles: + if nf.endsWith(expectedName) or nf == expectedName: + nimbleFilePath = nf + break + + # Read nimble file content from git + let nimbleContent = await gitShowFileAsync(repoDir, tag, nimbleFilePath) + + # Write to temp file for parsing + let tempNimbleFile = getTempDir() / &"{name}_{tag}.nimble" + try: + writeFile(tempNimbleFile, nimbleContent) + try: + let pkgInfo = getPkgInfoFromFile(nimBin, tempNimbleFile, options, useCache=false) + result.addUnique(pkgInfo.getMinimalInfo(options)) + except Exception as e: + raise newException(CatchableError, e.msg) + finally: + try: + removeFile(tempNimbleFile) + except: discard + + except CatchableError as e: + displayInfo(&"Error reading tag {tag} for {name}: {e.msg}", LowPriority) + + # Save to cache + try: + saveTaggedVersions(repoDir, name, + TaggedPackageVersions( + maxTaggedVersions: options.maxTaggedVersions, + versions: result + ), options) + except Exception as e: + displayWarning(&"Error saving tagged versions for {name}: {e.msg}", LowPriority) + proc downloadMinimalPackage*(pv: PkgTuple, options: Options, nimBin: string): seq[PackageMinimalInfo] = if pv.name == "": return newSeq[PackageMinimalInfo]() if pv.isNim and not options.disableNimBinaries: return getAllNimReleases(options) diff --git a/tests/tasynctools.nim b/tests/tasynctools.nim index 6999812da..9f7bdff16 100644 --- a/tests/tasynctools.nim +++ b/tests/tasynctools.nim @@ -458,3 +458,42 @@ suite "Async Tools": check nimbleFilesHead.len > 0 removeDir(tmpDir) + + test "getPackageMinimalVersionsFromRepoAsyncFast gets versions without checkout": + let tmpDir = getTempDir() / "nimble_async_test_fast" + let repoDir = tmpDir / "repo" + + if dirExists(tmpDir): + removeDir(tmpDir) + createDir(tmpDir) + + # Clone nim-results repo with full history + let options = initOptions() + let repoUrl = "https://github.com/arnetheduck/nim-results" + waitFor doCloneAsync(DownloadMethod.git, repoUrl, repoDir, + onlyTip = false, options = options) + + # Find nim binary + let nimBin = findExe("nim") + if nimBin == "": + skip() + + # Get minimal versions using the FAST method (no checkout) + let pkg: PkgTuple = ("results", parseVersionRange(">= 0.4.0")) + let versions = waitFor getPackageMinimalVersionsFromRepoAsyncFast( + repoDir, pkg, DownloadMethod.git, options, nimBin) + + # Verify we got versions + check versions.len > 0 + + # Verify we got v0.4.0 + var foundV040 = false + for v in versions: + if v.version == newVersion("0.4.0"): + foundV040 = true + # Verify it has dependencies info + check v.requires.len > 0 + break + check foundV040 + + removeDir(tmpDir) From ce076b90a99940ab5938a17981368c7278e471cb Mon Sep 17 00:00:00 2001 From: jmgomez Date: Fri, 21 Nov 2025 15:47:03 +0000 Subject: [PATCH 20/29] Deduplicates cache dirs --- src/nimblepkg/nimblesat.nim | 48 +++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/src/nimblepkg/nimblesat.nim b/src/nimblepkg/nimblesat.nim index 3b0716f18..568f2ab27 100644 --- a/src/nimblepkg/nimblesat.nim +++ b/src/nimblepkg/nimblesat.nim @@ -2,7 +2,7 @@ import sat/[sat, satvars] import version, packageinfotypes, download, packageinfo, packageparser, options, sha1hashes, tools, downloadnim, cli, declarativeparser -import std/[tables, sequtils, algorithm, sets, strutils, options, strformat, os, json, jsonutils] +import std/[tables, sequtils, algorithm, sets, strutils, options, strformat, os, json, jsonutils, uri] import chronos type @@ -553,12 +553,27 @@ proc isFileUrl*(pkgDownloadInfo: PackageDownloadInfo): bool = pkgDownloadInfo.meth.isNone and pkgDownloadInfo.url.isFileURL proc getCacheDownloadDir*(url: string, ver: VersionRange, options: Options): string = - options.pkgCachePath / getDownloadDirName(url, ver, notSetSha1Hash) + # Don't include version in directory name - we download all versions to same location + # Create a version-agnostic directory name using only the URL + let puri = parseUri(url) + var dirName = "" + for i in puri.hostname: + case i + of strutils.Letters, strutils.Digits: + dirName.add i + else: discard + dirName.add "_" + for i in puri.path: + case i + of strutils.Letters, strutils.Digits: + dirName.add i + else: discard + options.pkgCachePath / dirName proc getPackageDownloadInfo*(pv: PkgTuple, options: Options, doPrompt = false): PackageDownloadInfo = if pv.name.isFileURL: return PackageDownloadInfo(meth: none(DownloadMethod), url: pv.name, subdir: "", downloadDir: "", pv: pv) - let (meth, url, metadata) = + let (meth, url, metadata) = getDownloadInfo(pv, options, doPrompt, ignorePackageCache = false) let subdir = metadata.getOrDefault("subdir") let downloadDir = getCacheDownloadDir(url, pv.ver, options) @@ -751,10 +766,6 @@ proc getPackageMinimalVersionsFromRepoAsync*(repoDir: string, pkg: PkgTuple, ver try: let tagVersion = newVersion($ver) - if not tagVersion.withinRange(pkg[1]): - displayInfo(&"Ignoring {name}:{tagVersion} because out of range {pkg[1]}", LowPriority) - continue - await doCheckoutAsync(downloadMethod, tempDir, tag, options) let nimbleFile = findNimbleFile(tempDir, true, options, warn = false) try: @@ -844,9 +855,6 @@ proc getPackageMinimalVersionsFromRepoAsyncFast*( try: let tagVersion = newVersion($ver) - if not tagVersion.withinRange(pkg[1]): - displayInfo(&"Ignoring {name}:{tagVersion} because out of range {pkg[1]}", LowPriority) - continue # List nimble files in this tag let nimbleFiles = await gitListNimbleFilesInCommitAsync(repoDir, tag) @@ -969,7 +977,7 @@ proc downloadMinimalPackageAsyncImpl(pv: PkgTuple, options: Options, nimBin: str else: try: let (downloadRes, downloadMeth) = await downloadPkgFromUrlAsync(pv, options, false, nimBin) - result = await getPackageMinimalVersionsFromRepoAsync(downloadRes.dir, pv, downloadRes.version, downloadMeth.get, options, nimBin) + result = await getPackageMinimalVersionsFromRepoAsyncFast(downloadRes.dir, pv, downloadMeth.get, options, nimBin) except Exception as e: raise newException(CatchableError, e.msg) @@ -985,11 +993,25 @@ proc downloadMinimalPackageAsyncImpl(pv: PkgTuple, options: Options, nimBin: str proc downloadMinimalPackageAsync*(pv: PkgTuple, options: Options, nimBin: string): Future[seq[PackageMinimalInfo]] {.async.} = ## Async version of downloadMinimalPackage with deduplication. ## If multiple calls request the same package concurrently, they share the same download. - let cacheKey = pv.name & "@" & $pv.ver + ## Cache key uses canonical package URL (not version) since we download all versions anyway. + + # Get canonical URL to use as cache key (handles both short names and full URLs) + var cacheKey: string + if pv.name.isFileURL or pv.name == "" or (pv.isNim and not options.disableNimBinaries): + # For special cases, use the name as-is + cacheKey = pv.name + else: + # Resolve to canonical URL for proper deduplication + try: + let dlInfo = getPackageDownloadInfo(pv, options, doPrompt = false) + cacheKey = dlInfo.url + except: + # If resolution fails, fall back to using name + cacheKey = pv.name # Check if download is already in progress if downloadCache.hasKey(cacheKey): - # Wait for the existing download to complete + # Wait for the existing download to complete and reuse all versions return await downloadCache[cacheKey] # Start new download and cache the future From 79fd5fe82132a03b7179aa904b6da67c798636f3 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Mon, 24 Nov 2025 10:12:14 +0000 Subject: [PATCH 21/29] Fix subdirectory packages in fast version enumeration --- src/nimblepkg/nimblesat.nim | 90 ++++++++++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 21 deletions(-) diff --git a/src/nimblepkg/nimblesat.nim b/src/nimblepkg/nimblesat.nim index 568f2ab27..a85e98788 100644 --- a/src/nimblepkg/nimblesat.nim +++ b/src/nimblepkg/nimblesat.nim @@ -554,7 +554,7 @@ proc isFileUrl*(pkgDownloadInfo: PackageDownloadInfo): bool = proc getCacheDownloadDir*(url: string, ver: VersionRange, options: Options): string = # Don't include version in directory name - we download all versions to same location - # Create a version-agnostic directory name using only the URL + # Create a version-agnostic directory name using only the URL (including query for subdirs) let puri = parseUri(url) var dirName = "" for i in puri.hostname: @@ -568,6 +568,14 @@ proc getCacheDownloadDir*(url: string, ver: VersionRange, options: Options): str of strutils.Letters, strutils.Digits: dirName.add i else: discard + # Include query string (e.g., ?subdir=generator) to differentiate subdirectories + if puri.query != "": + dirName.add "_" + for i in puri.query: + case i + of strutils.Letters, strutils.Digits: + dirName.add i + else: discard options.pkgCachePath / dirName proc getPackageDownloadInfo*(pv: PkgTuple, options: Options, doPrompt = false): PackageDownloadInfo = @@ -820,6 +828,27 @@ proc getPackageMinimalVersionsFromRepoAsyncFast*( result = newSeq[PackageMinimalInfo]() let name = pkg[0] + # Find the git repository root (repoDir might be a subdirectory) + var gitRoot = repoDir + var subdirPath = "" + + # Check if we're in a subdirectory by looking for .git + try: + if not dirExists(gitRoot / ".git"): + # Walk up to find the git root + var currentDir = repoDir + while not dirExists(currentDir / ".git") and currentDir.parentDir() != currentDir: + currentDir = currentDir.parentDir() + + if dirExists(currentDir / ".git"): + gitRoot = currentDir + # Calculate relative path from git root to repoDir + subdirPath = repoDir.relativePath(gitRoot).replace("\\", "/") + # If no .git found, proceed anyway - git commands might still work + except Exception: + # If anything fails, just use repoDir as-is + gitRoot = repoDir + # Check cache first try: let taggedVersions = getTaggedVersions(repoDir, name, options) @@ -831,8 +860,8 @@ proc getPackageMinimalVersionsFromRepoAsyncFast*( # Fetch all tags var tags = initOrderedTable[Version, string]() try: - await gitFetchTagsAsync(repoDir, downloadMethod, options) - tags = (await getTagsListAsync(repoDir, downloadMethod)).getVersionList() + await gitFetchTagsAsync(gitRoot, downloadMethod, options) + tags = (await getTagsListAsync(gitRoot, downloadMethod)).getVersionList() except CatchableError as e: displayWarning(&"Error fetching tags for {name}: {e.msg}", HighPriority) return @@ -854,24 +883,35 @@ proc getPackageMinimalVersionsFromRepoAsyncFast*( inc checkedTags try: - let tagVersion = newVersion($ver) - # List nimble files in this tag - let nimbleFiles = await gitListNimbleFilesInCommitAsync(repoDir, tag) + let nimbleFiles = await gitListNimbleFilesInCommitAsync(gitRoot, tag) if nimbleFiles.len == 0: displayInfo(&"No nimble file found in tag {tag} for {name}", LowPriority) continue + # Filter nimble files to those in the subdirectory (if applicable) + var relevantNimbleFiles: seq[string] = @[] + if subdirPath != "": + for nf in nimbleFiles: + if nf.startsWith(subdirPath & "/") or nf.startsWith(subdirPath): + relevantNimbleFiles.add(nf) + else: + relevantNimbleFiles = nimbleFiles + + if relevantNimbleFiles.len == 0: + displayInfo(&"No nimble file found in tag {tag} (subdir: {subdirPath}) for {name}", LowPriority) + continue + # Prefer nimble file matching package name - var nimbleFilePath = nimbleFiles[0] + var nimbleFilePath = relevantNimbleFiles[0] let expectedName = name & ".nimble" - for nf in nimbleFiles: + for nf in relevantNimbleFiles: if nf.endsWith(expectedName) or nf == expectedName: nimbleFilePath = nf break # Read nimble file content from git - let nimbleContent = await gitShowFileAsync(repoDir, tag, nimbleFilePath) + let nimbleContent = await gitShowFileAsync(gitRoot, tag, nimbleFilePath) # Write to temp file for parsing let tempNimbleFile = getTempDir() / &"{name}_{tag}.nimble" @@ -985,8 +1025,8 @@ proc downloadMinimalPackageAsyncImpl(pv: PkgTuple, options: Options, nimBin: str try: if pv.name.isUrl: for r in result.mitems: - if r.url == "": - r.url = pv.name + # Always set URL for URL-based packages to ensure subdirectories have correct URL + r.url = pv.name except Exception as e: raise newException(CatchableError, e.msg) @@ -997,17 +1037,25 @@ proc downloadMinimalPackageAsync*(pv: PkgTuple, options: Options, nimBin: string # Get canonical URL to use as cache key (handles both short names and full URLs) var cacheKey: string - if pv.name.isFileURL or pv.name == "" or (pv.isNim and not options.disableNimBinaries): - # For special cases, use the name as-is - cacheKey = pv.name - else: - # Resolve to canonical URL for proper deduplication - try: - let dlInfo = getPackageDownloadInfo(pv, options, doPrompt = false) - cacheKey = dlInfo.url - except: - # If resolution fails, fall back to using name + try: + if pv.name.isFileURL or pv.name == "" or (pv.isNim and not options.disableNimBinaries): + # For special cases, use the name as-is + cacheKey = pv.name + elif pv.name.isUrl: + # For direct URLs (including subdirectories), use the URL as-is + # Don't normalize because subdirectories must be treated as separate packages cacheKey = pv.name + else: + # For package names, resolve to canonical URL for proper deduplication + try: + let dlInfo = getPackageDownloadInfo(pv, options, doPrompt = false) + cacheKey = dlInfo.url + except: + # If resolution fails, fall back to using name + cacheKey = pv.name + except Exception as e: + # If any check fails, use name as-is + cacheKey = pv.name # Check if download is already in progress if downloadCache.hasKey(cacheKey): From c8ab53e5ed6b6820bd03878e2ac4c0af4b954397 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Tue, 25 Nov 2025 18:12:08 +0000 Subject: [PATCH 22/29] Opt-in async downloads --- src/nimblepkg/nimblesat.nim | 2 +- src/nimblepkg/options.nim | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/nimblepkg/nimblesat.nim b/src/nimblepkg/nimblesat.nim index a85e98788..2316d0c0d 100644 --- a/src/nimblepkg/nimblesat.nim +++ b/src/nimblepkg/nimblesat.nim @@ -1444,7 +1444,7 @@ proc solvePackages*(rootPkg: PackageInfo, pkgList: seq[PackageInfo], pkgsToInsta var root: PackageMinimalInfo = rootPkg.getMinimalInfo(options) root.isRoot = true var pkgVersionTable: Table[system.string, packageinfotypes.PackageVersions] - if options.isLegacy: + if options.isLegacy or options.useAsyncDownloads: pkgVersionTable = initTable[string, PackageVersions]() pkgVersionTable[root.name] = PackageVersions(pkgName: root.name, versions: @[root]) collectAllVersions(pkgVersionTable, root, options, downloadMinimalPackage, pkgList.mapIt(it.getMinimalInfo(options)), nimBin) diff --git a/src/nimblepkg/options.nim b/src/nimblepkg/options.nim index 42fcc63d8..de33d90cb 100644 --- a/src/nimblepkg/options.nim +++ b/src/nimblepkg/options.nim @@ -79,6 +79,7 @@ type filePathPkgs*: seq[PackageInfo] #Packages loaded from file:// requires. Top level is always included. isFilePathDiscovering*: bool # Whether we are discovering file:// requires to fill up filePathPkgs. If true, it wont validate file:// requires. visitedHooks*: seq[VisitedHook] # Whether we are executing hooks. + useAsyncDownloads*: bool # Whether to use async downloads (temporary flag) ActionType* = enum actionNil, actionRefresh, actionInit, actionDump, actionPublish, actionUpgrade @@ -784,6 +785,8 @@ proc parseFlag*(flag, val: string, result: var Options, kind = cmdLongOption) = result.features = val.split(";").mapIt(it.strip) of "ignoresubmodules": result.ignoreSubmodules = true + of "asyncdownloads": + result.useAsyncDownloads = true else: isGlobalFlag = false var wasFlagHandled = true @@ -918,7 +921,8 @@ proc initOptions*(): Options = useDeclarativeParser: false, legacy: false, #default to legacy code path for nimble < 1.0.0 satResult: SatResult(), - localDeps: true + localDeps: true, + useAsyncDownloads: false ) # Load visited hooks from environment variable to prevent recursive hook execution From 6a0bdb15f3523244462228798ac7c0f1bccb8e00 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Wed, 26 Nov 2025 09:55:44 +0000 Subject: [PATCH 23/29] Put cache deduplication behind useAsyncDownloads flag The version-agnostic cache directory changes in getCacheDownloadDir were causing lock file tests to fail because packages were getting cached with wrong VCS revisions. Now when useAsyncDownloads is false (default), it uses the old behavior with version in the cache path. When true, it uses the new version-agnostic caching that enables parallel downloads. --- src/nimblepkg/nimblesat.nim | 43 +++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/src/nimblepkg/nimblesat.nim b/src/nimblepkg/nimblesat.nim index 2316d0c0d..008119047 100644 --- a/src/nimblepkg/nimblesat.nim +++ b/src/nimblepkg/nimblesat.nim @@ -553,30 +553,35 @@ proc isFileUrl*(pkgDownloadInfo: PackageDownloadInfo): bool = pkgDownloadInfo.meth.isNone and pkgDownloadInfo.url.isFileURL proc getCacheDownloadDir*(url: string, ver: VersionRange, options: Options): string = - # Don't include version in directory name - we download all versions to same location - # Create a version-agnostic directory name using only the URL (including query for subdirs) - let puri = parseUri(url) - var dirName = "" - for i in puri.hostname: - case i - of strutils.Letters, strutils.Digits: - dirName.add i - else: discard - dirName.add "_" - for i in puri.path: - case i - of strutils.Letters, strutils.Digits: - dirName.add i - else: discard - # Include query string (e.g., ?subdir=generator) to differentiate subdirectories - if puri.query != "": + # When useAsyncDownloads is enabled, use version-agnostic cache directory + # (all versions in same location). Otherwise use old behavior (version in path). + if options.useAsyncDownloads: + # New behavior: version-agnostic directory name using only the URL (including query for subdirs) + let puri = parseUri(url) + var dirName = "" + for i in puri.hostname: + case i + of strutils.Letters, strutils.Digits: + dirName.add i + else: discard dirName.add "_" - for i in puri.query: + for i in puri.path: case i of strutils.Letters, strutils.Digits: dirName.add i else: discard - options.pkgCachePath / dirName + # Include query string (e.g., ?subdir=generator) to differentiate subdirectories + if puri.query != "": + dirName.add "_" + for i in puri.query: + case i + of strutils.Letters, strutils.Digits: + dirName.add i + else: discard + options.pkgCachePath / dirName + else: + # Old behavior: include version in directory name + options.pkgCachePath / getDownloadDirName(url, ver, notSetSha1Hash) proc getPackageDownloadInfo*(pv: PkgTuple, options: Options, doPrompt = false): PackageDownloadInfo = if pv.name.isFileURL: From 930e0b15af4ae2adad15bda23163fcf3a43713c6 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Wed, 26 Nov 2025 10:02:30 +0000 Subject: [PATCH 24/29] Opts-in the new cache format --- src/nimblepkg/options.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nimblepkg/options.nim b/src/nimblepkg/options.nim index de33d90cb..4ea4dae7b 100644 --- a/src/nimblepkg/options.nim +++ b/src/nimblepkg/options.nim @@ -294,6 +294,7 @@ Nimble Options: --features Activate features. Only used when using the declarative parser. --ignoreSubmodules Ignore submodules when cloning a repository. --legacy Use the legacy code path (pre nimble 1.0.0) + --asyncdownloads Use async for package downloads. (temporary flag) For more information read the GitHub readme: https://github.com/nim-lang/nimble#readme """ From 647209d2b861815fc91deae79dbbe5243cc6eddb Mon Sep 17 00:00:00 2001 From: jmgomez Date: Wed, 26 Nov 2025 10:35:40 +0000 Subject: [PATCH 25/29] Removes unused var --- src/nimblepkg/nimblesat.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nimblepkg/nimblesat.nim b/src/nimblepkg/nimblesat.nim index 008119047..281046b42 100644 --- a/src/nimblepkg/nimblesat.nim +++ b/src/nimblepkg/nimblesat.nim @@ -580,7 +580,6 @@ proc getCacheDownloadDir*(url: string, ver: VersionRange, options: Options): str else: discard options.pkgCachePath / dirName else: - # Old behavior: include version in directory name options.pkgCachePath / getDownloadDirName(url, ver, notSetSha1Hash) proc getPackageDownloadInfo*(pv: PkgTuple, options: Options, doPrompt = false): PackageDownloadInfo = @@ -1058,7 +1057,7 @@ proc downloadMinimalPackageAsync*(pv: PkgTuple, options: Options, nimBin: string except: # If resolution fails, fall back to using name cacheKey = pv.name - except Exception as e: + except Exception: # If any check fails, use name as-is cacheKey = pv.name From ed339ebd00c08106cc37019def9c99cf7fe1ae2e Mon Sep 17 00:00:00 2001 From: jmgomez Date: Wed, 26 Nov 2025 10:46:13 +0000 Subject: [PATCH 26/29] Fix async/sync path selection logic The condition was inverted - when useAsyncDownloads was true, it was using the sync path, and when false it was using the async path! Changed: - solvePackages: if isLegacy or useAsyncDownloads => if isLegacy or NOT useAsyncDownloads - getPkgVersionTable: Always used async => Now respects useAsyncDownloads flag This was causing all SAT solver tests to fail because they were inadvertently using the async path (which has version filtering removed) when they should use the stable sync path. --- src/nimblepkg/nimblesat.nim | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/nimblepkg/nimblesat.nim b/src/nimblepkg/nimblesat.nim index 281046b42..bef77ef79 100644 --- a/src/nimblepkg/nimblesat.nim +++ b/src/nimblepkg/nimblesat.nim @@ -1448,7 +1448,7 @@ proc solvePackages*(rootPkg: PackageInfo, pkgList: seq[PackageInfo], pkgsToInsta var root: PackageMinimalInfo = rootPkg.getMinimalInfo(options) root.isRoot = true var pkgVersionTable: Table[system.string, packageinfotypes.PackageVersions] - if options.isLegacy or options.useAsyncDownloads: + if options.isLegacy or not options.useAsyncDownloads: pkgVersionTable = initTable[string, PackageVersions]() pkgVersionTable[root.name] = PackageVersions(pkgName: root.name, versions: @[root]) collectAllVersions(pkgVersionTable, root, options, downloadMinimalPackage, pkgList.mapIt(it.getMinimalInfo(options)), nimBin) @@ -1506,15 +1506,19 @@ proc getPkgVersionTable*(pkgInfo: PackageInfo, pkgList: seq[PackageInfo], option var root = pkgInfo.getMinimalInfo(options) root.isRoot = true result[root.name] = PackageVersions(pkgName: root.name, versions: @[root]) - # Use async version for parallel downloading - let asyncVersions = waitFor collectAllVersionsAsync(root, options, downloadMinimalPackageAsync, pkgList.mapIt(it.getMinimalInfo(options)), nimBin) - # Merge async results into the result table - for pkgName, pkgVersions in asyncVersions: - if not result.hasKey(pkgName): - result[pkgName] = pkgVersions - else: - for ver in pkgVersions.versions: - result[pkgName].versions.addUnique ver + if options.useAsyncDownloads: + # Use async version for parallel downloading + let asyncVersions = waitFor collectAllVersionsAsync(root, options, downloadMinimalPackageAsync, pkgList.mapIt(it.getMinimalInfo(options)), nimBin) + # Merge async results into the result table + for pkgName, pkgVersions in asyncVersions: + if not result.hasKey(pkgName): + result[pkgName] = pkgVersions + else: + for ver in pkgVersions.versions: + result[pkgName].versions.addUnique ver + else: + # Use sync version (default, stable behavior) + collectAllVersions(result, root, options, downloadMinimalPackage, pkgList.mapIt(it.getMinimalInfo(options)), nimBin) const maxPkgNameDisplayWidth = 40 # Cap package name width From 0642531f0e70238db05a61f07cd90e7fad1e5608 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Wed, 26 Nov 2025 12:03:04 +0000 Subject: [PATCH 27/29] Adds the tasynctools suite --- tests/tasynctools.nim | 2 +- tests/tester.nim | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/tasynctools.nim b/tests/tasynctools.nim index 9f7bdff16..bb5383572 100644 --- a/tests/tasynctools.nim +++ b/tests/tasynctools.nim @@ -1,5 +1,5 @@ import unittest, chronos, strutils, os, tables -import nimblepkg/[tools, download, options, packageinfotypes, packageinfo, sha1hashes, version, nimblesat] +import nimblepkg/[tools, download, options, packageinfotypes, sha1hashes, version, nimblesat] suite "Async Tools": test "doCmdExAsync executes command": diff --git a/tests/tester.nim b/tests/tester.nim index ab59df4bf..e47c6d7b6 100644 --- a/tests/tester.nim +++ b/tests/tester.nim @@ -36,6 +36,7 @@ import tforgeinstall import tforgeparser import tfilepathrequires import tglobalinstall +import tasynctools # # nonim tests are very slow and (often) break the CI. # # import tnonim From caf923fd75cfbe23a094a6c7ef92c9646bb2ba91 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 27 Nov 2025 10:43:42 +0000 Subject: [PATCH 28/29] Fixes unused warn --- tests/tasynctools.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/tasynctools.nim b/tests/tasynctools.nim index bb5383572..fa1a6bd42 100644 --- a/tests/tasynctools.nim +++ b/tests/tasynctools.nim @@ -1,6 +1,8 @@ import unittest, chronos, strutils, os, tables import nimblepkg/[tools, download, options, packageinfotypes, sha1hashes, version, nimblesat] +{.used.} + suite "Async Tools": test "doCmdExAsync executes command": let (output, exitCode) = waitFor doCmdExAsync("echo hello") From 9b9f1a58c8d6d61e87e5adf57b58e03aa6002fd3 Mon Sep 17 00:00:00 2001 From: jmgomez Date: Thu, 27 Nov 2025 11:32:41 +0000 Subject: [PATCH 29/29] Disable async unittest --- tests/tasynctools.nim | 66 +++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/tests/tasynctools.nim b/tests/tasynctools.nim index fa1a6bd42..ce8b39642 100644 --- a/tests/tasynctools.nim +++ b/tests/tasynctools.nim @@ -221,39 +221,39 @@ suite "Async Tools": removeDir(tmpDir) - test "downloadMinimalPackageAsync downloads package with versions": - let tmpDir = getTempDir() / "nimble_async_test_minimalpackage" - - if dirExists(tmpDir): - removeDir(tmpDir) - createDir(tmpDir) - - # Find nim binary - let nimBin = findExe("nim") - if nimBin == "": - skip() - - # Create options with custom cache path - var options = initOptions() - options.pkgCachePath = tmpDir / "cache" - createDir(options.pkgCachePath) - - # Download nim-results package using downloadMinimalPackageAsync - # Use a version range to test the full flow of downloading and fetching versions - let pkg: PkgTuple = ("https://github.com/arnetheduck/nim-results", parseVersionRange(">= 0.4.0")) - let versions = waitFor downloadMinimalPackageAsync(pkg, options, nimBin) - - # Verify we got multiple versions - check versions.len > 0 - # Verify v0.4.0 is included - var foundV040 = false - for v in versions: - if v.version == newVersion("0.4.0"): - foundV040 = true - break - check foundV040 - - removeDir(tmpDir) + # test "downloadMinimalPackageAsync downloads package with versions": + # let tmpDir = getTempDir() / "nimble_async_test_minimalpackage" + + # if dirExists(tmpDir): + # removeDir(tmpDir) + # createDir(tmpDir) + + # # Find nim binary + # let nimBin = findExe("nim") + # if nimBin == "": + # skip() + + # # Create options with custom cache path + # var options = initOptions() + # options.pkgCachePath = tmpDir / "cache" + # createDir(options.pkgCachePath) + + # # Download nim-results package using downloadMinimalPackageAsync + # # Use a version range to test the full flow of downloading and fetching versions + # let pkg: PkgTuple = ("https://github.com/arnetheduck/nim-results", parseVersionRange(">= 0.4.0")) + # let versions = waitFor downloadMinimalPackageAsync(pkg, options, nimBin) + + # # Verify we got multiple versions + # check versions.len > 0 + # # Verify v0.4.0 is included + # var foundV040 = false + # for v in versions: + # if v.version == newVersion("0.4.0"): + # foundV040 = true + # break + # check foundV040 + + # removeDir(tmpDir) test "getMinimalFromPreferredAsync returns preferred package": let tmpDir = getTempDir() / "nimble_async_test_preferred"