Skip to content

Commit 29bde61

Browse files
987Nabil987Nabil
committed
Add option to pass lib dependencies to coursier
This can be used to add custom url handlers to coursier, so scala-steward could for example reach s3 buckets, google cloud storage or artifactregistry. See https://get-coursier.io/docs/extra.html#extra-protocols and coursier/coursier#1987
1 parent 7ec418c commit 29bde61

File tree

6 files changed

+110
-34
lines changed

6 files changed

+110
-34
lines changed

‎docs/help.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ All command line arguments for the `scala-steward` application.
66
Usage:
77
scala-steward --workspace <file> --repos-file <uri> [--repos-file <uri>]... [--git-author-name <string>] --git-author-email <string> [--git-author-signing-key <string>] --git-ask-pass <file> [--sign-commits] [--forge-type <forge-type>] [--forge-api-host <uri>] --forge-login <string> [--do-not-fork] [--add-labels] [--ignore-opts-files] [--env-var <name=value>]... [--process-timeout <duration>] [--whitelist <string>]... [--read-only <string>]... [--enable-sandbox | --disable-sandbox] [--max-buffer-size <integer>] [--repo-config <uri>]... [--disable-default-repo-config] [--scalafix-migrations <uri>]... [--disable-default-scalafix-migrations] [--artifact-migrations <uri>]... [--disable-default-artifact-migrations] [--cache-ttl <duration>] [--bitbucket-use-default-reviewers] [--bitbucket-server-use-default-reviewers] [--gitlab-merge-when-pipeline-succeeds] [--gitlab-required-reviewers <integer>] [--gitlab-remove-source-branch] [--azure-repos-organization <string>] [--github-app-id <integer> --github-app-key-file <file>] [--url-checker-test-url <uri>]... [--default-maven-repo <string>] [--refresh-backoff-period <duration>]
88
scala-steward validate-repo-config
9+
Usage: scala-steward --workspace <file> --repos-file <file> [--git-author-name <string>] --git-author-email <string> [--git-author-signing-key <string>] --git-ask-pass <file> [--sign-commits] [--vcs-type <vcs-type>] [--vcs-api-host <uri>] --vcs-login <string> [--do-not-fork] [--ignore-opts-files] [--env-var <name=value>]... [--process-timeout <duration>] [--whitelist <string>]... [--read-only <string>]... [--enable-sandbox | --disable-sandbox] [--max-buffer-size <integer>] [--repo-config <uri>]... [--disable-default-repo-config] [--scalafix-migrations <uri>]... [--disable-default-scalafix-migrations] [--artifact-migrations <uri>]... [--disable-default-artifact-migrations] [--cache-ttl <duration>] [--bitbucket-server-use-default-reviewers] [--gitlab-merge-when-pipeline-succeeds] [--github-app-id <integer> --github-app-key-file <file>] [--url-checker-test-url <uri>] [--default-maven-repo <string>] [--refresh-backoff-period <duration>] [--coursier-dependencies <string>]...
910
1011
1112
@@ -94,6 +95,8 @@ Options and flags:
9495
default: https://repo1.maven.org/maven2/
9596
--refresh-backoff-period <duration>
9697
Period of time a failed build won't be triggered again; default: 0days
98+
--coursier-dependencies <string>
99+
Additional coursier dependencies in the format <groupId>:<artifactId>:<version>, i.e. additional URL handlers. (can be used multiple times)
97100
98101
Subcommands:
99102
validate-repo-config

‎modules/core/src/main/scala/org/scalasteward/core/application/Cli.scala‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -315,6 +315,13 @@ object Cli {
315315
private val azureReposCfg: Opts[AzureReposCfg] =
316316
azureReposOrganization.map(AzureReposCfg.apply)
317317

318+
private val coursierDependencies: Opts[List[String]] =
319+
options[String](
320+
"coursier-dependencies",
321+
"Additional coursier dependencies in the format <groupId>:<artifactId>:<version>, " +
322+
"i.e. additional URL handlers. (can be used multiple times)"
323+
).orEmpty
324+
318325
private val refreshBackoffPeriod: Opts[FiniteDuration] = {
319326
val default = 0.days
320327
val help = "Period of time a failed build won't be triggered again" +
@@ -355,7 +362,8 @@ object Cli {
355362
gitHubApp,
356363
urlCheckerTestUrls,
357364
defaultMavenRepo,
358-
refreshBackoffPeriod
365+
refreshBackoffPeriod,
366+
coursierDependencies
359367
).mapN(Config.apply).map(Usage.Regular.apply)
360368

361369
private val validateRepoConfig: Opts[Usage] =

‎modules/core/src/main/scala/org/scalasteward/core/application/Config.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,8 @@ final case class Config(
6363
githubApp: Option[GitHubApp],
6464
urlCheckerTestUrls: Nel[Uri],
6565
defaultResolver: Resolver,
66-
refreshBackoffPeriod: FiniteDuration
66+
refreshBackoffPeriod: FiniteDuration,
67+
coursierDependencies: List[String]
6768
) {
6869
def forgeSpecificCfg: ForgeSpecificCfg =
6970
forgeCfg.tpe match {

‎modules/core/src/main/scala/org/scalasteward/core/application/Context.scala‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import org.scalasteward.core.buildtool.mill.MillAlg
3030
import org.scalasteward.core.buildtool.sbt.SbtAlg
3131
import org.scalasteward.core.buildtool.scalacli.ScalaCliAlg
3232
import org.scalasteward.core.client.ClientConfiguration
33-
import org.scalasteward.core.coursier.{CoursierAlg, VersionsCache}
33+
import org.scalasteward.core.coursier.{CoursierAlg, CoursierDependenciesFetchAlg, VersionsCache}
3434
import org.scalasteward.core.data.Repo
3535
import org.scalasteward.core.edit.EditAlg
3636
import org.scalasteward.core.edit.hooks.HookExecutor
@@ -175,7 +175,9 @@ object Context {
175175
implicit val scalafixCli: ScalafixCli[F] = new ScalafixCli[F]
176176
implicit val scalafmtAlg: ScalafmtAlg[F] = new ScalafmtAlg[F](config.defaultResolver)
177177
implicit val selfCheckAlg: SelfCheckAlg[F] = new SelfCheckAlg[F](config)
178-
implicit val coursierAlg: CoursierAlg[F] = CoursierAlg.create[F]
178+
implicit val coursierDependenciesFetchAlg: CoursierDependenciesFetchAlg[F] =
179+
CoursierDependenciesFetchAlg.create[F]
180+
implicit val coursierAlg: CoursierAlg[F] = CoursierAlg.create[F](config)
179181
implicit val versionsCache: VersionsCache[F] =
180182
new VersionsCache[F](config.cacheTtl, versionsStore)
181183
implicit val updateAlg: UpdateAlg[F] = new UpdateAlg[F]

‎modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierAlg.scala‎

Lines changed: 48 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ import cats.implicits._
2222
import coursier.cache.{CachePolicy, FileCache}
2323
import coursier.core.{Authentication, Project}
2424
import coursier.{Fetch, Module, ModuleName, Organization}
25+
import coursier.{Fetch, Info, Module, ModuleName, Organization}
26+
import org.http4s.Uri
27+
import org.scalasteward.core.application.Config
2528
import org.scalasteward.core.data.Resolver.Credentials
2629
import org.scalasteward.core.data.{Dependency, Resolver, Version}
2730
import org.scalasteward.core.util.uri
@@ -37,16 +40,29 @@ trait CoursierAlg[F[_]] {
3740
}
3841

3942
object CoursierAlg {
40-
def create[F[_]](implicit
43+
def create[F[_]](config: Config)(implicit
4144
logger: Logger[F],
4245
parallel: Parallel[F],
43-
F: Async[F]
46+
F: Async[F],
47+
fetchAlg: CoursierDependenciesFetchAlg[F]
4448
): CoursierAlg[F] = {
45-
val fetch: Fetch[F] =
46-
Fetch[F](FileCache[F]())
49+
val fetch: F[Fetch[F]] = fetchAlg
50+
.classLoader(config.coursierDependencies)
51+
.map { loader =>
52+
Fetch[F](
53+
FileCache[F]().withClassLoaders(loader :: Nil)
54+
)
55+
}
4756

48-
val cacheNoTtl: FileCache[F] =
49-
FileCache[F]().withTtl(None).withCachePolicies(List(CachePolicy.Update))
57+
val cacheNoTtl: F[FileCache[F]] =
58+
fetchAlg
59+
.classLoader(config.coursierDependencies)
60+
.map { loader =>
61+
FileCache[F]()
62+
.withTtl(None)
63+
.withCachePolicies(List(CachePolicy.Update))
64+
.withClassLoaders(loader :: Nil)
65+
}
5066

5167
new CoursierAlg[F] {
5268
override def getMetadata(
@@ -64,38 +80,40 @@ object CoursierAlg {
6480
repositories: List[coursier.Repository],
6581
acc: DependencyMetadata
6682
): F[DependencyMetadata] = {
67-
val fetchArtifacts = fetch
68-
.withArtifactTypes(Set(coursier.Type.pom, coursier.Type.ivy))
69-
.withDependencies(List(dependency))
70-
.withRepositories(repositories)
71-
72-
fetchArtifacts.ioResult.attempt.flatMap {
73-
case Left(throwable) =>
74-
logger.debug(throwable)(s"Failed to fetch artifacts of $dependency").as(acc)
75-
case Right(result) =>
76-
val maybeProject = result.resolution.projectCache
77-
.get(dependency.moduleVersion)
78-
.map { case (_, project) => project }
79-
80-
maybeProject.fold(F.pure(acc)) { project =>
81-
val metadata = acc.enrichWith(metadataFrom(project))
82-
val recurse = Option.when(metadata.repoUrl.isEmpty)(())
83+
val fetchArtifacts = fetch.map(
84+
_.withArtifactTypes(Set(coursier.Type.pom, coursier.Type.ivy))
85+
.withDependencies(List(dependency))
86+
.withRepositories(repositories)
87+
)
88+
fetchArtifacts.flatMap {
89+
_.ioResult.attempt.flatMap {
90+
case Left(throwable) =>
91+
logger.debug(throwable)(s"Failed to fetch artifacts of $dependency").as(acc)
92+
case Right(result) =>
93+
val maybeProject = result.resolution.projectCache
94+
.get(dependency.moduleVersion)
95+
.map { case (_, project) => project }
96+
maybeProject.fold(F.pure(acc)) { project =>
97+
val metadata = acc.enrichWith(metadataFrom(project))
98+
val recurse = Option.when(metadata.repoUrl.isEmpty)(())
8399
(recurse >> parentOf(project)).fold(F.pure(metadata)) { parent =>
84-
getMetadataImpl(parent, repositories, metadata)
100+
getMetadataImpl(parent, repositories, metadata)
101+
}
85102
}
86-
}
103+
}
87104
}
88105
}
89106

90107
override def getVersions(dependency: Dependency, resolver: Resolver): F[List[Version]] =
91108
convertResolver(resolver).flatMap { repository =>
92109
val module = toCoursierModule(dependency)
93-
repository.versions(module, cacheNoTtl.fetch).run.flatMap {
94-
case Left(message) =>
95-
logger.debug(message) >> F.raiseError[List[Version]](new Throwable(message))
96-
case Right((versions, _)) =>
97-
F.pure(versions.available.map(Version.apply).sorted)
98-
}
110+
cacheNoTtl.flatMap { cacheNoTtl =>
111+
repository.versions(module, cacheNoTtl.fetch).run.flatMap {
112+
case Left(message) =>
113+
logger.debug(message) >> F.raiseError[List[Version]](new Throwable(message))
114+
case Right((versions, _)) => F.pure(versions.available.map(Version.apply).sorted)
115+
}
116+
}
99117
}
100118

101119
private def convertResolver(resolver: Resolver): F[coursier.Repository] =

‎modules/core/src/main/scala/org/scalasteward/core/coursier/CoursierDependenciesFetchAlg.scala‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* Copyright 2018-2022 Scala Steward contributors
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.scalasteward.core.coursier
18+
19+
import cats.Parallel
20+
import cats.effect._
21+
import cats.implicits._
22+
import coursier._
23+
import coursier.cache.FileCache
24+
25+
import java.net.URLClassLoader
26+
27+
/**
28+
* An interface to fetch dependencies for the coursier alg via coursier.
29+
*/
30+
trait CoursierDependenciesFetchAlg[F[_]] {
31+
def classLoader(dependencies: Seq[String]): F[ClassLoader]
32+
}
33+
34+
object CoursierDependenciesFetchAlg {
35+
def create[F[_]](implicit parallel: Parallel[F], F: Sync[F]): CoursierDependenciesFetchAlg[F] =
36+
(dependencies: Seq[String]) =>
37+
Fetch[F](FileCache[F]())
38+
.withDependencies(dependencies.map { dep =>
39+
val Array(org, name, version) = dep.split(':')
40+
Dependency(Module(Organization(org), ModuleName(name)), version)
41+
})
42+
.io
43+
.map(result => new URLClassLoader(result.map(_.toURI.toURL).toArray))
44+
}

0 commit comments

Comments
 (0)