Merge pull request #10237 from romanowski/scala3doc/fixes-in-scaladoc

Scala3doc/fixes in scaladoc

Author: romanowski

project/Build.scala

@@ -1480,6 +1480,7 @@ object Build {
             "org.jetbrains.kotlinx" % "kotlinx-html-jvm" % "0.7.2", // Needs update when dokka version changes
             "com.vladsch.flexmark" % "flexmark-all" % "0.42.12",
             "nl.big-o" % "liqp" % "0.6.7",
+            "org.jsoup" % "jsoup" % "1.13.1", // Needed to process .html files for static site
             "args4j" % "args4j" % "2.33",
             Dependencies.`jackson-dataformat-yaml`,
 

scala3doc/resources/dotty_res/hljs/highlight.pack.js

@@ -192,13 +192,6 @@ pre {
   border: 1px solid rgba(0, 0, 0, 0.1);
 }
 
-pre > code {
-  display: block;
-  padding: 0.5rem;
-  overflow-x: auto;
-  background: transparent;
-}
-
 /* admonitions */
 blockquote {
   padding: 0 1em;

scala3doc/scala3-docs/css/dottydoc.css

@@ -4,6 +4,9 @@ package site
 import org.jetbrains.dokka.model.DisplaySourceSet
 import org.jetbrains.dokka.model.properties.PropertyContainer
 import org.jetbrains.dokka.pages.{ContentNode, DCI, Style}
+import org.jetbrains.dokka.base.resolvers.local.LocationProvider
+import com.vladsch.flexmark.convert.html.FlexmarkHtmlParser
+import org.jsoup.Jsoup
 
 case class PartiallyRenderedContent(
   template: TemplateFile,
@@ -23,3 +26,8 @@ case class PartiallyRenderedContent(
     copy(getSourceSets = sourceSets)
 
   lazy val resolved = template.resolveToHtml(context)
+
+  def procsesHtml(linkTo: String => String): String =
+    val document = Jsoup.parse(resolved.code)
+    document.select("a").forEach(element => element.attr("href", linkTo(element.attr("href"))))// forrach does not work here
+    document.outerHtml()

scala3doc/src/dotty/dokka/site/PartiallyRenderedContent.scala

@@ -10,10 +10,7 @@ import org.jetbrains.dokka.model.Documentable
 import org.jetbrains.dokka.pages._
 import org.jetbrains.dokka.transformers.pages.PageTransformer
 
-case class LoadedTemplate(templateFile: TemplateFile, children: List[LoadedTemplate], file: File) {
-  def relativePath(root: File): String =
-    root.toPath().relativize(file.toPath()).toString().replace(File.separatorChar, '.')
-}
+case class LoadedTemplate(templateFile: TemplateFile, children: List[LoadedTemplate], file: File)
 
 case class StaticPageNode(
                            template: TemplateFile,

scala3doc/src/dotty/dokka/site/StaticPageNode.scala

@@ -3,7 +3,9 @@ package site
 
 import java.io.File
 import java.nio.file.Files
-
+import java.nio.file.FileVisitOption
+import java.nio.file.Path
+import java.nio.file.Paths
 
 import org.jetbrains.dokka.base.parsers.MarkdownParser
 import org.jetbrains.dokka.base.transformers.pages.comments.DocTagToContentConverter
@@ -15,14 +17,16 @@ import org.jetbrains.dokka.pages.{ContentKind, ContentNode, DCI, PageNode}
 import org.jetbrains.dokka.plugability.DokkaContext
 import org.jetbrains.dokka.pages.Style
 import org.jetbrains.dokka.model.DisplaySourceSet
+import util.Try
 
 import scala.collection.JavaConverters._
 
 class StaticSiteContext(val root: File, sourceSets: Set[SourceSetWrapper]):
 
   def indexPage():Option[StaticPageNode] =
     val files = List(new File(root, "index.html"), new File(root, "index.md")).filter { _.exists() }
-    if (files.size > 1) println(s"ERROR: Multiple root index pages found: ${files.map(_.getAbsolutePath)}") // TODO (#14): provide proper error handling
+    // TODO (https://github.com/lampepfl/scala3doc/issues/238): provide proper error handling
+    if (files.size > 1) println(s"ERROR: Multiple root index pages found: ${files.map(_.getAbsolutePath)}")
     files.flatMap(loadTemplate(_, isBlog = false)).headOption.map(templateToPage)
 
   lazy val layouts: Map[String, TemplateFile] =
@@ -37,7 +41,27 @@ class StaticSiteContext(val root: File, sourceSets: Set[SourceSetWrapper]):
 
   lazy val templates: Seq[LoadedTemplate] = sideBarConfig.fold(loadAllFiles())(_.map(loadSidebarContent))
 
-  lazy val pages = templates.map(templateToPage)
+  lazy val mainPages: Seq[StaticPageNode] = templates.map(templateToPage)
+
+  lazy val allPages: Seq[StaticPageNode] = sideBarConfig.fold(mainPages){ sidebar =>
+    def flattenPages(p: StaticPageNode): Set[Path] =
+      Set(p.template.file.toPath) ++ p.getChildren.asScala.collect { case p: StaticPageNode => flattenPages(p) }.flatten
+
+    val mainFiles = mainPages.toSet.flatMap(flattenPages)
+    val docsPath = root.toPath.resolve("docs")
+    val allPaths =
+      if !Files.exists(docsPath) then Nil
+      else Files.walk(docsPath, FileVisitOption.FOLLOW_LINKS).iterator().asScala.toList
+
+    val orphanedFiles = allPaths.filterNot(mainFiles.contains).filter { p =>
+        val name = p.getFileName.toString
+        name.endsWith(".md") || name.endsWith(".html")
+    }
+
+    val orphanedTemplates = orphanedFiles.flatMap(p => loadTemplate(p.toFile, isBlog = false))
+
+    mainPages ++ orphanedTemplates.map(templateToPage)
+  }
 
   private def isValidTemplate(file: File): Boolean =
     (file.isDirectory && !file.getName.startsWith("_")) ||
@@ -52,7 +76,8 @@ class StaticSiteContext(val root: File, sourceSets: Set[SourceSetWrapper]):
         val allFiles = topLevelFiles.filter(_.isDirectory).flatMap(_.listFiles())
         val (indexes, children) = allFiles.flatMap(loadTemplate(_)).partition(_.templateFile.isIndexPage())
         if (indexes.size > 1)
-          println(s"ERROR: Multiple index pages for $from found in ${indexes.map(_.file)}") // TODO (#14): provide proper error handling
+          // TODO (https://github.com/lampepfl/scala3doc/issues/238): provide proper error handling
+          println(s"ERROR: Multiple index pages for $from found in ${indexes.map(_.file)}")
 
         def loadIndexPage(): TemplateFile =
           val indexFiles = from.listFiles { file =>file.getName == "index.md" || file.getName == "index.html" }
@@ -68,7 +93,8 @@ class StaticSiteContext(val root: File, sourceSets: Set[SourceSetWrapper]):
         Some(LoadedTemplate(templateFile, children.toList, from))
       catch
           case e: RuntimeException =>
-            e.printStackTrace() // TODO (#14): provide proper error handling
+            // TODO (https://github.com/lampepfl/scala3doc/issues/238): provide proper error handling
+            e.printStackTrace()
             None
 
   def asContent(doctag: DocTag, dri: DRI) = new DocTagToContentConverter().buildContent(
@@ -83,10 +109,11 @@ class StaticSiteContext(val root: File, sourceSets: Set[SourceSetWrapper]):
     case Sidebar.Page(title, url) =>
       val isBlog = title == "Blog"
       val path = if isBlog then "blog" else url.stripSuffix(".html") + ".md"
-      val file = root.toPath.resolve(path) // Add support for.html files!
+      val file = root.toPath.resolve(path) // Add support for .html files!
       val LoadedTemplate(template, children, tFile) = loadTemplate(file.toFile, isBlog).get // Add proper logging if file does not exisits
       LoadedTemplate(template.copy(settings = template.settings + ("title" -> List(title))), children, tFile)
     case Sidebar.Category(title, nested) =>
+      // Add support for index.html/index.md files!
       val fakeFile = new File(root, title)
       LoadedTemplate(emptyTemplate(fakeFile), nested.map(loadSidebarContent), fakeFile)
 
@@ -95,9 +122,15 @@ class StaticSiteContext(val root: File, sourceSets: Set[SourceSetWrapper]):
     dir("docs").flatMap(_.listFiles()).flatMap(loadTemplate(_, isBlog = false))
       ++ dir("blog").flatMap(loadTemplate(_, isBlog = true))
 
+  def driForLink(template: TemplateFile, link: String): Try[DRI] = Try(driFor(
+        if link.startsWith("/") then root.toPath.resolve(link.drop(1))
+        else template.file.toPath.getParent().resolve(link)
+    ))
+
+  private def driFor(dest: Path): DRI = mkDRI(s"_.${root.toPath.relativize(dest)}")
+
   def templateToPage(myTemplate: LoadedTemplate): StaticPageNode =
-    def pathToDRI(path: String) = mkDRI(s"_.$path")
-    val dri = pathToDRI(myTemplate.relativePath(root))
+    val dri = driFor(myTemplate.file.toPath)
     val content = new PartiallyRenderedContent(
       myTemplate.templateFile,
       this,

scala3doc/src/dotty/dokka/site/StaticSiteContext.scala

@@ -3,6 +3,7 @@ package site
 
 import java.io.File
 import java.nio.file.Files
+import java.nio.file.FileVisitOption
 
 import org.jetbrains.dokka.base.renderers.html.{NavigationNode, NavigationPage}
 import org.jetbrains.dokka.links.DRI
@@ -25,18 +26,20 @@ class SiteResourceManager(ctx: Option[StaticSiteContext]) extends BaseStaticSite
     }.toSet
 
   override def transform(input: RootPageNode, ctx: StaticSiteContext): RootPageNode =
-    val imgPath = ctx.root.toPath().resolve("images")
+    val rootPath = ctx.root.toPath
+    val imgPath = rootPath.resolve("images")
     val images =
       if !Files.exists(imgPath) then Nil
       else
-        val stream = Files.walk(imgPath)filter(p => Files.isRegularFile(p) && p.getFileName().toString().endsWith(".svg"))
-        stream.iterator().asScala.toList.map(_.toString)
+        val allPaths = Files.walk(imgPath, FileVisitOption.FOLLOW_LINKS)
+        val files = allPaths.filter(Files.isRegularFile(_)).iterator().asScala
+        files.map(p => rootPath.relativize(p).toString).toList
 
-    val resources = listResources(input.getChildren.asScala.toList) ++ images
+    val resources = images ++ listResources(input.getChildren.asScala.toList)
     val resourcePages = resources.map { path =>
-      val content = Files.readAllLines(ctx.root.toPath.resolve(path)).asScala.mkString("\n")
-      new RendererSpecificResourcePage(path, JList(), new RenderingStrategy.Write(content))
-    }.toList
+      val strategy = new RenderingStrategy.Copy(rootPath.resolve(path).toString)
+      new RendererSpecificResourcePage(path, JList(), strategy)
+    }
 
     val modified = input.transformContentPagesTree {
       case it: StaticPageNode =>
@@ -90,8 +93,8 @@ class SitePagesCreator(ctx: Option[StaticSiteContext]) extends BaseStaticSitePro
   override def transform(input: RootPageNode, ctx: StaticSiteContext): RootPageNode =
     val (contentPage, others) = input.getChildren.asScala.toList.partition { _.isInstanceOf[ContentPage] }
     val modifiedModuleRoot = processRootPage(input, contentPage)
-    val (indexes, children) = ctx.pages.partition(_.template.isIndexPage())
-      // TODO (#14): provide proper error handling
+    val (indexes, children) = ctx.allPages.partition(_.template.isIndexPage())
+    // TODO (https://github.com/lampepfl/scala3doc/issues/238): provide proper error handling
     if (indexes.size > 1) println("ERROR: Multiple index pages found $indexes}")
 
     val rootContent = indexes.headOption.fold(ctx.asContent(Text(), mkDRI(extra = "root_content")).get(0))(_.getContent)
@@ -131,7 +134,7 @@ class RootIndexPageCreator(ctx: Option[StaticSiteContext]) extends BaseStaticSit
             input.getName,
             docsRootDRI,
             root.getSourceSets,
-            (ctx.pages.map(toNavigationNode) ++ api).asJava
+            (ctx.mainPages.map(toNavigationNode) ++ api).asJava
           )
         )
       }

scala3doc/src/dotty/dokka/site/processors.scala

@@ -1,4 +1,5 @@
-package dotty.dokka.site
+package dotty.dokka
+package site
 
 import java.io.File
 import java.nio.file.Files
@@ -22,7 +23,6 @@ case class RenderingContext(
                              properties: Map[String, Object],
                              layouts: Map[String, TemplateFile] = Map(),
                              resolving: Set[String] = Set(),
-                             markdownOptions: DataHolder = defaultMarkdownOptions,
                              resources: List[String] = Nil
                            ):
 
@@ -70,7 +70,9 @@ case class TemplateFile(
 
   def isIndexPage() = file.isFile && (file.getName == "index.md" || file.getName == "index.html")
 
-  def resolveToHtml(ctx: StaticSiteContext): ResolvedPage = resolveInner(RenderingContext(Map(), ctx.layouts))
+  def resolveToHtml(ctx: StaticSiteContext): ResolvedPage =
+    val props = Map("page" -> JMap("title" -> title()))
+    resolveInner(RenderingContext(props, ctx.layouts))
 
   private[site] def resolveInner(ctx: RenderingContext): ResolvedPage =
     if (ctx.resolving.contains(file.getAbsolutePath))
@@ -82,9 +84,10 @@ case class TemplateFile(
     // Library requires mutable maps..
     val mutableProperties = new java.util.HashMap[String, Object](ctx.properties.asJava)
     val rendered = Template.parse(this.rawCode).render(mutableProperties)
-    val code = if (!isHtml) rendered else
+    // We want to render markdown only if next template is html
+    val code = if (isHtml || layoutTemplate.exists(!_.isHtml)) rendered else
       val parser: Parser = Parser.builder().build()
-      HtmlRenderer.builder(ctx.markdownOptions).build().render(parser.parse(rendered))
+      HtmlRenderer.builder(defaultMarkdownOptions).build().render(parser.parse(rendered))
 
     val resources = listSetting("extraCSS") ++ listSetting("extraJS")
     layoutTemplate match

scala3doc/src/dotty/dokka/site/templates.scala

@@ -3,7 +3,6 @@ package dotty.dokka
 import dotty.dokka.model._
 import org.jetbrains.dokka.model._
 import org.jetbrains.dokka.links.DRI
-import org.jetbrains.dokka.base.resolvers.local.LocationProvider
 import org.jetbrains.dokka.pages._
 import dotty.dokka.model.api.Kind
 import HTML._

scala3doc/src/dotty/renderers/DotDiagramBuilder.scala

@@ -8,6 +8,7 @@ import org.jetbrains.dokka._
 import HTML._
 import collection.JavaConverters._
 import java.net.URI
+import java.net.URL
 import java.util.{List => JList, Set => JSet}
 import kotlinx.html.FlowContent
 import kotlinx.html.stream.StreamKt
@@ -21,6 +22,7 @@ import dotty.dokka.model.api.HierarchyGraph
 import org.jetbrains.dokka.base.resolvers.local.LocationProvider
 import dotty.dokka.site.StaticPageNode
 import dotty.dokka.site.PartiallyRenderedContent
+import scala.util.Try
 
 class SignatureRenderer(pageContext: ContentPage, sourceSetRestriciton: JSet[DisplaySourceSet], locationProvider: LocationProvider):
   def link(dri: DRI): Option[String] = Option(locationProvider.resolve(dri, sourceSetRestriciton, pageContext))
@@ -240,7 +242,13 @@ class ScalaHtmlRenderer(ctx: DokkaContext) extends HtmlRenderer(ctx) {
 
     page.getContent match
       case prc: PartiallyRenderedContent =>
-        withHtml(context, prc.resolved.code)
+        def processLocalLink(str: String): String =
+          // TODO (https://github.com/lampepfl/scala3doc/issues/238) error handling
+          prc.context.driForLink(prc.template, str).toOption
+            .flatMap(dri => Option(getLocationProvider.resolve(dri, sourceSets, page)))
+            .getOrElse(str)
+
+        withHtml(context, prc.procsesHtml(url => Try(URL(url)).fold(_ => processLocalLink(url), _ => url)))
       case content =>
         build(content, context, page, /*sourceSetRestriction=*/null)