Support resolving documentation links to content in other DocC archives (#710)

This adds experimental support for resolving documentation links and symbol
links to content in other DocC archives.

At a high level this external link support works in two steps:

  1. While building the documentation for the first module, 
     DocC emits two files, one that describe the link hierarchy and its possible
     disambiguations and one that contains minimal amount of content
     for every linkable page.
     
  2. Later, while building the documentation for the second module, 
     DocC is passed the documentation archive for the first module and reads
     these file and uses them to resolve links to content in the first archive.

> Important: There is more work needed to support this end-to-end.
>
> If you run this now you'll notice that the external links are 404s in the
> browser unless you host both archives on the same server.

rdar://114731067

Author: d-ronnqvist

Sources/SwiftDocC/Infrastructure/ConvertOutputConsumer.swift

@@ -46,11 +46,15 @@ public protocol ConvertOutputConsumer {
 
     /// Consumes build metadata created during a conversion.
     func consume(buildMetadata: BuildMetadata) throws
+    
+    /// Consumes a file representation of the local link resolution information.
+    func consume(linkResolutionInformation: SerializableLinkResolutionInformation) throws
 }
 
 // Default implementations that discard the documentation conversion products, for consumers that don't need these
 // values.
 public extension ConvertOutputConsumer {
     func consume(renderReferenceStore: RenderReferenceStore) throws {}
     func consume(buildMetadata: BuildMetadata) throws {}
+    func consume(linkResolutionInformation: SerializableLinkResolutionInformation) throws {}
 }

Sources/SwiftDocC/Infrastructure/DocumentationContext.swift

@@ -110,11 +110,8 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
         }
     }
 
-    /// A link resolver that resolves references by finding them in path hierarchy.
-    ///
-    /// The link resolver is `nil` until some documentation content is registered with the context.
-    /// It's safe to access the link resolver during symbol registration and at later points in the registration and conversion.
-    var hierarchyBasedLinkResolver: PathHierarchyBasedLinkResolver! = nil
+    /// A class that resolves documentation links by orchestrating calls to other link resolver implementations.
+    public var linkResolver = LinkResolver()
 
     /// The provider of documentation bundles for this context.
     var dataProvider: DocumentationContextDataProvider
@@ -200,6 +197,8 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
     /// A list of non-topic links that can be resolved.
     var nodeAnchorSections = [ResolvedTopicReference: AnchorSection]()
 
+    var externalCache = [ResolvedTopicReference: LinkResolver.ExternalEntity]()
+
     /// A list of all the problems that was encountered while registering and processing the documentation bundles in this context.
     public var problems: [Problem] {
         return diagnosticEngine.problems
@@ -361,7 +360,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
     ///   - dataProvider: The provider that removed this bundle.
     ///   - bundle: The bundle that was removed.
     public func dataProvider(_ dataProvider: DocumentationContextDataProvider, didRemoveBundle bundle: DocumentationBundle) throws {
-        hierarchyBasedLinkResolver?.unregisterBundle(identifier: bundle.identifier)
+        linkResolver.localResolver?.unregisterBundle(identifier: bundle.identifier)
 
         // Purge the reference cache for this bundle and disable reference caching for
         // this bundle moving forward.
@@ -1153,7 +1152,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
             var moduleReferences = [String: ResolvedTopicReference]()
 
             // Build references for all symbols in all of this module's symbol graphs.
-            let symbolReferences = hierarchyBasedLinkResolver.referencesForSymbols(in: symbolGraphLoader.unifiedGraphs, bundle: bundle, context: self)
+            let symbolReferences = linkResolver.localResolver.referencesForSymbols(in: symbolGraphLoader.unifiedGraphs, bundle: bundle, context: self)
 
             // Set the index and cache storage capacity to avoid ad-hoc storage resizing.
             symbolIndex.reserveCapacity(symbolReferences.count)
@@ -1270,7 +1269,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
             // Only add the symbol mapping now if the path hierarchy based resolver is the main implementation.
             // If it is only used for mismatch checking then we must wait until the documentation cache code path has traversed and updated all the colliding nodes.
             // Otherwise the mappings will save the unmodified references and the hierarchy based resolver won't find the expected parent nodes when resolving links.
-            hierarchyBasedLinkResolver.addMappingForSymbols(symbolIndex: symbolIndex)
+            linkResolver.localResolver.addMappingForSymbols(symbolIndex: symbolIndex)
 
             // Track the symbols that have multiple matching documentation extension files for diagnostics.
             var symbolsWithMultipleDocumentationExtensionMatches = [ResolvedTopicReference: [SemanticResult<Article>]]()
@@ -1817,7 +1816,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
             topicGraph.addNode(graphNode)
             documentationCache[reference] = documentation
 
-            hierarchyBasedLinkResolver.addRootArticle(article, anchorSections: documentation.anchorSections)
+            linkResolver.localResolver.addRootArticle(article, anchorSections: documentation.anchorSections)
             for anchor in documentation.anchorSections {
                 nodeAnchorSections[anchor.reference] = anchor
             }
@@ -1873,7 +1872,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
             let graphNode = TopicGraph.Node(reference: reference, kind: .article, source: .file(url: article.source), title: title)
             topicGraph.addNode(graphNode)
 
-            hierarchyBasedLinkResolver.addArticle(article, anchorSections: documentation.anchorSections)
+            linkResolver.localResolver.addArticle(article, anchorSections: documentation.anchorSections)
             for anchor in documentation.anchorSections {
                 nodeAnchorSections[anchor.reference] = anchor
             }
@@ -2078,6 +2077,17 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
             }
         }
 
+        discoveryGroup.async(queue: discoveryQueue) { [unowned self] in
+            do {
+                try linkResolver.loadExternalResolvers()
+            } catch {
+                // Pipe the error out of the dispatch queue.
+                discoveryError.sync({
+                    if $0 == nil { $0 = error }
+                })
+            }
+        }
+        
         discoveryGroup.wait()
 
         try shouldContinueRegistration()
@@ -2128,7 +2138,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
             options = globalOptions.first
         }
 
-        self.hierarchyBasedLinkResolver = hierarchyBasedResolver
+        self.linkResolver.localResolver = hierarchyBasedResolver
         hierarchyBasedResolver.addMappingForRoots(bundle: bundle)
         for tutorial in tutorials {
             hierarchyBasedResolver.addTutorial(tutorial)
@@ -2187,7 +2197,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
         }
 
         try shouldContinueRegistration()
-        var allCuratedReferences = try crawlSymbolCuration(in: hierarchyBasedLinkResolver.topLevelSymbols(), bundle: bundle)
+        var allCuratedReferences = try crawlSymbolCuration(in: linkResolver.localResolver.topLevelSymbols(), bundle: bundle)
 
         // Store the list of manually curated references if doc coverage is on.
         if shouldStoreManuallyCuratedReferences {
@@ -2222,7 +2232,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
         // Emit warnings for any remaining uncurated files.
         emitWarningsForUncuratedTopics()
 
-        hierarchyBasedLinkResolver.addAnchorForSymbols(symbolIndex: symbolIndex, documentationCache: documentationCache)
+        linkResolver.localResolver.addAnchorForSymbols(symbolIndex: symbolIndex, documentationCache: documentationCache)
 
         // Fifth, resolve links in nodes that are added solely via curation
         try preResolveExternalLinks(references: Array(allCuratedReferences), bundle: bundle)
@@ -2329,7 +2339,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
     /// - Returns: An ordered list of symbol references that have been added to the topic graph automatically.
     private func autoCurateSymbolsInTopicGraph(engine: DiagnosticEngine) -> [(child: ResolvedTopicReference, parent: ResolvedTopicReference)] {
         var automaticallyCuratedSymbols = [(ResolvedTopicReference, ResolvedTopicReference)]()
-        hierarchyBasedLinkResolver.traverseSymbolAndParentPairs { reference, parentReference in
+        linkResolver.localResolver.traverseSymbolAndParentPairs { reference, parentReference in
             guard let topicGraphNode = topicGraph.nodeWithReference(reference),
                   let topicGraphParentNode = topicGraph.nodeWithReference(parentReference),
                   // Check that the node hasn't got any parents from manual curation
@@ -2535,6 +2545,9 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
             let referenceWithoutFragment = reference.withFragment(nil)
             return try entity(with: referenceWithoutFragment).availableSourceLanguages
         } catch ContextError.notFound {
+            if let externalEntity = externalCache[reference] {
+                return externalEntity.sourceLanguages
+            }
             preconditionFailure("Reference does not have an associated documentation node.")
         } catch {
             fatalError("Unexpected error when retrieving source languages: \(error)")
@@ -2646,7 +2659,8 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
     public func resolve(_ reference: TopicReference, in parent: ResolvedTopicReference, fromSymbolLink isCurrentlyResolvingSymbolLink: Bool = false) -> TopicReferenceResolutionResult {
         switch reference {
         case .unresolved(let unresolvedReference):
-            return hierarchyBasedLinkResolver.resolve(unresolvedReference, in: parent, fromSymbolLink: isCurrentlyResolvingSymbolLink, context: self)
+            return linkResolver.resolve(unresolvedReference, in: parent, fromSymbolLink: isCurrentlyResolvingSymbolLink, context: self)
+            
         case .resolved(let resolved):
             // This reference is already resolved (either as a success or a failure), so don't change anything.
             return resolved

Sources/SwiftDocC/Infrastructure/DocumentationConverter.swift

@@ -330,14 +330,20 @@ public struct DocumentationConverter: DocumentationConverterProtocol {
                     }
 
                     if emitDigest {
-                        let nodeLinkSummaries = entity.externallyLinkableElementSummaries(context: context, renderNode: renderNode)
+                        let nodeLinkSummaries = entity.externallyLinkableElementSummaries(context: context, renderNode: renderNode, includeTaskGroups: true)
                         let nodeIndexingRecords = try renderNode.indexingRecords(onPage: identifier)
 
                         resultsGroup.async(queue: resultsSyncQueue) {
                             assets.merge(renderNode.assetReferences, uniquingKeysWith: +)
                             linkSummaries.append(contentsOf: nodeLinkSummaries)
                             indexingRecords.append(contentsOf: nodeIndexingRecords)
                         }
+                    } else if FeatureFlags.current.isExperimentalLinkHierarchySerializationEnabled {
+                        let nodeLinkSummaries = entity.externallyLinkableElementSummaries(context: context, renderNode: renderNode, includeTaskGroups: false)
+                        
+                        resultsGroup.async(queue: resultsSyncQueue) {
+                            linkSummaries.append(contentsOf: nodeLinkSummaries)
+                        }
                     }
                 } catch {
                     recordProblem(from: error, in: &results, withIdentifier: "render-node")
@@ -362,6 +368,19 @@ public struct DocumentationConverter: DocumentationConverterProtocol {
             }
         }
 
+        if FeatureFlags.current.isExperimentalLinkHierarchySerializationEnabled {
+            do {
+                let serializableLinkInformation = try context.linkResolver.localResolver.prepareForSerialization(bundleID: bundle.identifier)
+                try outputConsumer.consume(linkResolutionInformation: serializableLinkInformation)
+                
+                if !emitDigest {
+                    try outputConsumer.consume(linkableElementSummaries: linkSummaries)
+                }
+            } catch {
+                recordProblem(from: error, in: &conversionProblems, withIdentifier: "link-resolver")
+            }
+        }
+        
         if emitDigest {
             do {
                 try outputConsumer.consume(problems: context.problems + conversionProblems)

Sources/SwiftDocC/Infrastructure/DocumentationCurator.swift

@@ -115,7 +115,7 @@ struct DocumentationCurator {
         context.topicGraph.addNode(curatedNode)
 
         // Move the article from the article cache to the documentation
-        context.hierarchyBasedLinkResolver.addArticle(filename: articleFilename, reference: reference, anchorSections: documentationNode.anchorSections)
+        context.linkResolver.localResolver.addArticle(filename: articleFilename, reference: reference, anchorSections: documentationNode.anchorSections)
 
         context.documentationCache[reference] = documentationNode
         for anchor in documentationNode.anchorSections {