Merge pull request #1235 from artemcm/RevertIncrementalChangesRecent

Revert [Explicit Module Builds] Serialize the inter-module dependenc…  … …y graph as incremental build state"

Author: etcwilde

Sources/SwiftDriver/Driver/Driver.swift

@@ -235,6 +235,22 @@ public struct Driver {
   /// Only present when the driver will be writing the record.
   /// Only used for reading when compiling incrementally.
   @_spi(Testing) public let buildRecordInfo: BuildRecordInfo?
+
+  /// A build-record-relative path to the location of a serialized copy of the
+  /// driver's dependency graph.
+  ///
+  /// FIXME: This is a little ridiculous. We could probably just replace the
+  /// build record outright with a serialized format.
+  var driverDependencyGraphPath: VirtualPath? {
+    guard let recordInfo = self.buildRecordInfo else {
+      return nil
+    }
+    let filename = recordInfo.buildRecordPath.basenameWithoutExt
+    return recordInfo
+      .buildRecordPath
+      .parentDirectory
+      .appending(component: filename + ".priors")
+  }
 
   /// Whether to consider incremental compilation.
   let shouldAttemptIncrementalCompilation: Bool
@@ -1267,6 +1283,10 @@ extension Driver {
   public mutating func run(
     jobs: [Job]
   ) throws {
+    if parsedOptions.hasArgument(.v) {
+      try printVersion(outputStream: &stderrStream)
+    }
+
     let forceResponseFiles = parsedOptions.contains(.driverForceResponseFiles)
 
     // If we're only supposed to print the jobs, do so now.
@@ -1431,15 +1451,6 @@ extension Driver {
           buildRecordInfo?.removeBuildRecord()
           return
       }
-      do {
-        try incrementalCompilationState.writeInterModuleDependencyGraph(buildRecordInfo)
-      }
-      catch {
-        diagnosticEngine.emit(
-          .warning("next compile must run a full dependency scan; could not write inter-module dependency graph: \(error.localizedDescription)"))
-        buildRecordInfo?.removeInterModuleDependencyGraph()
-        return
-      }
     }
     buildRecordInfo?.writeBuildRecord(
       jobs,

Sources/SwiftDriver/ExplicitModuleBuilds/ClangVersionedDependencyResolution.swift

@@ -178,7 +178,7 @@ private extension InterModuleDependencyGraph {
 
   /// Update the set of all PCMArgs against which a given clang module was re-scanned
   mutating func updateCapturedPCMArgClangDependencies(using pcmArgSetMap:
-                                                      [ModuleDependencyId : Set<[String]>]
+                                                        [ModuleDependencyId : Set<[String]>]
   ) throws {
     for (moduleId, newPCMArgs) in pcmArgSetMap {
       guard let moduleInfo = modules[moduleId] else {

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift

@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 import class Foundation.JSONEncoder
-import struct Foundation.Data
 
 /// A map from a module identifier to its info
 public typealias ModuleInfoMap = [ModuleDependencyId: ModuleInfo]
@@ -276,7 +275,7 @@ public struct InterModuleDependencyGraph: Codable {
 }
 
 internal extension InterModuleDependencyGraph {
-  func toJSONData() throws -> Data {
+  func toJSONString() throws -> String {
     let encoder = JSONEncoder()
 #if os(Linux) || os(Android)
     encoder.outputFormatting = [.prettyPrinted]
@@ -285,11 +284,8 @@ internal extension InterModuleDependencyGraph {
       encoder.outputFormatting = [.prettyPrinted, .withoutEscapingSlashes]
     }
 #endif
-    return try encoder.encode(self)
-  }
-
-  func toJSONString() throws -> String {
-    return try String(data: toJSONData(), encoding: .utf8)!
+    let data = try encoder.encode(self)
+    return String(data: data, encoding: .utf8)!
   }
 }
 

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

@@ -41,116 +41,6 @@ extension Diagnostic.Message {
   }
 }
 
-@_spi(Testing) public extension Driver {
-  /// Scan the current module's input source-files to compute its direct and transitive
-  /// module dependencies.
-  mutating func gatherModuleDependencies()
-  throws -> InterModuleDependencyGraph {
-    var dependencyGraph = try performDependencyScan()
-
-    if parsedOptions.hasArgument(.printPreprocessedExplicitDependencyGraph) {
-      try stdoutStream <<< dependencyGraph.toJSONString()
-      stdoutStream.flush()
-    }
-
-    if let externalTargetDetails = externalTargetModuleDetailsMap {
-      // Resolve external dependencies in the dependency graph, if any.
-      try dependencyGraph.resolveExternalDependencies(for: externalTargetDetails)
-    }
-
-    // Re-scan Clang modules at all the targets they will be built against.
-    // This is currently disabled because we are investigating it being unnecessary
-    // try resolveVersionedClangDependencies(dependencyGraph: &dependencyGraph)
-
-    // Set dependency modules' paths to be saved in the module cache.
-    try resolveDependencyModulePaths(dependencyGraph: &dependencyGraph)
-
-    if parsedOptions.hasArgument(.printExplicitDependencyGraph) {
-      let outputFormat = parsedOptions.getLastArgument(.explicitDependencyGraphFormat)?.asSingle
-      if outputFormat == nil || outputFormat == "json" {
-        try stdoutStream <<< dependencyGraph.toJSONString()
-      } else if outputFormat == "dot" {
-        DOTModuleDependencyGraphSerializer(dependencyGraph).writeDOT(to: &stdoutStream)
-      }
-      stdoutStream.flush()
-    }
-
-    return dependencyGraph
-  }
-
-  /// Update the given inter-module dependency graph to set module paths to be within the module cache,
-  /// if one is present, and for Swift modules to use the context hash in the file name.
-  private mutating func resolveDependencyModulePaths(dependencyGraph: inout InterModuleDependencyGraph)
-  throws {
-    // If a module cache path is specified, update all module dependencies
-    // to be output into it.
-    if let moduleCachePath = parsedOptions.getLastArgument(.moduleCachePath)?.asSingle {
-      try resolveDependencyModulePathsRelativeToModuleCache(dependencyGraph: &dependencyGraph,
-                                                            moduleCachePath: moduleCachePath)
-    }
-
-    // Set the output path to include the module's context hash
-    try resolveDependencyModuleFileNamesWithContextHash(dependencyGraph: &dependencyGraph)
-  }
-
-  /// For Swift module dependencies, set the output path to include the module's context hash
-  private mutating func resolveDependencyModuleFileNamesWithContextHash(dependencyGraph: inout InterModuleDependencyGraph)
-  throws {
-    for (moduleId, moduleInfo) in dependencyGraph.modules {
-      // Output path on the main module is determined by the invocation arguments.
-      guard moduleId.moduleName != dependencyGraph.mainModuleName else {
-        continue
-      }
-
-      let plainPath = VirtualPath.lookup(dependencyGraph.modules[moduleId]!.modulePath.path)
-      if case .swift(let swiftDetails) = moduleInfo.details {
-        guard let contextHash = swiftDetails.contextHash else {
-          throw Driver.Error.missingContextHashOnSwiftDependency(moduleId.moduleName)
-        }
-        let updatedPath = plainPath.parentDirectory.appending(component: "\(plainPath.basenameWithoutExt)-\(contextHash).\(plainPath.extension!)")
-        dependencyGraph.modules[moduleId]!.modulePath = TextualVirtualPath(path: updatedPath.intern())
-      }
-      // TODO: Remove this once toolchain is updated
-      else if case .clang(let clangDetails) = moduleInfo.details {
-        if !moduleInfo.modulePath.path.description.contains(clangDetails.contextHash) {
-          let contextHash = clangDetails.contextHash
-          let updatedPath = plainPath.parentDirectory.appending(component: "\(plainPath.basenameWithoutExt)-\(contextHash).\(plainPath.extension!)")
-          dependencyGraph.modules[moduleId]!.modulePath = TextualVirtualPath(path: updatedPath.intern())
-        }
-      }
-    }
-  }
-
-  /// Resolve all paths to dependency binary module files to be relative to the module cache path.
-  private mutating func resolveDependencyModulePathsRelativeToModuleCache(dependencyGraph: inout InterModuleDependencyGraph,
-                                                                          moduleCachePath: String)
-  throws {
-    for (moduleId, moduleInfo) in dependencyGraph.modules {
-      // Output path on the main module is determined by the invocation arguments.
-      if case .swift(let name) = moduleId {
-        if name == dependencyGraph.mainModuleName {
-          continue
-        }
-        let modulePath = VirtualPath.lookup(moduleInfo.modulePath.path)
-        // Use VirtualPath to get the OS-specific path separators right.
-        let modulePathInCache =
-            try VirtualPath(path: moduleCachePath).appending(component: modulePath.basename)
-        dependencyGraph.modules[moduleId]!.modulePath =
-            TextualVirtualPath(path: modulePathInCache.intern())
-      }
-      // TODO: Remove this once toolchain is updated
-      else if case .clang(_) = moduleId {
-        let modulePath = VirtualPath.lookup(moduleInfo.modulePath.path)
-        // Use VirtualPath to get the OS-specific path separators right.
-        let modulePathInCache =
-            try VirtualPath(path: moduleCachePath).appending(component: modulePath.basename)
-        dependencyGraph.modules[moduleId]!.modulePath =
-            TextualVirtualPath(path: modulePathInCache.intern())
-      }
-    }
-  }
-}
-
 public extension Driver {
   /// Precompute the dependencies for a given Swift compilation, producing a
   /// dependency graph including all Swift and C module files and

Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift

@@ -10,9 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-import struct Foundation.Data
-import class Foundation.JSONDecoder
-
 import class TSCBasic.DiagnosticsEngine
 import protocol TSCBasic.FileSystem
 import struct TSCBasic.AbsolutePath
@@ -49,7 +46,6 @@ import class Dispatch.DispatchQueue
   @_spi(Testing) public let timeBeforeFirstJob: TimePoint
   let diagnosticEngine: DiagnosticsEngine
   let compilationInputModificationDates: [TypedVirtualPath: TimePoint]
-  private var explicitModuleDependencyGraph: InterModuleDependencyGraph? = nil
 
   private var finishedJobResults = [JobResult]()
   // A confinement queue that protects concurrent access to the
@@ -75,6 +71,7 @@ import class Dispatch.DispatchQueue
     self.compilationInputModificationDates = compilationInputModificationDates
   }
 
+
   convenience init?(
     actualSwiftVersion: String,
     compilerOutputType: FileType?,
@@ -196,13 +193,6 @@ import class Dispatch.DispatchQueue
     try? fileSystem.removeFileTree(absPath)
   }
 
-  func removeInterModuleDependencyGraph() {
-    guard let absPath = interModuleDependencyGraphPath.absolutePath else {
-      return
-    }
-    try? fileSystem.removeFileTree(absPath)
-  }
-
   /// Before writing to the dependencies file path, preserve any previous file
   /// that may have been there. No error handling -- this is just a nicety, it
   /// doesn't matter if it fails.
@@ -213,8 +203,9 @@ import class Dispatch.DispatchQueue
   }
 
 
-  // TODO: Incremental too many names, buildRecord BuildRecord outofdatemap
+// TODO: Incremental too many names, buildRecord BuildRecord outofdatemap
   func populateOutOfDateBuildRecord(
+    inputFiles: [TypedVirtualPath],
     reporter: IncrementalCompilationState.Reporter?
   ) -> BuildRecord? {
     let contents: String
@@ -254,29 +245,6 @@ import class Dispatch.DispatchQueue
     return outOfDateBuildRecord
   }
 
-  func readOutOfDateInterModuleDependencyGraph(
-    buildRecord: BuildRecord?,
-    reporter: IncrementalCompilationState.Reporter?
-  ) -> InterModuleDependencyGraph? {
-    // If a valid build record could not be produced, do not bother here
-    guard buildRecord != nil else {
-      reporter?.report("Incremental compilation did not attempt to read inter-module dependency graph.")
-      return nil
-    }
-
-    let decodedGraph: InterModuleDependencyGraph
-    do {
-      let contents = try fileSystem.readFileContents(interModuleDependencyGraphPath).cString
-      decodedGraph = try JSONDecoder().decode(InterModuleDependencyGraph.self,
-                                              from: Data(contents.utf8))
-    } catch {
-      return nil
-    }
-    reporter?.report("Read inter-module dependency graph", interModuleDependencyGraphPath)
-
-    return decodedGraph
-  }
-
   func jobFinished(job: Job, result: ProcessResult) {
     self.confinementQueue.sync {
       finishedJobResults.append(JobResult(job, result))
@@ -295,15 +263,6 @@ import class Dispatch.DispatchQueue
       .appending(component: filename + ".priors")
   }
 
-  /// A build-record-relative path to the location of a serialized copy of the
-  /// driver's inter-module dependency graph.
-  var interModuleDependencyGraphPath: VirtualPath {
-    let filename = buildRecordPath.basenameWithoutExt
-    return buildRecordPath
-      .parentDirectory
-      .appending(component: filename + ".moduledeps")
-  }
-
   /// Directory to emit dot files into
   var dotFileDirectory: VirtualPath {
     buildRecordPath.parentDirectory

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationProtectedState.swift

@@ -36,7 +36,7 @@ extension IncrementalCompilationState {
     fileprivate let moduleDependencyGraph: ModuleDependencyGraph
 
     fileprivate let reporter: Reporter?
-
+    
     init(skippedCompileGroups: [TypedVirtualPath: CompileJobGroup],
          _ moduleDependencyGraph: ModuleDependencyGraph,
          _ driver: inout Driver) {