Add an option for symbol graph to support long module names. (#83782)

Currently symbol graphs are always written in files that contain 1 to 2
module names. It's possible for Swift module names to be very long, so
combining 2 of them in file name like `module1@module2...` in the same
path component means the name can be too long for some file systems. The
new option `-symbol-graph-shorten-output-names` changes the symbol graph
output files to use a MD5 hash of the module name(s) as the filename and
outputs an additional JSON file with the original names mapped to the
real filename. The module names JSON can be used to construct a VFS
overlay with the original naming scheme.

fix #83723

I considered using vfsoverlay, which seems like a viable solution, but
the vfsoverlay options don't seem to apply to any of the outputs from
the compiler. When I set an overlay to remap the symbol graph file
outputs, the remapped external paths aren't used so the root problem of
too long file names remains.

Author: dylansturg

include/swift/Option/Options.td

@@ -1961,6 +1961,11 @@ def skip_protocol_implementations : Flag<["-"], "skip-protocol-implementations">
          NoInteractiveOption, SupplementaryOutput, HelpHidden]>,
   HelpText<"Skip emitting symbols that are implementations of protocol requirements or inherited from protocol extensions">;
 
+def symbol_graph_shorten_output_names : Flag<["-"], "symbol-graph-shorten-output-names">,
+  Flags<[SwiftSymbolGraphExtractOption, FrontendOption,
+         NoInteractiveOption, SupplementaryOutput, HelpHidden]>,
+  HelpText<"Shorten the symbol graph output file names by hashing the module name; outputs an additional JSON file with the map of hash to original name">;
+
 // swift-api-digester-only options
 def dump_sdk: Flag<["-", "--"], "dump-sdk">,
   Flags<[NoDriverOption, SwiftAPIDigesterOption]>,

include/swift/SymbolGraphGen/SymbolGraphOptions.h

@@ -77,6 +77,13 @@ struct SymbolGraphOptions {
 
   /// Whether `AvailabilityPlatforms` is an allow list or a block list.
   bool AvailabilityIsBlockList = false;
+
+  /// Whether to use shortened, by using a hash of the module names, file names
+  /// when writing symbol graph files to `OutputDir`.
+  /// An additional JSON file is written at `OutputDir` that contains a mapping
+  /// of the shortened file names to the module name(s) in the symbol graph
+  /// files.
+  bool ShortenOutputNames = false;
 };
 
 } // end namespace symbolgraphgen

lib/Driver/ToolChains.cpp

@@ -733,6 +733,7 @@ ToolChain::constructInvocation(const CompileJobAction &job,
   context.Args.AddLastArg(Arguments, options::OPT_emit_extension_block_symbols,
                           options::OPT_omit_extension_block_symbols);
   context.Args.AddLastArg(Arguments, options::OPT_symbol_graph_minimum_access_level);
+  context.Args.AddLastArg(Arguments, options::OPT_symbol_graph_shorten_output_names);
   context.Args.AddLastArg(Arguments, options::OPT_symbol_graph_skip_synthesized_members);
   context.Args.AddLastArg(Arguments, options::OPT_symbol_graph_skip_inherited_docs);
   context.Args.AddLastArg(Arguments, options::OPT_symbol_graph_pretty_print);
@@ -1272,6 +1273,7 @@ ToolChain::constructInvocation(const MergeModuleJobAction &job,
   context.Args.AddLastArg(Arguments, options::OPT_emit_extension_block_symbols,
                           options::OPT_omit_extension_block_symbols);
   context.Args.AddLastArg(Arguments, options::OPT_symbol_graph_minimum_access_level);
+  context.Args.AddLastArg(Arguments, options::OPT_symbol_graph_shorten_output_names);
 
   context.Args.AddLastArg(Arguments, options::OPT_import_bridging_header,
                           options::OPT_internal_import_bridging_header);

lib/DriverTool/swift_symbolgraph_extract_main.cpp

@@ -185,6 +185,7 @@ int swift_symbolgraph_extract_main(ArrayRef<const char *> Args,
   Options.SkipInheritedDocs = ParsedArgs.hasArg(OPT_skip_inherited_docs);
   Options.SkipProtocolImplementations = ParsedArgs.hasArg(OPT_skip_protocol_implementations);
   Options.IncludeSPISymbols = ParsedArgs.hasArg(OPT_include_spi_symbols);
+  Options.ShortenOutputNames = ParsedArgs.hasArg(OPT_symbol_graph_shorten_output_names);
   Options.EmitExtensionBlockSymbols =
       ParsedArgs.hasFlag(OPT_emit_extension_block_symbols,
                          OPT_omit_extension_block_symbols, /*default=*/false);

lib/Frontend/CompilerInvocation.cpp

@@ -2285,6 +2285,7 @@ static void ParseSymbolGraphArgs(symbolgraphgen::SymbolGraphOptions &Opts,
   Opts.SkipInheritedDocs = Args.hasArg(OPT_skip_inherited_docs);
   Opts.SkipProtocolImplementations = Args.hasArg(OPT_skip_protocol_implementations);
   Opts.IncludeSPISymbols = Args.hasArg(OPT_include_spi_symbols);
+  Opts.ShortenOutputNames = Args.hasArg(OPT_symbol_graph_shorten_output_names);
   Opts.EmitExtensionBlockSymbols =
       Args.hasFlag(OPT_emit_extension_block_symbols,
                    OPT_omit_extension_block_symbols, /*default=*/false);

lib/SymbolGraphGen/SymbolGraphGen.cpp

@@ -20,6 +20,7 @@
 #include "clang/Basic/Module.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/JSON.h"
+#include "llvm/Support/MD5.h"
 #include "llvm/Support/Path.h"
 
 #include "SymbolGraphASTWalker.h"
@@ -28,28 +29,82 @@ using namespace swift;
 using namespace symbolgraphgen;
 
 namespace {
-int serializeSymbolGraph(SymbolGraph &SG,
-                         const SymbolGraphOptions &Options) {
+
+/// Utility for managing symbol graph output file names.
+class SymbolGraphWriter {
+
+public:
+  SymbolGraphWriter(const SymbolGraphOptions &Options) : Options(Options) {}
+
+  bool
+  withOutputPath(DiagnosticEngine &Diags, llvm::vfs::OutputBackend &Backend,
+                 StringRef ModuleName,
+                 llvm::function_ref<bool(llvm::raw_pwrite_stream &)> Action) {
+    SmallString<32> FileName;
+    if (Options.ShortenOutputNames) {
+      llvm::MD5 Hash;
+      Hash.update(ModuleName);
+      llvm::MD5::MD5Result Result;
+      Hash.final(Result);
+      llvm::MD5::stringifyResult(Result, FileName);
+    } else {
+      FileName = ModuleName;
+    }
+    FileName.append(".symbols.json");
+    SymbolFileShortPaths.try_emplace(std::string(ModuleName), std::string(FileName));
+
+    SmallString<1024> OutputPath(Options.OutputDir);
+    llvm::sys::path::append(OutputPath, FileName);
+    return swift::withOutputPath(Diags, Backend, OutputPath, Action);
+  }
+
+  bool finalize(DiagnosticEngine &Diags, llvm::vfs::OutputBackend &Backend) {
+    if (!Options.ShortenOutputNames) {
+      return false; // No errors.
+    }
+
+    SmallString<1024> OutputPath(Options.OutputDir);
+    llvm::sys::path::append(OutputPath, "symbol_modules.json");
+    return swift::withOutputPath(
+        Diags, Backend, OutputPath, [&](raw_ostream &OS) {
+          llvm::json::OStream JSON(OS, Options.PrettyPrint ? 2 : 0);
+          JSON.object([&] {
+            // Insert the module to real filename mappings in a deterministic
+            // order. Enumerating the map returns the keys in order.
+            JSON.attributeObject("modules", [&] {
+              for (const auto &[K, V] : SymbolFileShortPaths) {
+                JSON.attribute(K, V);
+              }
+            });
+          });
+          return false;
+        });
+  }
+
+private:
+  const SymbolGraphOptions &Options;
+  std::map<std::string, std::string> SymbolFileShortPaths;
+};
+
+int serializeSymbolGraph(SymbolGraph &SG, const SymbolGraphOptions &Options,
+                         SymbolGraphWriter &Writer) {
   SmallString<256> FileName;
   FileName.append(getFullModuleName(&SG.M));
   if (SG.ExtendedModule.has_value()) {
     FileName.push_back('@');
 //    FileName.append(SG.ExtendedModule.value()->getNameStr());
     FileName.append(getFullModuleName(SG.ExtendedModule.value()));
   } else if (SG.DeclaringModule.has_value()) {
-    // Treat cross-import overlay modules as "extensions" of their declaring module
+    // Treat cross-import overlay modules as "extensions" of their declaring
+    // module
     FileName.push_back('@');
 //    FileName.append(SG.DeclaringModule.value()->getNameStr());
     FileName.append(getFullModuleName(SG.DeclaringModule.value()));
   }
-  FileName.append(".symbols.json");
-
-  SmallString<1024> OutputPath(Options.OutputDir);
-  llvm::sys::path::append(OutputPath, FileName);
 
-  return withOutputPath(
+  return Writer.withOutputPath(
       SG.M.getASTContext().Diags, SG.M.getASTContext().getOutputBackend(),
-      OutputPath, [&](raw_ostream &OS) {
+      FileName, [&](raw_ostream &OS) {
         llvm::json::OStream J(OS, Options.PrettyPrint ? 2 : 0);
         SG.serialize(J);
         return false;
@@ -145,14 +200,18 @@ int symbolgraphgen::emitSymbolGraphForModule(
 
   int Success = EXIT_SUCCESS;
 
-  Success |= serializeSymbolGraph(Walker.MainGraph, Options);
+  SymbolGraphWriter Writer(Options);
+  Success |= serializeSymbolGraph(Walker.MainGraph, Options, Writer);
 
   for (const auto &Entry : Walker.ExtendedModuleGraphs) {
     if (Entry.getValue()->empty()) {
       continue;
     }
-    Success |= serializeSymbolGraph(*Entry.getValue(), Options);
+    Success |= serializeSymbolGraph(*Entry.getValue(), Options, Writer);
   }
+  Success |=
+      Writer.finalize(Walker.MainGraph.M.getASTContext().Diags,
+                      Walker.MainGraph.M.getASTContext().getOutputBackend());
 
   return Success;
 }

test/SymbolGraph/EmitShortenedNamesMap.swift

@@ -0,0 +1,33 @@
+// RUN: %empty-directory(%t)
+// RUN: %target-build-swift %s -module-name EmitShortenedNames -emit-module -emit-module-path %t/EmitShortenedNames.swiftmodule -emit-symbol-graph -emit-symbol-graph-dir %t/ -symbol-graph-shorten-output-names
+// RUN: %FileCheck %s --input-file %t/9d0ee7243b44e35c186695a49461007b.symbols.json --check-prefix BASE
+// RUN: %FileCheck %s --input-file %t/27a36c452b5f5e2b488b7755cecf40c7.symbols.json --check-prefix EXT
+// RUN: %FileCheck %s --input-file %t/symbol_modules.json --check-prefix MAP
+
+// now try with the swift-symbolgraph-extract tool directly
+
+// RUN: %empty-directory(%t)
+// RUN: %target-build-swift %s -module-name EmitShortenedNames -emit-module -emit-module-path %t/EmitShortenedNames.swiftmodule
+// RUN: %target-swift-symbolgraph-extract -module-name EmitShortenedNames -I %t -output-dir %t -symbol-graph-shorten-output-names
+// RUN: %FileCheck %s --input-file %t/9d0ee7243b44e35c186695a49461007b.symbols.json --check-prefix BASE
+// RUN: %FileCheck %s --input-file %t/27a36c452b5f5e2b488b7755cecf40c7.symbols.json --check-prefix EXT
+// RUN: %FileCheck %s --input-file %t/symbol_modules.json --check-prefix MAP
+
+// now try with the swiftc driver
+
+// RUN: %empty-directory(%t)
+// RUN: %target-swiftc_driver %s -module-name EmitShortenedNames -emit-module -emit-module-path %t/EmitShortenedNames.swiftmodule -emit-symbol-graph -emit-symbol-graph-dir %t/ -symbol-graph-shorten-output-names
+// RUN: %FileCheck %s --input-file %t/9d0ee7243b44e35c186695a49461007b.symbols.json --check-prefix BASE
+// RUN: %FileCheck %s --input-file %t/27a36c452b5f5e2b488b7755cecf40c7.symbols.json --check-prefix EXT
+// RUN: %FileCheck %s --input-file %t/symbol_modules.json --check-prefix MAP
+
+public func foo() {}
+
+extension Sequence {
+    public func bar() {}
+}
+
+// BASE: "precise":"s:18EmitShortenedNames3fooyyF"
+// EXT: "precise":"s:ST18EmitShortenedNamesE3baryyF"
+// MAP: "EmitShortenedNames":"9d0ee7243b44e35c186695a49461007b.symbols.json",
+// MAP: "EmitShortenedNames@Swift":"27a36c452b5f5e2b488b7755cecf40c7.symbols.json"