Add feature to produce indexstore files for CC builds

Author: BalestraPatrick

src/main/java/com/google/devtools/build/lib/rules/cpp/ArtifactCategory.java

@@ -25,6 +25,7 @@ public enum ArtifactCategory {
   DYNAMIC_LIBRARY("lib", ".so", ".dylib", ".dll", ".wasm"),
   EXECUTABLE("", "", ".exe", ".wasm"),
   INTERFACE_LIBRARY("lib", ".ifso", ".tbd", ".if.lib", ".lib"),
+  INDEXSTORE_FILE("", ".indexstore"),
   PIC_FILE("", ".pic"),
   INCLUDED_FILE_LIST("", ".d"),
   SERIALIZED_DIAGNOSTICS_FILE("", ".dia"),

src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java

@@ -1468,8 +1468,8 @@ private Artifact createCompileActionTemplate(
     SpecialArtifact outputFiles =
         CppHelper.getCompileOutputTreeArtifact(
             actionConstructionContext, label, sourceArtifact, outputName, usePic);
-    // Dotd and dia file outputs are specified in the execution phase.
-    builder.setOutputs(outputFiles, /* dotdFile= */ null, /* diagnosticsFile= */ null);
+    // Dotd, dia and indexstore file outputs are specified in the execution phase.
+    builder.setOutputs(outputFiles, /* dotdFile= */ null, /* diagnosticsFile= */ null, /* indexstoreFiles */ null);
     builder.setVariables(
         setupCompileBuildVariables(
             builder,
@@ -1497,12 +1497,19 @@ private Artifact createCompileActionTemplate(
           CppHelper.getDiagnosticsOutputTreeArtifact(
               actionConstructionContext, label, sourceArtifact, outputName, usePic);
     }
+    SpecialArtifact indexstoreTreeArtifact = null;
+    if (builder.indexstoreFilesEnabled()) {
+      indexstoreTreeArtifact =
+          CppHelper.getIndexstoreOutputTreeArtifact(
+              actionConstructionContext, label, sourceArtifact, outputName);
+    }
     CppCompileActionTemplate actionTemplate =
         new CppCompileActionTemplate(
             sourceArtifact,
             outputFiles,
             dotdTreeArtifact,
             diagnosticsTreeArtifact,
+            indexstoreTreeArtifact,
             builder,
             ccToolchain,
             outputCategories,
@@ -1574,6 +1581,10 @@ private CcToolchainVariables setupCompileBuildVariables(
     if (builder.getDiagnosticsFile() != null) {
       diagnosticsFileExecPath = builder.getDiagnosticsFile().getExecPathString();
     }
+    String indexstoreFilesExecPath = null;
+    if (builder.getIndexstoreFiles() != null) {
+      indexstoreFilesExecPath = builder.getIndexstoreFiles().getExecPathString();
+    }
     if (needsFdoBuildVariables && fdoContext.hasArtifacts(cppConfiguration)) {
       // This modifies the passed-in builder, which is a surprising side-effect, and makes it unsafe
       // to call this method multiple times for the same builder.
@@ -1650,6 +1661,7 @@ private CcToolchainVariables setupCompileBuildVariables(
         getCopts(builder.getSourceFile(), sourceLabel),
         dotdFileExecPath,
         diagnosticsFileExecPath,
+        indexstoreFilesExecPath,
         usePic,
         ccCompilationContext.getExternalIncludeDirs(),
         additionalBuildVariables);

src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java

@@ -360,6 +360,7 @@ public CcToolchainVariables getCompileBuildVariables(
                     /* fdoStamp= */ null,
                     /* dotdFileExecPath= */ null,
                     /* diagnosticsFileExecPath= */ null,
+                    /* indexstoreFilesExecPath= */ null,
                     variablesExtensions,
                     /* additionalBuildVariables= */ ImmutableMap.of(),
                     /* directModuleMaps= */ ImmutableList.of(),

src/main/java/com/google/devtools/build/lib/rules/cpp/CompileBuildVariables.java

@@ -46,6 +46,8 @@ public enum CompileBuildVariables {
   DEPENDENCY_FILE("dependency_file"),
   /** Variable for the serialized diagnostics file path */
   SERIALIZED_DIAGNOSTICS_FILE("serialized_diagnostics_file"),
+  /** Variable for the indexstore file paths */
+  INDEXSTORE_FILES("indexstore_files"),
   /** Variable for the module file name. */
   MODULE_NAME("module_name"),
   /**
@@ -154,6 +156,7 @@ public static CcToolchainVariables setupVariablesOrReportRuleError(
       String fdoStamp,
       String dotdFileExecPath,
       String diagnosticsFileExecPath,
+      String indexstoreFilesExecPath,
       ImmutableList<VariablesExtension> variablesExtensions,
       ImmutableMap<String, String> additionalBuildVariables,
       Iterable<Artifact> directModuleMaps,
@@ -188,6 +191,7 @@ public static CcToolchainVariables setupVariablesOrReportRuleError(
           fdoStamp,
           dotdFileExecPath,
           diagnosticsFileExecPath,
+          indexstoreFilesExecPath,
           variablesExtensions,
           additionalBuildVariables,
           directModuleMaps,
@@ -224,6 +228,7 @@ public static CcToolchainVariables setupVariablesOrThrowEvalException(
       String fdoStamp,
       String dotdFileExecPath,
       String diagnosticsFileExecPath,
+      String indexstoreFilesExecPath,
       ImmutableList<VariablesExtension> variablesExtensions,
       ImmutableMap<String, String> additionalBuildVariables,
       Iterable<Artifact> directModuleMaps,
@@ -258,6 +263,7 @@ public static CcToolchainVariables setupVariablesOrThrowEvalException(
         fdoStamp,
         dotdFileExecPath,
         diagnosticsFileExecPath,
+        indexstoreFilesExecPath,
         variablesExtensions,
         additionalBuildVariables,
         directModuleMaps,
@@ -288,6 +294,7 @@ private static CcToolchainVariables setupVariables(
       String fdoStamp,
       String dotdFileExecPath,
       String diagnosticsFileExecPath,
+      String indexstoreFilesExecPath,
       ImmutableList<VariablesExtension> variablesExtensions,
       ImmutableMap<String, String> additionalBuildVariables,
       Iterable<Artifact> directModuleMaps,
@@ -327,6 +334,7 @@ private static CcToolchainVariables setupVariables(
         userCompileFlags,
         dotdFileExecPath,
         diagnosticsFileExecPath,
+        indexstoreFilesExecPath,
         usePic,
         ImmutableList.of(),
         ImmutableMap.of());
@@ -347,6 +355,7 @@ public static void setupSpecificVariables(
       Iterable<String> userCompileFlags,
       String dotdFileExecPath,
       String diagnosticsFileExecPath,
+      String indexstoreFilesExecPath,
       boolean usePic,
       ImmutableList<PathFragment> externalIncludeDirs,
       Map<String, String> additionalBuildVariables) {
@@ -372,6 +381,12 @@ public static void setupSpecificVariables(
           SERIALIZED_DIAGNOSTICS_FILE.getVariableName(), diagnosticsFileExecPath);
     }
 
+    // Set indexstore_files to enable <object>.indexstore files generation.
+    if (indexstoreFilesExecPath != null) {
+      buildVariables.addStringVariable(
+          INDEXSTORE_FILES.getVariableName(), indexstoreFilesExecPath);
+    }
+
     if (gcnoFile != null) {
       buildVariables.addStringVariable(GCOV_GCNO_FILE.getVariableName(), gcnoFile);
     }

src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java

@@ -223,6 +223,7 @@ public class CppCompileAction extends AbstractAction implements IncludeScannable
    * @param outputFile the object file that is written as result of the compilation
    * @param dotdFile the .d file that is generated as a side-effect of compilation
    * @param diagnosticsFile the .dia file that is generated as a side-effect of compilation
+   * @param indexstoreFiles the .indexstore files that are generated as a side-effect of compilation
    * @param gcnoFile the coverage notes that are written in coverage mode, can be null
    * @param dwoFile the .dwo output file where debug information is stored for Fission builds (null
    *     if Fission mode is disabled)
@@ -252,6 +253,7 @@ public class CppCompileAction extends AbstractAction implements IncludeScannable
       Artifact outputFile,
       @Nullable Artifact dotdFile,
       @Nullable Artifact diagnosticsFile,
+      @Nullable Artifact indexstoreFiles,
       @Nullable Artifact gcnoFile,
       @Nullable Artifact dwoFile,
       @Nullable Artifact ltoIndexingFile,
@@ -275,6 +277,7 @@ public class CppCompileAction extends AbstractAction implements IncludeScannable
             Preconditions.checkNotNull(outputFile, "outputFile"),
             dotdFile,
             diagnosticsFile,
+            indexstoreFiles,
             gcnoFile,
             dwoFile,
             ltoIndexingFile,
@@ -327,6 +330,7 @@ private static ImmutableSet<Artifact> collectOutputs(
       Artifact outputFile,
       @Nullable Artifact dotdFile,
       @Nullable Artifact diagnosticsFile,
+      @Nullable Artifact indexstoreFiles,
       @Nullable Artifact gcnoFile,
       @Nullable Artifact dwoFile,
       @Nullable Artifact ltoIndexingFile,
@@ -343,6 +347,9 @@ private static ImmutableSet<Artifact> collectOutputs(
     if (diagnosticsFile != null) {
       outputs.add(diagnosticsFile);
     }
+    if (indexstoreFiles != null) {
+      outputs.add(indexstoreFiles);
+    }
     if (dwoFile != null) {
       outputs.add(dwoFile);
     }

src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java

@@ -60,6 +60,7 @@ public class CppCompileActionBuilder {
   private Artifact ltoIndexingFile;
   private Artifact dotdFile;
   private Artifact diagnosticsFile;
+  private Artifact indexstoreFiles;
   private Artifact gcnoFile;
   private CcCompilationContext ccCompilationContext = CcCompilationContext.EMPTY;
   private final List<String> pluginOpts = new ArrayList<>();
@@ -320,6 +321,7 @@ public CppCompileAction buildAndVerify() throws UnconfiguredActionConfigExceptio
             outputFile,
             dotdFile,
             diagnosticsFile,
+            indexstoreFiles,
             gcnoFile,
             dwoFile,
             ltoIndexingFile,
@@ -493,12 +495,17 @@ public boolean serializedDiagnosticsFilesEnabled() {
     return featureConfiguration.isEnabled(CppRuleClasses.SERIALIZED_DIAGNOSTICS_FILE);
   }
 
+  public boolean indexstoreFilesEnabled() {
+    return featureConfiguration.isEnabled(CppRuleClasses.INDEXSTORE_FILES);
+  }
+
   @CanIgnoreReturnValue
   public CppCompileActionBuilder setOutputs(
-      Artifact outputFile, Artifact dotdFile, Artifact diagnosticsFile) {
+      Artifact outputFile, Artifact dotdFile, Artifact diagnosticsFile, Artifact indexstoreFiles) {
     this.outputFile = outputFile;
     this.dotdFile = dotdFile;
     this.diagnosticsFile = diagnosticsFile;
+    this.indexstoreFiles = indexstoreFiles;
     return this;
   }
 
@@ -533,6 +540,15 @@ public CppCompileActionBuilder setOutputs(
     } else {
       diagnosticsFile = null;
     }
+    if (indexstoreFilesEnabled()) {
+      String indexstoreFilesName =
+          CppHelper.getIndexstoreFilesName(ccToolchain, outputCategory, outputName);
+      indexstoreFiles =
+          CppHelper.getCompileOutputArtifact(
+              actionConstructionContext, label, indexstoreFilesName, configuration);
+    } else {
+      indexstoreFiles = null;
+    }
     return this;
   }
 
@@ -564,6 +580,10 @@ public Artifact getDiagnosticsFile() {
     return this.diagnosticsFile;
   }
 
+  public Artifact getIndexstoreFiles() {
+    return this.indexstoreFiles;
+  }
+
   @CanIgnoreReturnValue
   public CppCompileActionBuilder setGcnoFile(Artifact gcnoFile) {
     this.gcnoFile = gcnoFile;

src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionTemplate.java

@@ -48,6 +48,7 @@ public final class CppCompileActionTemplate extends ActionKeyCacher
   private final SpecialArtifact outputTreeArtifact;
   private final SpecialArtifact dotdTreeArtifact;
   private final SpecialArtifact diagnosticsTreeArtifact;
+  private final SpecialArtifact indexstoreTreeArtifact;
   private final CcToolchainProvider toolchain;
   private final ImmutableList<ArtifactCategory> categories;
   private final ActionOwner actionOwner;
@@ -61,6 +62,7 @@ public final class CppCompileActionTemplate extends ActionKeyCacher
    * @param outputTreeArtifact the TreeArtifact that contains compilation outputs.
    * @param dotdTreeArtifact the TreeArtifact that contains dotd files.
    * @param diagnosticsTreeArtifact the TreeArtifact that contains serialized diagnostics files.
+   * @param indexstoreTreeArtifact the TreeArtifact that contains indexstore files.
    * @param cppCompileActionBuilder An almost completely configured {@link CppCompileActionBuilder}
    *     without the input and output files set. It is used as a template to instantiate expanded
    *     {CppCompileAction}s.
@@ -74,6 +76,7 @@ public final class CppCompileActionTemplate extends ActionKeyCacher
       SpecialArtifact outputTreeArtifact,
       SpecialArtifact dotdTreeArtifact,
       SpecialArtifact diagnosticsTreeArtifact,
+      SpecialArtifact indexstoreTreeArtifact,
       CppCompileActionBuilder cppCompileActionBuilder,
       CcToolchainProvider toolchain,
       ImmutableList<ArtifactCategory> categories,
@@ -83,6 +86,7 @@ public final class CppCompileActionTemplate extends ActionKeyCacher
     this.outputTreeArtifact = outputTreeArtifact;
     this.dotdTreeArtifact = dotdTreeArtifact;
     this.diagnosticsTreeArtifact = diagnosticsTreeArtifact;
+    this.indexstoreTreeArtifact = indexstoreTreeArtifact;
     this.toolchain = toolchain;
     this.categories = categories;
     this.actionOwner = checkNotNull(actionOwner, outputTreeArtifact);
@@ -146,12 +150,19 @@ public ImmutableList<CppCompileAction> generateActionsForInputArtifacts(
             TreeFileArtifact.createTemplateExpansionOutput(
                 diagnosticsTreeArtifact, outputName + ".dia", artifactOwner);
       }
+      TreeFileArtifact indexstoreFilesArtifact = null;
+      if (indexstoreTreeArtifact != null) {
+        indexstoreFilesArtifact =
+            TreeFileArtifact.createTemplateExpansionOutput(
+                indexstoreTreeArtifact, outputName + ".indexstore", artifactOwner);
+      }
       expandedActions.add(
           createAction(
               inputTreeFileArtifact,
               outputTreeFileArtifact,
               dotdFileArtifact,
               diagnosticsFileArtifact,
+              indexstoreFilesArtifact,
               privateHeaders));
     }
 
@@ -201,13 +212,14 @@ private CppCompileAction createAction(
       TreeFileArtifact outputTreeFileArtifact,
       @Nullable Artifact dotdFileArtifact,
       @Nullable Artifact diagnosticsFileArtifact,
+      @Nullable Artifact indexstoreFilesArtifact,
       NestedSet<Artifact> privateHeaders)
       throws ActionExecutionException {
     CppCompileActionBuilder builder =
         new CppCompileActionBuilder(cppCompileActionBuilder)
             .setAdditionalPrunableHeaders(privateHeaders)
             .setSourceFile(sourceTreeFileArtifact)
-            .setOutputs(outputTreeFileArtifact, dotdFileArtifact, diagnosticsFileArtifact);
+            .setOutputs(outputTreeFileArtifact, dotdFileArtifact, diagnosticsFileArtifact, indexstoreFilesArtifact);
 
     CcToolchainVariables.Builder buildVariables =
         CcToolchainVariables.builder(cppCompileActionBuilder.getVariables());
@@ -227,6 +239,11 @@ private CppCompileAction createAction(
           CompileBuildVariables.SERIALIZED_DIAGNOSTICS_FILE.getVariableName(),
           diagnosticsFileArtifact.getExecPathString());
     }
+    if (indexstoreFilesArtifact != null) {
+      buildVariables.overrideStringVariable(
+          CompileBuildVariables.INDEXSTORE_FILES.getVariableName(),
+          indexstoreFilesArtifact.getExecPathString());
+    }
 
     builder.setVariables(buildVariables.build());
 

src/main/java/com/google/devtools/build/lib/rules/cpp/CppHelper.java

@@ -73,6 +73,7 @@ public class CppHelper {
   static final PathFragment PIC_DOTD_FILES = PathFragment.create("_pic_dotd");
   static final PathFragment DIA_FILES = PathFragment.create("_dia");
   static final PathFragment PIC_DIA_FILES = PathFragment.create("_pic_dia");
+  static final PathFragment INDEXSTORE_FILES = PathFragment.create("_indexstore");
 
   // TODO(bazel-team): should this use Link.SHARED_LIBRARY_FILETYPES?
   public static final FileTypeSet SHARED_LIBRARY_FILETYPES =
@@ -307,6 +308,13 @@ private static PathFragment getDiagnosticsDirectory(
         ruleLabel, usePic ? PIC_DIA_FILES : DIA_FILES, siblingRepositoryLayout);
   }
 
+  /** Returns the directory where indexstore files are created. */
+  private static PathFragment getIndexstoreDirectory(
+      Label ruleLabel, boolean siblingRepositoryLayout) {
+    return AnalysisUtils.getUniqueDirectory(
+        ruleLabel, INDEXSTORE_FILES, siblingRepositoryLayout);
+  }
+
   public static Artifact getLinkedArtifact(
       Label label,
       ActionConstructionContext actionConstructionContext,
@@ -583,6 +591,23 @@ public static SpecialArtifact getDiagnosticsOutputTreeArtifact(
         sourceTreeArtifact.getRoot());
   }
 
+  /**
+   * Returns the corresponding indexstore files TreeArtifact given the source
+   * TreeArtifact.
+   */
+  public static SpecialArtifact getIndexstoreOutputTreeArtifact(
+      ActionConstructionContext actionConstructionContext,
+      Label label,
+      Artifact sourceTreeArtifact,
+      String outputName) {
+    return actionConstructionContext.getTreeArtifact(
+        getIndexstoreDirectory(
+                label,
+                actionConstructionContext.getConfiguration().isSiblingRepositoryLayout())
+            .getRelative(outputName),
+        sourceTreeArtifact.getRoot());
+  }
+
   public static String getArtifactNameForCategory(
       CcToolchainProvider toolchain,
       ArtifactCategory category,
@@ -618,6 +643,18 @@ static String getDiagnosticsFileName(
         toolchain, ArtifactCategory.SERIALIZED_DIAGNOSTICS_FILE, baseName);
   }
 
+  static String getIndexstoreFilesName(
+      CcToolchainProvider toolchain, ArtifactCategory outputCategory, String outputName)
+      throws RuleErrorException {
+    String baseName =
+        outputCategory == ArtifactCategory.OBJECT_FILE
+                || outputCategory == ArtifactCategory.PROCESSED_HEADER
+            ? outputName
+            : getArtifactNameForCategory(toolchain, outputCategory, outputName);
+    return getArtifactNameForCategory(
+        toolchain, ArtifactCategory.INDEXSTORE_FILE, baseName);
+  }
+
   /**
    * Returns true if the build implied by the given config and toolchain uses --start-lib/--end-lib
    * ld options.