Add feature to produce indexstore files for CC builds

Author: BalestraPatrick

src/main/java/com/google/devtools/build/lib/rules/cpp/ArtifactCategory.java

@@ -25,6 +25,7 @@ public enum ArtifactCategory {
   DYNAMIC_LIBRARY("lib", ".so", ".dylib", ".dll", ".wasm"),
   EXECUTABLE("", "", ".exe", ".wasm"),
   INTERFACE_LIBRARY("lib", ".ifso", ".tbd", ".if.lib", ".lib"),
+  INDEXSTORE_FILE("", ".indexstore"),
   PIC_FILE("", ".pic"),
   INCLUDED_FILE_LIST("", ".d"),
   SERIALIZED_DIAGNOSTICS_FILE("", ".dia"),

src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java

@@ -1468,8 +1468,8 @@ private Artifact createCompileActionTemplate(
     SpecialArtifact outputFiles =
         CppHelper.getCompileOutputTreeArtifact(
             actionConstructionContext, label, sourceArtifact, outputName, usePic);
-    // Dotd and dia file outputs are specified in the execution phase.
-    builder.setOutputs(outputFiles, /* dotdFile= */ null, /* diagnosticsFile= */ null);
+    // Dotd, dia and indexstore file outputs are specified in the execution phase.
+    builder.setOutputs(outputFiles, /* dotdFile= */ null, /* diagnosticsFile= */ null, /* indexstoreFiles */ null);
     builder.setVariables(
         setupCompileBuildVariables(
             builder,
@@ -1497,12 +1497,19 @@ private Artifact createCompileActionTemplate(
           CppHelper.getDiagnosticsOutputTreeArtifact(
               actionConstructionContext, label, sourceArtifact, outputName, usePic);
     }
+    SpecialArtifact indexstoreTreeArtifact = null;
+    if (builder.indexstoreFilesEnabled()) {
+      indexstoreTreeArtifact =
+          CppHelper.getIndexstoreOutputTreeArtifact(
+              actionConstructionContext, label, sourceArtifact, outputName);
+    }
     CppCompileActionTemplate actionTemplate =
         new CppCompileActionTemplate(
             sourceArtifact,
             outputFiles,
             dotdTreeArtifact,
             diagnosticsTreeArtifact,
+            indexstoreTreeArtifact,
             builder,
             ccToolchain,
             outputCategories,
@@ -1574,6 +1581,10 @@ private CcToolchainVariables setupCompileBuildVariables(
     if (builder.getDiagnosticsFile() != null) {
       diagnosticsFileExecPath = builder.getDiagnosticsFile().getExecPathString();
     }
+    String indexstoreFilesExecPath = null;
+    if (builder.getIndexstoreFiles() != null) {
+      indexstoreFilesExecPath = builder.getIndexstoreFiles().getExecPathString();
+    }
     if (needsFdoBuildVariables && fdoContext.hasArtifacts(cppConfiguration)) {
       // This modifies the passed-in builder, which is a surprising side-effect, and makes it unsafe
       // to call this method multiple times for the same builder.
@@ -1650,6 +1661,7 @@ private CcToolchainVariables setupCompileBuildVariables(
         getCopts(builder.getSourceFile(), sourceLabel),
         dotdFileExecPath,
         diagnosticsFileExecPath,
+        indexstoreFilesExecPath,
         usePic,
         ccCompilationContext.getExternalIncludeDirs(),
         additionalBuildVariables);

src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java

@@ -360,6 +360,7 @@ public CcToolchainVariables getCompileBuildVariables(
                     /* fdoStamp= */ null,
                     /* dotdFileExecPath= */ null,
                     /* diagnosticsFileExecPath= */ null,
+                    /* indexstoreFilesExecPath= */ null,
                     variablesExtensions,
                     /* additionalBuildVariables= */ ImmutableMap.of(),
                     /* directModuleMaps= */ ImmutableList.of(),

src/main/java/com/google/devtools/build/lib/rules/cpp/CompileBuildVariables.java

@@ -46,6 +46,8 @@ public enum CompileBuildVariables {
   DEPENDENCY_FILE("dependency_file"),
   /** Variable for the serialized diagnostics file path */
   SERIALIZED_DIAGNOSTICS_FILE("serialized_diagnostics_file"),
+  /** Variable for the indexstore file paths */
+  INDEXSTORE_FILES("indexstore_files"),
   /** Variable for the module file name. */
   MODULE_NAME("module_name"),
   /**
@@ -154,6 +156,7 @@ public static CcToolchainVariables setupVariablesOrReportRuleError(
       String fdoStamp,
       String dotdFileExecPath,
       String diagnosticsFileExecPath,
+      String indexstoreFilesExecPath,
       ImmutableList<VariablesExtension> variablesExtensions,
       ImmutableMap<String, String> additionalBuildVariables,
       Iterable<Artifact> directModuleMaps,
@@ -188,6 +191,7 @@ public static CcToolchainVariables setupVariablesOrReportRuleError(
           fdoStamp,
           dotdFileExecPath,
           diagnosticsFileExecPath,
+          indexstoreFilesExecPath,
           variablesExtensions,
           additionalBuildVariables,
           directModuleMaps,
@@ -224,6 +228,7 @@ public static CcToolchainVariables setupVariablesOrThrowEvalException(
       String fdoStamp,
       String dotdFileExecPath,
       String diagnosticsFileExecPath,
+      String indexstoreFilesExecPath,
       ImmutableList<VariablesExtension> variablesExtensions,
       ImmutableMap<String, String> additionalBuildVariables,
       Iterable<Artifact> directModuleMaps,
@@ -258,6 +263,7 @@ public static CcToolchainVariables setupVariablesOrThrowEvalException(
         fdoStamp,
         dotdFileExecPath,
         diagnosticsFileExecPath,
+        indexstoreFilesExecPath,
         variablesExtensions,
         additionalBuildVariables,
         directModuleMaps,
@@ -288,6 +294,7 @@ private static CcToolchainVariables setupVariables(
       String fdoStamp,
       String dotdFileExecPath,
       String diagnosticsFileExecPath,
+      String indexstoreFilesExecPath,
       ImmutableList<VariablesExtension> variablesExtensions,
       ImmutableMap<String, String> additionalBuildVariables,
       Iterable<Artifact> directModuleMaps,
@@ -327,6 +334,7 @@ private static CcToolchainVariables setupVariables(
         userCompileFlags,
         dotdFileExecPath,
         diagnosticsFileExecPath,
+        indexstoreFilesExecPath,
         usePic,
         ImmutableList.of(),
         ImmutableMap.of());
@@ -347,6 +355,7 @@ public static void setupSpecificVariables(
       Iterable<String> userCompileFlags,
       String dotdFileExecPath,
       String diagnosticsFileExecPath,
+      String indexstoreFilesExecPath,
       boolean usePic,
       ImmutableList<PathFragment> externalIncludeDirs,
       Map<String, String> additionalBuildVariables) {
@@ -372,6 +381,12 @@ public static void setupSpecificVariables(
           SERIALIZED_DIAGNOSTICS_FILE.getVariableName(), diagnosticsFileExecPath);
     }
 
+    // Set indexstore_files to enable <object>.indexstore files generation.
+    if (indexstoreFilesExecPath != null) {
+      buildVariables.addStringVariable(
+          INDEXSTORE_FILES.getVariableName(), indexstoreFilesExecPath);
+    }
+
     if (gcnoFile != null) {
       buildVariables.addStringVariable(GCOV_GCNO_FILE.getVariableName(), gcnoFile);
     }

src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java

@@ -224,6 +224,7 @@ public class CppCompileAction extends AbstractAction implements IncludeScannable
    * @param outputFile the object file that is written as result of the compilation
    * @param dotdFile the .d file that is generated as a side-effect of compilation
    * @param diagnosticsFile the .dia file that is generated as a side-effect of compilation
+   * @param indexstoreFiles the .indexstore files that are generated as a side-effect of compilation
    * @param gcnoFile the coverage notes that are written in coverage mode, can be null
    * @param dwoFile the .dwo output file where debug information is stored for Fission builds (null
    *     if Fission mode is disabled)
@@ -253,6 +254,7 @@ public class CppCompileAction extends AbstractAction implements IncludeScannable
       Artifact outputFile,
       @Nullable Artifact dotdFile,
       @Nullable Artifact diagnosticsFile,
+      @Nullable Artifact indexstoreFiles,
       @Nullable Artifact gcnoFile,
       @Nullable Artifact dwoFile,
       @Nullable Artifact ltoIndexingFile,
@@ -276,6 +278,7 @@ public class CppCompileAction extends AbstractAction implements IncludeScannable
             outputFile,
             dotdFile,
             diagnosticsFile,
+            indexstoreFiles,
             gcnoFile,
             dwoFile,
             ltoIndexingFile,
@@ -305,6 +308,7 @@ public class CppCompileAction extends AbstractAction implements IncludeScannable
             actionName,
             dotdFile,
             diagnosticsFile,
+            indexstoreFiles,
             featureConfiguration,
             variables);
     this.executionInfo = executionInfo;
@@ -331,6 +335,7 @@ private static ImmutableSet<Artifact> collectOutputs(
       @Nullable Artifact outputFile,
       @Nullable Artifact dotdFile,
       @Nullable Artifact diagnosticsFile,
+      @Nullable Artifact indexstoreFiles,
       @Nullable Artifact gcnoFile,
       @Nullable Artifact dwoFile,
       @Nullable Artifact ltoIndexingFile,
@@ -350,6 +355,9 @@ private static ImmutableSet<Artifact> collectOutputs(
     if (diagnosticsFile != null) {
       outputs.add(diagnosticsFile);
     }
+    if (indexstoreFiles != null) {
+      outputs.add(indexstoreFiles);
+    }
     if (dwoFile != null) {
       outputs.add(dwoFile);
     }
@@ -365,6 +373,7 @@ static CompileCommandLine buildCommandLine(
       String actionName,
       Artifact dotdFile,
       Artifact diagnosticsFile,
+      Artifact indexstoreFiles,
       FeatureConfiguration featureConfiguration,
       CcToolchainVariables variables) {
     return CompileCommandLine.builder(sourceFile, coptsFilter, actionName, dotdFile)

src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java

@@ -60,6 +60,7 @@ public class CppCompileActionBuilder {
   private Artifact ltoIndexingFile;
   private Artifact dotdFile;
   private Artifact diagnosticsFile;
+  private Artifact indexstoreFiles;
   private Artifact gcnoFile;
   private CcCompilationContext ccCompilationContext = CcCompilationContext.EMPTY;
   private final List<String> pluginOpts = new ArrayList<>();
@@ -320,6 +321,7 @@ public CppCompileAction buildAndVerify() throws UnconfiguredActionConfigExceptio
             outputFile,
             dotdFile,
             diagnosticsFile,
+            indexstoreFiles,
             gcnoFile,
             dwoFile,
             ltoIndexingFile,
@@ -493,12 +495,17 @@ public boolean serializedDiagnosticsFilesEnabled() {
     return featureConfiguration.isEnabled(CppRuleClasses.SERIALIZED_DIAGNOSTICS_FILE);
   }
 
+  public boolean indexstoreFilesEnabled() {
+    return featureConfiguration.isEnabled(CppRuleClasses.INDEXSTORE_FILES);
+  }
+
   @CanIgnoreReturnValue
   public CppCompileActionBuilder setOutputs(
-      Artifact outputFile, Artifact dotdFile, Artifact diagnosticsFile) {
+      Artifact outputFile, Artifact dotdFile, Artifact diagnosticsFile, Artifact indexstoreFiles) {
     this.outputFile = outputFile;
     this.dotdFile = dotdFile;
     this.diagnosticsFile = diagnosticsFile;
+    this.indexstoreFiles = indexstoreFiles;
     return this;
   }
 
@@ -533,6 +540,15 @@ public CppCompileActionBuilder setOutputs(
     } else {
       diagnosticsFile = null;
     }
+    if (indexstoreFilesEnabled()) {
+      String indexstoreFilesName =
+          CppHelper.getIndexstoreFilesName(ccToolchain, outputCategory, outputName);
+      indexstoreFiles =
+          CppHelper.getCompileOutputArtifact(
+              actionConstructionContext, label, indexstoreFilesName, configuration);
+    } else {
+      indexstoreFiles = null;
+    }
     return this;
   }
 
@@ -564,6 +580,10 @@ public Artifact getDiagnosticsFile() {
     return this.diagnosticsFile;
   }
 
+  public Artifact getIndexstoreFiles() {
+    return this.indexstoreFiles;
+  }
+
   @CanIgnoreReturnValue
   public CppCompileActionBuilder setGcnoFile(Artifact gcnoFile) {
     this.gcnoFile = gcnoFile;

src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionTemplate.java

@@ -48,6 +48,7 @@ public final class CppCompileActionTemplate extends ActionKeyCacher
   private final SpecialArtifact outputTreeArtifact;
   private final SpecialArtifact dotdTreeArtifact;
   private final SpecialArtifact diagnosticsTreeArtifact;
+  private final SpecialArtifact indexstoreTreeArtifact;
   private final CcToolchainProvider toolchain;
   private final ImmutableList<ArtifactCategory> categories;
   private final ActionOwner actionOwner;
@@ -61,6 +62,7 @@ public final class CppCompileActionTemplate extends ActionKeyCacher
    * @param outputTreeArtifact the TreeArtifact that contains compilation outputs.
    * @param dotdTreeArtifact the TreeArtifact that contains dotd files.
    * @param diagnosticsTreeArtifact the TreeArtifact that contains serialized diagnostics files.
+   * @param indexstoreTreeArtifact the TreeArtifact that contains indexstore files.
    * @param cppCompileActionBuilder An almost completely configured {@link CppCompileActionBuilder}
    *     without the input and output files set. It is used as a template to instantiate expanded
    *     {CppCompileAction}s.
@@ -74,6 +76,7 @@ public final class CppCompileActionTemplate extends ActionKeyCacher
       SpecialArtifact outputTreeArtifact,
       SpecialArtifact dotdTreeArtifact,
       SpecialArtifact diagnosticsTreeArtifact,
+      SpecialArtifact indexstoreTreeArtifact,
       CppCompileActionBuilder cppCompileActionBuilder,
       CcToolchainProvider toolchain,
       ImmutableList<ArtifactCategory> categories,
@@ -83,6 +86,7 @@ public final class CppCompileActionTemplate extends ActionKeyCacher
     this.outputTreeArtifact = outputTreeArtifact;
     this.dotdTreeArtifact = dotdTreeArtifact;
     this.diagnosticsTreeArtifact = diagnosticsTreeArtifact;
+    this.indexstoreTreeArtifact = indexstoreTreeArtifact;
     this.toolchain = toolchain;
     this.categories = categories;
     this.actionOwner = checkNotNull(actionOwner, outputTreeArtifact);
@@ -146,12 +150,19 @@ public ImmutableList<CppCompileAction> generateActionsForInputArtifacts(
             TreeFileArtifact.createTemplateExpansionOutput(
                 diagnosticsTreeArtifact, outputName + ".dia", artifactOwner);
       }
+      TreeFileArtifact indexstoreFilesArtifact = null;
+      if (indexstoreTreeArtifact != null) {
+        indexstoreFilesArtifact =
+            TreeFileArtifact.createTemplateExpansionOutput(
+                indexstoreTreeArtifact, outputName + ".indexstore", artifactOwner);
+      }
       expandedActions.add(
           createAction(
               inputTreeFileArtifact,
               outputTreeFileArtifact,
               dotdFileArtifact,
               diagnosticsFileArtifact,
+              indexstoreFilesArtifact,
               privateHeaders));
     }
 
@@ -171,6 +182,7 @@ protected void computeKey(
             CppActionNames.CPP_COMPILE,
             dotdTreeArtifact,
             diagnosticsTreeArtifact,
+            indexstoreTreeArtifact,
             cppCompileActionBuilder.getFeatureConfiguration(),
             cppCompileActionBuilder.getVariables());
     CppCompileAction.computeKey(
@@ -202,13 +214,14 @@ private CppCompileAction createAction(
       TreeFileArtifact outputTreeFileArtifact,
       @Nullable Artifact dotdFileArtifact,
       @Nullable Artifact diagnosticsFileArtifact,
+      @Nullable Artifact indexstoreFilesArtifact,
       NestedSet<Artifact> privateHeaders)
       throws ActionExecutionException {
     CppCompileActionBuilder builder =
         new CppCompileActionBuilder(cppCompileActionBuilder)
             .setAdditionalPrunableHeaders(privateHeaders)
             .setSourceFile(sourceTreeFileArtifact)
-            .setOutputs(outputTreeFileArtifact, dotdFileArtifact, diagnosticsFileArtifact);
+            .setOutputs(outputTreeFileArtifact, dotdFileArtifact, diagnosticsFileArtifact, indexstoreFilesArtifact);
 
     CcToolchainVariables.Builder buildVariables =
         CcToolchainVariables.builder(cppCompileActionBuilder.getVariables());
@@ -228,6 +241,11 @@ private CppCompileAction createAction(
           CompileBuildVariables.SERIALIZED_DIAGNOSTICS_FILE.getVariableName(),
           diagnosticsFileArtifact.getExecPathString());
     }
+    if (indexstoreFilesArtifact != null) {
+      buildVariables.overrideStringVariable(
+          CompileBuildVariables.INDEXSTORE_FILES.getVariableName(),
+          indexstoreFilesArtifact.getExecPathString());
+    }
 
     builder.setVariables(buildVariables.build());