[3/5] support C++20 Modules, add deps-scanner and update toolchains

src/main/java/com/google/devtools/build/lib/rules/cpp/ArtifactCategory.java

@@ -32,6 +32,12 @@ public enum ArtifactCategory {
   OBJECT_FILE("", ".o", ".obj"),
   PIC_OBJECT_FILE("", ".pic.o"),
   CPP_MODULE("", ".pcm"),
+  CPP_MODULE_GCM("", ".gcm"),
+  CPP_MODULE_IFC("", ".ifc"),
+  CPP_MODULES_INFO("", ".CXXModules.json"),
+  CPP_MODULES_DDI("", ".ddi"),
+  CPP_MODULES_MODMAP("", ".modmap"),
+  CPP_MODULES_MODMAP_INPUT("", ".modmap.input"),
   GENERATED_ASSEMBLY("", ".s", ".asm"),
   PROCESSED_HEADER("", ".processed"),
   GENERATED_HEADER("", ".h"),

src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java

@@ -76,6 +76,9 @@ public final class CcCommon implements StarlarkValue {
           CppActionNames.CPP_HEADER_PARSING,
           CppActionNames.CPP_MODULE_COMPILE,
           CppActionNames.CPP_MODULE_CODEGEN,
+          CppActionNames.CPP_MODULE_DEPS_SCANNING,
+          CppActionNames.CPP20_MODULE_COMPILE,
+          CppActionNames.CPP20_MODULE_CODEGEN,
           CppActionNames.ASSEMBLE,
           CppActionNames.PREPROCESS_ASSEMBLE,
           CppActionNames.CLIF_MATCH,

src/main/java/com/google/devtools/build/lib/rules/cpp/CompileBuildVariables.java

@@ -79,6 +79,10 @@ public enum CompileBuildVariables {
    * @see CcCompilationContext#getFrameworkIncludeDirs().
    */
   FRAMEWORK_PATHS("framework_include_paths"),
+  /** Variable for the c++20 module modmap file name. */
+  CPP_MODULE_MODMAP_FILE("cpp_module_modmap_file"),
+  /** Variable for the c++20 module output file name. */
+  CPP_MODULE_OUTPUT_FILE("cpp_module_output_file"),
   /** Variable for the module map file name. */
   MODULE_MAP_FILE("module_map_file"),
   /** Variable for the dependent module map file name. */

src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionNames.java

@@ -36,6 +36,11 @@ public class CppActionNames {
   public static final String OBJCPP_COMPILE = "objc++-compile";
   /** A string constant for the c++ header parsing. */
   public static final String CPP_HEADER_PARSING = "c++-header-parsing";
+  /** A string constant for the c++20 modules deps scanning */
+  public static final String CPP_MODULE_DEPS_SCANNING = "c++-module-deps-scanning";
+  /** A string constant for the c++20 module compile action. */
+  public static final String CPP20_MODULE_COMPILE = "c++20-module-compile";
+  public static final String CPP20_MODULE_CODEGEN = "c++20-module-codegen";
   /**
    * A string constant for the c++ module compilation action. Note: currently we don't support C
    * module compilation.

src/main/java/com/google/devtools/build/lib/rules/cpp/CppFileTypes.java

@@ -285,7 +285,7 @@ public boolean apply(String path) {
   public static final FileType LLVM_PROFILE_ZIP = FileType.of(".zip");
 
   public static final FileType CPP_MODULE_MAP = FileType.of(".cppmap");
-  public static final FileType CPP_MODULE = FileType.of(".pcm");
+  public static final FileType CPP_MODULE = FileType.of(".pcm", ".gcm", ".ifc");
   public static final FileType OBJC_MODULE_MAP = FileType.of("module.modulemap");
 
   /** Predicate that matches all artifacts that can be used in an objc Clang module map. */

src/main/starlark/builtins_bzl/common/cc/action_names.bzl

@@ -35,6 +35,13 @@ CPP_MODULE_CODEGEN_ACTION_NAME = "c++-module-codegen"
 # Name of the C++ header parsing action.
 CPP_HEADER_PARSING_ACTION_NAME = "c++-header-parsing"
 
+# Name of the C++ deps scanning action.
+CPP_MODULE_DEPS_SCANNING_ACTION_NAME = "c++-module-deps-scanning"
+
+# Name of the C++20 module compile action.
+CPP20_MODULE_COMPILE_ACTION_NAME = "c++20-module-compile"
+CPP20_MODULE_CODEGEN_ACTION_NAME = "c++20-module-codegen"
+
 # Name of the C++ module compile action.
 CPP_MODULE_COMPILE_ACTION_NAME = "c++-module-compile"
 
@@ -104,6 +111,9 @@ ACTION_NAMES = struct(
     cc_flags_make_variable = CC_FLAGS_MAKE_VARIABLE_ACTION_NAME,
     cpp_module_codegen = CPP_MODULE_CODEGEN_ACTION_NAME,
     cpp_header_parsing = CPP_HEADER_PARSING_ACTION_NAME,
+    cpp_module_deps_scanning = CPP_MODULE_DEPS_SCANNING_ACTION_NAME,
+    cpp20_module_compile = CPP20_MODULE_COMPILE_ACTION_NAME,
+    cpp20_module_codegen = CPP20_MODULE_CODEGEN_ACTION_NAME,
     cpp_module_compile = CPP_MODULE_COMPILE_ACTION_NAME,
     assemble = ASSEMBLE_ACTION_NAME,
     preprocess_assemble = PREPROCESS_ASSEMBLE_ACTION_NAME,

src/main/starlark/builtins_bzl/common/cc/cc_helper_internal.bzl

@@ -46,6 +46,12 @@ artifact_category = struct(
     OBJECT_FILE = "OBJECT_FILE",
     PIC_OBJECT_FILE = "PIC_OBJECT_FILE",
     CPP_MODULE = "CPP_MODULE",
+    CPP_MODULE_GCM = "CPP_MODULE_GCM",
+    CPP_MODULE_IFC = "CPP_MODULE_IFC",
+    CPP_MODULES_INFO = "CPP_MODULES_INFO",
+    CPP_MODULES_DDI = "CPP_MODULES_DDI",
+    CPP_MODULES_MODMAP = "CPP_MODULES_MODMAP",
+    CPP_MODULES_MODMAP_INPUT = "CPP_MODULES_MODMAP_INPUT",
     GENERATED_ASSEMBLY = "GENERATED_ASSEMBLY",
     PROCESSED_HEADER = "PROCESSED_HEADER",
     GENERATED_HEADER = "GENERATED_HEADER",

tools/build_defs/cc/action_names.bzl

@@ -31,6 +31,13 @@ CPP_MODULE_CODEGEN_ACTION_NAME = "c++-module-codegen"
 # Name of the C++ header parsing action.
 CPP_HEADER_PARSING_ACTION_NAME = "c++-header-parsing"
 
+# Name of the C++ deps scanning action.
+CPP_MODULE_DEPS_SCANNING_ACTION_NAME = "c++-module-deps-scanning"
+
+# Name of the C++ module compile action.
+CPP20_MODULE_COMPILE_ACTION_NAME = "c++20-module-compile"
+CPP20_MODULE_CODEGEN_ACTION_NAME = "c++20-module-codegen"
+
 # Name of the C++ module compile action.
 CPP_MODULE_COMPILE_ACTION_NAME = "c++-module-compile"
 
@@ -101,6 +108,9 @@ ACTION_NAMES = struct(
     cc_flags_make_variable = CC_FLAGS_MAKE_VARIABLE_ACTION_NAME,
     cpp_module_codegen = CPP_MODULE_CODEGEN_ACTION_NAME,
     cpp_header_parsing = CPP_HEADER_PARSING_ACTION_NAME,
+    cpp_module_deps_scanning = CPP_MODULE_DEPS_SCANNING_ACTION_NAME,
+    cpp20_module_compile = CPP20_MODULE_COMPILE_ACTION_NAME,
+    cpp20_module_codegen = CPP20_MODULE_CODEGEN_ACTION_NAME,
     cpp_module_compile = CPP_MODULE_COMPILE_ACTION_NAME,
     assemble = ASSEMBLE_ACTION_NAME,
     preprocess_assemble = PREPROCESS_ASSEMBLE_ACTION_NAME,

tools/cpp/BUILD.tpl

@@ -59,6 +59,10 @@ filegroup(
     srcs = ["validate_static_library.sh"],
 )
 
+filegroup(
+    name = "deps_scanner_wrapper",
+    srcs = ["deps_scanner_wrapper.sh"],
+)
 filegroup(
     name = "compiler_deps",
     srcs = glob(["extra_tools/**"], allow_empty = True) + [%{cc_compiler_deps}],

tools/cpp/BUILD.windows.tpl

@@ -59,7 +59,22 @@ filegroup(
 
 filegroup(
     name = "msvc_compiler_files",
-    srcs = [":builtin_include_directory_paths_msvc"]
+    srcs = [":builtin_include_directory_paths_msvc"
+    , "%{msvc_deps_scanner_wrapper_path_x86}"
+    , "%{msvc_deps_scanner_wrapper_path_x64}"
+    , "%{msvc_deps_scanner_wrapper_path_arm}"
+    , "%{msvc_deps_scanner_wrapper_path_arm64}"
+    ]
+)
+
+filegroup(
+    name = "agg-ddi",
+    srcs = ["//tools/cpp/cpp20modules_tools:agg-ddi-bin"],
+)
+
+filegroup(
+    name = "gen-modmap",
+    srcs = ["//tools/cpp/cpp20modules_tools:gen-modmap-bin"],
 )
 
 # Hardcoded toolchain, legacy behaviour.
@@ -315,6 +330,7 @@ cc_toolchain_config(
         "objdump": "wrapper/bin/msvc_nop.bat",
         "strip": "wrapper/bin/msvc_nop.bat",
         "dumpbin": "%{msvc_dumpbin_path_x64}",
+        "cpp-module-deps-scanner": "%{msvc_deps_scanner_wrapper_path_x64}",
     },
     archiver_flags = ["/MACHINE:X64"],
     default_link_flags = ["/MACHINE:X64"],
@@ -383,6 +399,7 @@ cc_toolchain_config(
         "objdump": "wrapper/bin/msvc_nop.bat",
         "strip": "wrapper/bin/msvc_nop.bat",
         "dumpbin": "%{msvc_dumpbin_path_x86}",
+        "cpp-module-deps-scanner": "%{msvc_deps_scanner_wrapper_path_x86}",
     },
     archiver_flags = ["/MACHINE:X86"],
     default_link_flags = ["/MACHINE:X86"],
@@ -451,6 +468,7 @@ cc_toolchain_config(
         "objdump": "wrapper/bin/msvc_nop.bat",
         "strip": "wrapper/bin/msvc_nop.bat",
         "dumpbin": "%{msvc_dumpbin_path_arm}",
+        "cpp-module-deps-scanner": "%{msvc_deps_scanner_wrapper_path_arm}",
     },
     archiver_flags = ["/MACHINE:ARM"],
     default_link_flags = ["/MACHINE:ARM"],
@@ -519,6 +537,7 @@ cc_toolchain_config(
         "objdump": "wrapper/bin/msvc_nop.bat",
         "strip": "wrapper/bin/msvc_nop.bat",
         "dumpbin": "%{msvc_dumpbin_path_arm64}",
+        "cpp-module-deps-scanner": "%{msvc_deps_scanner_wrapper_path_arm64}",
     },
     archiver_flags = ["/MACHINE:ARM64"],
     default_link_flags = ["/MACHINE:ARM64"],

tools/cpp/clang_deps_scanner_wrapper.sh.tpl

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+#
+# Ship the environment to the C++ action
+#
+set -eu
+
+# Set-up the environment
+%{env}
+
+# Call the C++ compiler
+%{deps_scanner} -format=p1689 -- %{cc} "$@" >out.tmp && mv out.tmp $DEPS_SCANNER_OUTPUT_FILE

tools/cpp/gcc_deps_scanner_wrapper.sh.tpl

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# Ship the environment to the C++ action
+#
+set -eu
+
+# Set-up the environment
+%{env}
+
+# Call the C++ compiler
+
+%{cc} -E -x c++ -fmodules-ts -fdeps-file=out.tmp -fdeps-format=p1689r5 "$@" >out.tmp && mv out.tmp $DEPS_SCANNER_OUTPUT_FILE

tools/cpp/msvc_deps_scanner_wrapper.bat.tpl

@@ -0,0 +1,16 @@
+:: Copyright 2024 The Bazel Authors. All rights reserved.
+::
+:: Licensed under the Apache License, Version 2.0 (the "License");
+:: you may not use this file except in compliance with the License.
+:: You may obtain a copy of the License at
+::
+::    http://www.apache.org/licenses/LICENSE-2.0
+::
+:: Unless required by applicable law or agreed to in writing, software
+:: distributed under the License is distributed on an "AS IS" BASIS,
+:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+:: See the License for the specific language governing permissions and
+:: limitations under the License.
+
+@echo OFF
+"%{cc}" /scanDependencies- /TP %* >out.tmp && move out.tmp %DEPS_SCANNER_OUTPUT_FILE%

tools/cpp/unix_cc_configure.bzl

@@ -332,6 +332,8 @@ def configure_unix_toolchain(repository_ctx, cpu_value, overriden_tools):
         "@bazel_tools//tools/cpp:linux_cc_wrapper.sh.tpl",
         "@bazel_tools//tools/cpp:validate_static_library.sh.tpl",
         "@bazel_tools//tools/cpp:osx_cc_wrapper.sh.tpl",
+        "@bazel_tools//tools/cpp:clang_deps_scanner_wrapper.sh.tpl",
+        "@bazel_tools//tools/cpp:gcc_deps_scanner_wrapper.sh.tpl",
     ])
 
     repository_ctx.symlink(
@@ -387,8 +389,10 @@ def configure_unix_toolchain(repository_ctx, cpu_value, overriden_tools):
     if darwin:
         overriden_tools["gcc"] = "cc_wrapper.sh"
         overriden_tools["ar"] = _find_generic(repository_ctx, "libtool", "LIBTOOL", overriden_tools)
+
     auto_configure_warning_maybe(repository_ctx, "CC used: " + str(cc))
     tool_paths = _get_tool_paths(repository_ctx, overriden_tools)
+    tool_paths["cpp-module-deps-scanner"] = "deps_scanner_wrapper.sh"
 
     # The parse_header tool needs to be a wrapper around the compiler as it has
     # to touch the output file.
@@ -424,6 +428,24 @@ def configure_unix_toolchain(repository_ctx, cpu_value, overriden_tools):
             "%{env}": escape_string(get_env(repository_ctx)),
         },
     )
+    deps_scanner_wrapper_src = (
+        "@bazel_tools//tools/cpp:clang_deps_scanner_wrapper.sh.tpl" if is_clang else "@bazel_tools//tools/cpp:gcc_deps_scanner_wrapper.sh.tpl"
+    )
+    deps_scanner = "cpp-module-deps-scanner_not_found"
+    if is_clang:
+        cc_str = str(cc)
+        path_arr = cc_str.split("/")[:-1]
+        path_arr.append("clang-scan-deps")
+        deps_scanner = "/".join(path_arr)
+    repository_ctx.template(
+        "deps_scanner_wrapper.sh",
+        paths[deps_scanner_wrapper_src],
+        {
+            "%{cc}": escape_string(str(cc)),
+            "%{deps_scanner}": escape_string(deps_scanner),
+            "%{env}": escape_string(get_env(repository_ctx)),
+        },
+    )
 
     conly_opts = split_escaped(get_env_var(
         repository_ctx,
@@ -584,6 +606,7 @@ def configure_unix_toolchain(repository_ctx, cpu_value, overriden_tools):
             "%{cc_compiler_deps}": get_starlark_list([
                 ":builtin_include_directory_paths",
                 ":cc_wrapper",
+                ":deps_scanner_wrapper",
             ] + (
                 [":validate_static_library"] if "validate_static_library" in tool_paths else []
             )),