From 893e1cfb99104552ac14d79415242677be5dd767 Mon Sep 17 00:00:00 2001 From: PikachuHy Date: Fri, 17 May 2024 14:10:22 +0800 Subject: [PATCH 1/6] support C++20 Modules, add module_interfaces attr --- .../lib/rules/cpp/CcCompilationHelper.java | 22 ++++++++ .../build/lib/rules/cpp/CcModule.java | 10 +++- .../build/lib/rules/cpp/CppActionConfigs.java | 9 +++ .../build/lib/rules/cpp/CppRuleClasses.java | 2 + .../lib/starlarkbuildapi/cpp/CcModuleApi.java | 7 +++ .../starlark/builtins_bzl/common/cc/attrs.bzl | 16 ++++++ .../builtins_bzl/common/cc/cc_binary.bzl | 4 ++ .../builtins_bzl/common/cc/cc_common.bzl | 4 +- .../builtins_bzl/common/cc/cc_helper.bzl | 56 ++++++++++++++++--- .../builtins_bzl/common/cc/cc_library.bzl | 3 + 10 files changed, 121 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java index 54202e5e0f2b9e..9e2a98a550531d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java @@ -269,6 +269,7 @@ public CcCompilationContext getCcCompilationContext() { private final List additionalExportedHeaders = new ArrayList<>(); private final List additionalCppModuleMaps = new ArrayList<>(); private final LinkedHashMap compilationUnitSources = new LinkedHashMap<>(); + private final LinkedHashMap moduleInterfaceSources = new LinkedHashMap<>(); private ImmutableList copts = ImmutableList.of(); private CoptsFilter coptsFilter = CoptsFilter.alwaysPasses(); private final Set defines = new LinkedHashSet<>(); @@ -518,6 +519,27 @@ public CcCompilationHelper addSources(Artifact... sources) { return addSources(Arrays.asList(sources)); } + @CanIgnoreReturnValue + public CcCompilationHelper addModuleInterfaceSources(Collection sources) { + for (Artifact source : sources) { + addModuleInterfaceSource(source, label); + } + return this; + } + + @CanIgnoreReturnValue + public CcCompilationHelper addModuleInterfaceSources(Iterable> sources) { + for (Pair source : sources) { + addModuleInterfaceSource(source.first, source.second); + } + return this; + } + + private void addModuleInterfaceSource(Artifact source, Label label) { + Preconditions.checkNotNull(featureConfiguration); + moduleInterfaceSources.put(source, CppSource.create(source, label, CppSource.Type.SOURCE)); + } + /** Add the corresponding files as non-header, non-source input files. */ @CanIgnoreReturnValue public CcCompilationHelper addAdditionalInputs(Collection inputs) { diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java index 8954f9006aa47f..324087f36978af 100755 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java @@ -2023,6 +2023,7 @@ public Tuple compile( FeatureConfigurationForStarlark starlarkFeatureConfiguration, Info starlarkCcToolchainProvider, Sequence sourcesUnchecked, // expected + Sequence moduleInterfacesUnchecked, // expected Sequence publicHeadersUnchecked, // expected Sequence privateHeadersUnchecked, // expected Object textualHeadersStarlarkObject, @@ -2143,23 +2144,28 @@ public Tuple compile( configuration.getFragment(CppConfiguration.class))); boolean tuple = (!sourcesUnchecked.isEmpty() && sourcesUnchecked.get(0) instanceof Tuple) + || (!moduleInterfacesUnchecked.isEmpty() && moduleInterfacesUnchecked.get(0) instanceof Tuple) || (!publicHeadersUnchecked.isEmpty() && publicHeadersUnchecked.get(0) instanceof Tuple) || (!privateHeadersUnchecked.isEmpty() && privateHeadersUnchecked.get(0) instanceof Tuple); if (tuple) { ImmutableList> sources = convertSequenceTupleToPair(sourcesUnchecked); + ImmutableList> moduleInterfaces = convertSequenceTupleToPair(moduleInterfacesUnchecked); ImmutableList> publicHeaders = convertSequenceTupleToPair(publicHeadersUnchecked); ImmutableList> privateHeaders = convertSequenceTupleToPair(privateHeadersUnchecked); - helper.addPublicHeaders(publicHeaders).addPrivateHeaders(privateHeaders).addSources(sources); + helper.addPublicHeaders(publicHeaders).addPrivateHeaders(privateHeaders).addSources(sources) + .addModuleInterfaceSources(moduleInterfaces); } else { List sources = Sequence.cast(sourcesUnchecked, Artifact.class, "srcs"); + List moduleInterfaces = Sequence.cast(moduleInterfacesUnchecked, Artifact.class, "module_interfaces"); List publicHeaders = Sequence.cast(publicHeadersUnchecked, Artifact.class, "public_hdrs"); List privateHeaders = Sequence.cast(privateHeadersUnchecked, Artifact.class, "private_hdrs"); - helper.addPublicHeaders(publicHeaders).addPrivateHeaders(privateHeaders).addSources(sources); + helper.addPublicHeaders(publicHeaders).addPrivateHeaders(privateHeaders).addSources(sources) + .addModuleInterfaceSources(moduleInterfaces); } List includes = diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionConfigs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionConfigs.java index 77e1b89162dc69..17ca70c90cc90a 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionConfigs.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionConfigs.java @@ -1454,6 +1454,15 @@ public static ImmutableList getFeaturesToAppearLastInFeature " }", " }"))); } + if (!existingFeatureNames.contains("cpp20_module")) { + featureBuilder.add( + getFeature( + Joiner.on("\n") + .join( + " name: 'cpp20_module'", + " enabled: false" + ))); + } // unfiltered_compile_flags contain system include paths. These must be added // after the user provided options (present in legacy_compile_flags build // variable above), otherwise users adding include paths will not pick up their own diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java index fc5e215ae97801..6d5d16a0013685 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppRuleClasses.java @@ -107,6 +107,8 @@ public static ToolchainTypeRequirement ccToolchainTypeRequirement(RuleDefinition */ public static final String MODULE_MAPS = "module_maps"; + public static final String CPP20_MODULE = "cpp20_module"; + /** * A string constant for the random_seed feature. This is used by gcc and Clangfor the * randomization of symbol names that are in the anonymous namespace but have external linkage. diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp/CcModuleApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp/CcModuleApi.java index aaea454242da3f..a84eda38911140 100755 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp/CcModuleApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp/CcModuleApi.java @@ -110,6 +110,12 @@ default void compilerFlagExists() {} positional = false, named = true, defaultValue = "[]"), + @Param( + name = "module_interfaces", + doc = "The list of module interfaces source files to be compiled.", + positional = false, + named = true, + defaultValue = "[]"), @Param( name = "public_hdrs", doc = @@ -373,6 +379,7 @@ Tuple compile( FeatureConfigurationT starlarkFeatureConfiguration, Info starlarkCcToolchainProvider, Sequence sourcesUnchecked, // expected + Sequence moduleInterfacesUnchecked, // expected Sequence publicHeadersUnchecked, // expected Sequence privateHeadersUnchecked, // expected Object textualHeadersStarlarkObject, diff --git a/src/main/starlark/builtins_bzl/common/cc/attrs.bzl b/src/main/starlark/builtins_bzl/common/cc/attrs.bzl index c7e37177d0840a..30044ebbbe614c 100644 --- a/src/main/starlark/builtins_bzl/common/cc/attrs.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/attrs.bzl @@ -84,6 +84,22 @@ using the C/C++ compiler.

""", ), + "module_interfaces": attr.label_list( + allow_files = True, + flags = ["DIRECT_COMPILE_TIME_INPUT"], + doc = """ +The list of files are regarded as C++20 Modules Interface. + +

+C++ Standard has no restriction about module interface file extension +

    +
  • Clang use cppm
    • +
  • GCC can use any source file extension
    • +
  • MSVC use ixx
    • +
+

+ """, + ), "data": attr.label_list( allow_files = True, flags = ["SKIP_CONSTRAINTS_OVERRIDE"], diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_binary.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_binary.bzl index 743c70834658b3..eebeb62cd4e1a1 100644 --- a/src/main/starlark/builtins_bzl/common/cc/cc_binary.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/cc_binary.bzl @@ -486,6 +486,9 @@ def cc_binary_impl(ctx, additional_linkopts, force_linkstatic = False): requested_features = features, unsupported_features = disabled_features, ) + + cc_helper.check_cpp20_module(ctx, feature_configuration) + all_deps = ctx.attr.deps + semantics.get_cc_runtimes(ctx, _is_link_shared(ctx)) compilation_context_deps = [dep[CcInfo].compilation_context for dep in all_deps if CcInfo in dep] @@ -507,6 +510,7 @@ def cc_binary_impl(ctx, additional_linkopts, force_linkstatic = False): public_hdrs = cc_helper.get_public_hdrs(ctx), copts_filter = cc_helper.copts_filter(ctx, additional_make_variable_substitutions), srcs = cc_helper.get_srcs(ctx), + module_interfaces = cc_helper.get_module_interfaces(ctx), compilation_contexts = compilation_context_deps, code_coverage_enabled = cc_helper.is_code_coverage_enabled(ctx = ctx), ) diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_common.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_common.bzl index bdd40fa01303db..5be38f6e077c01 100644 --- a/src/main/starlark/builtins_bzl/common/cc/cc_common.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/cc_common.bzl @@ -657,6 +657,7 @@ def _compile( cc_toolchain, name, srcs = [], + module_interfaces = [], public_hdrs = [], private_hdrs = [], textual_hdrs = [], @@ -727,7 +728,7 @@ def _compile( if non_compilation_additional_inputs == _UNBOUND: non_compilation_additional_inputs = [] - has_tuple = _check_all_sources_contain_tuples_or_none_of_them([srcs, private_hdrs, public_hdrs]) + has_tuple = _check_all_sources_contain_tuples_or_none_of_them([srcs, module_interfaces, private_hdrs, public_hdrs]) if has_tuple: cc_common_internal.check_private_api(allowlist = _PRIVATE_STARLARKIFICATION_ALLOWLIST) @@ -737,6 +738,7 @@ def _compile( cc_toolchain = cc_toolchain, name = name, srcs = srcs, + module_interfaces = module_interfaces, public_hdrs = public_hdrs, private_hdrs = private_hdrs, textual_hdrs = textual_hdrs, diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl index 78a47e931191dc..26423e44992938 100644 --- a/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl @@ -944,16 +944,11 @@ def _map_to_list(m): result.append((k, v)) return result -# Returns a list of (Artifact, Label) tuples. Each tuple represents an input source -# file and the label of the rule that generates it (or the label of the source file itself if it -# is an input file). -def _get_srcs(ctx): - if not hasattr(ctx.attr, "srcs"): - return [] - # "srcs" attribute is a LABEL_LIST in cc_rules, which might also contain files. +# "srcs" attribute is a LABEL_LIST in cc_rules, which might also contain files. +def _calculate_artifact_label_map(srcs, attr): artifact_label_map = {} - for src in ctx.attr.srcs: + for src in srcs: if DefaultInfo in src: for artifact in src[DefaultInfo].files.to_list(): if "." + artifact.extension not in CC_HEADER: @@ -962,8 +957,43 @@ def _get_srcs(ctx): if old_label != None and not _are_labels_equal(old_label, src.label) and "." + artifact.extension in CC_AND_OBJC: fail( "Artifact '{}' is duplicated (through '{}' and '{}')".format(artifact, old_label, src), - attr = "srcs", + attr = attr, ) + return artifact_label_map +# Returns a list of (Artifact, Label) tuples. Each tuple represents an input source +# file and the label of the rule that generates it (or the label of the source file itself if it +# is an input file). +def _get_srcs(ctx): + if not hasattr(ctx.attr, "srcs"): + return [] + artifact_label_map = _calculate_artifact_label_map(ctx.attr.srcs, "srcs") + return _map_to_list(artifact_label_map) + +# Returns a list of (Artifact, Label) tuples. Each tuple represents an input source +# file and the label of the rule that generates it (or the label of the source file itself if it +# is an input file). +def _get_module_interfaces(ctx): + if not hasattr(ctx.attr, "module_interfaces"): + return [] + srcs_artifact_label_map = _calculate_artifact_label_map(ctx.attr.srcs, "srcs") + + artifact_label_map = {} + for src in ctx.attr.module_interfaces: + if DefaultInfo in src: + for artifact in src[DefaultInfo].files.to_list(): + old_label = artifact_label_map.get(artifact, None) + artifact_label_map[artifact] = src.label + if old_label != None and not _are_labels_equal(old_label, src.label): + fail( + "Artifact '{}' is duplicated (through '{}' and '{}')".format(artifact.path, old_label, src), + attr = "module_interfaces", + ) + label_from_srcs = srcs_artifact_label_map.get(artifact, None) + if label_from_srcs != None: + fail( + "Artifact '{}' is duplicated (through '{}' and '{}')".format(artifact.path, label_from_srcs, src), + attr = "module_interfaces", + ) return _map_to_list(artifact_label_map) # Returns a list of (Artifact, Label) tuples. Each tuple represents an input source @@ -1175,6 +1205,12 @@ def _should_use_pic(ctx, cc_toolchain, feature_configuration): cc_common.is_enabled(feature_configuration = feature_configuration, feature_name = "prefer_pic_for_opt_binaries") ) ) +def _check_cpp20_module(ctx, feature_configuration): + if len(ctx.files.module_interfaces) > 0 and not cc_common.is_enabled( + feature_configuration = feature_configuration, + feature_name = "cpp20_module", + ): + fail("to use C++20 Modules, the feature cpp20_module must be enabled") cc_helper = struct( CPP_TOOLCHAIN_TYPE = _CPP_TOOLCHAIN_TYPE, @@ -1225,6 +1261,7 @@ cc_helper = struct( get_local_defines_for_runfiles_lookup = _get_local_defines_for_runfiles_lookup, are_labels_equal = _are_labels_equal, get_srcs = _get_srcs, + get_module_interfaces = _get_module_interfaces, get_private_hdrs = _get_private_hdrs, get_public_hdrs = _get_public_hdrs, report_invalid_options = _report_invalid_options, @@ -1242,4 +1279,5 @@ cc_helper = struct( package_source_root = _package_source_root, tokenize = _tokenize, should_use_pic = _should_use_pic, + check_cpp20_module = _check_cpp20_module, ) diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_library.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_library.bzl index 7295719955e6dd..1ded2172d05f97 100755 --- a/src/main/starlark/builtins_bzl/common/cc/cc_library.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/cc_library.bzl @@ -37,6 +37,8 @@ def _cc_library_impl(ctx): unsupported_features = ctx.disabled_features, ) + cc_helper.check_cpp20_module(ctx, feature_configuration) + precompiled_files = cc_helper.build_precompiled_files(ctx = ctx) semantics.validate_attributes(ctx = ctx) @@ -63,6 +65,7 @@ def _cc_library_impl(ctx): copts_filter = cc_helper.copts_filter(ctx, additional_make_variable_substitutions), purpose = "cc_library-compile", srcs = cc_helper.get_srcs(ctx), + module_interfaces = cc_helper.get_module_interfaces(ctx), private_hdrs = cc_helper.get_private_hdrs(ctx), public_hdrs = cc_helper.get_public_hdrs(ctx), code_coverage_enabled = cc_helper.is_code_coverage_enabled(ctx), From 63073e83900a83d2b6bfe3eaf864146baa510538 Mon Sep 17 00:00:00 2001 From: PikachuHy Date: Fri, 17 May 2024 15:16:55 +0800 Subject: [PATCH 2/6] support C++20 Modules, add C++20 related actions --- .../build/lib/rules/cpp/ArtifactCategory.java | 4 + .../google/devtools/build/lib/rules/cpp/BUILD | 1 + .../build/lib/rules/cpp/CcCommon.java | 3 + .../lib/rules/cpp/CompileBuildVariables.java | 2 + .../rules/cpp/Cpp20ModuleDepMapAction.java | 222 ++++++++++++++ .../lib/rules/cpp/Cpp20ModuleHelper.java | 277 ++++++++++++++++++ .../lib/rules/cpp/Cpp20ModulesInfoAction.java | 132 +++++++++ .../build/lib/rules/cpp/CppActionNames.java | 5 + src/main/protobuf/failure_details.proto | 3 + .../builtins_bzl/common/cc/action_names.bzl | 10 + tools/build_defs/cc/action_names.bzl | 10 + 11 files changed, 669 insertions(+) create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/Cpp20ModuleDepMapAction.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/Cpp20ModuleHelper.java create mode 100644 src/main/java/com/google/devtools/build/lib/rules/cpp/Cpp20ModulesInfoAction.java diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/ArtifactCategory.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/ArtifactCategory.java index ca2a2b1552eca5..b129c111acaac2 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/ArtifactCategory.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/ArtifactCategory.java @@ -32,6 +32,10 @@ public enum ArtifactCategory { OBJECT_FILE("", ".o", ".obj"), PIC_OBJECT_FILE("", ".pic.o"), CPP_MODULE("", ".pcm"), + CPP20_MODULES_INFO("", ".CXXModules.json"), + CPP20_MODULE_DEP("", ".ddi"), + CPP20_MODULE_MAP("", ".modmap"), + CPP20_MODULE_MAP_INPUT("", ".modmap.input"), GENERATED_ASSEMBLY("", ".s", ".asm"), PROCESSED_HEADER("", ".processed"), GENERATED_HEADER("", ".h"), diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/BUILD b/src/main/java/com/google/devtools/build/lib/rules/cpp/BUILD index 209f89eec20054..ffdfcc96dc6bd2 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/BUILD +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/BUILD @@ -128,6 +128,7 @@ java_library( "//third_party:caffeine", "//third_party:checker_framework_annotations", "//third_party:flogger", + "//third_party:gson", "//third_party:guava", "//third_party:jsr305", "//third_party/protobuf:protobuf_java", diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java index a30c8b3a839b94..c545a59bb1c950 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCommon.java @@ -76,6 +76,9 @@ public final class CcCommon implements StarlarkValue { CppActionNames.CPP_HEADER_PARSING, CppActionNames.CPP_MODULE_COMPILE, CppActionNames.CPP_MODULE_CODEGEN, + CppActionNames.CPP20_DEPS_SCANNING, + CppActionNames.CPP20_MODULE_COMPILE, + CppActionNames.CPP20_MODULE_CODEGEN, CppActionNames.ASSEMBLE, CppActionNames.PREPROCESS_ASSEMBLE, CppActionNames.CLIF_MATCH, diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CompileBuildVariables.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CompileBuildVariables.java index ed1cdf2eb9d6fa..a34f4cd3ee6a08 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CompileBuildVariables.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CompileBuildVariables.java @@ -77,6 +77,8 @@ public enum CompileBuildVariables { * @see CcCompilationContext#getFrameworkIncludeDirs(). */ FRAMEWORK_PATHS("framework_include_paths"), + /** Variable for the c++20 module map file name. */ + CPP20_MODMAP_FILE("cpp20_modmap_file"), /** Variable for the module map file name. */ MODULE_MAP_FILE("module_map_file"), /** Variable for the dependent module map file name. */ diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/Cpp20ModuleDepMapAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/Cpp20ModuleDepMapAction.java new file mode 100644 index 00000000000000..c830c55d280564 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/Cpp20ModuleDepMapAction.java @@ -0,0 +1,222 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.devtools.build.lib.actions.*; +import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander; +import com.google.devtools.build.lib.analysis.actions.DeterministicWriter; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.server.FailureDetails; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; + +import javax.annotation.Nullable; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * Creates C++20 module map (.modmap) file. + * 1. the modules needed for compiling C++ Modules or ordinary sources + * are provided by the ddi file + * 2. all modules information provided by the modules info file + * 3. the output file .modmap collect all modules, including indirect dependencies + * e.g. a -> b -> c -> d + * the ddi file only record module a require module b + * However, in .modmap file, we need to put all modules (b, c, d) in it + * the format of .modmap file is + * -fmodule-file== + * -fmodule-file== + * ... + * 4. the output file .modmap.input collects all module files' paths + * it is convenient for CppCompileAction to handle dynamic modules input with the .modmap.input file + * otherwise, we need to parse the .modmap file and get all modules' path in CppCompileAction + * the format of .modmap file is + * + * + * ... + */ +@Immutable +public final class Cpp20ModuleDepMapAction extends AbstractAction { + private static final String GUID = "b02b043a-6c4e-4eeb-992a-09769b366fdd"; + Artifact ddiFile; + Artifact modulesInfoFile; + Artifact modmapFile; + Artifact modmapInputFile; + + public Cpp20ModuleDepMapAction( + ActionOwner owner, + Artifact ddiFile, + Artifact modulesInfoFile, + Artifact modmapFile, + Artifact modmapInputFile) { + super( + owner, + NestedSetBuilder.stableOrder().add(ddiFile).add(modulesInfoFile).build(), + ImmutableSet.of(modmapFile, modmapInputFile)); + this.ddiFile = ddiFile; + this.modulesInfoFile = modulesInfoFile; + this.modmapFile = modmapFile; + this.modmapInputFile = modmapInputFile; + } + + @Override + public ActionResult execute(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + try { + Map moduleMap = computeModuleMap(actionExecutionContext); + + var result1 = + writeOutputToFile( + actionExecutionContext, + out -> { + OutputStreamWriter content = + new OutputStreamWriter(out, StandardCharsets.ISO_8859_1); + for (Map.Entry entry : moduleMap.entrySet()) { + String moduleName = entry.getKey(); + String modulePath = entry.getValue(); + content.append("-f"); + content.append("module-file="); + content.append(moduleName); + content.append("="); + content.append(modulePath); + content.append('\n'); + } + content.flush(); + }, + modmapFile); + var result2 = + writeOutputToFile( + actionExecutionContext, + out -> { + OutputStreamWriter content = + new OutputStreamWriter(out, StandardCharsets.ISO_8859_1); + for (String modulePath : moduleMap.values()) { + content.append(modulePath); + content.append('\n'); + } + content.flush(); + }, + modmapInputFile); + var builder = new ImmutableList.Builder(); + builder.addAll(result1); + builder.addAll(result2); + return ActionResult.create(builder.build()); + } catch (ExecException e) { + throw ActionExecutionException.fromExecException(e, this); + } + } + + private Map computeModuleMap(ActionExecutionContext ctx) throws ExecException { + String ddi; + try { + ddi = FileSystemUtils.readContent(ctx.getInputPath(ddiFile), Charset.defaultCharset()); + } catch (IOException e) { + throw new EnvironmentalExecException( + e, + createFailureDetail( + String.format("read ddi file fail: %s", ddiFile.getExecPathString()), + FailureDetails.CppCompile.Code.DDI_FILE_READ_FAILURE)); + } + var moduleDep = Cpp20ModuleHelper.parseScanResult(ddi); + String modules; + try { + modules = + FileSystemUtils.readContent(ctx.getInputPath(modulesInfoFile), Charset.defaultCharset()); + } catch (IOException e) { + throw new EnvironmentalExecException( + e, + createFailureDetail( + String.format("read module info file fail: %s", modulesInfoFile.getExecPathString()), + FailureDetails.CppCompile.Code.MODULES_INFO_FILE_READ_FAILURE)); + } + + var cxxModules = Cpp20ModuleHelper.Cpp20ModulesInfo.fromJSON(modules); + Set moduleNameSet = new HashSet<>(); + Queue requireModuleQueue = new ArrayDeque<>(); + if (moduleDep.getRequireModules() != null) { + requireModuleQueue.addAll(moduleDep.getRequireModules()); + } + while (!requireModuleQueue.isEmpty()) { + String requireModuleName = requireModuleQueue.poll(); + moduleNameSet.add(requireModuleName); + var deps = cxxModules.getRequireModules(requireModuleName); + for (String dep : deps) { + if (moduleNameSet.contains(dep)) { + continue; + } + requireModuleQueue.add(dep); + } + } + Map moduleMap = new HashMap<>(moduleNameSet.size()); + for (String moduleName : moduleNameSet) { + String modulePath = cxxModules.getModulePath(moduleName); + if (modulePath == null) { + continue; + } + moduleMap.put(moduleName, modulePath); + } + return moduleMap; + } + + private ImmutableList writeOutputToFile( + ActionExecutionContext actionExecutionContext, + DeterministicWriter deterministicWriter, + Artifact output) + throws ExecException { + actionExecutionContext.getEventHandler().post(new RunningActionEvent(this, "local")); + Path outputPath = actionExecutionContext.getInputPath(output); + try { + try (OutputStream out = new BufferedOutputStream(outputPath.getOutputStream())) { + deterministicWriter.writeOutputFile(out); + } + } catch (IOException e) { + throw new EnvironmentalExecException( + e, FailureDetails.Execution.Code.FILE_WRITE_IO_EXCEPTION); + } + return ImmutableList.of(); + } + + @Override + public String getMnemonic() { + return "Cpp20ModuleDepMap"; + } + + @Override + protected void computeKey( + ActionKeyContext actionKeyContext, + @Nullable ArtifactExpander artifactExpander, + Fingerprint fp) { + fp.addString(GUID); + fp.addString(ddiFile.getExecPathString()); + fp.addString(modulesInfoFile.getExecPathString()); + fp.addString(modmapFile.getExecPathString()); + } + + private static FailureDetails.FailureDetail createFailureDetail( + String message, FailureDetails.CppCompile.Code detailedCode) { + return FailureDetails.FailureDetail.newBuilder() + .setMessage(message) + .setCppCompile(FailureDetails.CppCompile.newBuilder().setCode(detailedCode)) + .build(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/Cpp20ModuleHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/Cpp20ModuleHelper.java new file mode 100644 index 00000000000000..fa0cbdf3915497 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/Cpp20ModuleHelper.java @@ -0,0 +1,277 @@ +package com.google.devtools.build.lib.rules.cpp; + +import com.google.gson.Gson; +import com.google.gson.annotations.SerializedName; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.HashMap; + +/** + * to parse JSON content with format + * p1689 + * when discovering dependencies. + * all related class are put here + * for example, + * + * { + * "revision": 0, + * "rules": [ + * { + * "primary-output": "path/to/a.pcm", + * "provides": [ + * { + * "is-interface": true, + * "logical-name": "a", + * "source-path": "path/to/a.cppm" + * } + * ], + * "requires": [ + * { + * "logical-name": "b" + * } + * ] + * } + * ], + * "version": 1 + * } + * + * + * + */ +public class Cpp20ModuleHelper { + static class Require { + @SerializedName("logical-name") + private String logicalName; + + @SerializedName("source-path") + private String sourcePath; + + public String getLogicalName() { + return logicalName; + } + + public void setLogicalName(String logicalName) { + this.logicalName = logicalName; + } + + public String getSourcePath() { + return sourcePath; + } + + public void setSourcePath(String sourcePath) { + this.sourcePath = sourcePath; + } + } + + static class Provide { + @SerializedName("is-interface") + private Boolean interfaceModule; + @SerializedName("logical-name") + private String logicalName; + @SerializedName("source-path") + private String sourcePath; + + @Override + public String toString() { + Gson gson = new Gson(); + return gson.toJson(this); + } + + public Boolean getInterfaceModule() { + return interfaceModule; + } + + public void setInterfaceModule(Boolean interfaceModule) { + this.interfaceModule = interfaceModule; + } + + public String getLogicalName() { + return logicalName; + } + + public void setLogicalName(String logicalName) { + this.logicalName = logicalName; + } + + public String getSourcePath() { + return sourcePath; + } + + public void setSourcePath(String sourcePath) { + this.sourcePath = sourcePath; + } + } + + static class Rule { + @SerializedName("primary-output") + private String primaryOutput; + private List provides; + private List requires; + + @Override + public String toString() { + Gson gson = new Gson(); + return gson.toJson(this); + } + + public String getPrimaryOutput() { + return primaryOutput; + } + + public void setPrimaryOutput(String primaryOutput) { + this.primaryOutput = primaryOutput; + } + + public List getProvides() { + return provides; + } + + public void setProvides(List provides) { + this.provides = provides; + } + + public List getRequires() { + return requires; + } + + public void setRequires(List requires) { + this.requires = requires; + } + } + + static class Cpp20ModuleScanDepsResult { + private Integer revision; + private String version; + private List rules; + + @Override + public String toString() { + Gson gson = new Gson(); + return gson.toJson(this); + } + + public Integer getRevision() { + return revision; + } + + public void setRevision(Integer revision) { + this.revision = revision; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public List getRules() { + return rules; + } + + public void setRules(List rules) { + this.rules = rules; + } + } + + static class ModuleDep { + private boolean needProduceBMI = false; + private String moduleName; + private List requireModules; + + public boolean isNeedProduceBMI() { + return needProduceBMI; + } + + public void setNeedProduceBMI(boolean needProduceBMI) { + this.needProduceBMI = needProduceBMI; + } + + public String getModuleName() { + return moduleName; + } + + public void setModuleName(String moduleName) { + this.moduleName = moduleName; + } + + public List getRequireModules() { + return requireModules; + } + + public void setRequireModules(List requireModules) { + this.requireModules = requireModules; + } + } + + static class Cpp20ModulesInfo { + Cpp20ModulesInfo() { + modules = new HashMap<>(); + usages = new HashMap<>(); + } + void merge(Cpp20ModulesInfo other) { + this.modules.putAll(other.modules); + this.usages.putAll(other.usages); + } + private final Map modules; + private final Map> usages; + + public List getRequireModules(String moduleName) { + return usages.getOrDefault(moduleName, List.of()); + } + public void addRequireModule(String moduleName, String requireModuleName) { + usages.computeIfAbsent(moduleName, it -> new ArrayList<>()); + usages.get(moduleName).add(requireModuleName); + } + + public String getModulePath(String moduleName) { + return modules.getOrDefault(moduleName, null); + } + public void putModulePath(String moduleName, String modulePath) { + modules.put(moduleName, modulePath); + } + static Cpp20ModulesInfo fromJSON(String json) { + Gson gson = new Gson(); + return gson.fromJson(json, Cpp20ModulesInfo.class); + } + } + static ModuleDep parseScanResult(String out) { + Gson gson = new Gson(); + Cpp20ModuleScanDepsResult dep = gson.fromJson(out, Cpp20ModuleScanDepsResult.class); + if (dep == null) { + throw new RuntimeException("call clang-scan-deps error"); + } + if (dep.getRules() == null) { + throw new RuntimeException("call clang-scan-deps error"); + } + if (dep.getRules().size() != 1) { + throw new RuntimeException("expect only 1 rule, but got " + dep.getRules()); + } + ModuleDep moduleDep = new ModuleDep(); + Rule rule = dep.getRules().get(0); + if (rule.getProvides() != null && !rule.getProvides().isEmpty()) { + if (rule.getProvides().size() != 1) { + throw new RuntimeException("expect only 1 rule, but got " + rule.getProvides()); + } + Provide provide = rule.getProvides().get(0); + moduleDep.setModuleName(provide.getLogicalName()); + moduleDep.setNeedProduceBMI(true); + } + if (rule.getRequires() != null) { + var requireModules = new ArrayList(rule.getRequires().size()); + for (int i = 0; i < rule.getRequires().size(); i++) { + Require require = rule.getRequires().get(i); + requireModules.add(require.getLogicalName()); + } + moduleDep.setRequireModules(requireModules); + } + return moduleDep; + } + public static boolean isCpp20ModuleCompilationAction(String actionName) { + return CppActionNames.CPP20_MODULE_COMPILE.equals(actionName) + || CppActionNames.CPP20_MODULE_CODEGEN.equals(actionName); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/Cpp20ModulesInfoAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/Cpp20ModulesInfoAction.java new file mode 100644 index 00000000000000..332a7d61c16b61 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/Cpp20ModulesInfoAction.java @@ -0,0 +1,132 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.rules.cpp; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionKeyContext; +import com.google.devtools.build.lib.actions.ActionOwner; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander; +import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction; +import com.google.devtools.build.lib.analysis.actions.DeterministicWriter; +import com.google.devtools.build.lib.collect.nestedset.NestedSet; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.Pair; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import javax.annotation.Nullable; +import java.io.OutputStreamWriter; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +/** + * Creates C++20 Modules info file (.CXXModules.json) + * e.g. + * { + * "modules": { + * "a": "path/to/bmi/of/module/a", + * "b": "path/to/bmi/of/module/b", + * }, + * "usages": { + * "a": ["b"] + * } + * } + * Each target has only one to record all modules information + * 1. modules: the map of module-name -> module bmi + * 2. usages: the direct dependencies of each module + */ +@Immutable +public final class Cpp20ModulesInfoAction extends AbstractFileWriteAction { + + private static final String GUID = "dd122dd8-72c2-4c98-b343-fac327adb923"; + NestedSet ddiFiles; + ImmutableList> pcmAndDdiPairList; + NestedSet modulesInfoFiles; + public Cpp20ModulesInfoAction( + ActionOwner owner, + NestedSet ddiFiles, + ImmutableList> pcmAndDdiPairList, + NestedSet modulesInfoFiles, + Artifact modulesInfoFile + ) { + super( + owner, + collectInput(ddiFiles, modulesInfoFiles), + modulesInfoFile, + /*makeExecutable=*/ true); + this.ddiFiles = ddiFiles; + this.pcmAndDdiPairList = pcmAndDdiPairList; + this.modulesInfoFiles = modulesInfoFiles; + } + private static NestedSet collectInput(NestedSet ddiFiles, NestedSet modulesInfoFiles) { + return NestedSetBuilder.stableOrder() + .addTransitive(ddiFiles) + .addTransitive(modulesInfoFiles) + .build(); + } + @Override + public DeterministicWriter newDeterministicWriter(ActionExecutionContext ctx) { + return out -> { + var cxxModules = new Cpp20ModuleHelper.Cpp20ModulesInfo(); + OutputStreamWriter content = new OutputStreamWriter(out, StandardCharsets.ISO_8859_1); + for (Pair pair : pcmAndDdiPairList) { + Artifact pcmFile = pair.first; + Artifact ddiFile = pair.second; + String s = FileSystemUtils.readContent(ctx.getInputPath(ddiFile), Charset.defaultCharset()); + var moduleDep = Cpp20ModuleHelper.parseScanResult(s); + if (moduleDep.isNeedProduceBMI() && pcmFile != null) { + cxxModules.putModulePath(moduleDep.getModuleName(), pcmFile.getExecPathString()); + } + if (moduleDep.getRequireModules()!=null) { + for (String requireModule : moduleDep.getRequireModules()) { + cxxModules.addRequireModule(moduleDep.getModuleName(), requireModule); + } + } + } + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + for (Artifact depModulesInfoFile : modulesInfoFiles.toList()) { + String modulesInfoContent = FileSystemUtils.readContent(ctx.getInputPath(depModulesInfoFile), Charset.defaultCharset()); + Cpp20ModuleHelper.Cpp20ModulesInfo depCpp20ModulesInfo = Cpp20ModuleHelper.Cpp20ModulesInfo.fromJSON(modulesInfoContent); + cxxModules.merge(depCpp20ModulesInfo); + } + String json = gson.toJson(cxxModules); + content.append(json); + content.flush(); + }; + } + + @Override + public String getMnemonic() { + return "Cpp20ModulesInfo"; + } + + @Override + protected void computeKey( + ActionKeyContext actionKeyContext, + @Nullable ArtifactExpander artifactExpander, + Fingerprint fp) { + fp.addString(GUID); + ImmutableList ddiFileList = ddiFiles.toList(); + fp.addInt(ddiFileList.size()); + for (Artifact artifact : ddiFileList) { + fp.addString(artifact.getExecPathString()); + } + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionNames.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionNames.java index ab4881340abd16..fc7ba2c31dc0c7 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionNames.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionNames.java @@ -36,6 +36,11 @@ public class CppActionNames { public static final String OBJCPP_COMPILE = "objc++-compile"; /** A string constant for the c++ header parsing. */ public static final String CPP_HEADER_PARSING = "c++-header-parsing"; + /** A string constant for the c++20 modules deps scanning */ + public static final String CPP20_DEPS_SCANNING = "c++20-deps-scanning"; + /** A string constant for the c++20 module compile action. */ + public static final String CPP20_MODULE_COMPILE = "c++20-module-compile"; + public static final String CPP20_MODULE_CODEGEN = "c++20-module-codegen"; /** * A string constant for the c++ module compilation action. Note: currently we don't support C * module compilation. diff --git a/src/main/protobuf/failure_details.proto b/src/main/protobuf/failure_details.proto index 30cb43c5847e8d..c645d78a74b76b 100644 --- a/src/main/protobuf/failure_details.proto +++ b/src/main/protobuf/failure_details.proto @@ -1080,6 +1080,9 @@ message CppCompile { D_FILE_PARSE_FAILURE = 9 [(metadata) = { exit_code: 1 }]; COVERAGE_NOTES_CREATION_FAILURE = 10 [(metadata) = { exit_code: 1 }]; MODULE_EXPANSION_MISSING_DATA = 11 [(metadata) = { exit_code: 1 }]; + DDI_FILE_READ_FAILURE = 12 [(metadata) = { exit_code: 36 }]; + MODULES_INFO_FILE_READ_FAILURE = 13 [(metadata) = { exit_code: 36 }]; + MODMAP_INPUT_FILE_READ_FAILURE = 14 [(metadata) = { exit_code: 36 }]; } Code code = 1; diff --git a/src/main/starlark/builtins_bzl/common/cc/action_names.bzl b/src/main/starlark/builtins_bzl/common/cc/action_names.bzl index 620ac7ffd7becb..c66383459ffde3 100644 --- a/src/main/starlark/builtins_bzl/common/cc/action_names.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/action_names.bzl @@ -35,6 +35,13 @@ CPP_MODULE_CODEGEN_ACTION_NAME = "c++-module-codegen" # Name of the C++ header parsing action. CPP_HEADER_PARSING_ACTION_NAME = "c++-header-parsing" +# Name of the C++ deps scanning action. +CPP20_DEPS_SCANNING_ACTION_NAME = "c++20-deps-scanning" + +# Name of the C++20 module compile action. +CPP20_MODULE_COMPILE_ACTION_NAME = "c++20-module-compile" +CPP20_MODULE_CODEGEN_ACTION_NAME = "c++20-module-codegen" + # Name of the C++ module compile action. CPP_MODULE_COMPILE_ACTION_NAME = "c++-module-compile" @@ -102,6 +109,9 @@ ACTION_NAMES = struct( cc_flags_make_variable = CC_FLAGS_MAKE_VARIABLE_ACTION_NAME, cpp_module_codegen = CPP_MODULE_CODEGEN_ACTION_NAME, cpp_header_parsing = CPP_HEADER_PARSING_ACTION_NAME, + cpp20_deps_scanning = CPP20_DEPS_SCANNING_ACTION_NAME, + cpp20_module_compile = CPP20_MODULE_COMPILE_ACTION_NAME, + cpp20_module_codegen = CPP20_MODULE_CODEGEN_ACTION_NAME, cpp_module_compile = CPP_MODULE_COMPILE_ACTION_NAME, assemble = ASSEMBLE_ACTION_NAME, preprocess_assemble = PREPROCESS_ASSEMBLE_ACTION_NAME, diff --git a/tools/build_defs/cc/action_names.bzl b/tools/build_defs/cc/action_names.bzl index f377ecafb19660..aa5e4828cc4aab 100644 --- a/tools/build_defs/cc/action_names.bzl +++ b/tools/build_defs/cc/action_names.bzl @@ -31,6 +31,13 @@ CPP_MODULE_CODEGEN_ACTION_NAME = "c++-module-codegen" # Name of the C++ header parsing action. CPP_HEADER_PARSING_ACTION_NAME = "c++-header-parsing" +# Name of the C++ deps scanning action. +CPP20_DEPS_SCANNING_ACTION_NAME = "c++20-deps-scanning" + +# Name of the C++ module compile action. +CPP20_MODULE_COMPILE_ACTION_NAME = "c++20-module-compile" +CPP20_MODULE_CODEGEN_ACTION_NAME = "c++20-module-codegen" + # Name of the C++ module compile action. CPP_MODULE_COMPILE_ACTION_NAME = "c++-module-compile" @@ -98,6 +105,9 @@ ACTION_NAMES = struct( cc_flags_make_variable = CC_FLAGS_MAKE_VARIABLE_ACTION_NAME, cpp_module_codegen = CPP_MODULE_CODEGEN_ACTION_NAME, cpp_header_parsing = CPP_HEADER_PARSING_ACTION_NAME, + cpp20_deps_scanning = CPP20_DEPS_SCANNING_ACTION_NAME, + cpp20_module_compile = CPP20_MODULE_COMPILE_ACTION_NAME, + cpp20_module_codegen = CPP20_MODULE_CODEGEN_ACTION_NAME, cpp_module_compile = CPP_MODULE_COMPILE_ACTION_NAME, assemble = ASSEMBLE_ACTION_NAME, preprocess_assemble = PREPROCESS_ASSEMBLE_ACTION_NAME, From 721c87fcfc096e25937a8629af383df106721e8a Mon Sep 17 00:00:00 2001 From: PikachuHy Date: Fri, 17 May 2024 16:00:14 +0800 Subject: [PATCH 3/6] support C++20 Modules, add deps-scanner --- .../build/lib/rules/cpp/CcModule.java | 5 + .../build/lib/rules/cpp/CppActionConfigs.java | 153 +++++++++++++++++- .../build/lib/rules/cpp/CppConfiguration.java | 1 + .../devtools/build/lib/analysis/mock/BUILD | 1 + .../build/lib/analysis/mock/action_names.bzl | 131 +++++++++++++++ .../lib/analysis/mock/cc_toolchain_config.bzl | 37 +++++ .../android/AndroidNdkRepositoryTest.java | 5 - .../AndroidNdkCrosstoolsTest.java | 3 +- .../util/BazelMockAndroidSupport.java | 2 + .../lib/packages/util/MockCcSupport.java | 16 +- .../util/mock/osx_cc_toolchain_config.bzl | 8 + .../rules/cpp/CcToolchainProviderTest.java | 19 ++- .../build/lib/rules/cpp/CcToolchainTest.java | 1 + .../lib/rules/cpp/CppLinkActionTest.java | 1 + .../lib/rules/cpp/LinkCommandLineTest.java | 1 + tools/cpp/BUILD.tpl | 5 + tools/cpp/armeabi_cc_toolchain_config.bzl | 1 + tools/cpp/clang_deps_scanner_wrapper.sh.tpl | 52 ++++++ tools/cpp/gcc_deps_scanner_wrapper.sh.tpl | 12 ++ tools/cpp/unix_cc_configure.bzl | 24 +++ tools/cpp/unix_cc_toolchain_config.bzl | 64 ++++++++ 21 files changed, 522 insertions(+), 20 deletions(-) create mode 100644 src/test/java/com/google/devtools/build/lib/analysis/mock/action_names.bzl create mode 100644 tools/cpp/clang_deps_scanner_wrapper.sh.tpl create mode 100644 tools/cpp/gcc_deps_scanner_wrapper.sh.tpl diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java index 324087f36978af..0055ceaa929bd7 100755 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java @@ -1189,6 +1189,7 @@ public CcToolchainConfigInfo ccToolchainConfigInfoFromStarlark( String linkerToolPath = "DUMMY_LINKER_TOOL"; String arToolPath = "DUMMY_AR_TOOL"; String stripToolPath = "DUMMY_STRIP_TOOL"; + String depsScannerToolPath = "DUMMY_DEPS_SCANNER_TOOL"; for (Pair tool : toolPathList) { if (tool.first.equals(CppConfiguration.Tool.GCC.getNamePart())) { gccToolPath = tool.second; @@ -1207,6 +1208,9 @@ public CcToolchainConfigInfo ccToolchainConfigInfoFromStarlark( if (tool.first.equals(CppConfiguration.Tool.STRIP.getNamePart())) { stripToolPath = tool.second; } + if (tool.first.equals(CppConfiguration.Tool.DEPS_SCANNER.getNamePart())) { + depsScannerToolPath = tool.second; + } } ImmutableList.Builder legacyFeaturesBuilder = ImmutableList.builder(); @@ -1268,6 +1272,7 @@ public CcToolchainConfigInfo ccToolchainConfigInfoFromStarlark( gccToolPath, arToolPath, stripToolPath, + depsScannerToolPath, /* supportsInterfaceSharedLibraries= */ false, actionConfigNames)) { legacyActionConfigBuilder.add(new ActionConfig(actionConfig)); diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionConfigs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionConfigs.java index 17ca70c90cc90a..df242c7f52f302 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionConfigs.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionConfigs.java @@ -72,6 +72,9 @@ public static ImmutableList getLegacyFeatures( " action: 'c++-header-parsing'", " action: 'c++-module-compile'", " action: 'c++-module-codegen'", + " action: 'c++20-deps-scanning'", + " action: 'c++20-module-compile'", + " action: 'c++20-module-codegen'", " action: 'lto-backend'", " action: 'clif-match'", " flag_group {", @@ -102,7 +105,9 @@ public static ImmutableList getLegacyFeatures( " action: 'preprocess-assemble'", " action: 'c-compile'", " action: 'c++-compile'", - " action: 'c++-module-compile'", + " action: 'c++-module-codegen'", + " action: 'c++20-deps-scanning'", + " action: 'c++20-module-codegen'", " action: 'objc-compile'", " action: 'objc++-compile'", " action: 'c++-header-parsing'", @@ -133,6 +138,9 @@ public static ImmutableList getLegacyFeatures( " action: 'c++-compile'", " action: 'c++-module-codegen'", " action: 'c++-module-compile'", + " action: 'c++20-deps-scanning'", + " action: 'c++20-module-compile'", + " action: 'c++20-module-codegen'", " flag_group {", " expand_if_all_available: 'output_file'", " flag: '-frandom-seed=%{output_file}'", @@ -154,6 +162,9 @@ public static ImmutableList getLegacyFeatures( " action: 'c++-compile'", " action: 'c++-module-codegen'", " action: 'c++-module-compile'", + " action: 'c++20-deps-scanning'", + " action: 'c++20-module-compile'", + " action: 'c++20-module-codegen'", " flag_group {", " expand_if_all_available: 'pic'", " flag: '-fPIC'", @@ -172,6 +183,7 @@ public static ImmutableList getLegacyFeatures( " action: 'c-compile'", " action: 'c++-compile'", " action: 'c++-module-codegen'", + " action: 'c++20-module-compile'", " flag_group {", " expand_if_all_available: 'per_object_debug_info_file'", " flag: '-gsplit-dwarf'", @@ -193,6 +205,8 @@ public static ImmutableList getLegacyFeatures( " action: 'c++-compile'", " action: 'c++-header-parsing'", " action: 'c++-module-compile'", + " action: 'c++20-deps-scanning'", + " action: 'c++20-module-compile'", " action: 'clif-match'", " flag_group {", " iterate_over: 'preprocessor_defines'", @@ -214,6 +228,8 @@ public static ImmutableList getLegacyFeatures( " action: 'c++-compile'", " action: 'c++-header-parsing'", " action: 'c++-module-compile'", + " action: 'c++20-deps-scanning'", + " action: 'c++20-module-compile'", " action: 'clif-match'", " action: 'objc-compile'", " action: 'objc++-compile'", @@ -239,6 +255,8 @@ public static ImmutableList getLegacyFeatures( " action: 'c++-compile'", " action: 'c++-header-parsing'", " action: 'c++-module-compile'", + " action: 'c++20-deps-scanning'", + " action: 'c++20-module-compile'", " action: 'clif-match'", " action: 'objc-compile'", " action: 'objc++-compile'", @@ -956,6 +974,7 @@ public static ImmutableList getLegacyFeatures( " action: 'c-compile'", " action: 'c++-compile'", " action: 'c++-module-compile'", + " action: 'c++20-module-compile'", " action: 'objc-compile'", " action: 'objc++-compile'", " flag_group {", @@ -989,6 +1008,7 @@ public static ImmutableList getLegacyFeatures( " action: 'c-compile'", " action: 'c++-compile'", " action: 'c++-module-compile'", + " action: 'c++20-module-compile'", " action: 'objc-compile'", " action: 'objc++-compile'", " action: 'objc-executable'", @@ -1029,6 +1049,7 @@ public static ImmutableList getLegacyActionConfigs( String gccToolPath, String arToolPath, String stripToolPath, + String depsScannerToolPath, boolean supportsInterfaceSharedLibraries, ImmutableSet existingActionConfigNames) { try { @@ -1186,6 +1207,59 @@ public static ImmutableList getLegacyActionConfigs( " implies: 'compiler_input_flags'", " implies: 'compiler_output_flags'"))); } + + if (!existingActionConfigNames.contains(CppActionNames.CPP20_DEPS_SCANNING)) { + actionConfigBuilder.add( + getActionConfig( + Joiner.on("\n") + .join( + " config_name: 'c++20-deps-scanning'", + " action_name: 'c++20-deps-scanning'", + " tool {", + " tool_path: '" + depsScannerToolPath + "'", + " }", + " implies: 'legacy_compile_flags'", + " implies: 'user_compile_flags'", + " implies: 'sysroot'", + " implies: 'unfiltered_compile_flags'", + " implies: 'compiler_input_flags'", + " implies: 'compiler_output_flags'"))); + } + if (!existingActionConfigNames.contains(CppActionNames.CPP20_MODULE_COMPILE)) { + actionConfigBuilder.add( + getActionConfig( + Joiner.on("\n") + .join( + " config_name: 'c++20-module-compile'", + " action_name: 'c++20-module-compile'", + " tool {", + " tool_path: '" + gccToolPath + "'", + " }", + " implies: 'cpp20_module_compile'", + " implies: 'legacy_compile_flags'", + " implies: 'user_compile_flags'", + " implies: 'sysroot'", + " implies: 'unfiltered_compile_flags'", + " implies: 'compiler_input_flags'", + " implies: 'compiler_output_flags'"))); + } + if (!existingActionConfigNames.contains(CppActionNames.CPP20_MODULE_CODEGEN)) { + actionConfigBuilder.add( + getActionConfig( + Joiner.on("\n") + .join( + " config_name: 'c++20-module-codegen'", + " action_name: 'c++20-module-codegen'", + " tool {", + " tool_path: '" + gccToolPath + "'", + " }", + " implies: 'legacy_compile_flags'", + " implies: 'user_compile_flags'", + " implies: 'sysroot'", + " implies: 'unfiltered_compile_flags'", + " implies: 'compiler_input_flags'", + " implies: 'compiler_output_flags'"))); + } if (!existingActionConfigNames.contains(CppActionNames.CPP_LINK_EXECUTABLE)) { actionConfigBuilder.add( getActionConfig( @@ -1417,6 +1491,9 @@ public static ImmutableList getFeaturesToAppearLastInFeature " action: 'c++-header-parsing'", " action: 'c++-module-compile'", " action: 'c++-module-codegen'", + " action: 'c++20-deps-scanning'", + " action: 'c++20-module-compile'", + " action: 'c++20-module-codegen'", " action: 'lto-backend'", " action: 'clif-match'", " flag_group {", @@ -1440,6 +1517,8 @@ public static ImmutableList getFeaturesToAppearLastInFeature " action: 'c++-compile'", " action: 'c++-header-parsing'", " action: 'c++-module-compile'", + " action: 'c++20-deps-scanning'", + " action: 'c++20-module-compile'", " action: 'c++-link-executable'", " action: 'c++-link-dynamic-library'", " action: 'c++-link-nodeps-dynamic-library'", @@ -1456,12 +1535,45 @@ public static ImmutableList getFeaturesToAppearLastInFeature } if (!existingFeatureNames.contains("cpp20_module")) { featureBuilder.add( - getFeature( - Joiner.on("\n") - .join( - " name: 'cpp20_module'", - " enabled: false" - ))); + getFeature( + Joiner.on("\n") + .join( + " name: 'cpp20_module'", + " enabled: false" + ))); + } + if (!existingFeatureNames.contains("cpp20_modmap_file")) { + featureBuilder.add( + getFeature( + Joiner.on("\n") + .join( + " name: 'cpp20_modmap_file'", + " enabled: true", + " flag_set {", + " action: 'c++-compile'", + " action: 'c++20-module-compile'", + " action: 'c++20-module-codegen'", + " flag_group {", + " expand_if_all_available: 'cpp20_modmap_file'", + " flag: '@%{cpp20_modmap_file}'", + " }", + " }"))); + } + if (!existingFeatureNames.contains("cpp20_module_compile")) { + featureBuilder.add( + getFeature( + Joiner.on("\n") + .join( + " name: 'cpp20_module_compile'", + " enabled: true", + " flag_set {", + " action: 'c++20-module-compile'", + " flag_group {", + " flag: '--precompile'", + " flag: '-x'", + " flag: 'c++-module'", + " }", + " }"))); } // unfiltered_compile_flags contain system include paths. These must be added // after the user provided options (present in legacy_compile_flags build @@ -1483,6 +1595,9 @@ public static ImmutableList getFeaturesToAppearLastInFeature " action: 'c++-header-parsing'", " action: 'c++-module-compile'", " action: 'c++-module-codegen'", + " action: 'c++20-deps-scanning'", + " action: 'c++20-module-compile'", + " action: 'c++20-module-codegen'", " action: 'lto-backend'", " action: 'clif-match'", " flag_group {", @@ -1492,6 +1607,24 @@ public static ImmutableList getFeaturesToAppearLastInFeature " }", " }"))); } + if (!existingFeatureNames.contains("cpp20_modmap_file")) { + String compileModmapFile = " flag: '@%{cpp20_modmap_file}'"; + featureBuilder.add( + getFeature( + Joiner.on("\n") + .join( + " name: 'compile_modmap_file'", + " enabled: true", + " flag_set {", + " action: 'c++-compile'", + " action: 'c++20-module-compile'", + " action: 'c++20-module-codegen'", + " flag_group {", + " expand_if_all_available: 'cpp20_modmap_file'", + compileModmapFile, + " }", + " }"))); + } if (!existingFeatureNames.contains("linker_param_file")) { String dynamicLibraryParamFile = " flag: '@%{linker_param_file}'"; featureBuilder.add( @@ -1534,6 +1667,9 @@ public static ImmutableList getFeaturesToAppearLastInFeature " action: 'linkstamp-compile'", " action: 'c++-module-compile'", " action: 'c++-module-codegen'", + " action: 'c++20-deps-scanning'", + " action: 'c++20-module-compile'", + " action: 'c++20-module-codegen'", " action: 'objc-compile'", " action: 'objc++-compile'", " action: 'c++-header-parsing'", @@ -1561,6 +1697,9 @@ public static ImmutableList getFeaturesToAppearLastInFeature " action: 'linkstamp-compile'", " action: 'c++-module-compile'", " action: 'c++-module-codegen'", + " action: 'c++20-deps-scanning'", + " action: 'c++20-module-compile'", + " action: 'c++20-module-codegen'", " action: 'objc-compile'", " action: 'objc++-compile'", " action: 'c++-header-parsing'", diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java index 2a316f21d8e8eb..bac5e1319746bf 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java @@ -76,6 +76,7 @@ public enum Tool { OBJDUMP("objdump"), STRIP("strip"), DWP("dwp"), + DEPS_SCANNER("deps-scanner"), LLVM_PROFDATA("llvm-profdata"); private final String namePart; diff --git a/src/test/java/com/google/devtools/build/lib/analysis/mock/BUILD b/src/test/java/com/google/devtools/build/lib/analysis/mock/BUILD index bf38cae77bacdb..2a886b4a63220d 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/mock/BUILD +++ b/src/test/java/com/google/devtools/build/lib/analysis/mock/BUILD @@ -25,6 +25,7 @@ java_library( ], resources = [ "cc_toolchain_config.bzl", + "action_names.bzl", "//tools/build_defs/cc:action_names.bzl", "//tools/cpp:cc_toolchain_config_lib.bzl", ], diff --git a/src/test/java/com/google/devtools/build/lib/analysis/mock/action_names.bzl b/src/test/java/com/google/devtools/build/lib/analysis/mock/action_names.bzl new file mode 100644 index 00000000000000..aa5e4828cc4aab --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/analysis/mock/action_names.bzl @@ -0,0 +1,131 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Constants for action names used for C++ rules.""" + +# Name for the C compilation action. +C_COMPILE_ACTION_NAME = "c-compile" + +# Name of the C++ compilation action. +CPP_COMPILE_ACTION_NAME = "c++-compile" + +# Name of the linkstamp-compile action. +LINKSTAMP_COMPILE_ACTION_NAME = "linkstamp-compile" + +# Name of the action used to compute CC_FLAGS make variable. +CC_FLAGS_MAKE_VARIABLE_ACTION_NAME = "cc-flags-make-variable" + +# Name of the C++ module codegen action. +CPP_MODULE_CODEGEN_ACTION_NAME = "c++-module-codegen" + +# Name of the C++ header parsing action. +CPP_HEADER_PARSING_ACTION_NAME = "c++-header-parsing" + +# Name of the C++ deps scanning action. +CPP20_DEPS_SCANNING_ACTION_NAME = "c++20-deps-scanning" + +# Name of the C++ module compile action. +CPP20_MODULE_COMPILE_ACTION_NAME = "c++20-module-compile" +CPP20_MODULE_CODEGEN_ACTION_NAME = "c++20-module-codegen" + +# Name of the C++ module compile action. +CPP_MODULE_COMPILE_ACTION_NAME = "c++-module-compile" + +# Name of the assembler action. +ASSEMBLE_ACTION_NAME = "assemble" + +# Name of the assembly preprocessing action. +PREPROCESS_ASSEMBLE_ACTION_NAME = "preprocess-assemble" + +LLVM_COV = "llvm-cov" + +# Name of the action producing ThinLto index. +LTO_INDEXING_ACTION_NAME = "lto-indexing" + +# Name of the action producing ThinLto index for executable. +LTO_INDEX_FOR_EXECUTABLE_ACTION_NAME = "lto-index-for-executable" + +# Name of the action producing ThinLto index for dynamic library. +LTO_INDEX_FOR_DYNAMIC_LIBRARY_ACTION_NAME = "lto-index-for-dynamic-library" + +# Name of the action producing ThinLto index for nodeps dynamic library. +LTO_INDEX_FOR_NODEPS_DYNAMIC_LIBRARY_ACTION_NAME = "lto-index-for-nodeps-dynamic-library" + +# Name of the action compiling lto bitcodes into native objects. +LTO_BACKEND_ACTION_NAME = "lto-backend" + +# Name of the link action producing executable binary. +CPP_LINK_EXECUTABLE_ACTION_NAME = "c++-link-executable" + +# Name of the link action producing dynamic library. +CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME = "c++-link-dynamic-library" + +# Name of the link action producing dynamic library that doesn't include it's +# transitive dependencies. +CPP_LINK_NODEPS_DYNAMIC_LIBRARY_ACTION_NAME = "c++-link-nodeps-dynamic-library" + +# Name of the archiving action producing static library. +CPP_LINK_STATIC_LIBRARY_ACTION_NAME = "c++-link-static-library" + +# Name of the action stripping the binary. +STRIP_ACTION_NAME = "strip" + +# A string constant for the objc compilation action. +OBJC_COMPILE_ACTION_NAME = "objc-compile" + +# A string constant for the objc++ compile action. +OBJCPP_COMPILE_ACTION_NAME = "objc++-compile" + +# A string constant for the objc executable link action. +OBJC_EXECUTABLE_ACTION_NAME = "objc-executable" + +# A string constant for the objc fully-link link action. +OBJC_FULLY_LINK_ACTION_NAME = "objc-fully-link" + +# A string constant for the clif actions. +CLIF_MATCH_ACTION_NAME = "clif-match" + +# A string constant for the obj copy actions. +OBJ_COPY_ACTION_NAME = "objcopy_embed_data" + +ACTION_NAMES = struct( + c_compile = C_COMPILE_ACTION_NAME, + cpp_compile = CPP_COMPILE_ACTION_NAME, + linkstamp_compile = LINKSTAMP_COMPILE_ACTION_NAME, + cc_flags_make_variable = CC_FLAGS_MAKE_VARIABLE_ACTION_NAME, + cpp_module_codegen = CPP_MODULE_CODEGEN_ACTION_NAME, + cpp_header_parsing = CPP_HEADER_PARSING_ACTION_NAME, + cpp20_deps_scanning = CPP20_DEPS_SCANNING_ACTION_NAME, + cpp20_module_compile = CPP20_MODULE_COMPILE_ACTION_NAME, + cpp20_module_codegen = CPP20_MODULE_CODEGEN_ACTION_NAME, + cpp_module_compile = CPP_MODULE_COMPILE_ACTION_NAME, + assemble = ASSEMBLE_ACTION_NAME, + preprocess_assemble = PREPROCESS_ASSEMBLE_ACTION_NAME, + llvm_cov = LLVM_COV, + lto_indexing = LTO_INDEXING_ACTION_NAME, + lto_backend = LTO_BACKEND_ACTION_NAME, + lto_index_for_executable = LTO_INDEX_FOR_EXECUTABLE_ACTION_NAME, + lto_index_for_dynamic_library = LTO_INDEX_FOR_DYNAMIC_LIBRARY_ACTION_NAME, + lto_index_for_nodeps_dynamic_library = LTO_INDEX_FOR_NODEPS_DYNAMIC_LIBRARY_ACTION_NAME, + cpp_link_executable = CPP_LINK_EXECUTABLE_ACTION_NAME, + cpp_link_dynamic_library = CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME, + cpp_link_nodeps_dynamic_library = CPP_LINK_NODEPS_DYNAMIC_LIBRARY_ACTION_NAME, + cpp_link_static_library = CPP_LINK_STATIC_LIBRARY_ACTION_NAME, + strip = STRIP_ACTION_NAME, + objc_compile = OBJC_COMPILE_ACTION_NAME, + objc_executable = OBJC_EXECUTABLE_ACTION_NAME, + objc_fully_link = OBJC_FULLY_LINK_ACTION_NAME, + objcpp_compile = OBJCPP_COMPILE_ACTION_NAME, + clif_match = CLIF_MATCH_ACTION_NAME, + objcopy_embed_data = OBJ_COPY_ACTION_NAME, +) diff --git a/src/test/java/com/google/devtools/build/lib/analysis/mock/cc_toolchain_config.bzl b/src/test/java/com/google/devtools/build/lib/analysis/mock/cc_toolchain_config.bzl index 7e831624f186d7..1e0689e8fdf7f8 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/mock/cc_toolchain_config.bzl +++ b/src/test/java/com/google/devtools/build/lib/analysis/mock/cc_toolchain_config.bzl @@ -32,6 +32,7 @@ load( ) _FEATURE_NAMES = struct( + cpp20_module = "cpp20_module", generate_pdb_file = "generate_pdb_file", no_legacy_features = "no_legacy_features", do_not_split_linking_cmdline = "do_not_split_linking_cmdline", @@ -120,6 +121,34 @@ _FEATURE_NAMES = struct( generate_linkmap = "generate_linkmap", ) + # Tell bazel we support C++20 Modules now +_cpp20_module = feature( + name = "cpp20_module", + # set default value to False + # to enable the feature + # use --features=cpp20_module + # or add cpp20_module to features attr + enabled = False, + ) +_cpp20_modmap_file_feature = feature( + name = "cpp20_modmap_file", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp20_module_compile, + ACTION_NAMES.cpp20_module_codegen, + ], + flag_groups = [ + flag_group( + flags = ["@%{cpp20_modmap_file}"], + expand_if_available = "cpp20_modmap_file", + ), + ], + ), + ], + enabled = True, + ) _no_copts_tokenization_feature = feature(name = _FEATURE_NAMES.no_copts_tokenization) _disable_pbh_feature = feature(name = _FEATURE_NAMES.disable_pbh) @@ -147,6 +176,7 @@ _define_with_space = feature( ACTION_NAMES.cpp_compile, ACTION_NAMES.linkstamp_compile, ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp20_deps_scanning, ACTION_NAMES.cpp_module_compile, ACTION_NAMES.cpp_module_codegen, ACTION_NAMES.clif_match, @@ -457,6 +487,7 @@ _user_compile_flags_feature = feature( ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile, ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp20_deps_scanning, ACTION_NAMES.cpp_module_compile, ACTION_NAMES.cpp_module_codegen, ACTION_NAMES.lto_backend, @@ -1330,6 +1361,7 @@ _generate_linkmap_feature = feature( ) _feature_name_to_feature = { + _FEATURE_NAMES.cpp20_module: _cpp20_module, _FEATURE_NAMES.no_legacy_features: _no_legacy_features_feature, _FEATURE_NAMES.do_not_split_linking_cmdline: _do_not_split_linking_cmdline_feature, _FEATURE_NAMES.supports_dynamic_linker: _supports_dynamic_linker_feature, @@ -1511,6 +1543,7 @@ def _impl(ctx): ACTION_NAMES.linkstamp_compile, ACTION_NAMES.cpp_compile, ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp20_deps_scanning, ACTION_NAMES.cpp_module_compile, ACTION_NAMES.cpp_module_codegen, ACTION_NAMES.lto_backend, @@ -1592,6 +1625,10 @@ def _impl(ctx): name = "llvm-profdata", path = "/usr/bin/mock-llvm-profdata", ), + tool_path( + name = "deps-scanner", + path = "/usr/bin/mock-deps-scanner" + ) ] else: tool_paths = [_get_tool_path(name, path) for name, path in ctx.attr.tool_paths.items()] diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryTest.java index b4eab52c34bb5e..29e31a4fbd7e27 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryTest.java @@ -61,11 +61,6 @@ public void setup() throws Exception { CcToolchainConfig.builder() .withToolchainTargetConstraints("@platforms//os:android", "@platforms//cpu:armv7")); scratch.overwriteFile("embedded_tools/tools/build_defs/cc/BUILD"); - scratch.overwriteFile( - "embedded_tools/tools/build_defs/cc/action_names.bzl", - ResourceLoader.readFromResources( - TestConstants.RULES_CC_REPOSITORY_EXECROOT + "cc/action_names.bzl")); - scratch.overwriteFile( "embedded_tools/tools/cpp/cc_toolchain_config_lib.bzl", ResourceLoader.readFromResources( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/AndroidNdkCrosstoolsTest.java b/src/test/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/AndroidNdkCrosstoolsTest.java index dd3f6009f7d912..e5f3ddc7978fb5 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/AndroidNdkCrosstoolsTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/rules/android/ndkcrosstools/AndroidNdkCrosstoolsTest.java @@ -163,7 +163,8 @@ public void testPathsExist() throws Exception { // TODO(tmsriram): Not all crosstools contain llvm-profdata tool yet, remove // the check once llvm-profdata becomes always available. if (toolpath.getPath().contains("llvm-profdata") - || toolpath.getPath().contains("llvm-cov")) { + || toolpath.getPath().contains("llvm-cov") + || toolpath.getPath().contains("deps-scanner")) { continue; } assertThat(ndkFiles).contains(toolpath.getPath()); diff --git a/src/test/java/com/google/devtools/build/lib/packages/util/BazelMockAndroidSupport.java b/src/test/java/com/google/devtools/build/lib/packages/util/BazelMockAndroidSupport.java index 619aa4ff8e6d85..622c88ee9cfd7c 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/util/BazelMockAndroidSupport.java +++ b/src/test/java/com/google/devtools/build/lib/packages/util/BazelMockAndroidSupport.java @@ -58,6 +58,7 @@ public static CcToolchainConfig.Builder x86Config() { Pair.of("objcopy", "x86/bin/i686-linux-android-objcopy"), Pair.of("objdump", "x86/bin/i686-linux-android-objdump"), Pair.of("strip", "x86/bin/i686-linux-android-strip"), + Pair.of("deps-scanner", "x86/bin/i686-linux-android-deps-scanner"), Pair.of("ld-bfd", "x86/bin/i686-linux-android-ld.bfd"), Pair.of("ld-gold", "x86/bin/i686-linux-android-ld.gold")) .withToolchainTargetConstraints( @@ -86,6 +87,7 @@ public static CcToolchainConfig.Builder armeabiV7a() { Pair.of("objcopy", "arm/bin/arm-linux-androideabi-objcopy"), Pair.of("objdump", "arm/bin/arm-linux-androideabi-objdump"), Pair.of("strip", "arm/bin/arm-linux-androideabi-strip"), + Pair.of("deps-scanner", "arm/bin/arm-linux-androideabi-deps-scanner"), Pair.of("ld-bfd", "arm/bin/arm-linux-androideabi-ld.bfd"), Pair.of("ld-gold", "arm/bin/arm-linux-androideabi-ld.gold")) .withToolchainTargetConstraints( diff --git a/src/test/java/com/google/devtools/build/lib/packages/util/MockCcSupport.java b/src/test/java/com/google/devtools/build/lib/packages/util/MockCcSupport.java index dd8361e4207934..0cf5bbaa0201ee 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/util/MockCcSupport.java +++ b/src/test/java/com/google/devtools/build/lib/packages/util/MockCcSupport.java @@ -271,10 +271,19 @@ protected void setupRulesCc(MockToolsConfig config) throws IOException { TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/cpp/cc_toolchain_config_lib.bzl", ResourceLoader.readFromResources( TestConstants.RULES_CC_REPOSITORY_EXECROOT + "cc/cc_toolchain_config_lib.bzl")); + // to support C++20 Modules + // many action names have been added + // However, the rules_cc is not update now, resulting test failed. + // e.g. Error: 'struct' value has no field or method 'cpp20_deps_scanning' + // to fix the error, another action_names.bzl is created for testing + // Attention: keep the file content same as + // src/main/starlark/builtins_bzl/common/cc/action_names.bzl config.overwrite( TestConstants.TOOLS_REPOSITORY_SCRATCH + "tools/build_defs/cc/action_names.bzl", + readCcActionNamesFile() + ); ResourceLoader.readFromResources( - TestConstants.RULES_CC_REPOSITORY_EXECROOT + "cc/action_names.bzl")); + TestConstants.RULES_CC_REPOSITORY_EXECROOT + "cc/action_names.bzl"); config.create( TestConstants.RULES_CC_REPOSITORY_EXECROOT + "BUILD", "genrule(name='license', cmd='exit 0', outs=['dummy_license'])"); @@ -341,6 +350,11 @@ protected String readCcToolchainConfigFile() throws IOException { "com/google/devtools/build/lib/analysis/mock/cc_toolchain_config.bzl"); } + protected String readCcActionNamesFile() throws IOException { + return ResourceLoader.readFromResources( + "com/google/devtools/build/lib/analysis/mock/action_names.bzl"); + } + public abstract Label getMockCrosstoolLabel(); protected abstract ImmutableList getCrosstoolArchs(); diff --git a/src/test/java/com/google/devtools/build/lib/packages/util/mock/osx_cc_toolchain_config.bzl b/src/test/java/com/google/devtools/build/lib/packages/util/mock/osx_cc_toolchain_config.bzl index d102d5a2606e0b..8ddc35f5eca7ec 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/util/mock/osx_cc_toolchain_config.bzl +++ b/src/test/java/com/google/devtools/build/lib/packages/util/mock/osx_cc_toolchain_config.bzl @@ -7277,6 +7277,7 @@ def _impl(ctx): tool_path(name = "strip", path = "/strip"), tool_path(name = "dwp", path = "/usr/bin/dwp"), tool_path(name = "objdump", path = "/usr/bin/objdump"), + tool_path(name = "deps-scanner", path = "/usr/bin/deps-scanner"), ] elif (ctx.attr.cpu == "ios_arm64" or ctx.attr.cpu == "ios_armv7"): @@ -7290,6 +7291,7 @@ def _impl(ctx): tool_path(name = "strip", path = "ios/strip"), tool_path(name = "dwp", path = "/usr/bin/dwp"), tool_path(name = "objdump", path = "/usr/bin/objdump"), + tool_path(name = "deps-scanner", path = "/usr/bin/deps-scanner"), ] elif (ctx.attr.cpu == "ios_i386" or ctx.attr.cpu == "ios_x86_64"): @@ -7303,6 +7305,7 @@ def _impl(ctx): tool_path(name = "strip", path = "iossim/strip"), tool_path(name = "dwp", path = "/usr/bin/dwp"), tool_path(name = "objdump", path = "/usr/bin/objdump"), + tool_path(name = "deps-scanner", path = "/usr/bin/deps-scanner"), ] elif (ctx.attr.cpu == "darwin_x86_64"): tool_paths = [ @@ -7315,6 +7318,7 @@ def _impl(ctx): tool_path(name = "strip", path = "mac/strip"), tool_path(name = "dwp", path = "/usr/bin/dwp"), tool_path(name = "objdump", path = "/usr/bin/objdump"), + tool_path(name = "deps-scanner", path = "/usr/bin/deps-scanner"), ] elif (ctx.attr.cpu == "tvos_arm64"): tool_paths = [ @@ -7327,6 +7331,7 @@ def _impl(ctx): tool_path(name = "strip", path = "tvos/strip"), tool_path(name = "dwp", path = "/usr/bin/dwp"), tool_path(name = "objdump", path = "/usr/bin/objdump"), + tool_path(name = "deps-scanner", path = "/usr/bin/deps-scanner"), ] elif (ctx.attr.cpu == "tvos_x86_64"): tool_paths = [ @@ -7339,6 +7344,7 @@ def _impl(ctx): tool_path(name = "strip", path = "tvsim/strip"), tool_path(name = "dwp", path = "/usr/bin/dwp"), tool_path(name = "objdump", path = "/usr/bin/objdump"), + tool_path(name = "deps-scanner", path = "/usr/bin/deps-scanner"), ] elif (ctx.attr.cpu == "watchos_armv7k" or ctx.attr.cpu == "watchos_arm64_32"): @@ -7352,6 +7358,7 @@ def _impl(ctx): tool_path(name = "strip", path = "watchos/strip"), tool_path(name = "dwp", path = "/usr/bin/dwp"), tool_path(name = "objdump", path = "/usr/bin/objdump"), + tool_path(name = "deps-scanner", path = "/usr/bin/deps-scanner"), ] elif (ctx.attr.cpu == "watchos_i386" or ctx.attr.cpu == "watchos_x86_64"): @@ -7365,6 +7372,7 @@ def _impl(ctx): tool_path(name = "strip", path = "watchsim/strip"), tool_path(name = "dwp", path = "/usr/bin/dwp"), tool_path(name = "objdump", path = "/usr/bin/objdump"), + tool_path(name = "deps-scanner", path = "/usr/bin/deps-scanner"), ] else: tool_paths = [] diff --git a/src/test/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProviderTest.java b/src/test/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProviderTest.java index edbd830b55df26..983bd9823e6796 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProviderTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/cpp/CcToolchainProviderTest.java @@ -169,6 +169,7 @@ def _impl(ctx): tool_path(name = "objdump", path = "some/objdump"), tool_path(name = "strip", path = "some/strip"), tool_path(name = "dwp", path = "some/dwp"), + tool_path(name = "deps-scanner", path = "some/deps-scanner"), ], cc_target_os = "os", builtin_sysroot = "sysroot", @@ -249,7 +250,8 @@ public void testGcovToolNotDefined() throws Exception { Pair.of("ld", "ld"), Pair.of("nm", "nm"), Pair.of("objdump", "objdump"), - Pair.of("strip", "strip")) + Pair.of("strip", "strip"), + Pair.of("deps-scanner", "deps-scanner")) .build() .getCcToolchainConfigRule()); analysisMock.ccSupport().setupCcToolchainConfig(mockToolsConfig, CcToolchainConfig.builder()); @@ -293,7 +295,8 @@ public void testGcovToolDefined() throws Exception { Pair.of("ld", "ld"), Pair.of("nm", "nm"), Pair.of("objdump", "objdump"), - Pair.of("strip", "strip")) + Pair.of("strip", "strip"), + Pair.of("deps-scanner", "deps-scanner")) .build() .getCcToolchainConfigRule()); analysisMock.ccSupport().setupCcToolchainConfig(mockToolsConfig, CcToolchainConfig.builder()); @@ -322,7 +325,8 @@ public void testGcovNotDefined() throws Exception { Pair.of("ld", "ld"), Pair.of("nm", "nm"), Pair.of("objdump", "objdump"), - Pair.of("strip", "strip")); + Pair.of("strip", "strip"), + Pair.of("deps-scanner", "deps-scanner")); scratch.file( "a/BUILD", "load(':cc_toolchain_config.bzl', 'cc_toolchain_config')", @@ -412,7 +416,8 @@ public void testLlvmCoverageToolsDefined() throws Exception { Pair.of("llvm-profdata", "path-to-llvm-profdata"), Pair.of("nm", "nm"), Pair.of("objdump", "objdump"), - Pair.of("strip", "strip")); + Pair.of("strip", "strip"), + Pair.of("deps-scanner", "deps-scanner")); scratch.file( "a/BUILD", "load(':cc_toolchain_config.bzl', 'cc_toolchain_config')", @@ -467,7 +472,8 @@ public void testLlvmCoverageToolsNotDefined() throws Exception { Pair.of("ld", "ld"), Pair.of("nm", "nm"), Pair.of("objdump", "objdump"), - Pair.of("strip", "strip")); + Pair.of("strip", "strip"), + Pair.of("deps-scanner", "deps-scanner")); scratch.file( "a/BUILD", "load(':cc_toolchain_config.bzl', 'cc_toolchain_config')", @@ -575,8 +581,9 @@ public void testConfigWithMissingToolDefs() throws Exception { Pair.of("gcov", "gcov"), Pair.of("ld", "ld"), Pair.of("nm", "nm"), - Pair.of("objdump", "objdump") + Pair.of("objdump", "objdump"), // Pair.of("strip", "strip") + Pair.of("deps-scanner", "deps-scanner") ) .build() .getCcToolchainConfigRule()); diff --git a/src/test/java/com/google/devtools/build/lib/rules/cpp/CcToolchainTest.java b/src/test/java/com/google/devtools/build/lib/rules/cpp/CcToolchainTest.java index 8af3d00ddb912a..899ed0020157da 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/cpp/CcToolchainTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/cpp/CcToolchainTest.java @@ -742,6 +742,7 @@ def _impl(ctx): tool_path(name = "strip", path = "/some/path"), tool_path(name = "dwp", path = "/some/path"), tool_path(name = "llvm_profdata", path = "/some/path"), + tool_path(name = "deps-scanner", path = "/some/path"), ], cc_target_os = "os", builtin_sysroot = "sysroot", diff --git a/src/test/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionTest.java b/src/test/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionTest.java index 9c2185e4d6133d..a8417a38e2234e 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/cpp/CppLinkActionTest.java @@ -163,6 +163,7 @@ private static FeatureConfiguration getMockFeatureConfiguration( "gcc_tool", "ar_tool", "strip_tool", + "deps_scanner_tool", /* supportsInterfaceSharedLibraries= */ false, /* existingActionConfigNames= */ ImmutableSet.of()); diff --git a/src/test/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLineTest.java b/src/test/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLineTest.java index a745afc5409a91..02e0451aef5758 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLineTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/cpp/LinkCommandLineTest.java @@ -91,6 +91,7 @@ private static FeatureConfiguration getMockFeatureConfiguration() throws Excepti "MOCK_GCC_TOOL", "MOCK_AR_TOOL", "MOCK_STRIP_TOOL", + "MOCK_DEPS_SCANNER_TOOL", /* supportsInterfaceSharedLibraries= */ false, /* existingActionConfigNames= */ ImmutableSet.of()); diff --git a/tools/cpp/BUILD.tpl b/tools/cpp/BUILD.tpl index f99995d2b923eb..a8ab542745f7b6 100644 --- a/tools/cpp/BUILD.tpl +++ b/tools/cpp/BUILD.tpl @@ -54,6 +54,11 @@ filegroup( srcs = ["cc_wrapper.sh"], ) +filegroup( + name = "deps_scanner_wrapper", + srcs = ["deps_scanner_wrapper.sh"], +) + filegroup( name = "compiler_deps", srcs = glob(["extra_tools/**"], allow_empty = True) + [%{cc_compiler_deps}], diff --git a/tools/cpp/armeabi_cc_toolchain_config.bzl b/tools/cpp/armeabi_cc_toolchain_config.bzl index 72ef48ae6d6dfc..3d518999c56ecd 100644 --- a/tools/cpp/armeabi_cc_toolchain_config.bzl +++ b/tools/cpp/armeabi_cc_toolchain_config.bzl @@ -53,6 +53,7 @@ def _impl(ctx): tool_path(name = "objcopy", path = "/bin/false"), tool_path(name = "objdump", path = "/bin/false"), tool_path(name = "strip", path = "/bin/false"), + tool_path(name = "deps-scanner", path = "/bin/false"), ] return cc_common.create_cc_toolchain_config_info( diff --git a/tools/cpp/clang_deps_scanner_wrapper.sh.tpl b/tools/cpp/clang_deps_scanner_wrapper.sh.tpl new file mode 100644 index 00000000000000..63a14ed6761451 --- /dev/null +++ b/tools/cpp/clang_deps_scanner_wrapper.sh.tpl @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Ship the environment to the C++ action +# +set -eu + +# Set-up the environment +%{env} + +# Call the C++ compiler +%{deps_scanner} -format=p1689 -- %{cc} "$@" > $DEPS_SCANNER_OUTPUT_FILE + +# workaround for clang-scan-deps BUG +# the .d file generated by clang-scan-deps is inconsistent with the one generated by clang. +# the source file path in .d file is absolute path, and Bazel regards the source file as extra dependency. +# see: https://github.com/llvm/llvm-project/issues/69439 +# use two functions handle Linux and MacOS +# the difference is as follows +# 1. how to get string length +# 2. how to use sed to modify .d file +fix_bash() { +output_file_path=$(readlink -f $DEPS_SCANNER_OUTPUT_FILE) +output_file_length=$(expr length $output_file_path) +output_file_short_length=$(expr length $DEPS_SCANNER_OUTPUT_FILE) +current_path_length=$(expr $output_file_length - $output_file_short_length) +current_path=${output_file_path:0:$current_path_length} +dotd_file_length=$(expr $output_file_short_length - 2) +dotd_file_path=${DEPS_SCANNER_OUTPUT_FILE:0:$dotd_file_length} +sed_arg="s|$current_path||g" +sed -i $sed_arg $dotd_file_path +} +fix_zsh() { +output_file_path=$(readlink -f $DEPS_SCANNER_OUTPUT_FILE) +output_file_length=${#output_file_path} +output_file_short_length=${#DEPS_SCANNER_OUTPUT_FILE} +current_path_length=$(expr $output_file_length - $output_file_short_length) +current_path=${output_file_path:0:$current_path_length} +dotd_file_length=$(expr $output_file_short_length - 2) +dotd_file_path=${DEPS_SCANNER_OUTPUT_FILE:0:$dotd_file_length} +sed_arg="s|$current_path||g" +sed -i '' $sed_arg $dotd_file_path +} + +system_type=$(uname -s) +if [ "$system_type" = "Linux" ]; then + fix_bash +elif [ "$system_type" = "Darwin" ]; then + fix_zsh +else + echo "The system is not supported now: $system_type" + exit 1 +fi diff --git a/tools/cpp/gcc_deps_scanner_wrapper.sh.tpl b/tools/cpp/gcc_deps_scanner_wrapper.sh.tpl new file mode 100644 index 00000000000000..2893e5bc4f7667 --- /dev/null +++ b/tools/cpp/gcc_deps_scanner_wrapper.sh.tpl @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Ship the environment to the C++ action +# +set -eu + +# Set-up the environment +%{env} + +# Call the C++ compiler +echo "gcc not supported now" +exit 1 diff --git a/tools/cpp/unix_cc_configure.bzl b/tools/cpp/unix_cc_configure.bzl index 3f162bdeb12551..09f169027b5a77 100644 --- a/tools/cpp/unix_cc_configure.bzl +++ b/tools/cpp/unix_cc_configure.bzl @@ -91,6 +91,7 @@ def _get_tool_paths(repository_ctx, overriden_tools): "objcopy", "objdump", "strip", + "deps-scanner", ] }.items()) @@ -330,6 +331,8 @@ def configure_unix_toolchain(repository_ctx, cpu_value, overriden_tools): "@bazel_tools//tools/cpp:unix_cc_toolchain_config.bzl", "@bazel_tools//tools/cpp:linux_cc_wrapper.sh.tpl", "@bazel_tools//tools/cpp:osx_cc_wrapper.sh.tpl", + "@bazel_tools//tools/cpp:clang_deps_scanner_wrapper.sh.tpl", + "@bazel_tools//tools/cpp:gcc_deps_scanner_wrapper.sh.tpl", ]) repository_ctx.symlink( @@ -385,6 +388,8 @@ def configure_unix_toolchain(repository_ctx, cpu_value, overriden_tools): if darwin: overriden_tools["gcc"] = "cc_wrapper.sh" overriden_tools["ar"] = _find_generic(repository_ctx, "libtool", "LIBTOOL", overriden_tools) + + overriden_tools["deps-scanner"] = "deps_scanner_wrapper.sh" auto_configure_warning_maybe(repository_ctx, "CC used: " + str(cc)) tool_paths = _get_tool_paths(repository_ctx, overriden_tools) @@ -409,6 +414,24 @@ def configure_unix_toolchain(repository_ctx, cpu_value, overriden_tools): "%{env}": escape_string(get_env(repository_ctx)), }, ) + deps_scanner_wrapper_src = ( + "@bazel_tools//tools/cpp:clang_deps_scanner_wrapper.sh.tpl" if is_clang else "@bazel_tools//tools/cpp:gcc_deps_scanner_wrapper.sh.tpl" + ) + deps_scanner = "" + if is_clang: + cc_str = str(cc) + path_arr = cc_str.split("/")[:-1] + path_arr.append("clang-scan-deps") + deps_scanner = "/".join(path_arr) + repository_ctx.template( + "deps_scanner_wrapper.sh", + paths[deps_scanner_wrapper_src], + { + "%{cc}": escape_string(str(cc)), + "%{deps_scanner}": escape_string(deps_scanner), + "%{env}": escape_string(get_env(repository_ctx)), + }, + ) conly_opts = split_escaped(get_env_var( repository_ctx, @@ -553,6 +576,7 @@ def configure_unix_toolchain(repository_ctx, cpu_value, overriden_tools): "%{cc_compiler_deps}": get_starlark_list([ ":builtin_include_directory_paths", ":cc_wrapper", + ":deps_scanner_wrapper", ]), "%{compiler}": escape_string(get_env_var( repository_ctx, diff --git a/tools/cpp/unix_cc_toolchain_config.bzl b/tools/cpp/unix_cc_toolchain_config.bzl index a361e7b02e2ef6..9be877f2a29315 100644 --- a/tools/cpp/unix_cc_toolchain_config.bzl +++ b/tools/cpp/unix_cc_toolchain_config.bzl @@ -146,6 +146,9 @@ all_compile_actions = [ ACTION_NAMES.cpp_header_parsing, ACTION_NAMES.cpp_module_compile, ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp20_deps_scanning, + ACTION_NAMES.cpp20_module_compile, + ACTION_NAMES.cpp20_module_codegen, ACTION_NAMES.clif_match, ACTION_NAMES.lto_backend, ] @@ -156,6 +159,9 @@ all_cpp_compile_actions = [ ACTION_NAMES.cpp_header_parsing, ACTION_NAMES.cpp_module_compile, ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp20_deps_scanning, + ACTION_NAMES.cpp20_module_compile, + ACTION_NAMES.cpp20_module_codegen, ACTION_NAMES.clif_match, ] @@ -166,6 +172,8 @@ preprocessor_compile_actions = [ ACTION_NAMES.preprocess_assemble, ACTION_NAMES.cpp_header_parsing, ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp20_deps_scanning, + ACTION_NAMES.cpp20_module_compile, ACTION_NAMES.clif_match, ] @@ -176,6 +184,7 @@ codegen_compile_actions = [ ACTION_NAMES.assemble, ACTION_NAMES.preprocess_assemble, ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp20_module_codegen, ACTION_NAMES.lto_backend, ] @@ -386,6 +395,9 @@ def _impl(ctx): ACTION_NAMES.cpp_header_parsing, ACTION_NAMES.cpp_module_compile, ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp20_deps_scanning, + ACTION_NAMES.cpp20_module_compile, + ACTION_NAMES.cpp20_module_codegen, ACTION_NAMES.lto_backend, ACTION_NAMES.clif_match, ] + all_link_actions + lto_index_actions, @@ -500,6 +512,8 @@ def _impl(ctx): ACTION_NAMES.cpp_compile, ACTION_NAMES.cpp_module_codegen, ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp20_module_compile, + ACTION_NAMES.cpp20_module_codegen, ], flag_groups = [ flag_group(flags = ["-fPIC"], expand_if_available = "pic"), @@ -519,6 +533,7 @@ def _impl(ctx): ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile, ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp20_module_codegen, ], flag_groups = [ flag_group( @@ -542,6 +557,9 @@ def _impl(ctx): ACTION_NAMES.cpp_compile, ACTION_NAMES.cpp_header_parsing, ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp20_deps_scanning, + ACTION_NAMES.cpp20_module_compile, + ACTION_NAMES.cpp20_module_codegen, ACTION_NAMES.clif_match, ], flag_groups = [ @@ -698,6 +716,9 @@ def _impl(ctx): ACTION_NAMES.cpp_compile, ACTION_NAMES.cpp_module_codegen, ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp20_deps_scanning, + ACTION_NAMES.cpp20_module_compile, + ACTION_NAMES.cpp20_module_codegen, ], flag_groups = [ flag_group( @@ -721,6 +742,8 @@ def _impl(ctx): ACTION_NAMES.cpp_compile, ACTION_NAMES.cpp_header_parsing, ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp20_deps_scanning, + ACTION_NAMES.cpp20_module_compile, ACTION_NAMES.clif_match, ACTION_NAMES.objc_compile, ACTION_NAMES.objcpp_compile, @@ -792,6 +815,8 @@ def _impl(ctx): ACTION_NAMES.cpp_compile, ACTION_NAMES.cpp_header_parsing, ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp20_deps_scanning, + ACTION_NAMES.cpp20_module_compile, ACTION_NAMES.clif_match, ACTION_NAMES.objc_compile, ACTION_NAMES.objcpp_compile, @@ -825,6 +850,9 @@ def _impl(ctx): ACTION_NAMES.cpp_compile, ACTION_NAMES.cpp_header_parsing, ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp20_deps_scanning, + ACTION_NAMES.cpp20_module_compile, + ACTION_NAMES.cpp20_module_codegen, ACTION_NAMES.clif_match, ACTION_NAMES.objc_compile, ACTION_NAMES.objcpp_compile, @@ -1158,6 +1186,8 @@ def _impl(ctx): ACTION_NAMES.objc_compile, ACTION_NAMES.objcpp_compile, ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp20_deps_scanning, + ACTION_NAMES.cpp20_module_compile, ACTION_NAMES.clif_match, ], flag_groups = [ @@ -1183,6 +1213,7 @@ def _impl(ctx): ACTION_NAMES.objc_compile, ACTION_NAMES.objcpp_compile, ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp20_deps_scanning, ACTION_NAMES.clif_match, ], flag_groups = [ @@ -1427,11 +1458,42 @@ def _impl(ctx): ], ) + # Tell bazel we support C++20 Modules now + cpp20_module = feature( + name = "cpp20_module", + # set default value to False + # to enable the feature + # use --features=cpp20_module + # or add cpp20_module to features attr + enabled = False, + ) + cpp20_modmap_file_feature = feature( + name = "cpp20_modmap_file", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp20_module_compile, + ACTION_NAMES.cpp20_module_codegen, + ], + flag_groups = [ + flag_group( + flags = ["@%{cpp20_modmap_file}"], + expand_if_available = "cpp20_modmap_file", + ), + ], + ), + ], + enabled = True, + ) + # TODO(#8303): Mac crosstool should also declare every feature. if is_linux: # Linux artifact name patterns are the default. artifact_name_patterns = [] features = [ + cpp20_module, + cpp20_modmap_file_feature, dependency_file_feature, serialized_diagnostics_file_feature, random_seed_feature, @@ -1499,6 +1561,8 @@ def _impl(ctx): ), ] features = [ + cpp20_module, + cpp20_modmap_file_feature, macos_minimum_os_feature, macos_default_link_flags_feature, libtool_feature, From abdd7b4bc7ae08c9fcf4100d281791098fbea046 Mon Sep 17 00:00:00 2001 From: PikachuHy Date: Mon, 20 May 2024 17:10:11 +0800 Subject: [PATCH 4/6] support C++20 Modules, implement compilation --- .../lib/rules/cpp/CcCompilationContext.java | 92 ++++++++++ .../lib/rules/cpp/CcCompilationOutputs.java | 72 ++++++++ .../build/lib/rules/cpp/CppCompileAction.java | 158 ++++++++++++++++-- .../rules/cpp/CppCompileActionBuilder.java | 99 +++++++---- 4 files changed, 378 insertions(+), 43 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationContext.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationContext.java index f071d1a47508d5..1ce4c46bae1624 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationContext.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationContext.java @@ -79,6 +79,11 @@ public final class CcCompilationContext implements CcCompilationContextApi transitiveModules; private final NestedSet transitivePicModules; + private final NestedSet pcmFiles; + private final NestedSet picPcmFiles; + private final NestedSet modulesInfoFiles; + private final NestedSet picModulesInfoFiles; + private final CppModuleMap cppModuleMap; private final boolean propagateModuleMapAsActionInput; @@ -111,6 +116,10 @@ private CcCompilationContext( HeaderInfo headerInfo, NestedSet transitiveModules, NestedSet transitivePicModules, + NestedSet pcmFiles, + NestedSet picPcmFiles, + NestedSet modulesInfoFiles, + NestedSet picModulesInfoFiles, ImmutableList directModuleMaps, ImmutableList exportingModuleMaps, CppModuleMap cppModuleMap, @@ -125,6 +134,10 @@ private CcCompilationContext( this.headerInfo = headerInfo; this.transitiveModules = transitiveModules; this.transitivePicModules = transitivePicModules; + this.pcmFiles = pcmFiles; + this.picPcmFiles = picPcmFiles; + this.modulesInfoFiles = modulesInfoFiles; + this.picModulesInfoFiles = picModulesInfoFiles; this.cppModuleMap = cppModuleMap; this.nonCodeInputs = nonCodeInputs; this.compilationPrerequisites = compilationPrerequisites; @@ -550,6 +563,14 @@ public NestedSet getTransitiveModules(boolean usePic) { return usePic ? transitivePicModules : transitiveModules; } + public NestedSet getPcmFiles(boolean usePic) { + return usePic ? picPcmFiles: pcmFiles; + } + + public NestedSet getModulesInfoFiles(boolean usePic) { + return usePic ? picModulesInfoFiles : modulesInfoFiles; + } + @Override public Depset getStarlarkTransitiveModules(boolean usePic, StarlarkThread thread) throws EvalException { @@ -627,6 +648,44 @@ ImmutableList getNonTransitiveDefines() { return commandLineCcCompilationContext.localDefines; } + public static CcCompilationContext createWithCpp20Modules( + CcCompilationContext ccCompilationContext, + NestedSet pcmFiles, + NestedSet picPcmFiles, + ImmutableList modulesInfoFiles, + ImmutableList picModulesInfoFiles) { + var pcmFilesBuilder = NestedSetBuilder + .fromNestedSet(ccCompilationContext.pcmFiles) + .addTransitive(pcmFiles); + var picPcmFilesBuilder = NestedSetBuilder + .fromNestedSet(ccCompilationContext.picPcmFiles) + .addTransitive(picPcmFiles); + var modulesInfoFilesBuilder = NestedSetBuilder + .fromNestedSet(ccCompilationContext.modulesInfoFiles) + .addAll(modulesInfoFiles); + var picModulesInfoFilesBuilder = NestedSetBuilder + .fromNestedSet(ccCompilationContext.picModulesInfoFiles) + .addAll(picModulesInfoFiles); + return new CcCompilationContext( + ccCompilationContext.commandLineCcCompilationContext, + ccCompilationContext.compilationPrerequisites, + ccCompilationContext.declaredIncludeSrcs, + ccCompilationContext.nonCodeInputs, + ccCompilationContext.headerInfo, + ccCompilationContext.transitiveModules, + ccCompilationContext.transitivePicModules, + pcmFilesBuilder.build(), + picPcmFilesBuilder.build(), + modulesInfoFilesBuilder.build(), + picModulesInfoFilesBuilder.build(), + ccCompilationContext.directModuleMaps, + ccCompilationContext.exportingModuleMaps, + ccCompilationContext.cppModuleMap, + ccCompilationContext.propagateModuleMapAsActionInput, + ccCompilationContext.virtualToOriginalHeaders, + ccCompilationContext.headerTokens); + } + /** * Returns a {@code CcCompilationContext} that is based on a given {@code CcCompilationContext}, * with {@code extraHeaderTokens} added to the header tokens. @@ -644,6 +703,10 @@ public static CcCompilationContext createWithExtraHeaderTokens( ccCompilationContext.headerInfo, ccCompilationContext.transitiveModules, ccCompilationContext.transitivePicModules, + ccCompilationContext.pcmFiles, + ccCompilationContext.picPcmFiles, + ccCompilationContext.modulesInfoFiles, + ccCompilationContext.picModulesInfoFiles, ccCompilationContext.directModuleMaps, ccCompilationContext.exportingModuleMaps, ccCompilationContext.cppModuleMap, @@ -765,6 +828,10 @@ private void mergeDependentCcCompilationContext( TransitiveSetHelper allDefines, NestedSetBuilder transitiveModules, NestedSetBuilder transitivePicModules, + NestedSetBuilder pcmFiles, + NestedSetBuilder picPcmFiles, + NestedSetBuilder modulesInfoFiles, + NestedSetBuilder picModulesInfoFiles, Set directModuleMaps) { Preconditions.checkNotNull(otherCcCompilationContext); compilationPrerequisites.addTransitive( @@ -785,6 +852,11 @@ private void mergeDependentCcCompilationContext( addIfNotNull(transitivePicModules, otherCcCompilationContext.headerInfo.picHeaderModule); addIfNotNull(transitivePicModules, otherCcCompilationContext.headerInfo.separatePicModule); + pcmFiles.addTransitive(otherCcCompilationContext.pcmFiles); + picPcmFiles.addTransitive(otherCcCompilationContext.picPcmFiles); + modulesInfoFiles.addTransitive(otherCcCompilationContext.modulesInfoFiles); + picModulesInfoFiles.addTransitive(otherCcCompilationContext.picModulesInfoFiles); + nonCodeInputs.addTransitive(otherCcCompilationContext.nonCodeInputs); // All module maps of direct dependencies are inputs to the current compile independently of @@ -842,6 +914,10 @@ private void mergeDependentCcCompilationContexts( TransitiveSetHelper allDefines, NestedSetBuilder transitiveModules, NestedSetBuilder transitivePicModules, + NestedSetBuilder pcmFiles, + NestedSetBuilder picPcmFiles, + NestedSetBuilder modulesInfoFiles, + NestedSetBuilder picModulesInfoFiles, Set directModuleMaps, Set exportingModuleMaps) { for (CcCompilationContext ccCompilationContext : @@ -851,6 +927,10 @@ private void mergeDependentCcCompilationContexts( allDefines, transitiveModules, transitivePicModules, + pcmFiles, + picPcmFiles, + modulesInfoFiles, + picModulesInfoFiles, directModuleMaps); } @@ -1079,6 +1159,10 @@ public CcCompilationContext build() { TransitiveSetHelper allDefines = new TransitiveSetHelper<>(); NestedSetBuilder transitiveModules = NestedSetBuilder.stableOrder(); NestedSetBuilder transitivePicModules = NestedSetBuilder.stableOrder(); + NestedSetBuilder pcmFiles = NestedSetBuilder.stableOrder(); + NestedSetBuilder picPcmFiles = NestedSetBuilder.stableOrder(); + NestedSetBuilder modulesInfoFiles = NestedSetBuilder.stableOrder(); + NestedSetBuilder picModulesInfoFiles = NestedSetBuilder.stableOrder(); Set directModuleMaps = new LinkedHashSet<>(); Set exportingModuleMaps = new LinkedHashSet<>(); mergeDependentCcCompilationContexts( @@ -1087,6 +1171,10 @@ public CcCompilationContext build() { allDefines, transitiveModules, transitivePicModules, + pcmFiles, + picPcmFiles, + modulesInfoFiles, + picModulesInfoFiles, directModuleMaps, exportingModuleMaps); @@ -1110,6 +1198,10 @@ public CcCompilationContext build() { headerInfo, transitiveModules.build(), transitivePicModules.build(), + pcmFiles.build(), + picPcmFiles.build(), + modulesInfoFiles.build(), + picModulesInfoFiles.build(), ImmutableList.copyOf(directModuleMaps), ImmutableList.copyOf(exportingModuleMaps), cppModuleMap, diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java index b65048cb29cd62..34702095b3dcb8 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationOutputs.java @@ -44,6 +44,18 @@ public class CcCompilationOutputs implements CcCompilationOutputsApi { */ private final ImmutableList picObjectFiles; + /** + * All .pcm files built by the target. + */ + private final NestedSet pcmFiles; + + /** + * All .pic.pcm files built by the target. + */ + private final NestedSet picPcmFiles; + + private final ImmutableList modulesInfoFiles; + private final ImmutableList picModulesInfoFiles; /** * Maps all .o bitcode files coming from a ThinLTO C(++) compilation under our control to * information needed by the LTO indexing and backend steps. @@ -82,6 +94,10 @@ public class CcCompilationOutputs implements CcCompilationOutputsApi { private CcCompilationOutputs( ImmutableList objectFiles, ImmutableList picObjectFiles, + NestedSet pcmFiles, + NestedSet picPcmFiles, + ImmutableList modulesInfoFiles, + ImmutableList picModulesInfoFiles, LtoCompilationContext ltoCompilationContext, ImmutableList dwoFiles, ImmutableList picDwoFiles, @@ -92,6 +108,10 @@ private CcCompilationOutputs( ImmutableList moduleFiles) { this.objectFiles = objectFiles; this.picObjectFiles = picObjectFiles; + this.pcmFiles = pcmFiles; + this.picPcmFiles = picPcmFiles; + this.modulesInfoFiles = modulesInfoFiles; + this.picModulesInfoFiles = picModulesInfoFiles; this.ltoCompilationContext = ltoCompilationContext; this.dwoFiles = dwoFiles; this.picDwoFiles = picDwoFiles; @@ -111,6 +131,19 @@ public ImmutableList getObjectFiles(boolean usePic) { return usePic ? picObjectFiles : objectFiles; } + /** + * Returns an unmodifiable view of the .pcm or .pic.pcm files set. + * + * @param usePic whether to return .pic.pcm files + */ + public NestedSet getPcmFiles(boolean usePic) { + return usePic ? picPcmFiles : pcmFiles; + } + + public ImmutableList getModulesInfoFiles(boolean usePic) { + return usePic ? picModulesInfoFiles : modulesInfoFiles; + } + @Override public Sequence getStarlarkObjects() throws EvalException { return StarlarkList.immutableCopyOf(getObjectFiles(/* usePic= */ false)); @@ -244,6 +277,10 @@ public static Builder builder() { public static final class Builder { private final Set objectFiles = new LinkedHashSet<>(); private final Set picObjectFiles = new LinkedHashSet<>(); + private final NestedSetBuilder pcmFiles = NestedSetBuilder.stableOrder(); + private final NestedSetBuilder picPcmFiles = NestedSetBuilder.stableOrder(); + private final Set modulesInfoFiles = new LinkedHashSet<>(); + private final Set picModulesInfoFiles = new LinkedHashSet<>(); private final LtoCompilationContext.Builder ltoCompilationContext = new LtoCompilationContext.Builder(); private final Set dwoFiles = new LinkedHashSet<>(); @@ -262,6 +299,10 @@ public CcCompilationOutputs build() { return new CcCompilationOutputs( ImmutableList.copyOf(objectFiles), ImmutableList.copyOf(picObjectFiles), + pcmFiles.build(), + picPcmFiles.build(), + ImmutableList.copyOf(modulesInfoFiles), + ImmutableList.copyOf(picModulesInfoFiles), ltoCompilationContext.build(), ImmutableList.copyOf(dwoFiles), ImmutableList.copyOf(picDwoFiles), @@ -276,6 +317,10 @@ public CcCompilationOutputs build() { public Builder merge(CcCompilationOutputs outputs) { this.objectFiles.addAll(outputs.objectFiles); this.picObjectFiles.addAll(outputs.picObjectFiles); + this.pcmFiles.addTransitive(outputs.pcmFiles); + this.picPcmFiles.addTransitive(outputs.picPcmFiles); + this.modulesInfoFiles.addAll(outputs.modulesInfoFiles); + this.picModulesInfoFiles.addAll(outputs.picModulesInfoFiles); this.dwoFiles.addAll(outputs.dwoFiles); this.picDwoFiles.addAll(outputs.picDwoFiles); this.gcnoFiles.addAll(outputs.gcnoFiles); @@ -298,6 +343,20 @@ public Builder addObjectFile(Artifact artifact) { return this; } + /** Adds a pcm file. */ + @CanIgnoreReturnValue + public Builder addPcmFile(Artifact.DerivedArtifact artifact) { + pcmFiles.add(artifact); + return this; + } + + /** Adds a modules info file. */ + @CanIgnoreReturnValue + public Builder addModulesInfoFile(Artifact artifact) { + modulesInfoFiles.add(artifact); + return this; + } + @CanIgnoreReturnValue public Builder addObjectFiles(Iterable artifacts) { for (Artifact artifact : artifacts) { @@ -315,6 +374,19 @@ public Builder addPicObjectFile(Artifact artifact) { return this; } + /** Adds a pic pcm file. */ + @CanIgnoreReturnValue + public Builder addPicPcmFile(Artifact.DerivedArtifact artifact) { + picPcmFiles.add(artifact); + return this; + } + /** Adds a pic modules info file. */ + @CanIgnoreReturnValue + public Builder addPicModulesInfoFile(Artifact artifact) { + picModulesInfoFiles.add(artifact); + return this; + } + @CanIgnoreReturnValue public Builder addLtoBitcodeFile( Artifact fullBitcode, Artifact ltoIndexingBitcode, ImmutableList copts) { diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java index 534ed1ada2452a..c7efba5d2cd94d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileAction.java @@ -94,14 +94,17 @@ import com.google.devtools.build.skyframe.SkyframeLookupResult; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; +import java.util.Objects; import java.util.List; import java.util.Map; +import java.util.HashMap; import java.util.Set; import java.util.UUID; import java.util.function.Predicate; @@ -120,6 +123,7 @@ public class CppCompileAction extends AbstractAction implements IncludeScannable @VisibleForTesting static final String OBJC_COMPILE_MNEMONIC = "ObjcCompile"; @Nullable private final Artifact gcnoFile; + @Nullable private final Artifact dotdFile; private final Artifact sourceFile; private final BuildConfigurationValue configuration; private final NestedSet mandatoryInputs; @@ -173,6 +177,7 @@ public class CppCompileAction extends AbstractAction implements IncludeScannable * first. */ private Set usedModules; + private Set usedCpp20Modules; private boolean inputsDiscovered = false; @@ -193,6 +198,8 @@ public class CppCompileAction extends AbstractAction implements IncludeScannable private ParamFileActionInput paramFileActionInput; private PathFragment paramFilePath; + private final NestedSet pcmFiles; + private final Artifact modmapInputFile; /** * Creates a new action to compile C/C++ source files. * @@ -248,7 +255,10 @@ public class CppCompileAction extends AbstractAction implements IncludeScannable CppSemantics cppSemantics, ImmutableList builtInIncludeDirectories, @Nullable Artifact grepIncludes, - ImmutableList additionalOutputs) { + ImmutableList additionalOutputs, + NestedSet pcmFiles, + Artifact modmapInputFile + ) { super( owner, mandatoryInputs, @@ -259,8 +269,11 @@ public class CppCompileAction extends AbstractAction implements IncludeScannable gcnoFile, dwoFile, ltoIndexingFile, - additionalOutputs)); + additionalOutputs, + featureConfiguration, + actionName)); this.gcnoFile = gcnoFile; + this.dotdFile = dotdFile; this.sourceFile = sourceFile; this.shareable = shareable; this.configuration = configuration; @@ -315,7 +328,13 @@ public class CppCompileAction extends AbstractAction implements IncludeScannable if (separateModule != null && !separateModule.equals(getPrimaryOutput())) { allowedDerivedInputsBuilder.add(separateModule); } + if (pcmFiles != null) { + allowedDerivedInputsBuilder.addTransitive(pcmFiles); + } allowedDerivedInputs = allowedDerivedInputsBuilder.build(); + + this.pcmFiles = pcmFiles; + this.modmapInputFile = modmapInputFile; } private static ImmutableSet collectOutputs( @@ -325,7 +344,10 @@ private static ImmutableSet collectOutputs( @Nullable Artifact gcnoFile, @Nullable Artifact dwoFile, @Nullable Artifact ltoIndexingFile, - ImmutableList additionalOutputs) { + ImmutableList additionalOutputs, + FeatureConfiguration featureConfiguration, + String actionName + ) { ImmutableSet.Builder outputs = ImmutableSet.builder(); outputs.add(outputFile); if (gcnoFile != null) { @@ -333,7 +355,26 @@ private static ImmutableSet collectOutputs( } outputs.addAll(additionalOutputs); if (dotdFile != null) { - outputs.add(dotdFile); + if (featureConfiguration.isEnabled(CppRuleClasses.CPP20_MODULE)) { + switch (actionName) { + // if cpp20_module enabled, only c++20-deps-scanning will produce .d file + // other actions will reuse the .d file from c++20-deps-scanning + case CppActionNames.CPP_COMPILE: + case CppActionNames.CPP20_MODULE_COMPILE: + case CppActionNames.CPP20_MODULE_CODEGEN: + { + break; + } + case CppActionNames.CPP20_DEPS_SCANNING: + // other source files. e.g. c + default: { + outputs.add(dotdFile); + } + } + } + else { + outputs.add(dotdFile); + } } if (diagnosticsFile != null) { outputs.add(diagnosticsFile); @@ -435,7 +476,8 @@ private void clearAdditionalInputs() { @Override public boolean discoversInputs() { - return shouldScanIncludes || getDotdFile() != null || shouldParseShowIncludes(); + return Cpp20ModuleHelper.isCpp20ModuleCompilationAction(actionName) || + shouldScanIncludes || getDotdFile() != null || shouldParseShowIncludes(); } @Override @@ -502,7 +544,9 @@ public void prepareInputDiscovery() { @Override public NestedSet discoverInputs(ActionExecutionContext actionExecutionContext) throws ActionExecutionException, InterruptedException { - Preconditions.checkArgument(!sourceFile.isFileType(CppFileTypes.CPP_MODULE)); + if (!Cpp20ModuleHelper.isCpp20ModuleCompilationAction(actionName)) { + Preconditions.checkArgument(!sourceFile.isFileType(CppFileTypes.CPP_MODULE)); + } if (additionalInputs == null) { List options; @@ -516,6 +560,10 @@ public NestedSet discoverInputs(ActionExecutionContext actionExecution DetailedExitCode code = createDetailedExitCode(message, Code.COMMAND_GENERATION_FAILURE); throw new ActionExecutionException(message, this, /*catastrophe=*/ false, code); } + usedCpp20Modules = computeUsedCpp20Modules(actionExecutionContext); + if (usedCpp20Modules == null) { + return null; + } commandLineKey = computeCommandLineKey(options); ImmutableList systemIncludeDirs = getSystemIncludeDirs(options); boolean siblingLayout = @@ -531,6 +579,7 @@ public NestedSet discoverInputs(ActionExecutionContext actionExecution additionalInputs = NestedSetBuilder.fromNestedSet(ccCompilationContext.getDeclaredIncludeSrcs()) .addTransitive(additionalPrunableHeaders) + .addAll(usedCpp20Modules) .build(); if (needsIncludeValidation) { verifyActionIncludePaths(systemIncludeDirs, siblingLayout); @@ -603,7 +652,8 @@ public NestedSet discoverInputs(ActionExecutionContext actionExecution additionalInputs = NestedSetBuilder.fromNestedSet(additionalInputs).addTransitive(discoveredModules).build(); - if (getPrimaryOutput().isFileType(CppFileTypes.CPP_MODULE)) { + if (getPrimaryOutput().isFileType(CppFileTypes.CPP_MODULE) + && !Cpp20ModuleHelper.isCpp20ModuleCompilationAction(actionName)) { this.discoveredModules = discoveredModules; } usedModules = null; @@ -660,7 +710,7 @@ public NestedSet getDiscoveredModules() { /** Returns the path where the compiler should put the discovered dependency information. */ public Artifact getDotdFile() { - return compileCommandLine.getDotdFile(); + return dotdFile; } public CcCompilationContext getCcCompilationContext() { @@ -1167,7 +1217,8 @@ public NestedSet getAllowedDerivedInputs() { @Override public synchronized void updateInputs(NestedSet inputs) { super.updateInputs(inputs); - if (getPrimaryOutput().isFileType(CppFileTypes.CPP_MODULE)) { + if (getPrimaryOutput().isFileType(CppFileTypes.CPP_MODULE) + && !Cpp20ModuleHelper.isCpp20ModuleCompilationAction(actionName)) { discoveredModules = NestedSetBuilder.wrap( Order.STABLE_ORDER, @@ -1178,10 +1229,14 @@ public synchronized void updateInputs(NestedSet inputs) { @Override protected String getRawProgressMessage() { - return (actionName.equals(CppActionNames.CPP_HEADER_ANALYSIS) - ? "Header analysis for " - : "Compiling ") - + getSourceFile().prettyPrint(); + switch (actionName) { + case CppActionNames.CPP_HEADER_ANALYSIS: + return "Header analysis for " + getSourceFile().prettyPrint(); + case CppActionNames.CPP20_DEPS_SCANNING: + return "Deps scanning for " + getSourceFile().prettyPrint(); + default: + return "Compiling " + getSourceFile().prettyPrint(); + } } /** Returns explicitly listed header files. */ @@ -1429,6 +1484,12 @@ public ActionResult execute(ActionExecutionContext actionExecutionContext) siblingRepositoryLayout); dotDContents = null; // Garbage collect in-memory .d contents. + if (usedCpp20Modules != null) { + discoveredInputs = NestedSetBuilder.stableOrder() + .addAll(usedCpp20Modules) + .addTransitive(discoveredInputs) + .build(); + } updateActionInputs(discoveredInputs); // hdrs_check: This cannot be switched off for C++ build actions, @@ -1492,6 +1553,64 @@ private boolean shouldParseShowIncludes() { return featureConfiguration.isEnabled(CppRuleClasses.PARSE_SHOWINCLUDES); } + /** + * Dynamic dependencies handle for C++20 Modules + * We use restart mechanism to ensure all required module files have been generated. + */ + private Set computeUsedCpp20Modules(ActionExecutionContext actionExecutionContext) + throws ActionExecutionException, InterruptedException { + // if cpp20_module not enable, skip + if (!featureConfiguration.isEnabled(CppRuleClasses.CPP20_MODULE)) { + return Set.of(); + } + // c++20-deps-scanning use source file and header files only + if (Objects.equals(CppActionNames.CPP20_DEPS_SCANNING, actionName)) { + return Set.of(); + } + if (!Cpp20ModuleHelper.isCpp20ModuleCompilationAction(actionName) + && !CppFileTypes.CPP_SOURCE.matches(sourceFile.getExecPath())) { + return Set.of(); + } + Set usedModules = new HashSet<>(); + ImmutableList modulePathList; + try { + modulePathList = FileSystemUtils.readLines( + actionExecutionContext.getInputPath(modmapInputFile), Charset.defaultCharset()); + } catch (IOException e) { + String message = + String.format( + "failed to read modmap input: %s", + modmapInputFile.getExecPathString()); + DetailedExitCode code = createDetailedExitCode(message, Code.MODMAP_INPUT_FILE_READ_FAILURE); + throw new ActionExecutionException(message, this, /*catastrophe=*/ false, code); + } + var pcmFileSet = pcmFiles.toSet(); + Map pcmFileMap = new HashMap<>(pcmFileSet.size()); + for (DerivedArtifact pcmFile : pcmFileSet) { + pcmFileMap.put(pcmFile.getExecPathString(), pcmFile); + } + for (String modulePath : modulePathList) { + if (pcmFileMap.containsKey(modulePath)) { + usedModules.add(pcmFileMap.get(modulePath)); + } + } + // use restart mechanism + var env = actionExecutionContext.getEnvironmentForDiscoveringInputs(); + var skyKeys = Collections2.transform(usedModules, DerivedArtifact::getGeneratingActionKey); + var actionExecutionValues = env.getValuesAndExceptions(skyKeys); + if (env.valuesMissing()) { + return null; + } + for (DerivedArtifact module: usedModules) { + Preconditions.checkState(module.isFileType(CppFileTypes.CPP_MODULE), "Non-module? %s", module); + var skyValue = actionExecutionValues.get(module.getGeneratingActionKey()); + if (skyValue == null) { + return null; + } + } + return usedModules; + } + Spawn createSpawn(Path execRoot, Map clientEnv) throws ActionExecutionException { // Intentionally not adding {@link CppCompileAction#inputsForInvalidation}, those are not needed // for execution. @@ -1549,10 +1668,19 @@ Spawn createSpawn(Path execRoot, Map clientEnv) throws ActionExe } try { + var envOld = getEffectiveEnvironment(clientEnv); + Map environment = Maps.newLinkedHashMapWithExpectedSize(envOld.size() + 1); + environment.putAll(envOld); + if (Objects.equals(CppActionNames.CPP20_DEPS_SCANNING, actionName)) { + // export DEPS_SCANNER_OUTPUT_FILE + // redirect clang-scan-deps output to outputFile + // due to clang-scan-deps has no option like `-o` to specify the output file + environment.put("DEPS_SCANNER_OUTPUT_FILE", getPrimaryOutput().getExecPathString()); + } return new SimpleSpawn( this, ImmutableList.copyOf(getArguments()), - getEffectiveEnvironment(clientEnv), + ImmutableMap.copyOf(environment), executionInfo.buildOrThrow(), /* filesetMappings= */ ImmutableMap.of(), inputs, @@ -1761,6 +1889,8 @@ static String actionNameToMnemonic( : CPP_COMPILE_MNEMONIC + suffix; case CppActionNames.CPP_HEADER_ANALYSIS: return "CppHeaderAnalysis"; + case CppActionNames.CPP20_DEPS_SCANNING: + return "CppDepsScanning"; default: return CPP_COMPILE_MNEMONIC; } diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java index 6234a14b47ccdc..b1fae79b0f0887 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppCompileActionBuilder.java @@ -52,6 +52,9 @@ public final class CppCompileActionBuilder { private Artifact sourceFile; private final NestedSetBuilder mandatoryInputsBuilder; private Artifact outputFile; + private Artifact modmapFile; + private Artifact modmapInputFile; + private NestedSet pcmFiles; private Artifact dwoFile; private Artifact ltoIndexingFile; private Artifact dotdFile; @@ -129,6 +132,9 @@ public CppCompileActionBuilder(CppCompileActionBuilder other) { this.ccToolchain = other.ccToolchain; this.actionName = other.actionName; this.additionalOutputs = other.additionalOutputs; + this.pcmFiles = other.pcmFiles; + this.modmapFile = other.modmapFile; + this.modmapInputFile = other.modmapInputFile; } public CppCompileActionBuilder setSourceFile(Artifact sourceFile) { @@ -287,35 +293,40 @@ public CppCompileAction buildAndVerify() throws UnconfiguredActionConfigExceptio actionName, featureConfiguration, cppConfiguration.useCppCompileHeaderMnemonic())); // Copying the collections is needed to make the builder reusable. - return new CppCompileAction( - owner, - featureConfiguration, - variables, - sourceFile, - configuration, - shareable, - shouldScanIncludes, - usePic, - useHeaderModules, - realMandatoryInputs, - realMandatorySpawnInputs, - getBuiltinIncludeFiles(), - prunableHeaders, - outputFile, - dotdFile, - diagnosticsFile, - gcnoFile, - dwoFile, - ltoIndexingFile, - ccCompilationContext, - coptsFilter, - ImmutableList.copyOf(additionalIncludeScanningRoots), - ImmutableMap.copyOf(executionInfo), - actionName, - cppSemantics, - getBuiltinIncludeDirectories(), - ccToolchain.getGrepIncludes(), - additionalOutputs); + CppCompileAction action; + action = + new CppCompileAction( + owner, + featureConfiguration, + variables, + sourceFile, + configuration, + shareable, + shouldScanIncludes, + usePic, + useHeaderModules, + realMandatoryInputs, + realMandatorySpawnInputs, + getBuiltinIncludeFiles(), + prunableHeaders, + outputFile, + dotdFile, + diagnosticsFile, + gcnoFile, + dwoFile, + ltoIndexingFile, + ccCompilationContext, + coptsFilter, + ImmutableList.copyOf(additionalIncludeScanningRoots), + ImmutableMap.copyOf(executionInfo), + actionName, + cppSemantics, + getBuiltinIncludeDirectories(), + ccToolchain.getGrepIncludes(), + additionalOutputs, + pcmFiles, + modmapInputFile); + return action; } private ImmutableList getBuiltinIncludeFiles() throws EvalException { @@ -354,6 +365,19 @@ NestedSet buildMandatoryInputs() throws EvalException { realMandatoryInputsBuilder.addTransitive(ccCompilationContext.getDeclaredIncludeSrcs()); realMandatoryInputsBuilder.addTransitive(additionalPrunableHeaders); } + if (CppActionNames.CPP20_DEPS_SCANNING.equals(actionName)) { + // scan deps, do nothing + } + else if (Cpp20ModuleHelper.isCpp20ModuleCompilationAction(actionName) + || CppFileTypes.CPP_SOURCE.matches(sourceFile.getExecPath())) { + // C++20 module compile and codegen + // or C++ source compile + if (featureConfiguration.isEnabled(CppRuleClasses.CPP20_MODULE)) { + Preconditions.checkNotNull(modmapFile); + Preconditions.checkNotNull(modmapInputFile); + realMandatoryInputsBuilder.add(modmapFile).add(modmapInputFile); + } + } return realMandatoryInputsBuilder.build(); } @@ -616,6 +640,23 @@ public CppCompileActionBuilder setAdditionalPrunableHeaders( return this; } + @CanIgnoreReturnValue + public CppCompileActionBuilder setModmapFile(Artifact modmapFile) { + this.modmapFile = modmapFile; + return this; + } + + @CanIgnoreReturnValue + public CppCompileActionBuilder setModmapInputFile(Artifact modmapInputFile) { + this.modmapInputFile = modmapInputFile; + return this; + } + + @CanIgnoreReturnValue + public CppCompileActionBuilder setPcmFiles(NestedSet pcmFiles) { + this.pcmFiles = pcmFiles; + return this; + } ImmutableList getBuiltinIncludeDirectories() throws EvalException { return ccToolchain.getBuiltInIncludeDirectories(); } From c25bfb5101c598d0125ac8dbed6c1f8ab9201ac0 Mon Sep 17 00:00:00 2001 From: PikachuHy Date: Mon, 20 May 2024 17:09:52 +0800 Subject: [PATCH 5/6] support C++20 Modules, construct build graph --- .../lib/rules/cpp/CcCompilationHelper.java | 465 +++++++++++++++++- 1 file changed, 463 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java index 9e2a98a550531d..7c0c6baf652012 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcCompilationHelper.java @@ -34,6 +34,7 @@ import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; +import com.google.devtools.build.lib.collect.nestedset.Order; import com.google.devtools.build.lib.packages.BuildType; import com.google.devtools.build.lib.packages.RuleClass.ConfiguredTargetFactory.RuleErrorException; import com.google.devtools.build.lib.rules.cpp.CcCommon.CoptsFilter; @@ -58,6 +59,7 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.Nullable; import net.starlark.java.eval.EvalException; import net.starlark.java.eval.Starlark; @@ -863,8 +865,21 @@ public CompilationInfo compile(RuleContext ruleContext) // Create compile actions (both PIC and no-PIC). try { - CcCompilationOutputs ccOutputs = createCcCompileActions(); - + CcCompilationOutputs ccOutputs; + if (featureConfiguration.isEnabled(CppRuleClasses.CPP20_MODULE)) { + // Handle C++20 Module compile + ccOutputs = createCcCompileActionsWithCpp20Module(); + publicCompilationContext = + CcCompilationContext.createWithCpp20Modules( + publicCompilationContext, + ccOutputs.getPcmFiles(false), + ccOutputs.getPcmFiles(true), + ccOutputs.getModulesInfoFiles(false), + ccOutputs.getModulesInfoFiles(true)); + } else { + // Create compile actions (both PIC and no-PIC). + ccOutputs = createCcCompileActions(); + } if (cppConfiguration.processHeadersInDependencies()) { return new CompilationInfo( CcCompilationContext.createWithExtraHeaderTokens( @@ -990,6 +1005,429 @@ private ImmutableSet getSourceArtifactsByType( return result.build(); } + private CcCompilationOutputs createCcCompileActionsWithCpp20Module() + throws RuleErrorException, EvalException, InterruptedException { + Preconditions.checkState( + featureConfiguration.isEnabled(CppRuleClasses.CPP20_MODULE), + "to use C++20 Modules, the feature cpp20_module must be enabled"); + Preconditions.checkNotNull(ccCompilationContext); + CcCompilationOutputs.Builder result = CcCompilationOutputs.builder(); + // merge module interfaces and ordinary sources + Map sourcesMap = new LinkedHashMap<>(); + sourcesMap.putAll(compilationUnitSources); + sourcesMap.putAll(moduleInterfaceSources); + ImmutableMap outputNameMap = + calculateOutputNameMapByType(sourcesMap, /* prefixDir= */ null); + if (generateNoPicAction) { + createCcCompileActionsWithCpp20ModuleHelper(result, /* usePic= */ false, outputNameMap); + } + if (generatePicAction) { + createCcCompileActionsWithCpp20ModuleHelper(result, /* usePic= */ true, outputNameMap); + } + return result.build(); + } + + private void createCcCompileActionsWithCpp20ModuleHelper( + CcCompilationOutputs.Builder result, + boolean usePic, + ImmutableMap outputNameMap) + throws RuleErrorException, EvalException, InterruptedException { + NestedSetBuilder pcmSetBuilder = NestedSetBuilder.stableOrder(); + NestedSetBuilder ddiSetBuilder = NestedSetBuilder.stableOrder(); + ImmutableList.Builder> pcmAndDdiPairListBuilder = + ImmutableList.builder(); + + // declare .CXXModules.json forward + // all modules information is put here + String modulesInfoFileName = label.getName(); + if (usePic) { + modulesInfoFileName = + CppHelper.getArtifactNameForCategory( + ccToolchain, ArtifactCategory.PIC_FILE, modulesInfoFileName); + } + Artifact modulesInfoFile = + CppHelper.getCompileOutputArtifact( + actionConstructionContext, + label, + CppHelper.getArtifactNameForCategory( + ccToolchain, ArtifactCategory.CPP20_MODULES_INFO, modulesInfoFileName), + configuration); + if (usePic) { + result.addPicModulesInfoFile(modulesInfoFile); + } else { + result.addModulesInfoFile(modulesInfoFile); + } + // the builder list contains + // 1. compile c++ source (e.g. .cc -> .o) compilationUnitSources.size() + // 2. compile c++ module (e.g. .cppm -> .pcm) moduleInterfaceSources.size() + // 3. compile c++ module (e.g. .cpm -> .o) moduleInterfaceSources.size() + List builderList = + new ArrayList<>(compilationUnitSources.size() + 2 * moduleInterfaceSources.size()); + + createCppCompileActionBuilder( + result, + usePic, + outputNameMap, + pcmSetBuilder, + ddiSetBuilder, + pcmAndDdiPairListBuilder, + modulesInfoFile, + builderList, + compilationUnitSources, + /* isModuleInterface= */ false); + createCppCompileActionBuilder( + result, + usePic, + outputNameMap, + pcmSetBuilder, + ddiSetBuilder, + pcmAndDdiPairListBuilder, + modulesInfoFile, + builderList, + moduleInterfaceSources, + /* isModuleInterface= */ true); + // create .CXXModules.json + var modulesInfoAction = + new Cpp20ModulesInfoAction( + actionConstructionContext.getActionOwner(), + ddiSetBuilder.build(), + pcmAndDdiPairListBuilder.build(), + ccCompilationContext.getModulesInfoFiles(usePic), + modulesInfoFile); + actionConstructionContext.registerAction(modulesInfoAction); + var depPcmFiles = ccCompilationContext.getPcmFiles(usePic); + if (depPcmFiles != null) { + pcmSetBuilder.addTransitive(depPcmFiles); + } + var pcmFiles = pcmSetBuilder.build(); + for (CppCompileActionBuilder builder : builderList) { + builder.setPcmFiles(pcmFiles); + CppCompileAction cppCompileAction = builder.buildOrThrowRuleError(ruleErrorConsumer); + actionConstructionContext.registerAction(cppCompileAction); + } + } + + private void createCppCompileActionBuilder( + CcCompilationOutputs.Builder result, + boolean usePic, + ImmutableMap outputNameMap, + NestedSetBuilder pcmSetBuilder, + NestedSetBuilder ddiSetBuilder, + ImmutableList.Builder> pcmAndDdiPairListBuilder, + Artifact modulesInfoFile, + List builderList, + Map sourceMap, + boolean isModuleInterface) + throws RuleErrorException, EvalException, InterruptedException { + for (CppSource source : sourceMap.values()) { + CppCompileActionBuilder builder; + Label sourceLabel = source.getLabel(); + Artifact sourceArtifact = source.getSource(); + PathFragment sourcePath = sourceArtifact.getExecPath(); + String outputName = outputNameMap.get(sourceArtifact); + if (usePic) { + outputName = + CppHelper.getArtifactNameForCategory( + ccToolchain, ArtifactCategory.PIC_FILE, outputName); + } + builder = initializeCompileAction(sourceArtifact); + // declare .d file forward + // reuse .d file produced by scan dependencies (c++20-deps-scanning) + Artifact dotdFile; + if (builder.dotdFilesEnabled() + && builder.useDotdFile(sourceArtifact)) { + String dotdFileName = CppHelper.getArtifactNameForCategory( + ccToolchain, ArtifactCategory.INCLUDED_FILE_LIST, outputName); + dotdFile = + CppHelper.getCompileOutputArtifact( + actionConstructionContext, label, dotdFileName, configuration); + } else { + dotdFile = null; + } + // scan dependencies + // both module source files and ordinary c++ source files are needed + // others skip (e.g. C or assembler) + if (isModuleInterface || CppFileTypes.CPP_SOURCE.matches(sourcePath)) { + // dependencies information are put in .ddi file + // the format is https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p1689r5.html + Artifact ddiFile = + CppHelper.getCompileOutputArtifact( + actionConstructionContext, + label, + CppHelper.getArtifactNameForCategory( + ccToolchain, ArtifactCategory.CPP20_MODULE_DEP, outputName), + configuration); + ddiSetBuilder.add(ddiFile); + // all -fmodule-file== flags are put in .modmap file + var modmapFile = + CppHelper.getCompileOutputArtifact( + actionConstructionContext, + label, + CppHelper.getArtifactNameForCategory( + ccToolchain, ArtifactCategory.CPP20_MODULE_MAP, outputName), + configuration); + // all path/to/bmi are put in .modmap.input file, + // which is convenient to get all bmi in CppCompileAction + var modmapInputFile = + CppHelper.getCompileOutputArtifact( + actionConstructionContext, + label, + CppHelper.getArtifactNameForCategory( + ccToolchain, ArtifactCategory.CPP20_MODULE_MAP_INPUT, outputName), + configuration); + Cpp20ModuleDepMapAction cpp20ModuleDepMapAction = + new Cpp20ModuleDepMapAction( + actionConstructionContext.getActionOwner(), + ddiFile, + modulesInfoFile, + modmapFile, + modmapInputFile); + actionConstructionContext.registerAction(cpp20ModuleDepMapAction); + + if (isModuleInterface) { + // use two-phase compilation + // e.g. + // 1. .cppm -> .pcm + // 2. .pcm -> .o + // echo phase need one builder + // so the builder will change + // and the new builder return + builder = + createCpp20ModuleCompileActionBuilder( + builder, + sourceLabel, + outputName, + result, + sourceArtifact, + usePic, + pcmSetBuilder, + pcmAndDdiPairListBuilder, + builderList, + ddiFile, + dotdFile, + modmapFile, + modmapInputFile); + } else { + createCompileActionBuilder( + builder, + sourceLabel, + outputName, + result, + sourceArtifact, + usePic, + ddiFile, + dotdFile, + modmapFile, + /* isCpp20Module= */ false); + } + builder.setModmapFile(modmapFile); + builder.setModmapInputFile(modmapInputFile); + } else { + createCompileActionBuilder( + builder, + sourceLabel, + outputName, + result, + sourceArtifact, + usePic, + /* ddiFile= */ null, + dotdFile, + /* modmapFile= */ null, + /* isCpp20Module= */ false); + } + semantics.finalizeCompileActionBuilder( + configuration, featureConfiguration, builder); + builderList.add(builder); + } + } + + private CppCompileActionBuilder createCpp20ModuleCompileActionBuilder( + CppCompileActionBuilder moduleCompileBuilder, + Label sourceLabel, + String outputName, + CcCompilationOutputs.Builder result, + Artifact sourceArtifact, + boolean usePic, + NestedSetBuilder pcmSetBuilder, + ImmutableList.Builder> pcmAndDdiPairListBuilder, + List builderList, + Artifact ddiFile, + Artifact dotdFile, + Artifact modmapFile, + Artifact modmapInputFile) + throws RuleErrorException, EvalException, InterruptedException { + Artifact diagnosticsFile; + var outputCategory = ArtifactCategory.CPP_MODULE; + if (moduleCompileBuilder.serializedDiagnosticsFilesEnabled()) { + String diagnosticsFileName = + CppHelper.getDiagnosticsFileName(ccToolchain, outputCategory, outputName); + diagnosticsFile = + CppHelper.getCompileOutputArtifact( + actionConstructionContext, label, diagnosticsFileName, configuration); + } else { + diagnosticsFile = null; + } + String pcmFileName = + CppHelper.getArtifactNameForCategory(ccToolchain, outputCategory, outputName); + PathFragment objectDir = + CppHelper.getObjDirectory(label, configuration.isSiblingRepositoryLayout()); + // the DerivedArtifact type is required when using restart mechanism + Artifact.DerivedArtifact pcmFile = + actionConstructionContext.getDerivedArtifact( + objectDir.getRelative(pcmFileName), + configuration.getBinDirectory(label.getRepository())); + pcmSetBuilder.add(pcmFile); + pcmAndDdiPairListBuilder.add(Pair.of(pcmFile, ddiFile)); + if (usePic) { + result.addPicPcmFile(pcmFile); + } else { + result.addPcmFile(pcmFile); + } + moduleCompileBuilder.setOutputs(pcmFile, dotdFile, diagnosticsFile); + moduleCompileBuilder.setModmapFile(modmapFile); + moduleCompileBuilder.setModmapInputFile(modmapInputFile); + moduleCompileBuilder.setActionName(CppActionNames.CPP20_MODULE_COMPILE); + PathFragment ccRelativeName = sourceArtifact.getRootRelativePath(); + var variables = + setupCompileBuildVariables( + moduleCompileBuilder, + sourceLabel, + usePic, + /* needsFdoBuildVariables= */ ccRelativeName != null, + cppModuleMap, + /* enableCoverage= */ false, + null, + false, + null, + null, + /* additionalBuildVariables= */ ImmutableMap.of( + CompileBuildVariables.CPP20_MODMAP_FILE.getVariableName(), + modmapFile.getExecPathString())); + moduleCompileBuilder.setVariables(variables); + semantics.finalizeCompileActionBuilder( + configuration, featureConfiguration, moduleCompileBuilder); + builderList.add(moduleCompileBuilder); + createScanDepsAction(variables, sourceArtifact, ddiFile, dotdFile); + // after compile module interface file to pcm file + // we compile pcm file to object file + var moduleCodegenBuilder = initializeCompileAction(pcmFile); + moduleCodegenBuilder.addMandatoryInputs(List.of(sourceArtifact)); + createCompileActionBuilder( + moduleCodegenBuilder, + sourceLabel, + outputName, + result, + pcmFile, + usePic, + /* ddiFile= */ null, + dotdFile, + modmapFile, + /* isCpp20Module= */ true); + return moduleCodegenBuilder; + } + + private void createCompileActionBuilder( + CppCompileActionBuilder builder, + Label sourceLabel, + String outputName, + CcCompilationOutputs.Builder result, + Artifact sourceArtifact, + boolean usePic, + Artifact ddiFile, + Artifact dotdFile, + Artifact modmapFile, + boolean isCpp20Module) + throws RuleErrorException, EvalException, InterruptedException { + if (isCpp20Module) { + builder.setActionName(CppActionNames.CPP20_MODULE_CODEGEN); + } else { + // do nothing + // fallback to default action name + } + boolean enableCoverage = isCodeCoverageEnabled; + boolean generateDwo = + ccToolchain.shouldCreatePerObjectDebugInfo(featureConfiguration, cppConfiguration); + boolean bitcodeOutput = + featureConfiguration.isEnabled(CppRuleClasses.THIN_LTO) + && CppFileTypes.LTO_SOURCE.matches(sourceArtifact.getFilename()); + PathFragment ccRelativeName = sourceArtifact.getRootRelativePath(); + + Artifact outputFile = CppHelper.getCompileOutputArtifact( + actionConstructionContext, + label, + CppHelper.getArtifactNameForCategory(ccToolchain, ArtifactCategory.OBJECT_FILE, outputName), + configuration + ); + Artifact diagnosticsFile; + if (builder.serializedDiagnosticsFilesEnabled()) { + String diagnosticsFileName = + CppHelper.getDiagnosticsFileName(ccToolchain, ArtifactCategory.OBJECT_FILE, outputName); + diagnosticsFile = + CppHelper.getCompileOutputArtifact( + actionConstructionContext, label, diagnosticsFileName, configuration); + } + else { + diagnosticsFile = null; + } + builder.setOutputs(outputFile, dotdFile, diagnosticsFile); + String gcnoFileName = + CppHelper.getArtifactNameForCategory( + ccToolchain, ArtifactCategory.COVERAGE_DATA_FILE, outputName); + Artifact gcnoFile = + enableCoverage + ? CppHelper.getCompileOutputArtifact( + actionConstructionContext, label, gcnoFileName, configuration) + : null; + Artifact dwoFile = generateDwo && !bitcodeOutput ? getDwoFile(outputFile) : null; + Artifact ltoIndexingFile = bitcodeOutput ? getLtoIndexingFile(outputFile) : null; + // compile arguments are produced by the same CompileBuildVariables + var variables = + setupCompileBuildVariables( + builder, + sourceLabel, + /* usePic= */ usePic, + /* needsFdoBuildVariables= */ ccRelativeName != null, + cppModuleMap, + /* enableCoverage= */ false, + gcnoFile, + generateDwo, + dwoFile, + ltoIndexingFile, + /* additionalBuildVariables= */ modmapFile == null + ? ImmutableMap.of() + : ImmutableMap.of( + CompileBuildVariables.CPP20_MODMAP_FILE.getVariableName(), + modmapFile.getExecPathString())); + + if (ddiFile != null) { + createScanDepsAction(variables, sourceArtifact, ddiFile, dotdFile); + } + builder.setVariables(variables); + builder.setGcnoFile(gcnoFile); + builder.setDwoFile(dwoFile); + builder.setLtoIndexingFile(ltoIndexingFile); + + result.addTemps( + createTempsActions( + sourceArtifact, sourceLabel, outputName, builder, usePic, ccRelativeName)); + if (usePic) { + result.addPicObjectFile(outputFile); + if (dwoFile != null) { + result.addPicDwoFile(dwoFile); + } + if (gcnoFile != null) { + result.addPicGcnoFile(gcnoFile); + } + } else { + result.addObjectFile(outputFile); + if (dwoFile != null) { + result.addDwoFile(dwoFile); + } + if (gcnoFile != null) { + result.addGcnoFile(gcnoFile); + } + } + } /** * Constructs the C++ compiler actions. It generally creates one action for every specified source * file. It takes into account coverage, and PIC, in addition to using the settings specified on @@ -997,6 +1435,9 @@ private ImmutableSet getSourceArtifactsByType( */ private CcCompilationOutputs createCcCompileActions() throws RuleErrorException, EvalException, InterruptedException { + Preconditions.checkState( + moduleInterfaceSources.isEmpty(), + "to use C++20 Modules, the feature cpp20_module must be enabled"); CcCompilationOutputs.Builder result = CcCompilationOutputs.builder(); Preconditions.checkNotNull(ccCompilationContext); @@ -1516,6 +1957,22 @@ private void createHeaderAction( result.addHeaderTokenFile(tokenFile); } + private void createScanDepsAction( + CcToolchainVariables variables, Artifact sourceArtifact, Artifact ddiFile, Artifact dotdFile) + throws RuleErrorException, EvalException { + var scanDepsBuilder = initializeCompileAction(sourceArtifact); + scanDepsBuilder.setActionName(CppActionNames.CPP20_DEPS_SCANNING); + scanDepsBuilder.setOutputs(ddiFile, dotdFile, null); + // only c++20-deps-scanning add .d file + var buildVariables = CcToolchainVariables.builder(variables) + .addStringVariable(CompileBuildVariables.DEPENDENCY_FILE.getVariableName(), dotdFile.getExecPathString()); + scanDepsBuilder.setVariables(buildVariables.build()); + semantics.finalizeCompileActionBuilder( + configuration, featureConfiguration, scanDepsBuilder); + var scanDepsAction = scanDepsBuilder.buildOrThrowRuleError(ruleErrorConsumer); + actionConstructionContext.registerAction(scanDepsAction); + } + private ImmutableList createModuleAction( CcCompilationOutputs.Builder result, CppModuleMap cppModuleMap) throws RuleErrorException, EvalException, InterruptedException { @@ -1865,4 +2322,8 @@ private ImmutableList createTempsActions( return ImmutableList.of(dAction.getPrimaryOutput(), sdAction.getPrimaryOutput()); } + private NestedSet getModuleInterfaceSourceFiles() { + var moduleInterfaceFiles = moduleInterfaceSources.values().stream().map(CppSource::getSource).collect(Collectors.toList()); + return NestedSetBuilder.wrap(Order.STABLE_ORDER, moduleInterfaceFiles); + } } From f07e7e28bac15a4387782b2f4c1f0097b895df50 Mon Sep 17 00:00:00 2001 From: PikachuHy Date: Mon, 20 May 2024 17:10:21 +0800 Subject: [PATCH 6/6] support C++20 Modules, add tests --- .../google/devtools/build/lib/rules/cpp/BUILD | 16 ++ .../lib/rules/cpp/Cpp20ModuleCompileTest.java | 158 ++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 src/test/java/com/google/devtools/build/lib/rules/cpp/Cpp20ModuleCompileTest.java diff --git a/src/test/java/com/google/devtools/build/lib/rules/cpp/BUILD b/src/test/java/com/google/devtools/build/lib/rules/cpp/BUILD index a4eece61b81b1e..9ea9c7d75777c3 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/cpp/BUILD +++ b/src/test/java/com/google/devtools/build/lib/rules/cpp/BUILD @@ -742,3 +742,19 @@ java_test( "//third_party:truth", ], ) + +java_test( + name = "Cpp20ModuleCompileTest", + srcs = ["Cpp20ModuleCompileTest.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib/actions:artifacts", + "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster", + "//src/main/java/com/google/devtools/build/lib/analysis:configured_target", + "//src/main/java/com/google/devtools/build/lib/rules/cpp", + "//src/test/java/com/google/devtools/build/lib/actions/util", + "//src/test/java/com/google/devtools/build/lib/analysis/util", + "//src/test/java/com/google/devtools/build/lib/packages:testutil", + "//third_party:junit4", + "@io_bazel//third_party:truth", + ], +) diff --git a/src/test/java/com/google/devtools/build/lib/rules/cpp/Cpp20ModuleCompileTest.java b/src/test/java/com/google/devtools/build/lib/rules/cpp/Cpp20ModuleCompileTest.java new file mode 100644 index 00000000000000..b80043a985fea0 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/rules/cpp/Cpp20ModuleCompileTest.java @@ -0,0 +1,158 @@ +package com.google.devtools.build.lib.rules.cpp; +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.actions.util.ActionsTestUtil.baseArtifactNames; + +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.OutputGroupInfo; +import com.google.devtools.build.lib.analysis.actions.SpawnAction; +import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; +import com.google.devtools.build.lib.packages.util.Crosstool; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class Cpp20ModuleCompileTest extends BuildViewTestCase { + private void enableCpp20Module() throws Exception { + getAnalysisMock().ccSupport().setupCcToolchainConfig( + mockToolsConfig, Crosstool.CcToolchainConfig.builder().withFeatures( + CppRuleClasses.CPP20_MODULE)); + } + // if we use module_interfaces + // we need to enable cpp20_module feature + @Test + public void testCpp20ModuleFeatureNotEnabled1() throws Exception { + reporter.removeHandler(failFastHandler); + scratch.file("foo/BUILD", + "cc_library(name='foo', module_interfaces=['foo.cppm'])"); + scratch.file("foo/foo.cppm", "foo"); + getConfiguredTarget("//foo:foo"); + assertContainsEvent( + "to use C++20 Modules, the feature cpp20_module must be enabled"); + } + + // when use C++20 Modules, the compile model changed + // to keep backward-compatibility + // the feature cpp20_module is not enabled by default + // we need to enable the feature in rule attr features + @Test + public void testCpp20ModuleFeatureNotEnabled2() throws Exception { + enableCpp20Module(); + reporter.removeHandler(failFastHandler); + scratch.file("foo/BUILD", + "cc_library(name='foo', module_interfaces=['foo.cppm'])"); + scratch.file("foo/foo.cppm", "foo"); + getConfiguredTarget("//foo:foo"); + assertContainsEvent( + "to use C++20 Modules, the feature cpp20_module must be enabled"); + } + + // add the feature cpp20_module to toolchain + // add the feature cpp20_module to rule attr features + // now we can play with C++20 Modules + @Test + public void testCpp20ModuleFeatureEnabled() throws Exception { + enableCpp20Module(); + reporter.removeHandler(failFastHandler); + scratch.file( + "foo/BUILD", + "cc_library(name='foo', module_interfaces=['foo.cppm'], features=['cpp20_module'])"); + scratch.file("foo/foo.cppm", "foo"); + getConfiguredTarget("//foo:foo"); + assertDoesNotContainEvent( + "to use C++20 Modules, the feature cpp20_module must be enabled"); + } + + // module_interfaces cannot have two same .cc files + @Test + public void testSameCcFileTwice1() throws Exception { + enableCpp20Module(); + scratch.file( + "a/BUILD", + "cc_library(name='a', module_interfaces=['a1', 'a2'], features=['cpp20_module'])", + "filegroup(name='a1', srcs=['a.cc'])", + "filegroup(name='a2', srcs=['a.cc'])"); + reporter.removeHandler(failFastHandler); + getConfiguredTarget("//a:a"); + assertContainsEvent("Artifact 'a/a.cc' is duplicated"); + } + + // if one file has already in srcs, + // it can not put to module_interfaces + // due to module_interfaces is a special srcs that produce bmi files + @Test + public void testSameCcFileTwice2() throws Exception { + enableCpp20Module(); + scratch.file( + "a/BUILD", + "cc_library(name='a', srcs=['a1'], module_interfaces=['a2'], features=['cpp20_module'])", + "filegroup(name='a1', srcs=['a.cc'])", + "filegroup(name='a2', srcs=['a.cc'])"); + reporter.removeHandler(failFastHandler); + getConfiguredTarget("//a:a"); + assertContainsEvent("Artifact 'a/a.cc' is duplicated"); + } + @Test + public void testObjectFile() throws Exception { + enableCpp20Module(); + useConfiguration("--cpu=k8"); + ConfiguredTarget archiveInSrcsTest = scratchConfiguredTarget( + "cc_in_module_interfaces", "cc_in_module_interfaces_test", + "cc_test(name = 'cc_in_module_interfaces_test',", + " module_interfaces = ['foo.cc'],", + " features = ['cpp20_module'],", ")"); + List artifactNames = + baseArtifactNames(getLinkerInputs(archiveInSrcsTest)); + assertThat(artifactNames).contains("foo.o"); + } + + private Iterable getLinkerInputs(ConfiguredTarget target) { + Artifact executable = getExecutable(target); + SpawnAction linkAction = (SpawnAction)getGeneratingAction(executable); + return linkAction.getInputs().toList(); + } + + // use --precompile to compile .cppm to .pcm + @Test + public void testModuleCompileOption() throws Exception { + enableCpp20Module(); + useConfiguration("--cpu=k8"); + scratch.file("a/BUILD", "cc_library(", "name='foo', ", + "features = ['cpp20_module'], ", + "module_interfaces=['foo.cppm']", ")"); + ConfiguredTarget target = getConfiguredTarget("//a:foo"); + List compilationSteps = + actionsTestUtil().findTransitivePrerequisitesOf( + getFilesToBuild(target).toList().get(0), CppCompileAction.class); + assertThat(compilationSteps.get(1).getArguments()).contains("--precompile"); + } + + @Test + public void testModuleCompileOnly() throws Exception { + enableCpp20Module(); + useConfiguration("--cpu=k8"); + scratch.file("a/BUILD", "cc_library(", "name='foo', ", + "features = ['cpp20_module'], ", + "module_interfaces=['foo.cppm']", ")"); + ConfiguredTarget target = getConfiguredTarget("//a:foo"); + var outputs = + getOutputGroup(target, OutputGroupInfo.FILES_TO_COMPILE).toList(); + assertThat(outputs).isNotEmpty(); + + assertThat(getArtifactByExecPathSuffix(target, "/foo.o")).isNotNull(); + assertThat(getArtifactByExecPathSuffix(target, "/foo.pcm.o")).isNull(); + } + + protected static Artifact getArtifactByExecPathSuffix(ConfiguredTarget target, + String path) { + for (Artifact artifact : + getOutputGroup(target, OutputGroupInfo.FILES_TO_COMPILE).toList()) { + if (artifact.getExecPathString().endsWith(path)) { + return artifact; + } + } + return null; + } +}