From 6f287f2dd8dc157bfe513586d0bded59b1f33a7a Mon Sep 17 00:00:00 2001 From: Mike Bland Date: Thu, 24 Oct 2024 03:29:10 -0400 Subject: [PATCH] Update repo name handling for Bzlmod compatibility (#1621) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add repo name macros for Bzlmod compatibility Part of #1482. These helper macros fix various repository name related errors when building under Bzlmod, while remaining backwards compatible with `WORKSPACE`. Without Bzlmod, these macros return original repo names. With Bzlmod enabled, they avoid the problems described below. I've prepared a change for bazelbuild/bazel-skylib containing these macros with full unit tests. If the maintainers accept that change, we can bump our bazel_skylib version to use the macros from there, and remove the `bzlmod.bzl` file. --- Also includes a couple of other minor touch-ups: - Updated the `runtime_deps` attribute in `repositories()` to add the Scala version suffix, just like `deps`. - Added a `fail()` message to `repositories()` to make it more clear which Scala version dictionary is missing an artifact. - Removed unnecessary internal uses of the `@io_bazel_rules_scala` repo name, applying `Label()` where necessary. - Updated the construction of `dep_providers` in `_default_dep_providers` to remove the repo name, reduce duplication, and make the upcoming toolchain update slightly cleaner. --- Before this change, `repositories()` would originally emit `BUILD` targets including canonical repo names: ```py scala_import( name = "_main~scala_deps~io_bazel_rules_scala_scala_compiler", jars = ["scala-compiler-2.12.18.jar"], ) ``` resulting in errors like: ```txt ERROR: .../_main~_repo_rules~io_bazel_rules_scala/scala/BUILD: no such target '@@_main~scala_deps~io_bazel_rules_scala_scala_compiler//:io_bazel_rules_scala_scala_compiler': target 'io_bazel_rules_scala_scala_compiler' not declared in package '' defined by .../_main~scala_deps~io_bazel_rules_scala_scala_compiler/BUILD and referenced by '@@_main~_repo_rules~io_bazel_rules_scala//scala:default_toolchain_scala_compile_classpath_provider' ``` --- Attaching resources from custom repos to targets under Bzlmod, like in `scalarules.test.resources.ScalaLibResourcesFromExternalDepTest`, would break with: ```txt $ bazel test //test/src/main/scala/scalarules/test/resources:all 1) Scala library depending on resources from external resource-only jar::allow to load resources(scalarules.test.resources.ScalaLibResourcesFromExternalDepTest) java.lang.NullPointerException at scalarules.test.resources.ScalaLibResourcesFromExternalDepTest.get(ScalaLibResourcesFromExternalDepTest.scala:17) at scalarules.test.resources.ScalaLibResourcesFromExternalDepTest.$anonfun$new$3(ScalaLibResourcesFromExternalDepTest.scala:11) at scalarules.test.resources.ScalaLibResourcesFromExternalDepTest.$anonfun$new$2(ScalaLibResourcesFromExternalDepTest.scala:11) ``` `_update_external_target_path` in `resources.bzl` fixes this problem. --- Fixes `test_strict_deps_filter_included_target` from `test/shell/test_strict_dependency.sh` when run under Bzlmod. The `dependency_tracking_strict_deps_patterns` attribute of //test_expect_failure/missing_direct_deps/filtering:plus_one_strict_deps_filter_a_impl contains patterns starting with `@//`. However, in `_phase_dependency()` from `scala/private/phases/phase_dependency.bzl`, these patterns were compared against a stringified Label. Under Bazel < 7.1.0, this works for root target Labels. Under Bazel >= 7.1.0, this breaks for root target Labels under Bzlmod, which start with `@@//`. `adjust_main_repo_prefix` updates the patterns accordingly in `_partition_patterns` from `scala_toolchain.bzl`. `apparent_repo_label_string` makes `_phase_dependency()` more resilient when comparing target Labels against filters containing external apparent repo names. --- Fixes the `alias` targets generated by `_jvm_import_external` from `scala_maven_import_external.bzl` by setting the `target` to the correct apparent repo name. Added `apparent_repo_name(repository_ctx.name)` to `_jvm_import_external` to avoid this familiar error when running `dt_patches/test_dt_patches` tests: ```txt $ bazel build //... ERROR: .../external/_main~compiler_source_repos~scala_reflect/BUILD: no such target '@@_main~compiler_source_repos~scala_reflect//:scala_reflect': target 'scala_reflect' not declared in package '' defined by .../external/_main~compiler_source_repos~scala_reflect/BUILD ERROR: .../dt_patches/test_dt_patches/BUILD:11:22: no such target '@@_main~compiler_source_repos~scala_reflect//:scala_reflect': target 'scala_reflect' not declared in package '' defined by .../external/_main~compiler_source_repos~scala_reflect/BUILD and referenced by '//:dt_scala_toolchain_scala_compile_classpath_provider' ERROR: Analysis of target '//:dt_scala_toolchain_scala_compile_classpath_provider' failed; build aborted: Analysis failed ``` --- As for why we need these macros, we can't rely on hacking the specific canonical repository name format: > Repos generated by extensions have canonical names in the form of > `module_repo_canonical_name+extension_name+repo_name`. Note that the > canonical name format is not an API you should depend on — it's > subject to change at any time. > > - https://bazel.build/external/extension#repository_names_and_visibility The change to no longer encode module versions in canonical repo names in Bazel 7.1.0 is a recent example of Bazel maintainers altering the format: - https://github.com/bazelbuild/bazel/pull/21316 And the maintainers recently replaced the `~` delimiter with `+` in the upcoming Bazel 8 release due to build performance issues on Windows: - https://github.com/bazelbuild/bazel/issues/22865 The core `apparent_repo_name` function assumes the only valid repo name characters are letters, numbers, '_', '-', and '.'. This is valid so long as this condition holds: - https://github.com/bazelbuild/bazel/blob/7.3.2/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryName.java#L159-L162 * Bazel 5 updates from bazelbuild/bazel-skylib#548 I don't think we really need them, as I don't think we support Bazel 5. But better to have and not need, I guess. * Fix _MAIN_REPO_PREFIX in bzlmod.bzl Backported from bazelbuild/bazel-skylib#548, whereby I realized `Label(//:all)` would not produce the main repo prefix when imported into other repos. * Expand dependency tracking patterns using Label Replaced `apparent_repo_label_string` and `adjust_main_repo_prefix` based on an idea from @fmeum during his review of bazelbuild/bazel-skylib#548. Added `_expand_patterns`, which uses `native.package_relative_label` to expand the `dependency_tracking_*_deps_patterns` attributes to full, correct `Label` strings. All `test/shell/test_{strict,unused}_dependency.sh` test cases pass. * Fix `scala_toolchains` lint error * Use `apparent_repo_name` only on `repository_ctx` Repurposed `apparent_repo_name` to only work for `repository_ctx` objects, not repository or module names in general. Removed `generated_rule_name` from `repositories.bzl`, since it's no longer necessary. Technically we could eliminate `apparent_repo_name` by making `generated_rule_name` a mandatory attribute of `_jvm_import_external`. However, this feels ultimately clunky and unnecessary. This update to `apparent_repo_name` required removing `_update_external_target_path` and updating `_target_path_by_default_prefixes` to remove `external/` prefixes. This represents a breaking change for files referencing `external/` paths, but the quick fix is to delete that prefix in the code. This matches the behavior in the same function regarding `resources/` and `java/` prefixes. * Update `_jvm_import_external` JAR alias target Changes the target for the `//jar:jar` alias in `_jvm_import_scala` to the top level repo target directly (`//:%s`) instead of the repo name (`@%s`). Functionally the same, but seems a bit cleaner than referencing the target as though it were external to the repo. * Fix typo in apparent_repo_name comment Caught by @simuons in #1621, thankfully. I hate sloppy comments, and hate it more when I write them! --- scala/private/macros/bzlmod.bzl | 21 ++++++++ scala/private/phases/phase_dependency.bzl | 6 +-- scala/private/resources.bzl | 7 +++ scala/scala_maven_import_external.bzl | 10 ++-- scala/scala_toolchain.bzl | 51 ++++++++++++++----- ...ScalaLibResourcesFromExternalDepTest.scala | 2 +- ...alaLibResourcesFromExternalScalaTest.scala | 2 +- third_party/repositories/repositories.bzl | 19 +++++-- 8 files changed, 92 insertions(+), 26 deletions(-) create mode 100644 scala/private/macros/bzlmod.bzl diff --git a/scala/private/macros/bzlmod.bzl b/scala/private/macros/bzlmod.bzl new file mode 100644 index 000000000..db8e8385f --- /dev/null +++ b/scala/private/macros/bzlmod.bzl @@ -0,0 +1,21 @@ +"""Utilities for working with Bazel modules""" + +def apparent_repo_name(repository_ctx): + """Generates a repository's apparent name from a repository_ctx object. + + Args: + repository_ctx: a repository_ctx object + + Returns: + An apparent repo name derived from repository_ctx.name + """ + repo_name = repository_ctx.name + + # Based on this pattern from the Bazel source: + # com.google.devtools.build.lib.cmdline.RepositoryName.VALID_REPO_NAME + for i in range(len(repo_name) - 1, -1, -1): + c = repo_name[i] + if not (c.isalnum() or c in "_-."): + return repo_name[i + 1:] + + return repo_name diff --git a/scala/private/phases/phase_dependency.bzl b/scala/private/phases/phase_dependency.bzl index d8ae91e7e..ccf629270 100644 --- a/scala/private/phases/phase_dependency.bzl +++ b/scala/private/phases/phase_dependency.bzl @@ -1,13 +1,13 @@ # Gathers information about dependency mode and analysis load( - "@io_bazel_rules_scala//scala/private:dependency.bzl", + "//scala/private:dependency.bzl", "get_compiler_deps_mode", "get_strict_deps_mode", "new_dependency_info", ) load( - "@io_bazel_rules_scala//scala/private:paths.bzl", + "//scala/private:paths.bzl", _get_files_with_extension = "get_files_with_extension", _java_extension = "java_extension", ) @@ -35,7 +35,7 @@ def _phase_dependency( p, unused_deps_always_off, strict_deps_always_off): - toolchain = ctx.toolchains["@io_bazel_rules_scala//scala:toolchain_type"] + toolchain = ctx.toolchains[Label("//scala:toolchain_type")] target_label = str(ctx.label) diff --git a/scala/private/resources.bzl b/scala/private/resources.bzl index 1db14a822..eaaf12bcb 100644 --- a/scala/private/resources.bzl +++ b/scala/private/resources.bzl @@ -44,6 +44,13 @@ def _target_path_by_default_prefixes(resource): if rel_path: return rel_path + # Looking inside an external repository. Trim off both the "external/" and + # the repository name components. Especially important under Bzlmod, because + # the canonical repository name may change between versions. + (dir_1, dir_2, rel_path) = path.partition("external/") + if rel_path: + return rel_path[rel_path.index("/"):] + # Both short_path and path have quirks we wish to avoid, in short_path there are times where # it is prefixed by `../` instead of `external/`. And in .path it will instead return the entire # bazel-out/... path, which is also wanting to be avoided. So instead, we return the short-path if diff --git a/scala/scala_maven_import_external.bzl b/scala/scala_maven_import_external.bzl index e69388ced..1a7bf8145 100644 --- a/scala/scala_maven_import_external.bzl +++ b/scala/scala_maven_import_external.bzl @@ -35,6 +35,7 @@ the following macros are defined below that utilize jvm_import_external: - java_import_external - to demonstrate that the original functionality of `java_import_external` stayed intact. """ +load("//scala/private:macros/bzlmod.bzl", "apparent_repo_name") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "read_netrc", "read_user_netrc", "use_netrc") # https://github.com/bazelbuild/bazel/issues/13709#issuecomment-1336699672 @@ -64,19 +65,20 @@ def _jvm_import_external(repository_ctx): if (repository_ctx.attr.generated_linkable_rule_name and not repository_ctx.attr.neverlink): fail("Only use generated_linkable_rule_name if neverlink is set") - name = repository_ctx.attr.generated_rule_name or repository_ctx.name + repo_name = apparent_repo_name(repository_ctx) + name = repository_ctx.attr.generated_rule_name or repo_name urls = repository_ctx.attr.jar_urls if repository_ctx.attr.jar_sha256: print("'jar_sha256' is deprecated. Please use 'artifact_sha256'") sha = repository_ctx.attr.jar_sha256 or repository_ctx.attr.artifact_sha256 - path = repository_ctx.name + ".jar" + path = repo_name + ".jar" for url in urls: if url.endswith(".jar"): path = url[url.rindex("/") + 1:] break srcurls = repository_ctx.attr.srcjar_urls srcsha = repository_ctx.attr.srcjar_sha256 - srcpath = repository_ctx.name + "-src.jar" if srcurls else "" + srcpath = repo_name + "-src.jar" if srcurls else "" coordinates = repository_ctx.attr.coordinates for url in srcurls: if url.endswith(".jar"): @@ -136,7 +138,7 @@ def _jvm_import_external(repository_ctx): "", "alias(", " name = \"jar\",", - " actual = \"@%s\"," % repository_ctx.name, + " actual = \"//:%s\"," % repo_name, ")", "", ])) diff --git a/scala/scala_toolchain.bzl b/scala/scala_toolchain.bzl index a09e03f73..60da0783a 100644 --- a/scala/scala_toolchain.bzl +++ b/scala/scala_toolchain.bzl @@ -1,7 +1,4 @@ -load( - "@io_bazel_rules_scala//scala:providers.bzl", - _DepsInfo = "DepsInfo", -) +load("//scala:providers.bzl", _DepsInfo = "DepsInfo") load( "@io_bazel_rules_scala_config//:config.bzl", "ENABLE_COMPILER_DEPENDENCY_TRACKING", @@ -108,17 +105,17 @@ def _scala_toolchain_impl(ctx): def _default_dep_providers(): dep_providers = [ - "@io_bazel_rules_scala//scala:scala_xml_provider", - "@io_bazel_rules_scala//scala:parser_combinators_provider", - "@io_bazel_rules_scala//scala:scala_compile_classpath_provider", - "@io_bazel_rules_scala//scala:scala_library_classpath_provider", - "@io_bazel_rules_scala//scala:scala_macro_classpath_provider", + "scala_xml", + "parser_combinators", + "scala_compile_classpath", + "scala_library_classpath", + "scala_macro_classpath", ] - if SCALA_MAJOR_VERSION.startswith("2"): - dep_providers.append("@io_bazel_rules_scala//scala:semanticdb_provider") - return dep_providers + if SCALA_MAJOR_VERSION.startswith("2."): + dep_providers.append("semanticdb") + return [Label("//scala:%s_provider" % p) for p in dep_providers] -scala_toolchain = rule( +_scala_toolchain = rule( _scala_toolchain_impl, attrs = { "scalacopts": attr.string_list(), @@ -179,3 +176,31 @@ scala_toolchain = rule( }, fragments = ["java"], ) + +def _expand_patterns(patterns): + """Expands string patterns to match actual Label values.""" + result = [] + + for p in patterns: + exclude = p.startswith("-") + p = p.lstrip("-") + expanded = str(native.package_relative_label(p)) if p else "" + + # If the original pattern doesn't contain ":", match any target + # beginning with the pattern prefix. + if expanded and ":" not in p: + expanded = expanded[:expanded.rindex(":")] + + result.append(("-" if exclude else "") + expanded) + + return result + +def scala_toolchain(**kwargs): + """Creates a Scala toolchain target.""" + strict = kwargs.pop("dependency_tracking_strict_deps_patterns", [""]) + unused = kwargs.pop("dependency_tracking_unused_deps_patterns", [""]) + _scala_toolchain( + dependency_tracking_strict_deps_patterns = _expand_patterns(strict), + dependency_tracking_unused_deps_patterns = _expand_patterns(unused), + **kwargs + ) diff --git a/test/src/main/scala/scalarules/test/resources/ScalaLibResourcesFromExternalDepTest.scala b/test/src/main/scala/scalarules/test/resources/ScalaLibResourcesFromExternalDepTest.scala index 10e4564f0..088ec71a3 100644 --- a/test/src/main/scala/scalarules/test/resources/ScalaLibResourcesFromExternalDepTest.scala +++ b/test/src/main/scala/scalarules/test/resources/ScalaLibResourcesFromExternalDepTest.scala @@ -8,7 +8,7 @@ class ScalaLibResourcesFromExternalDepTest extends SpecWithJUnit { "allow to load resources" >> { val expectedString = String.format("A resource%n"); //Using platform dependent newline (%n) - get("/external/test_new_local_repo/resource.txt") must beEqualTo(expectedString) + get("/resource.txt") must beEqualTo(expectedString) } } diff --git a/test/src/main/scala/scalarules/test/resources/ScalaLibResourcesFromExternalScalaTest.scala b/test/src/main/scala/scalarules/test/resources/ScalaLibResourcesFromExternalScalaTest.scala index 99a9e8e7f..599ca7edf 100644 --- a/test/src/main/scala/scalarules/test/resources/ScalaLibResourcesFromExternalScalaTest.scala +++ b/test/src/main/scala/scalarules/test/resources/ScalaLibResourcesFromExternalScalaTest.scala @@ -6,7 +6,7 @@ class ScalaLibResourcesFromExternalScalaTest extends AnyFunSuite { test("Scala library depending on resources from external resource-only jar should allow to load resources") { val expectedString = String.format("A resource%n"); //Using platform dependent newline (%n) - assert(get("/external/test_new_local_repo/resource.txt") === expectedString) + assert(get("/resource.txt") === expectedString) } private def get(s: String): String = diff --git a/third_party/repositories/repositories.bzl b/third_party/repositories/repositories.bzl index 88239c012..1e0529c1e 100644 --- a/third_party/repositories/repositories.bzl +++ b/third_party/repositories/repositories.bzl @@ -1,3 +1,4 @@ +load("//scala/private:macros/bzlmod.bzl", "apparent_repo_name") load( "//third_party/repositories:scala_2_11.bzl", _artifacts_2_11 = "artifacts", @@ -102,14 +103,24 @@ def repositories( default_artifacts = artifacts_by_major_scala_version[major_scala_version] artifacts = dict(default_artifacts.items() + overriden_artifacts.items()) for id in for_artifact_ids: + if id not in artifacts: + fail("artifact %s not in third_party/repositories/scala_%s.bzl" % ( + id, + major_scala_version.replace(".", "_"), + )) + + artifact_repo_name = id + suffix _scala_maven_import_external( - name = id + suffix, + name = artifact_repo_name, artifact = artifacts[id]["artifact"], artifact_sha256 = artifacts[id]["sha256"], licenses = ["notice"], server_urls = maven_servers, deps = [dep + suffix for dep in artifacts[id].get("deps", [])], - runtime_deps = artifacts[id].get("runtime_deps", []), + runtime_deps = [ + dep + suffix + for dep in artifacts[id].get("runtime_deps", []) + ], testonly_ = artifacts[id].get("testonly", False), fetch_sources = fetch_sources, ) @@ -118,13 +129,13 @@ def repositories( # See: https://github.com/bazelbuild/rules_scala/pull/1573 # Hopefully we can deprecate and remove it one day. if suffix and scala_version == SCALA_VERSION: - _alias_repository(name = id, target = id + suffix) + _alias_repository(name = id, target = artifact_repo_name) def _alias_repository_impl(rctx): """ Builds a repository containing just two aliases to the Scala Maven artifacts in the `target` repository. """ format_kwargs = { - "name": rctx.name, + "name": apparent_repo_name(rctx), "target": rctx.attr.target, } rctx.file("BUILD", """alias(