test(bzlmod): support toolchain tests on bzlmod

This makes the necessary changes to the toolchains acceptance tests and also moves the tests to the `//tests` directory as this is the current convention. Fixes bazelbuild#1469.
aignas · Oct 18, 2023 · d89376d · d89376d
1 parent ede4fd4
commit d89376d
Show file tree

Hide file tree

Showing 14 changed files with 105 additions and 46 deletions.
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml
@@ -100,10 +100,7 @@ tasks:
     # TODO: Change to "rolling" once
     # https://github.com/bazelbuild/bazel/commit/f3aafea59ae021c6a12086cb2cd34c5fa782faf1
     # is available in rolling.
-    # NOTE @aignas 2023-10-17: as of https://github.com/bazelbuild/bazel/issues/19838
-    # this is the last known-to-work bazel version, use `last_green` or `rolling` once the
-    # issue is fixed.
-    bazel: "2b8219042c132483e0af39ef20d67dfd6442af01"
+    bazel: "last_green"
     environment:
       RULES_PYTHON_ENABLE_PYSTAR: "1"
     test_flags:
@@ -118,10 +115,7 @@ tasks:
     # TODO: Change to "rolling" once
     # https://github.com/bazelbuild/bazel/commit/f3aafea59ae021c6a12086cb2cd34c5fa782faf1
     # is available in rolling.
-    # NOTE @aignas 2023-10-17: as of https://github.com/bazelbuild/bazel/issues/19838
-    # this is the last known-to-work bazel version, use `last_green` or `rolling` once the
-    # issue is fixed.
-    bazel: "2b8219042c132483e0af39ef20d67dfd6442af01"
+    bazel: "last_green"
     environment:
       RULES_PYTHON_ENABLE_PYSTAR: "1"
     test_flags:

diff --git a/BUILD.bazel b/BUILD.bazel
@@ -40,8 +40,8 @@ filegroup(
     ],
     visibility = [
         "//examples:__pkg__",
-        "//python/tests/toolchains:__pkg__",
         "//tests:__pkg__",
+        "//tests/toolchains:__pkg__",
     ],
 )
 

diff --git a/python/BUILD.bazel b/python/BUILD.bazel
@@ -37,6 +37,7 @@ filegroup(
         "//python/config_settings:distribution",
         "//python/constraints:distribution",
         "//python/entry_points:distribution",
+        "//python/extensions:distribution",
         "//python/private:distribution",
         "//python/runfiles:distribution",
     ],

diff --git a/python/extensions/BUILD.bazel b/python/extensions/BUILD.bazel
@@ -19,5 +19,5 @@ licenses(["notice"])
 filegroup(
     name = "distribution",
     srcs = glob(["**"]),
-    visibility = ["//extensions:__pkg__"],
+    visibility = ["//python:__pkg__"],
 )
diff --git a/python/tests/toolchains/BUILD.bazel → tests/toolchains/BUILD.bazel b/python/tests/toolchains/BUILD.bazel → tests/toolchains/BUILD.bazel
diff --git a/python/tests/toolchains/defs.bzl → tests/toolchains/defs.bzl b/python/tests/toolchains/defs.bzl → tests/toolchains/defs.bzl
@@ -16,6 +16,7 @@
 """
 
 load("//python:versions.bzl", "PLATFORMS", "TOOL_VERSIONS")
+load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")  # buildifier: disable=bzl-visibility
 
 _WINDOWS_RUNNER_TEMPLATE = """\
 @ECHO OFF
@@ -24,36 +25,56 @@ powershell.exe -c "& ./{interpreter_path} {run_acceptance_test_py}"
 """
 
 def _acceptance_test_impl(ctx):
-    workspace = ctx.actions.declare_file("/".join([ctx.attr.python_version, "WORKSPACE"]))
-    ctx.actions.expand_template(
-        template = ctx.file._workspace_tmpl,
-        output = workspace,
-        substitutions = {"%python_version%": ctx.attr.python_version},
-    )
+    files = []
+
+    if BZLMOD_ENABLED:
+        module_bazel = ctx.actions.declare_file("/".join([ctx.attr.python_version, "MODULE.bazel"]))
+        ctx.actions.expand_template(
+            template = ctx.file._module_bazel_tmpl,
+            output = module_bazel,
+            substitutions = {"%python_version%": ctx.attr.python_version},
+        )
+        files.append(module_bazel)
+
+        workspace = ctx.actions.declare_file("/".join([ctx.attr.python_version, "WORKSPACE"]))
+        ctx.actions.write(workspace, "")
+        files.append(workspace)
+    else:
+        workspace = ctx.actions.declare_file("/".join([ctx.attr.python_version, "WORKSPACE"]))
+        ctx.actions.expand_template(
+            template = ctx.file._workspace_tmpl,
+            output = workspace,
+            substitutions = {"%python_version%": ctx.attr.python_version},
+        )
+        files.append(workspace)
 
     build_bazel = ctx.actions.declare_file("/".join([ctx.attr.python_version, "BUILD.bazel"]))
     ctx.actions.expand_template(
         template = ctx.file._build_bazel_tmpl,
         output = build_bazel,
         substitutions = {"%python_version%": ctx.attr.python_version},
     )
+    files.append(build_bazel)
 
     python_version_test = ctx.actions.declare_file("/".join([ctx.attr.python_version, "python_version_test.py"]))
     ctx.actions.symlink(
         target_file = ctx.file._python_version_test,
         output = python_version_test,
     )
+    files.append(python_version_test)
 
     run_acceptance_test_py = ctx.actions.declare_file("/".join([ctx.attr.python_version, "run_acceptance_test.py"]))
     ctx.actions.expand_template(
         template = ctx.file._run_acceptance_test_tmpl,
         output = run_acceptance_test_py,
         substitutions = {
+            "%is_bzlmod%": str(BZLMOD_ENABLED),
             "%is_windows%": str(ctx.attr.is_windows),
             "%python_version%": ctx.attr.python_version,
             "%test_location%": "/".join([ctx.attr.test_location, ctx.attr.python_version]),
         },
     )
+    files.append(run_acceptance_test_py)
 
     toolchain = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"]
     py3_runtime = toolchain.py3_runtime
@@ -81,14 +102,9 @@ def _acceptance_test_impl(ctx):
             ),
             is_executable = True,
         )
+    files.append(executable)
+    files.extend(ctx.files._distribution)
 
-    files = [
-        build_bazel,
-        executable,
-        python_version_test,
-        run_acceptance_test_py,
-        workspace,
-    ] + ctx.files._distribution
     return [DefaultInfo(
         executable = executable,
         files = depset(
@@ -120,26 +136,31 @@ _acceptance_test = rule(
         "_build_bazel_tmpl": attr.label(
             doc = "The BUILD.bazel template.",
             allow_single_file = True,
-            default = Label("//python/tests/toolchains/workspace_template:BUILD.bazel.tmpl"),
+            default = Label("//tests/toolchains/workspace_template:BUILD.bazel.tmpl"),
         ),
         "_distribution": attr.label(
             doc = "The rules_python source distribution.",
             default = Label("//:distribution"),
         ),
+        "_module_bazel_tmpl": attr.label(
+            doc = "The MODULE.bazel template.",
+            allow_single_file = True,
+            default = Label("//tests/toolchains/workspace_template:MODULE.bazel.tmpl"),
+        ),
         "_python_version_test": attr.label(
             doc = "The python_version_test.py used to test the Python version.",
             allow_single_file = True,
-            default = Label("//python/tests/toolchains/workspace_template:python_version_test.py"),
+            default = Label("//tests/toolchains/workspace_template:python_version_test.py"),
         ),
         "_run_acceptance_test_tmpl": attr.label(
             doc = "The run_acceptance_test.py template.",
             allow_single_file = True,
-            default = Label("//python/tests/toolchains:run_acceptance_test.py.tmpl"),
+            default = Label("//tests/toolchains:run_acceptance_test.py.tmpl"),
         ),
         "_workspace_tmpl": attr.label(
             doc = "The WORKSPACE template.",
             allow_single_file = True,
-            default = Label("//python/tests/toolchains/workspace_template:WORKSPACE.tmpl"),
+            default = Label("//tests/toolchains/workspace_template:WORKSPACE.tmpl"),
         ),
     },
     test = True,

diff --git a/...ts/toolchains/run_acceptance_test.py.tmpl → tests/toolchains/run_acceptance_test.py.tmpl b/...ts/toolchains/run_acceptance_test.py.tmpl → tests/toolchains/run_acceptance_test.py.tmpl
@@ -15,6 +15,7 @@
 import os
 import subprocess
 import unittest
+import pathlib
 
 class TestPythonVersion(unittest.TestCase):
     @classmethod
@@ -41,30 +42,52 @@ class TestPythonVersion(unittest.TestCase):
         # Bazelisk requires a cache directory be set
         os.environ["XDG_CACHE_HOME"] = os.path.join(test_tmpdir, "xdg-cache-home")
 
-        # Unset this so this works when called by Bazel's latest Bazel build
-        # pipeline. It sets the following combination, which interfere with each other:
-        # * --sandbox_tmpfs_path=/tmp
-        # * --test_env=USE_BAZEL_VERSION
-        # * USE_BAZEL_VERSION=/tmp/<something>
-        os.environ.pop("USE_BAZEL_VERSION", None)
+        env_to_clear = [
+            # Unset this so this works when called by Bazel's latest Bazel build
+            # pipeline. It sets the following combination, which interfere with each other:
+            # * --sandbox_tmpfs_path=/tmp
+            # * --test_env=USE_BAZEL_VERSION
+            # * USE_BAZEL_VERSION=/tmp/<something>
+            "USE_BAZEL_VERSION",
+            "RUNFILES_DIR",
+            "RUNFILES_MANIFEST_FILE",
+            "RUNFILES_MANIFEST_ONLY",
+            "TEST_SRCDIR"
+        ]
+        for e in env_to_clear:
+            os.environ.pop(e, None)
 
-        with open(".bazelrc", "w") as bazelrc:
-            bazelrc.write(
-                os.linesep.join(
-                    [
-                        'build --override_repository rules_python="{}"'.format(
-                            rules_python_path.replace("\\", "/")
-                        ),
-                        "build --test_output=errors",
-                    ]
-                )
+        bazelrclines = [
+            "build --test_output=errors",
+        ]
+
+        if %is_bzlmod%:
+            bazelrclines.extend(
+                [
+                    'build --override_module rules_python="{}"'.format(
+                        rules_python_path.replace("\\", "/")
+                    ),
+                    "common --enable_bzlmod",
+                ]
+            )
+        else:
+            bazelrclines.extend(
+                [
+                    'build --override_repository rules_python="{}"'.format(
+                        rules_python_path.replace("\\", "/")
+                    ),
+                    "common --noexperimental_enable_bzlmod",
+                ]
             )
 
+        bazelrc = pathlib.Path(".bazelrc")
+        bazelrc.write_text(os.linesep.join(bazelrclines))
+
     def test_match_toolchain(self):
         output = subprocess.check_output(
-          f"bazel run --announce_rc @python//:python3 -- --version",
-          shell = True, # Shell needed to look up via PATH
-          text=True,
+            f"bazel run --announce_rc @python//:python3 -- --version",
+            shell = True, # Shell needed to look up via PATH
+            text=True,
         ).strip()
         self.assertEqual(output, "Python %python_version%")
 

diff --git a/python/tests/toolchains/versions_test.bzl → tests/toolchains/versions_test.bzl b/python/tests/toolchains/versions_test.bzl → tests/toolchains/versions_test.bzl
diff --git a/...toolchains/workspace_template/BUILD.bazel → ...toolchains/workspace_template/BUILD.bazel b/...toolchains/workspace_template/BUILD.bazel → ...toolchains/workspace_template/BUILD.bazel
@@ -1,5 +1,6 @@
 exports_files([
     "BUILD.bazel.tmpl",
+    "MODULE.bazel.tmpl",
     "WORKSPACE.tmpl",
     "python_version_test.py",
 ])
diff --git a/...hains/workspace_template/BUILD.bazel.tmpl → ...hains/workspace_template/BUILD.bazel.tmpl b/...hains/workspace_template/BUILD.bazel.tmpl → ...hains/workspace_template/BUILD.bazel.tmpl
diff --git a/tests/toolchains/workspace_template/MODULE.bazel.tmpl b/tests/toolchains/workspace_template/MODULE.bazel.tmpl
@@ -0,0 +1,19 @@
+module(
+    name = "module_test",
+    version = "0.0.0",
+    compatibility_level = 1,
+)
+
+bazel_dep(name = "bazel_skylib", version = "1.3.0")
+bazel_dep(name = "rules_python", version = "0.0.0")
+local_path_override(
+    module_name = "rules_python",
+    path = "",
+)
+
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+python.toolchain(
+    is_default = True,
+    python_version = "%python_version%",
+)
+use_repo(python, "python_versions", python = "python_%python_version%".replace(".", "_"))
diff --git a/...s/toolchains/workspace_template/README.md → ...s/toolchains/workspace_template/README.md b/...s/toolchains/workspace_template/README.md → ...s/toolchains/workspace_template/README.md
diff --git a/...lchains/workspace_template/WORKSPACE.tmpl → ...lchains/workspace_template/WORKSPACE.tmpl b/...lchains/workspace_template/WORKSPACE.tmpl → ...lchains/workspace_template/WORKSPACE.tmpl
diff --git a/...workspace_template/python_version_test.py → ...workspace_template/python_version_test.py b/...workspace_template/python_version_test.py → ...workspace_template/python_version_test.py