diff --git a/MODULE.bazel b/MODULE.bazel index 5d778393e9..efff7333de 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -14,7 +14,7 @@ bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf" bazel_dep(name = "stardoc", version = "0.6.2", dev_dependency = True, repo_name = "io_bazel_stardoc") -internal_deps = use_extension("@rules_python//python/extensions/private:internal_deps.bzl", "internal_deps") +internal_deps = use_extension("@rules_python//python/private/bzlmod:internal_deps.bzl", "internal_deps") internal_deps.install() use_repo( internal_deps, diff --git a/python/extensions/pip.bzl b/python/extensions/pip.bzl index a0559ffe97..a69ee34ae7 100644 --- a/python/extensions/pip.bzl +++ b/python/extensions/pip.bzl @@ -14,443 +14,6 @@ "pip module extension for use with bzlmod" -load("@bazel_features//:features.bzl", "bazel_features") -load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS") -load( - "//python/pip_install:pip_repository.bzl", - "locked_requirements_label", - "pip_hub_repository_bzlmod", - "pip_repository_attrs", - "use_isolated", - "whl_library", -) -load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") -load("//python/private:full_version.bzl", "full_version") -load("//python/private:normalize_name.bzl", "normalize_name") -load("//python/private:version_label.bzl", "version_label") +load("//python/private/bzlmod:pip.bzl", _pip = "pip") -def _whl_mods_impl(mctx): - """Implementation of the pip.whl_mods tag class. - - This creates the JSON files used to modify the creation of different wheels. -""" - whl_mods_dict = {} - for mod in mctx.modules: - for whl_mod_attr in mod.tags.whl_mods: - if whl_mod_attr.hub_name not in whl_mods_dict.keys(): - whl_mods_dict[whl_mod_attr.hub_name] = {whl_mod_attr.whl_name: whl_mod_attr} - elif whl_mod_attr.whl_name in whl_mods_dict[whl_mod_attr.hub_name].keys(): - # We cannot have the same wheel name in the same hub, as we - # will create the same JSON file name. - fail("""\ -Found same whl_name '{}' in the same hub '{}', please use a different hub_name.""".format( - whl_mod_attr.whl_name, - whl_mod_attr.hub_name, - )) - else: - whl_mods_dict[whl_mod_attr.hub_name][whl_mod_attr.whl_name] = whl_mod_attr - - for hub_name, whl_maps in whl_mods_dict.items(): - whl_mods = {} - - # create a struct that we can pass to the _whl_mods_repo rule - # to create the different JSON files. - for whl_name, mods in whl_maps.items(): - build_content = mods.additive_build_content - if mods.additive_build_content_file != None and mods.additive_build_content != "": - fail("""\ -You cannot use both the additive_build_content and additive_build_content_file arguments at the same time. -""") - elif mods.additive_build_content_file != None: - build_content = mctx.read(mods.additive_build_content_file) - - whl_mods[whl_name] = json.encode(struct( - additive_build_content = build_content, - copy_files = mods.copy_files, - copy_executables = mods.copy_executables, - data = mods.data, - data_exclude_glob = mods.data_exclude_glob, - srcs_exclude_glob = mods.srcs_exclude_glob, - )) - - _whl_mods_repo( - name = hub_name, - whl_mods = whl_mods, - ) - -def _create_whl_repos(module_ctx, pip_attr, whl_map): - python_interpreter_target = pip_attr.python_interpreter_target - - # if we do not have the python_interpreter set in the attributes - # we programmatically find it. - hub_name = pip_attr.hub_name - if python_interpreter_target == None: - python_name = "python_" + version_label(pip_attr.python_version, sep = "_") - if python_name not in INTERPRETER_LABELS.keys(): - fail(( - "Unable to find interpreter for pip hub '{hub_name}' for " + - "python_version={version}: Make sure a corresponding " + - '`python.toolchain(python_version="{version}")` call exists' - ).format( - hub_name = hub_name, - version = pip_attr.python_version, - )) - python_interpreter_target = INTERPRETER_LABELS[python_name] - - pip_name = "{}_{}".format( - hub_name, - version_label(pip_attr.python_version), - ) - requrements_lock = locked_requirements_label(module_ctx, pip_attr) - - # Parse the requirements file directly in starlark to get the information - # needed for the whl_libary declarations below. - requirements_lock_content = module_ctx.read(requrements_lock) - parse_result = parse_requirements(requirements_lock_content) - requirements = parse_result.requirements - extra_pip_args = pip_attr.extra_pip_args + parse_result.options - - if hub_name not in whl_map: - whl_map[hub_name] = {} - - whl_modifications = {} - if pip_attr.whl_modifications != None: - for mod, whl_name in pip_attr.whl_modifications.items(): - whl_modifications[whl_name] = mod - - # Create a new wheel library for each of the different whls - for whl_name, requirement_line in requirements: - # We are not using the "sanitized name" because the user - # would need to guess what name we modified the whl name - # to. - annotation = whl_modifications.get(whl_name) - whl_name = normalize_name(whl_name) - whl_library( - name = "%s_%s" % (pip_name, whl_name), - requirement = requirement_line, - repo = pip_name, - repo_prefix = pip_name + "_", - annotation = annotation, - python_interpreter = pip_attr.python_interpreter, - python_interpreter_target = python_interpreter_target, - quiet = pip_attr.quiet, - timeout = pip_attr.timeout, - isolated = use_isolated(module_ctx, pip_attr), - extra_pip_args = extra_pip_args, - download_only = pip_attr.download_only, - pip_data_exclude = pip_attr.pip_data_exclude, - enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs, - environment = pip_attr.environment, - ) - - if whl_name not in whl_map[hub_name]: - whl_map[hub_name][whl_name] = {} - - whl_map[hub_name][whl_name][full_version(pip_attr.python_version)] = pip_name + "_" - -def _pip_impl(module_ctx): - """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories. - - This implementation iterates through all of the `pip.parse` calls and creates - different pip hub repositories based on the "hub_name". Each of the - pip calls create spoke repos that uses a specific Python interpreter. - - In a MODULES.bazel file we have: - - pip.parse( - hub_name = "pip", - python_version = 3.9, - requirements_lock = "//:requirements_lock_3_9.txt", - requirements_windows = "//:requirements_windows_3_9.txt", - ) - pip.parse( - hub_name = "pip", - python_version = 3.10, - requirements_lock = "//:requirements_lock_3_10.txt", - requirements_windows = "//:requirements_windows_3_10.txt", - ) - - For instance, we have a hub with the name of "pip". - A repository named the following is created. It is actually called last when - all of the pip spokes are collected. - - - @@rules_python~override~pip~pip - - As shown in the example code above we have the following. - Two different pip.parse statements exist in MODULE.bazel provide the hub_name "pip". - These definitions create two different pip spoke repositories that are - related to the hub "pip". - One spoke uses Python 3.9 and the other uses Python 3.10. This code automatically - determines the Python version and the interpreter. - Both of these pip spokes contain requirements files that includes websocket - and its dependencies. - - We also need repositories for the wheels that the different pip spokes contain. - For each Python version a different wheel repository is created. In our example - each pip spoke had a requirements file that contained websockets. We - then create two different wheel repositories that are named the following. - - - @@rules_python~override~pip~pip_39_websockets - - @@rules_python~override~pip~pip_310_websockets - - And if the wheel has any other dependencies subsequent wheels are created in the same fashion. - - The hub repository has aliases for `pkg`, `data`, etc, which have a select that resolves to - a spoke repository depending on the Python version. - - Also we may have more than one hub as defined in a MODULES.bazel file. So we could have multiple - hubs pointing to various different pip spokes. - - Some other business rules notes. A hub can only have one spoke per Python version. We cannot - have a hub named "pip" that has two spokes that use the Python 3.9 interpreter. Second - we cannot have the same hub name used in sub-modules. The hub name has to be globally - unique. - - This implementation also handles the creation of whl_modification JSON files that are used - during the creation of wheel libraries. These JSON files used via the annotations argument - when calling wheel_installer.py. - - Args: - module_ctx: module contents - """ - - # Build all of the wheel modifications if the tag class is called. - _whl_mods_impl(module_ctx) - - # Used to track all the different pip hubs and the spoke pip Python - # versions. - pip_hub_map = {} - - # Keeps track of all the hub's whl repos across the different versions. - # dict[hub, dict[whl, dict[version, str pip]]] - # Where hub, whl, and pip are the repo names - hub_whl_map = {} - - for mod in module_ctx.modules: - for pip_attr in mod.tags.parse: - hub_name = pip_attr.hub_name - if hub_name not in pip_hub_map: - pip_hub_map[pip_attr.hub_name] = struct( - module_name = mod.name, - python_versions = [pip_attr.python_version], - ) - elif pip_hub_map[hub_name].module_name != mod.name: - # We cannot have two hubs with the same name in different - # modules. - fail(( - "Duplicate cross-module pip hub named '{hub}': pip hub " + - "names must be unique across modules. First defined " + - "by module '{first_module}', second attempted by " + - "module '{second_module}'" - ).format( - hub = hub_name, - first_module = pip_hub_map[hub_name].module_name, - second_module = mod.name, - )) - - elif pip_attr.python_version in pip_hub_map[hub_name].python_versions: - fail(( - "Duplicate pip python version '{version}' for hub " + - "'{hub}' in module '{module}': the Python versions " + - "used for a hub must be unique" - ).format( - hub = hub_name, - module = mod.name, - version = pip_attr.python_version, - )) - else: - pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version) - - _create_whl_repos(module_ctx, pip_attr, hub_whl_map) - - for hub_name, whl_map in hub_whl_map.items(): - pip_hub_repository_bzlmod( - name = hub_name, - repo_name = hub_name, - whl_map = whl_map, - default_version = full_version(DEFAULT_PYTHON_VERSION), - ) - -def _pip_parse_ext_attrs(): - attrs = dict({ - "hub_name": attr.string( - mandatory = True, - doc = """ -The name of the repo pip dependencies will be accessible from. - -This name must be unique between modules; unless your module is guaranteed to -always be the root module, it's highly recommended to include your module name -in the hub name. Repo mapping, `use_repo(..., pip="my_modules_pip_deps")`, can -be used for shorter local names within your module. - -Within a module, the same `hub_name` can be specified to group different Python -versions of pip dependencies under one repository name. This allows using a -Python version-agnostic name when referring to pip dependencies; the -correct version will be automatically selected. - -Typically, a module will only have a single hub of pip dependencies, but this -is not required. Each hub is a separate resolution of pip dependencies. This -means if different programs need different versions of some library, separate -hubs can be created, and each program can use its respective hub's targets. -Targets from different hubs should not be used together. -""", - ), - "python_version": attr.string( - mandatory = True, - doc = """ -The Python version to use for resolving the pip dependencies, in Major.Minor -format (e.g. "3.11"). Patch level granularity (e.g. "3.11.1") is not supported. -If not specified, then the default Python version (as set by the root module or -rules_python) will be used. - -The version specified here must have a corresponding `python.toolchain()` -configured. -""", - ), - "whl_modifications": attr.label_keyed_string_dict( - mandatory = False, - doc = """\ -A dict of labels to wheel names that is typically generated by the whl_modifications. -The labels are JSON config files describing the modifications. -""", - ), - }, **pip_repository_attrs) - - # Like the pip_repository rule, we end up setting this manually so - # don't allow users to override it. - attrs.pop("repo_prefix") - - # incompatible_generate_aliases is always True in bzlmod - attrs.pop("incompatible_generate_aliases") - - return attrs - -def _whl_mod_attrs(): - attrs = { - "additive_build_content": attr.string( - doc = "(str, optional): Raw text to add to the generated `BUILD` file of a package.", - ), - "additive_build_content_file": attr.label( - doc = """\ -(label, optional): path to a BUILD file to add to the generated -`BUILD` file of a package. You cannot use both additive_build_content and additive_build_content_file -arguments at the same time.""", - ), - "copy_executables": attr.string_dict( - doc = """\ -(dict, optional): A mapping of `src` and `out` files for -[@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as -executable.""", - ), - "copy_files": attr.string_dict( - doc = """\ -(dict, optional): A mapping of `src` and `out` files for -[@bazel_skylib//rules:copy_file.bzl][cf]""", - ), - "data": attr.string_list( - doc = """\ -(list, optional): A list of labels to add as `data` dependencies to -the generated `py_library` target.""", - ), - "data_exclude_glob": attr.string_list( - doc = """\ -(list, optional): A list of exclude glob patterns to add as `data` to -the generated `py_library` target.""", - ), - "hub_name": attr.string( - doc = """\ -Name of the whl modification, hub we use this name to set the modifications for -pip.parse. If you have different pip hubs you can use a different name, -otherwise it is best practice to just use one. - -You cannot have the same `hub_name` in different modules. You can reuse the same -name in the same module for different wheels that you put in the same hub, but you -cannot have a child module that uses the same `hub_name`. -""", - mandatory = True, - ), - "srcs_exclude_glob": attr.string_list( - doc = """\ -(list, optional): A list of labels to add as `srcs` to the generated -`py_library` target.""", - ), - "whl_name": attr.string( - doc = "The whl name that the modifications are used for.", - mandatory = True, - ), - } - return attrs - -def _extension_extra_args(): - args = {} - - if bazel_features.external_deps.module_extension_has_os_arch_dependent: - args = args | { - "arch_dependent": True, - "os_dependent": True, - } - - return args - -pip = module_extension( - doc = """\ -This extension is used to make dependencies from pip available. - -pip.parse: -To use, call `pip.parse()` and specify `hub_name` and your requirements file. -Dependencies will be downloaded and made available in a repo named after the -`hub_name` argument. - -Each `pip.parse()` call configures a particular Python version. Multiple calls -can be made to configure different Python versions, and will be grouped by -the `hub_name` argument. This allows the same logical name, e.g. `@pip//numpy` -to automatically resolve to different, Python version-specific, libraries. - -pip.whl_mods: -This tag class is used to help create JSON files to describe modifications to -the BUILD files for wheels. -""", - implementation = _pip_impl, - tag_classes = { - "parse": tag_class( - attrs = _pip_parse_ext_attrs(), - doc = """\ -This tag class is used to create a pip hub and all of the spokes that are part of that hub. -This tag class reuses most of the pip attributes that are found in -@rules_python//python/pip_install:pip_repository.bzl. -The exceptions are it does not use the args 'repo_prefix', -and 'incompatible_generate_aliases'. We set the repository prefix -for the user and the alias arg is always True in bzlmod. -""", - ), - "whl_mods": tag_class( - attrs = _whl_mod_attrs(), - doc = """\ -This tag class is used to create JSON file that are used when calling wheel_builder.py. These -JSON files contain instructions on how to modify a wheel's project. Each of the attributes -create different modifications based on the type of attribute. Previously to bzlmod these -JSON files where referred to as annotations, and were renamed to whl_modifications in this -extension. -""", - ), - }, - **_extension_extra_args() -) - -def _whl_mods_repo_impl(rctx): - rctx.file("BUILD.bazel", "") - for whl_name, mods in rctx.attr.whl_mods.items(): - rctx.file("{}.json".format(whl_name), mods) - -_whl_mods_repo = repository_rule( - doc = """\ -This rule creates json files based on the whl_mods attribute. -""", - implementation = _whl_mods_repo_impl, - attrs = { - "whl_mods": attr.string_dict( - mandatory = True, - doc = "JSON endcoded string that is provided to wheel_builder.py", - ), - }, -) +pip = _pip diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl index c7c2c82c05..5428b7542e 100644 --- a/python/extensions/python.bzl +++ b/python/extensions/python.bzl @@ -14,253 +14,6 @@ "Python toolchain module extensions for use with bzlmod" -load("//python:repositories.bzl", "python_register_toolchains") -load("//python/extensions/private:pythons_hub.bzl", "hub_repo") -load("//python/private:toolchains_repo.bzl", "multi_toolchain_aliases") +load("//python/private/bzlmod:python.bzl", _python = "python") -# This limit can be increased essentially arbitrarily, but doing so will cause a rebuild of all -# targets using any of these toolchains due to the changed repository name. -_MAX_NUM_TOOLCHAINS = 9999 -_TOOLCHAIN_INDEX_PAD_LENGTH = len(str(_MAX_NUM_TOOLCHAINS)) - -def _toolchain_prefix(index, name): - """Prefixes the given name with the index, padded with zeros to ensure lexicographic sorting. - - Examples: - _toolchain_prefix( 2, "foo") == "_0002_foo_" - _toolchain_prefix(2000, "foo") == "_2000_foo_" - """ - return "_{}_{}_".format(_left_pad_zero(index, _TOOLCHAIN_INDEX_PAD_LENGTH), name) - -def _left_pad_zero(index, length): - if index < 0: - fail("index must be non-negative") - return ("0" * length + str(index))[-length:] - -# Printing a warning msg not debugging, so we have to disable -# the buildifier check. -# buildifier: disable=print -def _print_warn(msg): - print("WARNING:", msg) - -def _python_register_toolchains(name, toolchain_attr, version_constraint): - """Calls python_register_toolchains and returns a struct used to collect the toolchains. - """ - python_register_toolchains( - name = name, - python_version = toolchain_attr.python_version, - register_coverage_tool = toolchain_attr.configure_coverage_tool, - ignore_root_user_error = toolchain_attr.ignore_root_user_error, - set_python_version_constraint = version_constraint, - ) - return struct( - python_version = toolchain_attr.python_version, - set_python_version_constraint = str(version_constraint), - name = name, - ) - -def _python_impl(module_ctx): - # The toolchain info structs to register, in the order to register them in. - toolchains = [] - - # We store the default toolchain separately to ensure it is the last - # toolchain added to toolchains. - default_toolchain = None - - # Map of string Major.Minor to the toolchain name and module name - global_toolchain_versions = {} - - for mod in module_ctx.modules: - module_toolchain_versions = [] - - for toolchain_attr in mod.tags.toolchain: - toolchain_version = toolchain_attr.python_version - toolchain_name = "python_" + toolchain_version.replace(".", "_") - - # Duplicate versions within a module indicate a misconfigured module. - if toolchain_version in module_toolchain_versions: - _fail_duplicate_module_toolchain_version(toolchain_version, mod.name) - module_toolchain_versions.append(toolchain_version) - - # Ignore version collisions in the global scope because there isn't - # much else that can be done. Modules don't know and can't control - # what other modules do, so the first in the dependency graph wins. - if toolchain_version in global_toolchain_versions: - # If the python version is explicitly provided by the root - # module, they should not be warned for choosing the same - # version that rules_python provides as default. - first = global_toolchain_versions[toolchain_version] - if mod.name != "rules_python" or not first.is_root: - _warn_duplicate_global_toolchain_version( - toolchain_version, - first = first, - second_toolchain_name = toolchain_name, - second_module_name = mod.name, - ) - continue - global_toolchain_versions[toolchain_version] = struct( - toolchain_name = toolchain_name, - module_name = mod.name, - is_root = mod.is_root, - ) - - # Only the root module and rules_python are allowed to specify the default - # toolchain for a couple reasons: - # * It prevents submodules from specifying different defaults and only - # one of them winning. - # * rules_python needs to set a soft default in case the root module doesn't, - # e.g. if the root module doesn't use Python itself. - # * The root module is allowed to override the rules_python default. - if mod.is_root: - # A single toolchain is treated as the default because it's unambiguous. - is_default = toolchain_attr.is_default or len(mod.tags.toolchain) == 1 - elif mod.name == "rules_python" and not default_toolchain: - # We don't do the len() check because we want the default that rules_python - # sets to be clearly visible. - is_default = toolchain_attr.is_default - else: - is_default = False - - # We have already found one default toolchain, and we can only have - # one. - if is_default and default_toolchain != None: - _fail_multiple_default_toolchains( - first = default_toolchain.name, - second = toolchain_name, - ) - - toolchain_info = _python_register_toolchains( - toolchain_name, - toolchain_attr, - version_constraint = not is_default, - ) - - if is_default: - default_toolchain = toolchain_info - else: - toolchains.append(toolchain_info) - - # A default toolchain is required so that the non-version-specific rules - # are able to match a toolchain. - if default_toolchain == None: - fail("No default Python toolchain configured. Is rules_python missing `is_default=True`?") - - # The last toolchain in the BUILD file is set as the default - # toolchain. We need the default last. - toolchains.append(default_toolchain) - - if len(toolchains) > _MAX_NUM_TOOLCHAINS: - fail("more than {} python versions are not supported".format(_MAX_NUM_TOOLCHAINS)) - - # Create the pythons_hub repo for the interpreter meta data and the - # the various toolchains. - hub_repo( - name = "pythons_hub", - default_python_version = default_toolchain.python_version, - toolchain_prefixes = [ - _toolchain_prefix(index, toolchain.name) - for index, toolchain in enumerate(toolchains) - ], - toolchain_python_versions = [t.python_version for t in toolchains], - toolchain_set_python_version_constraints = [t.set_python_version_constraint for t in toolchains], - toolchain_user_repository_names = [t.name for t in toolchains], - ) - - # This is require in order to support multiple version py_test - # and py_binary - multi_toolchain_aliases( - name = "python_versions", - python_versions = { - version: entry.toolchain_name - for version, entry in global_toolchain_versions.items() - }, - ) - -def _fail_duplicate_module_toolchain_version(version, module): - fail(("Duplicate module toolchain version: module '{module}' attempted " + - "to use version '{version}' multiple times in itself").format( - version = version, - module = module, - )) - -def _warn_duplicate_global_toolchain_version(version, first, second_toolchain_name, second_module_name): - _print_warn(( - "Ignoring toolchain '{second_toolchain}' from module '{second_module}': " + - "Toolchain '{first_toolchain}' from module '{first_module}' " + - "already registered Python version {version} and has precedence" - ).format( - first_toolchain = first.toolchain_name, - first_module = first.module_name, - second_module = second_module_name, - second_toolchain = second_toolchain_name, - version = version, - )) - -def _fail_multiple_default_toolchains(first, second): - fail(("Multiple default toolchains: only one toolchain " + - "can have is_default=True. First default " + - "was toolchain '{first}'. Second was '{second}'").format( - first = first, - second = second, - )) - -python = module_extension( - doc = """Bzlmod extension that is used to register Python toolchains. -""", - implementation = _python_impl, - tag_classes = { - "toolchain": tag_class( - doc = """Tag class used to register Python toolchains. -Use this tag class to register one or more Python toolchains. This class -is also potentially called by sub modules. The following covers different -business rules and use cases. - -Toolchains in the Root Module - -This class registers all toolchains in the root module. - -Toolchains in Sub Modules - -It will create a toolchain that is in a sub module, if the toolchain -of the same name does not exist in the root module. The extension stops name -clashing between toolchains in the root module and toolchains in sub modules. -You cannot configure more than one toolchain as the default toolchain. - -Toolchain set as the default version - -This extension will not create a toolchain that exists in a sub module, -if the sub module toolchain is marked as the default version. If you have -more than one toolchain in your root module, you need to set one of the -toolchains as the default version. If there is only one toolchain it -is set as the default toolchain. - -Toolchain repository name - -A toolchain's repository name uses the format `python_{major}_{minor}`, e.g. -`python_3_10`. The `major` and `minor` components are -`major` and `minor` are the Python version from the `python_version` attribute. -""", - attrs = { - "configure_coverage_tool": attr.bool( - mandatory = False, - doc = "Whether or not to configure the default coverage tool for the toolchains.", - ), - "ignore_root_user_error": attr.bool( - default = False, - doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.", - mandatory = False, - ), - "is_default": attr.bool( - mandatory = False, - doc = "Whether the toolchain is the default version", - ), - "python_version": attr.string( - mandatory = True, - doc = "The Python version, in `major.minor` format, e.g " + - "'3.12', to create a toolchain for. Patch level " + - "granularity (e.g. '3.12.1') is not supported.", - ), - }, - ), - }, -) +python = _python diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel index c071033384..271cad5547 100644 --- a/python/pip_install/BUILD.bazel +++ b/python/pip_install/BUILD.bazel @@ -38,6 +38,7 @@ bzl_library( "//python/private:render_pkg_aliases_bzl", "//python/private:toolchains_repo_bzl", "//python/private:which_bzl", + "//python/private/bzlmod:pip_repository_bzl", ], ) diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index ea8b9eb5ac..5f829a9683 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -25,6 +25,7 @@ load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases") load("//python/private:toolchains_repo.bzl", "get_host_os_arch") load("//python/private:which.bzl", "which_with_fail") +load("//python/private/bzlmod:pip_repository.bzl", _pip_hub_repository_bzlmod = "pip_repository") CPPFLAGS = "CPPFLAGS" @@ -32,6 +33,9 @@ COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools" _WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point" +# Kept for not creating merge conflicts with PR#1476, can be removed later. +pip_hub_repository_bzlmod = _pip_hub_repository_bzlmod + def _construct_pypath(rctx): """Helper function to construct a PYTHONPATH. @@ -267,68 +271,6 @@ A requirements_lock attribute must be specified, or a platform-specific lockfile """) return requirements_txt -def _pip_hub_repository_bzlmod_impl(rctx): - bzl_packages = rctx.attr.whl_map.keys() - aliases = render_pkg_aliases( - repo_name = rctx.attr.repo_name, - rules_python = rctx.attr._template.workspace_name, - default_version = rctx.attr.default_version, - whl_map = rctx.attr.whl_map, - ) - for path, contents in aliases.items(): - rctx.file(path, contents) - - # NOTE: we are using the canonical name with the double '@' in order to - # always uniquely identify a repository, as the labels are being passed as - # a string and the resolution of the label happens at the call-site of the - # `requirement`, et al. macros. - macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name) - - rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) - rctx.template("requirements.bzl", rctx.attr._template, substitutions = { - "%%ALL_DATA_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, "data") - for p in bzl_packages - ]), - "%%ALL_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, p) - for p in bzl_packages - ]), - "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, "whl") - for p in bzl_packages - ]), - "%%MACRO_TMPL%%": macro_tmpl, - "%%NAME%%": rctx.attr.name, - }) - -pip_hub_repository_bzlmod_attrs = { - "default_version": attr.string( - mandatory = True, - doc = """\ -This is the default python version in the format of X.Y.Z. This should match -what is setup by the 'python' extension using the 'is_default = True' -setting.""", - ), - "repo_name": attr.string( - mandatory = True, - doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", - ), - "whl_map": attr.string_list_dict( - mandatory = True, - doc = "The wheel map where values are python versions", - ), - "_template": attr.label( - default = ":pip_repository_requirements_bzlmod.bzl.tmpl", - ), -} - -pip_hub_repository_bzlmod = repository_rule( - attrs = pip_hub_repository_bzlmod_attrs, - doc = """A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY.""", - implementation = _pip_hub_repository_bzlmod_impl, -) - def _pip_repository_impl(rctx): requirements_txt = locked_requirements_label(rctx, rctx.attr) content = rctx.read(requirements_txt) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index beda50f923..b8b8e51308 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -27,6 +27,7 @@ licenses(["notice"]) filegroup( name = "distribution", srcs = glob(["**"]) + [ + "//python/private/bzlmod:distribution", "//python/private/common:distribution", "//python/private/proto:distribution", "//tools/build_defs/python/private:distribution", diff --git a/python/extensions/private/BUILD.bazel b/python/private/bzlmod/BUILD.bazel similarity index 68% rename from python/extensions/private/BUILD.bazel rename to python/private/bzlmod/BUILD.bazel index f367b71a78..fc8449ecaf 100644 --- a/python/extensions/private/BUILD.bazel +++ b/python/private/bzlmod/BUILD.bazel @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + package(default_visibility = ["//visibility:private"]) licenses(["notice"]) @@ -19,5 +21,15 @@ licenses(["notice"]) filegroup( name = "distribution", srcs = glob(["**"]), - visibility = ["//python/extensions/private:__pkg__"], + visibility = ["//python/private:__pkg__"], +) + +bzl_library( + name = "pip_repository_bzl", + srcs = ["pip_repository.bzl"], + visibility = ["//:__subpackages__"], + deps = [ + "//python/private:render_pkg_aliases_bzl", + "//python/private:text_util_bzl", + ], ) diff --git a/python/extensions/private/internal_deps.bzl b/python/private/bzlmod/internal_deps.bzl similarity index 100% rename from python/extensions/private/internal_deps.bzl rename to python/private/bzlmod/internal_deps.bzl diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl new file mode 100644 index 0000000000..3630648f1e --- /dev/null +++ b/python/private/bzlmod/pip.bzl @@ -0,0 +1,456 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"pip module extension for use with bzlmod" + +load("@bazel_features//:features.bzl", "bazel_features") +load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS") +load( + "//python/pip_install:pip_repository.bzl", + "locked_requirements_label", + "pip_repository_attrs", + "use_isolated", + "whl_library", +) +load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") +load("//python/private:full_version.bzl", "full_version") +load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:version_label.bzl", "version_label") +load(":pip_repository.bzl", "pip_repository") + +def _whl_mods_impl(mctx): + """Implementation of the pip.whl_mods tag class. + + This creates the JSON files used to modify the creation of different wheels. +""" + whl_mods_dict = {} + for mod in mctx.modules: + for whl_mod_attr in mod.tags.whl_mods: + if whl_mod_attr.hub_name not in whl_mods_dict.keys(): + whl_mods_dict[whl_mod_attr.hub_name] = {whl_mod_attr.whl_name: whl_mod_attr} + elif whl_mod_attr.whl_name in whl_mods_dict[whl_mod_attr.hub_name].keys(): + # We cannot have the same wheel name in the same hub, as we + # will create the same JSON file name. + fail("""\ +Found same whl_name '{}' in the same hub '{}', please use a different hub_name.""".format( + whl_mod_attr.whl_name, + whl_mod_attr.hub_name, + )) + else: + whl_mods_dict[whl_mod_attr.hub_name][whl_mod_attr.whl_name] = whl_mod_attr + + for hub_name, whl_maps in whl_mods_dict.items(): + whl_mods = {} + + # create a struct that we can pass to the _whl_mods_repo rule + # to create the different JSON files. + for whl_name, mods in whl_maps.items(): + build_content = mods.additive_build_content + if mods.additive_build_content_file != None and mods.additive_build_content != "": + fail("""\ +You cannot use both the additive_build_content and additive_build_content_file arguments at the same time. +""") + elif mods.additive_build_content_file != None: + build_content = mctx.read(mods.additive_build_content_file) + + whl_mods[whl_name] = json.encode(struct( + additive_build_content = build_content, + copy_files = mods.copy_files, + copy_executables = mods.copy_executables, + data = mods.data, + data_exclude_glob = mods.data_exclude_glob, + srcs_exclude_glob = mods.srcs_exclude_glob, + )) + + _whl_mods_repo( + name = hub_name, + whl_mods = whl_mods, + ) + +def _create_whl_repos(module_ctx, pip_attr, whl_map): + python_interpreter_target = pip_attr.python_interpreter_target + + # if we do not have the python_interpreter set in the attributes + # we programmatically find it. + hub_name = pip_attr.hub_name + if python_interpreter_target == None: + python_name = "python_" + version_label(pip_attr.python_version, sep = "_") + if python_name not in INTERPRETER_LABELS.keys(): + fail(( + "Unable to find interpreter for pip hub '{hub_name}' for " + + "python_version={version}: Make sure a corresponding " + + '`python.toolchain(python_version="{version}")` call exists' + ).format( + hub_name = hub_name, + version = pip_attr.python_version, + )) + python_interpreter_target = INTERPRETER_LABELS[python_name] + + pip_name = "{}_{}".format( + hub_name, + version_label(pip_attr.python_version), + ) + requrements_lock = locked_requirements_label(module_ctx, pip_attr) + + # Parse the requirements file directly in starlark to get the information + # needed for the whl_libary declarations below. + requirements_lock_content = module_ctx.read(requrements_lock) + parse_result = parse_requirements(requirements_lock_content) + requirements = parse_result.requirements + extra_pip_args = pip_attr.extra_pip_args + parse_result.options + + if hub_name not in whl_map: + whl_map[hub_name] = {} + + whl_modifications = {} + if pip_attr.whl_modifications != None: + for mod, whl_name in pip_attr.whl_modifications.items(): + whl_modifications[whl_name] = mod + + # Create a new wheel library for each of the different whls + for whl_name, requirement_line in requirements: + # We are not using the "sanitized name" because the user + # would need to guess what name we modified the whl name + # to. + annotation = whl_modifications.get(whl_name) + whl_name = normalize_name(whl_name) + whl_library( + name = "%s_%s" % (pip_name, whl_name), + requirement = requirement_line, + repo = pip_name, + repo_prefix = pip_name + "_", + annotation = annotation, + python_interpreter = pip_attr.python_interpreter, + python_interpreter_target = python_interpreter_target, + quiet = pip_attr.quiet, + timeout = pip_attr.timeout, + isolated = use_isolated(module_ctx, pip_attr), + extra_pip_args = extra_pip_args, + download_only = pip_attr.download_only, + pip_data_exclude = pip_attr.pip_data_exclude, + enable_implicit_namespace_pkgs = pip_attr.enable_implicit_namespace_pkgs, + environment = pip_attr.environment, + ) + + if whl_name not in whl_map[hub_name]: + whl_map[hub_name][whl_name] = {} + + whl_map[hub_name][whl_name][full_version(pip_attr.python_version)] = pip_name + "_" + +def _pip_impl(module_ctx): + """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories. + + This implementation iterates through all of the `pip.parse` calls and creates + different pip hub repositories based on the "hub_name". Each of the + pip calls create spoke repos that uses a specific Python interpreter. + + In a MODULES.bazel file we have: + + pip.parse( + hub_name = "pip", + python_version = 3.9, + requirements_lock = "//:requirements_lock_3_9.txt", + requirements_windows = "//:requirements_windows_3_9.txt", + ) + pip.parse( + hub_name = "pip", + python_version = 3.10, + requirements_lock = "//:requirements_lock_3_10.txt", + requirements_windows = "//:requirements_windows_3_10.txt", + ) + + For instance, we have a hub with the name of "pip". + A repository named the following is created. It is actually called last when + all of the pip spokes are collected. + + - @@rules_python~override~pip~pip + + As shown in the example code above we have the following. + Two different pip.parse statements exist in MODULE.bazel provide the hub_name "pip". + These definitions create two different pip spoke repositories that are + related to the hub "pip". + One spoke uses Python 3.9 and the other uses Python 3.10. This code automatically + determines the Python version and the interpreter. + Both of these pip spokes contain requirements files that includes websocket + and its dependencies. + + We also need repositories for the wheels that the different pip spokes contain. + For each Python version a different wheel repository is created. In our example + each pip spoke had a requirements file that contained websockets. We + then create two different wheel repositories that are named the following. + + - @@rules_python~override~pip~pip_39_websockets + - @@rules_python~override~pip~pip_310_websockets + + And if the wheel has any other dependencies subsequent wheels are created in the same fashion. + + The hub repository has aliases for `pkg`, `data`, etc, which have a select that resolves to + a spoke repository depending on the Python version. + + Also we may have more than one hub as defined in a MODULES.bazel file. So we could have multiple + hubs pointing to various different pip spokes. + + Some other business rules notes. A hub can only have one spoke per Python version. We cannot + have a hub named "pip" that has two spokes that use the Python 3.9 interpreter. Second + we cannot have the same hub name used in sub-modules. The hub name has to be globally + unique. + + This implementation also handles the creation of whl_modification JSON files that are used + during the creation of wheel libraries. These JSON files used via the annotations argument + when calling wheel_installer.py. + + Args: + module_ctx: module contents + """ + + # Build all of the wheel modifications if the tag class is called. + _whl_mods_impl(module_ctx) + + # Used to track all the different pip hubs and the spoke pip Python + # versions. + pip_hub_map = {} + + # Keeps track of all the hub's whl repos across the different versions. + # dict[hub, dict[whl, dict[version, str pip]]] + # Where hub, whl, and pip are the repo names + hub_whl_map = {} + + for mod in module_ctx.modules: + for pip_attr in mod.tags.parse: + hub_name = pip_attr.hub_name + if hub_name not in pip_hub_map: + pip_hub_map[pip_attr.hub_name] = struct( + module_name = mod.name, + python_versions = [pip_attr.python_version], + ) + elif pip_hub_map[hub_name].module_name != mod.name: + # We cannot have two hubs with the same name in different + # modules. + fail(( + "Duplicate cross-module pip hub named '{hub}': pip hub " + + "names must be unique across modules. First defined " + + "by module '{first_module}', second attempted by " + + "module '{second_module}'" + ).format( + hub = hub_name, + first_module = pip_hub_map[hub_name].module_name, + second_module = mod.name, + )) + + elif pip_attr.python_version in pip_hub_map[hub_name].python_versions: + fail(( + "Duplicate pip python version '{version}' for hub " + + "'{hub}' in module '{module}': the Python versions " + + "used for a hub must be unique" + ).format( + hub = hub_name, + module = mod.name, + version = pip_attr.python_version, + )) + else: + pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version) + + _create_whl_repos(module_ctx, pip_attr, hub_whl_map) + + for hub_name, whl_map in hub_whl_map.items(): + pip_repository( + name = hub_name, + repo_name = hub_name, + whl_map = whl_map, + default_version = full_version(DEFAULT_PYTHON_VERSION), + ) + +def _pip_parse_ext_attrs(): + attrs = dict({ + "hub_name": attr.string( + mandatory = True, + doc = """ +The name of the repo pip dependencies will be accessible from. + +This name must be unique between modules; unless your module is guaranteed to +always be the root module, it's highly recommended to include your module name +in the hub name. Repo mapping, `use_repo(..., pip="my_modules_pip_deps")`, can +be used for shorter local names within your module. + +Within a module, the same `hub_name` can be specified to group different Python +versions of pip dependencies under one repository name. This allows using a +Python version-agnostic name when referring to pip dependencies; the +correct version will be automatically selected. + +Typically, a module will only have a single hub of pip dependencies, but this +is not required. Each hub is a separate resolution of pip dependencies. This +means if different programs need different versions of some library, separate +hubs can be created, and each program can use its respective hub's targets. +Targets from different hubs should not be used together. +""", + ), + "python_version": attr.string( + mandatory = True, + doc = """ +The Python version to use for resolving the pip dependencies, in Major.Minor +format (e.g. "3.11"). Patch level granularity (e.g. "3.11.1") is not supported. +If not specified, then the default Python version (as set by the root module or +rules_python) will be used. + +The version specified here must have a corresponding `python.toolchain()` +configured. +""", + ), + "whl_modifications": attr.label_keyed_string_dict( + mandatory = False, + doc = """\ +A dict of labels to wheel names that is typically generated by the whl_modifications. +The labels are JSON config files describing the modifications. +""", + ), + }, **pip_repository_attrs) + + # Like the pip_repository rule, we end up setting this manually so + # don't allow users to override it. + attrs.pop("repo_prefix") + + # incompatible_generate_aliases is always True in bzlmod + attrs.pop("incompatible_generate_aliases") + + return attrs + +def _whl_mod_attrs(): + attrs = { + "additive_build_content": attr.string( + doc = "(str, optional): Raw text to add to the generated `BUILD` file of a package.", + ), + "additive_build_content_file": attr.label( + doc = """\ +(label, optional): path to a BUILD file to add to the generated +`BUILD` file of a package. You cannot use both additive_build_content and additive_build_content_file +arguments at the same time.""", + ), + "copy_executables": attr.string_dict( + doc = """\ +(dict, optional): A mapping of `src` and `out` files for +[@bazel_skylib//rules:copy_file.bzl][cf]. Targets generated here will also be flagged as +executable.""", + ), + "copy_files": attr.string_dict( + doc = """\ +(dict, optional): A mapping of `src` and `out` files for +[@bazel_skylib//rules:copy_file.bzl][cf]""", + ), + "data": attr.string_list( + doc = """\ +(list, optional): A list of labels to add as `data` dependencies to +the generated `py_library` target.""", + ), + "data_exclude_glob": attr.string_list( + doc = """\ +(list, optional): A list of exclude glob patterns to add as `data` to +the generated `py_library` target.""", + ), + "hub_name": attr.string( + doc = """\ +Name of the whl modification, hub we use this name to set the modifications for +pip.parse. If you have different pip hubs you can use a different name, +otherwise it is best practice to just use one. + +You cannot have the same `hub_name` in different modules. You can reuse the same +name in the same module for different wheels that you put in the same hub, but you +cannot have a child module that uses the same `hub_name`. +""", + mandatory = True, + ), + "srcs_exclude_glob": attr.string_list( + doc = """\ +(list, optional): A list of labels to add as `srcs` to the generated +`py_library` target.""", + ), + "whl_name": attr.string( + doc = "The whl name that the modifications are used for.", + mandatory = True, + ), + } + return attrs + +def _extension_extra_args(): + args = {} + + if bazel_features.external_deps.module_extension_has_os_arch_dependent: + args = args | { + "arch_dependent": True, + "os_dependent": True, + } + + return args + +pip = module_extension( + doc = """\ +This extension is used to make dependencies from pip available. + +pip.parse: +To use, call `pip.parse()` and specify `hub_name` and your requirements file. +Dependencies will be downloaded and made available in a repo named after the +`hub_name` argument. + +Each `pip.parse()` call configures a particular Python version. Multiple calls +can be made to configure different Python versions, and will be grouped by +the `hub_name` argument. This allows the same logical name, e.g. `@pip//numpy` +to automatically resolve to different, Python version-specific, libraries. + +pip.whl_mods: +This tag class is used to help create JSON files to describe modifications to +the BUILD files for wheels. +""", + implementation = _pip_impl, + tag_classes = { + "parse": tag_class( + attrs = _pip_parse_ext_attrs(), + doc = """\ +This tag class is used to create a pip hub and all of the spokes that are part of that hub. +This tag class reuses most of the pip attributes that are found in +@rules_python//python/pip_install:pip_repository.bzl. +The exceptions are it does not use the args 'repo_prefix', +and 'incompatible_generate_aliases'. We set the repository prefix +for the user and the alias arg is always True in bzlmod. +""", + ), + "whl_mods": tag_class( + attrs = _whl_mod_attrs(), + doc = """\ +This tag class is used to create JSON file that are used when calling wheel_builder.py. These +JSON files contain instructions on how to modify a wheel's project. Each of the attributes +create different modifications based on the type of attribute. Previously to bzlmod these +JSON files where referred to as annotations, and were renamed to whl_modifications in this +extension. +""", + ), + }, + **_extension_extra_args() +) + +def _whl_mods_repo_impl(rctx): + rctx.file("BUILD.bazel", "") + for whl_name, mods in rctx.attr.whl_mods.items(): + rctx.file("{}.json".format(whl_name), mods) + +_whl_mods_repo = repository_rule( + doc = """\ +This rule creates json files based on the whl_mods attribute. +""", + implementation = _whl_mods_repo_impl, + attrs = { + "whl_mods": attr.string_dict( + mandatory = True, + doc = "JSON endcoded string that is provided to wheel_builder.py", + ), + }, +) diff --git a/python/private/bzlmod/pip_repository.bzl b/python/private/bzlmod/pip_repository.bzl new file mode 100644 index 0000000000..f5bb46feaa --- /dev/null +++ b/python/private/bzlmod/pip_repository.bzl @@ -0,0 +1,87 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases") +load("//python/private:text_util.bzl", "render") + +_BUILD_FILE_CONTENTS = """\ +package(default_visibility = ["//visibility:public"]) + +# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it +exports_files(["requirements.bzl"]) +""" + +def _pip_repository_impl(rctx): + bzl_packages = rctx.attr.whl_map.keys() + aliases = render_pkg_aliases( + repo_name = rctx.attr.repo_name, + rules_python = rctx.attr._template.workspace_name, + default_version = rctx.attr.default_version, + whl_map = rctx.attr.whl_map, + ) + for path, contents in aliases.items(): + rctx.file(path, contents) + + # NOTE: we are using the canonical name with the double '@' in order to + # always uniquely identify a repository, as the labels are being passed as + # a string and the resolution of the label happens at the call-site of the + # `requirement`, et al. macros. + macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name) + + rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) + rctx.template("requirements.bzl", rctx.attr._template, substitutions = { + "%%ALL_DATA_REQUIREMENTS%%": render.list([ + macro_tmpl.format(p, "data") + for p in bzl_packages + ]), + "%%ALL_REQUIREMENTS%%": render.list([ + macro_tmpl.format(p, p) + for p in bzl_packages + ]), + "%%ALL_WHL_REQUIREMENTS%%": render.list([ + macro_tmpl.format(p, "whl") + for p in bzl_packages + ]), + "%%MACRO_TMPL%%": macro_tmpl, + "%%NAME%%": rctx.attr.name, + }) + +pip_repository_attrs = { + "default_version": attr.string( + mandatory = True, + doc = """\ +This is the default python version in the format of X.Y.Z. This should match +what is setup by the 'python' extension using the 'is_default = True' +setting.""", + ), + "repo_name": attr.string( + mandatory = True, + doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", + ), + "whl_map": attr.string_list_dict( + mandatory = True, + doc = "The wheel map where values are python versions", + ), + "_template": attr.label( + default = ":requirements.bzl.tmpl", + ), +} + +pip_repository = repository_rule( + attrs = pip_repository_attrs, + doc = """A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY.""", + implementation = _pip_repository_impl, +) diff --git a/python/private/bzlmod/python.bzl b/python/private/bzlmod/python.bzl new file mode 100644 index 0000000000..be5c083d3d --- /dev/null +++ b/python/private/bzlmod/python.bzl @@ -0,0 +1,266 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"Python toolchain module extensions for use with bzlmod" + +load("//python:repositories.bzl", "python_register_toolchains") +load("//python/private:toolchains_repo.bzl", "multi_toolchain_aliases") +load(":pythons_hub.bzl", "hub_repo") + +# This limit can be increased essentially arbitrarily, but doing so will cause a rebuild of all +# targets using any of these toolchains due to the changed repository name. +_MAX_NUM_TOOLCHAINS = 9999 +_TOOLCHAIN_INDEX_PAD_LENGTH = len(str(_MAX_NUM_TOOLCHAINS)) + +def _toolchain_prefix(index, name): + """Prefixes the given name with the index, padded with zeros to ensure lexicographic sorting. + + Examples: + _toolchain_prefix( 2, "foo") == "_0002_foo_" + _toolchain_prefix(2000, "foo") == "_2000_foo_" + """ + return "_{}_{}_".format(_left_pad_zero(index, _TOOLCHAIN_INDEX_PAD_LENGTH), name) + +def _left_pad_zero(index, length): + if index < 0: + fail("index must be non-negative") + return ("0" * length + str(index))[-length:] + +# Printing a warning msg not debugging, so we have to disable +# the buildifier check. +# buildifier: disable=print +def _print_warn(msg): + print("WARNING:", msg) + +def _python_register_toolchains(name, toolchain_attr, version_constraint): + """Calls python_register_toolchains and returns a struct used to collect the toolchains. + """ + python_register_toolchains( + name = name, + python_version = toolchain_attr.python_version, + register_coverage_tool = toolchain_attr.configure_coverage_tool, + ignore_root_user_error = toolchain_attr.ignore_root_user_error, + set_python_version_constraint = version_constraint, + ) + return struct( + python_version = toolchain_attr.python_version, + set_python_version_constraint = str(version_constraint), + name = name, + ) + +def _python_impl(module_ctx): + # The toolchain info structs to register, in the order to register them in. + toolchains = [] + + # We store the default toolchain separately to ensure it is the last + # toolchain added to toolchains. + default_toolchain = None + + # Map of string Major.Minor to the toolchain name and module name + global_toolchain_versions = {} + + for mod in module_ctx.modules: + module_toolchain_versions = [] + + for toolchain_attr in mod.tags.toolchain: + toolchain_version = toolchain_attr.python_version + toolchain_name = "python_" + toolchain_version.replace(".", "_") + + # Duplicate versions within a module indicate a misconfigured module. + if toolchain_version in module_toolchain_versions: + _fail_duplicate_module_toolchain_version(toolchain_version, mod.name) + module_toolchain_versions.append(toolchain_version) + + # Ignore version collisions in the global scope because there isn't + # much else that can be done. Modules don't know and can't control + # what other modules do, so the first in the dependency graph wins. + if toolchain_version in global_toolchain_versions: + # If the python version is explicitly provided by the root + # module, they should not be warned for choosing the same + # version that rules_python provides as default. + first = global_toolchain_versions[toolchain_version] + if mod.name != "rules_python" or not first.is_root: + _warn_duplicate_global_toolchain_version( + toolchain_version, + first = first, + second_toolchain_name = toolchain_name, + second_module_name = mod.name, + ) + continue + global_toolchain_versions[toolchain_version] = struct( + toolchain_name = toolchain_name, + module_name = mod.name, + is_root = mod.is_root, + ) + + # Only the root module and rules_python are allowed to specify the default + # toolchain for a couple reasons: + # * It prevents submodules from specifying different defaults and only + # one of them winning. + # * rules_python needs to set a soft default in case the root module doesn't, + # e.g. if the root module doesn't use Python itself. + # * The root module is allowed to override the rules_python default. + if mod.is_root: + # A single toolchain is treated as the default because it's unambiguous. + is_default = toolchain_attr.is_default or len(mod.tags.toolchain) == 1 + elif mod.name == "rules_python" and not default_toolchain: + # We don't do the len() check because we want the default that rules_python + # sets to be clearly visible. + is_default = toolchain_attr.is_default + else: + is_default = False + + # We have already found one default toolchain, and we can only have + # one. + if is_default and default_toolchain != None: + _fail_multiple_default_toolchains( + first = default_toolchain.name, + second = toolchain_name, + ) + + toolchain_info = _python_register_toolchains( + toolchain_name, + toolchain_attr, + version_constraint = not is_default, + ) + + if is_default: + default_toolchain = toolchain_info + else: + toolchains.append(toolchain_info) + + # A default toolchain is required so that the non-version-specific rules + # are able to match a toolchain. + if default_toolchain == None: + fail("No default Python toolchain configured. Is rules_python missing `is_default=True`?") + + # The last toolchain in the BUILD file is set as the default + # toolchain. We need the default last. + toolchains.append(default_toolchain) + + if len(toolchains) > _MAX_NUM_TOOLCHAINS: + fail("more than {} python versions are not supported".format(_MAX_NUM_TOOLCHAINS)) + + # Create the pythons_hub repo for the interpreter meta data and the + # the various toolchains. + hub_repo( + name = "pythons_hub", + default_python_version = default_toolchain.python_version, + toolchain_prefixes = [ + _toolchain_prefix(index, toolchain.name) + for index, toolchain in enumerate(toolchains) + ], + toolchain_python_versions = [t.python_version for t in toolchains], + toolchain_set_python_version_constraints = [t.set_python_version_constraint for t in toolchains], + toolchain_user_repository_names = [t.name for t in toolchains], + ) + + # This is require in order to support multiple version py_test + # and py_binary + multi_toolchain_aliases( + name = "python_versions", + python_versions = { + version: entry.toolchain_name + for version, entry in global_toolchain_versions.items() + }, + ) + +def _fail_duplicate_module_toolchain_version(version, module): + fail(("Duplicate module toolchain version: module '{module}' attempted " + + "to use version '{version}' multiple times in itself").format( + version = version, + module = module, + )) + +def _warn_duplicate_global_toolchain_version(version, first, second_toolchain_name, second_module_name): + _print_warn(( + "Ignoring toolchain '{second_toolchain}' from module '{second_module}': " + + "Toolchain '{first_toolchain}' from module '{first_module}' " + + "already registered Python version {version} and has precedence" + ).format( + first_toolchain = first.toolchain_name, + first_module = first.module_name, + second_module = second_module_name, + second_toolchain = second_toolchain_name, + version = version, + )) + +def _fail_multiple_default_toolchains(first, second): + fail(("Multiple default toolchains: only one toolchain " + + "can have is_default=True. First default " + + "was toolchain '{first}'. Second was '{second}'").format( + first = first, + second = second, + )) + +python = module_extension( + doc = """Bzlmod extension that is used to register Python toolchains. +""", + implementation = _python_impl, + tag_classes = { + "toolchain": tag_class( + doc = """Tag class used to register Python toolchains. +Use this tag class to register one or more Python toolchains. This class +is also potentially called by sub modules. The following covers different +business rules and use cases. + +Toolchains in the Root Module + +This class registers all toolchains in the root module. + +Toolchains in Sub Modules + +It will create a toolchain that is in a sub module, if the toolchain +of the same name does not exist in the root module. The extension stops name +clashing between toolchains in the root module and toolchains in sub modules. +You cannot configure more than one toolchain as the default toolchain. + +Toolchain set as the default version + +This extension will not create a toolchain that exists in a sub module, +if the sub module toolchain is marked as the default version. If you have +more than one toolchain in your root module, you need to set one of the +toolchains as the default version. If there is only one toolchain it +is set as the default toolchain. + +Toolchain repository name + +A toolchain's repository name uses the format `python_{major}_{minor}`, e.g. +`python_3_10`. The `major` and `minor` components are +`major` and `minor` are the Python version from the `python_version` attribute. +""", + attrs = { + "configure_coverage_tool": attr.bool( + mandatory = False, + doc = "Whether or not to configure the default coverage tool for the toolchains.", + ), + "ignore_root_user_error": attr.bool( + default = False, + doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.", + mandatory = False, + ), + "is_default": attr.bool( + mandatory = False, + doc = "Whether the toolchain is the default version", + ), + "python_version": attr.string( + mandatory = True, + doc = "The Python version, in `major.minor` format, e.g " + + "'3.12', to create a toolchain for. Patch level " + + "granularity (e.g. '3.12.1') is not supported.", + ), + }, + ), + }, +) diff --git a/python/extensions/private/pythons_hub.bzl b/python/private/bzlmod/pythons_hub.bzl similarity index 100% rename from python/extensions/private/pythons_hub.bzl rename to python/private/bzlmod/pythons_hub.bzl diff --git a/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl b/python/private/bzlmod/requirements.bzl.tmpl similarity index 100% rename from python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl rename to python/private/bzlmod/requirements.bzl.tmpl diff --git a/python/private/text_util.bzl b/python/private/text_util.bzl index 3d72b8d676..da67001ce8 100644 --- a/python/private/text_util.bzl +++ b/python/private/text_util.bzl @@ -57,9 +57,20 @@ def _render_select(selects, *, no_match_error = None): return "select({})".format(args) +def _render_list(items): + return "\n".join([ + "[", + _indent("\n".join([ + "{},".format(repr(item)) + for item in items + ])), + "]", + ]) + render = struct( - indent = _indent, alias = _render_alias, dict = _render_dict, + indent = _indent, + list = _render_list, select = _render_select, )