diff options
author | Ignas Anikevicius <240938+aignas@users.noreply.github.com> | 2024-03-18 16:16:26 +0900 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-03-18 07:16:26 +0000 |
commit | c9c2768ac38040edc1d56f74a0db4f8538c7ef5f (patch) | |
tree | f4bd4ed6f799c02f8543766ce3c2b79fd90df15c | |
parent | bdb2aa2e5c0ca435918037aa3568c0b0c4c9c1ab (diff) | |
download | bazelbuild-rules_python-c9c2768ac38040edc1d56f74a0db4f8538c7ef5f.tar.gz |
internal(config_settings): make config_setting creation reusable (#1750)
The PR #1743 explored the idea of creating extra config settings for
each target platform that our toolchain is targetting, however that has
a drawback of not being usable in `bzlmod` if someone built Python for
a platform that we don't provide a toolchain for and tried to use the
`pip.parse` machinery with that by providing the
`python_interpreter_target`. That is a niche usecase, but `rules_python`
is a core ruleset that should only provide abstractions/helpers that
work in all cases or make it possible to extend things.
This explores a way to decouple the definition of the available
`config_settings` values and how they are constructed by adding an extra
`is_python_config_setting` macro, that could be used to declare the
config settings from within the `pip.parse` hub repo. This makes the
work in #1744 to support whl-only hub repos more self-contained.
Supersedes #1743.
---------
Co-authored-by: Richard Levasseur <rlevasseur@google.com>
-rw-r--r-- | python/config_settings/BUILD.bazel | 2 | ||||
-rw-r--r-- | python/config_settings/config_settings.bzl | 115 | ||||
-rw-r--r-- | python/private/BUILD.bazel | 9 | ||||
-rw-r--r-- | python/private/config_settings.bzl | 190 | ||||
-rw-r--r-- | tests/config_settings/construct_config_settings_tests.bzl | 102 |
5 files changed, 303 insertions, 115 deletions
diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index 4f12ef4..a017f97 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -1,4 +1,3 @@ -load("//python:versions.bzl", "TOOL_VERSIONS") load(":config_settings.bzl", "construct_config_settings") filegroup( @@ -12,5 +11,4 @@ filegroup( construct_config_settings( name = "construct_config_settings", - python_versions = TOOL_VERSIONS.keys(), ) diff --git a/python/config_settings/config_settings.bzl b/python/config_settings/config_settings.bzl index 9e6bbd6..f1d2ff0 100644 --- a/python/config_settings/config_settings.bzl +++ b/python/config_settings/config_settings.bzl @@ -15,105 +15,16 @@ """This module is used to construct the config settings in the BUILD file in this same package. """ -load("@bazel_skylib//lib:selects.bzl", "selects") -load("@bazel_skylib//rules:common_settings.bzl", "string_flag") -load("//python:versions.bzl", "MINOR_MAPPING") - -def construct_config_settings(name, python_versions): - """Constructs a set of configs for all Python versions. - - Args: - name: str, unused; only specified to satisfy buildifier lint checks - and allow programatic modification of the target. - python_versions: list of all (x.y.z) Python versions supported by rules_python. - """ - - # Maps e.g. "3.8" -> ["3.8.1", "3.8.2", etc] - minor_to_micro_versions = {} - - allowed_flag_values = [] - for micro_version in python_versions: - minor, _, _ = micro_version.rpartition(".") - minor_to_micro_versions.setdefault(minor, []).append(micro_version) - allowed_flag_values.append(micro_version) - - allowed_flag_values.extend(list(minor_to_micro_versions)) - - string_flag( - name = "python_version", - # TODO: The default here should somehow match the MODULE config. Until - # then, use the empty string to indicate an unknown version. This - # also prevents version-unaware targets from inadvertently matching - # a select condition when they shouldn't. - build_setting_default = "", - values = [""] + sorted(allowed_flag_values), - visibility = ["//visibility:public"], - ) - - for minor_version, micro_versions in minor_to_micro_versions.items(): - # This matches the raw flag value, e.g. --//python/config_settings:python_version=3.8 - # It's private because matching the concept of e.g. "3.8" value is done - # using the `is_python_X.Y` config setting group, which is aware of the - # minor versions that could match instead. - equals_minor_version_name = "_python_version_flag_equals_" + minor_version - native.config_setting( - name = equals_minor_version_name, - flag_values = {":python_version": minor_version}, - ) - matches_minor_version_names = [equals_minor_version_name] - - default_micro_version = MINOR_MAPPING[minor_version] - - for micro_version in micro_versions: - is_micro_version_name = "is_python_" + micro_version - if default_micro_version != micro_version: - native.config_setting( - name = is_micro_version_name, - flag_values = {":python_version": micro_version}, - visibility = ["//visibility:public"], - ) - matches_minor_version_names.append(is_micro_version_name) - continue - - # Ensure that is_python_3.9.8 is matched if python_version is set - # to 3.9 if MINOR_MAPPING points to 3.9.8 - equals_micro_name = "_python_version_flag_equals_" + micro_version - native.config_setting( - name = equals_micro_name, - flag_values = {":python_version": micro_version}, - ) - - # An alias pointing to an underscore-prefixed config_setting_group - # is used because config_setting_group creates - # `is_{minor}_N` targets, which are easily confused with the - # `is_{minor}.{micro}` (dot) targets. - selects.config_setting_group( - name = "_" + is_micro_version_name, - match_any = [ - equals_micro_name, - equals_minor_version_name, - ], - ) - native.alias( - name = is_micro_version_name, - actual = "_" + is_micro_version_name, - visibility = ["//visibility:public"], - ) - matches_minor_version_names.append(equals_micro_name) - - # This is prefixed with an underscore to prevent confusion due to how - # config_setting_group is implemented and how our micro-version targets - # are named. config_setting_group will generate targets like - # "is_python_3.10_1" (where the `_N` suffix is len(match_any). - # Meanwhile, the micro-version tarets are named "is_python_3.10.1" -- - # just a single dot vs underscore character difference. - selects.config_setting_group( - name = "_is_python_" + minor_version, - match_any = matches_minor_version_names, - ) - - native.alias( - name = "is_python_" + minor_version, - actual = "_is_python_" + minor_version, - visibility = ["//visibility:public"], - ) +load( + "//python/private:config_settings.bzl", + _construct_config_settings = "construct_config_settings", + _is_python_config_setting = "is_python_config_setting", +) + +# This is exposed only for cases where the pip hub repo needs to use this rule +# to define hub-repo scoped config_settings for platform specific wheel +# support. +is_python_config_setting = _is_python_config_setting + +# This is exposed for usage in rules_python only. +construct_config_settings = _construct_config_settings diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 221c3b7..d3d6e76 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -69,6 +69,15 @@ bzl_library( ) bzl_library( + name = "config_settings_bzl", + srcs = ["config_settings.bzl"], + deps = [ + "//python:versions_bzl", + "@bazel_skylib//lib:selects", + ], +) + +bzl_library( name = "coverage_deps_bzl", srcs = ["coverage_deps.bzl"], deps = [ diff --git a/python/private/config_settings.bzl b/python/private/config_settings.bzl new file mode 100644 index 0000000..75f88de --- /dev/null +++ b/python/private/config_settings.bzl @@ -0,0 +1,190 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This module is used to construct the config settings in the BUILD file in this same package. +""" + +load("@bazel_skylib//lib:selects.bzl", "selects") +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") +load("//python:versions.bzl", "MINOR_MAPPING", "TOOL_VERSIONS") + +_PYTHON_VERSION_FLAG = str(Label("//python/config_settings:python_version")) + +def _ver_key(s): + major, _, s = s.partition(".") + minor, _, s = s.partition(".") + micro, _, s = s.partition(".") + return (int(major), int(minor), int(micro)) + +def _flag_values(python_versions): + """Construct a map of python_version to a list of toolchain values. + + This mapping maps the concept of a config setting to a list of compatible toolchain versions. + For using this in the code, the VERSION_FLAG_VALUES should be used instead. + + Args: + python_versions: list of strings; all X.Y.Z python versions + + Returns: + A `map[str, list[str]]`. Each key is a python_version flag value. Each value + is a list of the python_version flag values that should match when for the + `key`. For example: + ``` + "3.8" -> ["3.8", "3.8.1", "3.8.2", ..., "3.8.19"] # All 3.8 versions + "3.8.2" -> ["3.8.2"] # Only 3.8.2 + "3.8.19" -> ["3.8.19", "3.8"] # The latest version should also match 3.8 so + as when the `3.8` toolchain is used we just use the latest `3.8` toolchain. + this makes the `select("is_python_3.8.19")` work no matter how the user + specifies the latest python version to use. + ``` + """ + ret = {} + + for micro_version in sorted(python_versions, key = _ver_key): + minor_version, _, _ = micro_version.rpartition(".") + + # This matches the raw flag value, e.g. --//python/config_settings:python_version=3.8 + # It's private because matching the concept of e.g. "3.8" value is done + # using the `is_python_X.Y` config setting group, which is aware of the + # minor versions that could match instead. + ret.setdefault(minor_version, [minor_version]).append(micro_version) + + # Ensure that is_python_3.9.8 is matched if python_version is set + # to 3.9 if MINOR_MAPPING points to 3.9.8 + default_micro_version = MINOR_MAPPING[minor_version] + ret[micro_version] = [micro_version, minor_version] if default_micro_version == micro_version else [micro_version] + + return ret + +VERSION_FLAG_VALUES = _flag_values(TOOL_VERSIONS.keys()) + +def is_python_config_setting(name, *, python_version, reuse_conditions = None, **kwargs): + """Create a config setting for matching 'python_version' configuration flag. + + This function is mainly intended for internal use within the `whl_library` and `pip_parse` + machinery. + + The matching of the 'python_version' flag depends on the value passed in + `python_version` and here is the example for `3.8` (but the same applies + to other python versions present in @//python:versions.bzl#TOOL_VERSIONS): + * "3.8" -> ["3.8", "3.8.1", "3.8.2", ..., "3.8.19"] # All 3.8 versions + * "3.8.2" -> ["3.8.2"] # Only 3.8.2 + * "3.8.19" -> ["3.8.19", "3.8"] # The latest version should also match 3.8 so + as when the `3.8` toolchain is used we just use the latest `3.8` toolchain. + this makes the `select("is_python_3.8.19")` work no matter how the user + specifies the latest python version to use. + + Args: + name: name for the target that will be created to be used in select statements. + python_version: The python_version to be passed in the `flag_values` in the + `config_setting`. Depending on the version, the matching python version list + can be as described above. + reuse_conditions: A dict of version to version label for which we should + reuse config_setting targets instead of creating them from scratch. This + is useful when using is_python_config_setting multiple times in the + same package with the same `major.minor` python versions. + **kwargs: extra kwargs passed to the `config_setting`. + """ + if python_version not in name: + fail("The name '{}' must have the python version '{}' in it".format(name, python_version)) + + if python_version not in VERSION_FLAG_VALUES: + fail("The 'python_version' must be known to 'rules_python', choose from the values: {}".format(VERSION_FLAG_VALUES.keys())) + + python_versions = VERSION_FLAG_VALUES[python_version] + if len(python_versions) == 1: + native.config_setting( + name = name, + flag_values = { + _PYTHON_VERSION_FLAG: python_version, + }, + **kwargs + ) + return + + reuse_conditions = reuse_conditions or {} + create_config_settings = { + "_{}".format(name).replace(python_version, version): {_PYTHON_VERSION_FLAG: version} + for version in python_versions + if not reuse_conditions or version not in reuse_conditions + } + match_any = list(create_config_settings.keys()) + for version, condition in reuse_conditions.items(): + if len(VERSION_FLAG_VALUES[version]) == 1: + match_any.append(condition) + continue + + # Convert the name to an internal label that this function would create, + # so that we are hitting the config_setting and not the config_setting_group. + condition = Label(condition) + if hasattr(condition, "same_package_label"): + condition = condition.same_package_label("_" + condition.name) + else: + condition = condition.relative("_" + condition.name) + + match_any.append(condition) + + for name_, flag_values_ in create_config_settings.items(): + native.config_setting( + name = name_, + flag_values = flag_values_, + **kwargs + ) + + # An alias pointing to an underscore-prefixed config_setting_group + # is used because config_setting_group creates + # `is_{version}_N` targets, which are easily confused with the + # `is_{minor}.{micro}` (dot) targets. + selects.config_setting_group( + name = "_{}_group".format(name), + match_any = match_any, + visibility = ["//visibility:private"], + ) + native.alias( + name = name, + actual = "_{}_group".format(name), + visibility = kwargs.get("visibility", []), + ) + +def construct_config_settings(name = None): # buildifier: disable=function-docstring + """Create a 'python_version' config flag and construct all config settings used in rules_python. + + This mainly includes the targets that are used in the toolchain and pip hub + repositories that only match on the 'python_version' flag values. + + Args: + name(str): A dummy name value that is no-op for now. + """ + string_flag( + name = "python_version", + # TODO: The default here should somehow match the MODULE config. Until + # then, use the empty string to indicate an unknown version. This + # also prevents version-unaware targets from inadvertently matching + # a select condition when they shouldn't. + build_setting_default = "", + values = [""] + VERSION_FLAG_VALUES.keys(), + visibility = ["//visibility:public"], + ) + + for version, matching_versions in VERSION_FLAG_VALUES.items(): + is_python_config_setting( + name = "is_python_{}".format(version), + python_version = version, + reuse_conditions = { + v: native.package_relative_label("is_python_{}".format(v)) + for v in matching_versions + if v != version + }, + visibility = ["//visibility:public"], + ) diff --git a/tests/config_settings/construct_config_settings_tests.bzl b/tests/config_settings/construct_config_settings_tests.bzl index 61beb9c..b1b2e06 100644 --- a/tests/config_settings/construct_config_settings_tests.bzl +++ b/tests/config_settings/construct_config_settings_tests.bzl @@ -18,6 +18,7 @@ load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:truth.bzl", "subjects") load("@rules_testing//lib:util.bzl", rt_util = "util") +load("//python/config_settings:config_settings.bzl", "is_python_config_setting") _tests = [] @@ -28,25 +29,51 @@ def _subject_impl(ctx): _subject = rule( implementation = _subject_impl, attrs = { + "match_cpu": attr.string(), "match_micro": attr.string(), "match_minor": attr.string(), + "match_os": attr.string(), + "match_os_cpu": attr.string(), "no_match": attr.string(), "no_match_micro": attr.string(), }, ) def _test_minor_version_matching(name): + minor_matches = { + # Having it here ensures that we can mix and match config settings defined in + # the repo and elsewhere + str(Label("//python/config_settings:is_python_3.11")): "matched-3.11", + "//conditions:default": "matched-default", + } + minor_cpu_matches = { + str(Label(":is_python_3.11_aarch64")): "matched-3.11-aarch64", + str(Label(":is_python_3.11_ppc")): "matched-3.11-ppc", + str(Label(":is_python_3.11_s390x")): "matched-3.11-s390x", + str(Label(":is_python_3.11_x86_64")): "matched-3.11-x86_64", + } + minor_os_matches = { + str(Label(":is_python_3.11_linux")): "matched-3.11-linux", + str(Label(":is_python_3.11_osx")): "matched-3.11-osx", + str(Label(":is_python_3.11_windows")): "matched-3.11-windows", + } + minor_os_cpu_matches = { + str(Label(":is_python_3.11_linux_aarch64")): "matched-3.11-linux-aarch64", + str(Label(":is_python_3.11_linux_ppc")): "matched-3.11-linux-ppc", + str(Label(":is_python_3.11_linux_s390x")): "matched-3.11-linux-s390x", + str(Label(":is_python_3.11_linux_x86_64")): "matched-3.11-linux-x86_64", + str(Label(":is_python_3.11_osx_aarch64")): "matched-3.11-osx-aarch64", + str(Label(":is_python_3.11_osx_x86_64")): "matched-3.11-osx-x86_64", + str(Label(":is_python_3.11_windows_x86_64")): "matched-3.11-windows-x86_64", + } + rt_util.helper_target( _subject, name = name + "_subject", - match_minor = select({ - "//python/config_settings:is_python_3.11": "matched-3.11", - "//conditions:default": "matched-default", - }), - match_micro = select({ - "//python/config_settings:is_python_3.11": "matched-3.11", - "//conditions:default": "matched-default", - }), + match_minor = select(minor_matches), + match_cpu = select(minor_matches | minor_cpu_matches), + match_os = select(minor_matches | minor_os_matches), + match_os_cpu = select(minor_matches | minor_cpu_matches | minor_os_matches | minor_os_cpu_matches), no_match = select({ "//python/config_settings:is_python_3.12": "matched-3.12", "//conditions:default": "matched-default", @@ -59,16 +86,23 @@ def _test_minor_version_matching(name): impl = _test_minor_version_matching_impl, config_settings = { str(Label("//python/config_settings:python_version")): "3.11.1", + "//command_line_option:platforms": str(Label("//tests/config_settings:linux_aarch64")), }, ) def _test_minor_version_matching_impl(env, target): target = env.expect.that_target(target) + target.attr("match_cpu", factory = subjects.str).equals( + "matched-3.11-aarch64", + ) target.attr("match_minor", factory = subjects.str).equals( "matched-3.11", ) - target.attr("match_micro", factory = subjects.str).equals( - "matched-3.11", + target.attr("match_os", factory = subjects.str).equals( + "matched-3.11-linux", + ) + target.attr("match_os_cpu", factory = subjects.str).equals( + "matched-3.11-linux-aarch64", ) target.attr("no_match", factory = subjects.str).equals( "matched-default", @@ -124,8 +158,54 @@ def _test_latest_micro_version_matching_impl(env, target): _tests.append(_test_latest_micro_version_matching) -def construct_config_settings_test_suite(name): +def construct_config_settings_test_suite(name): # buildifier: disable=function-docstring + # We have CI runners running on a great deal of the platforms from the list below, + # hence use all of them within tests. + for os in ["linux", "osx", "windows"]: + is_python_config_setting( + name = "is_python_3.11_" + os, + constraint_values = [ + "@platforms//os:" + os, + ], + python_version = "3.11", + ) + + for cpu in ["s390x", "ppc", "x86_64", "aarch64"]: + is_python_config_setting( + name = "is_python_3.11_" + cpu, + constraint_values = [ + "@platforms//cpu:" + cpu, + ], + python_version = "3.11", + ) + + for (os, cpu) in [ + ("linux", "aarch64"), + ("linux", "ppc"), + ("linux", "s390x"), + ("linux", "x86_64"), + ("osx", "aarch64"), + ("osx", "x86_64"), + ("windows", "x86_64"), + ]: + is_python_config_setting( + name = "is_python_3.11_{}_{}".format(os, cpu), + constraint_values = [ + "@platforms//cpu:" + cpu, + "@platforms//os:" + os, + ], + python_version = "3.11", + ) + test_suite( name = name, tests = _tests, ) + + native.platform( + name = "linux_aarch64", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ], + ) |