From 3ef53b2098194d6754702dffa6defc24b0fd5566 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:07:15 +0900 Subject: [PATCH 01/20] refactor(pypi): select packages from the pyproject.toml dependency groups Read the dependencies, the optional dependencies and the PEP 735 dependency groups from the pyproject.toml and add a helper to select the packages using dependency group specifiers. --- python/private/pypi/BUILD.bazel | 9 ++ python/private/pypi/dependency_groups.bzl | 142 ++++++++++++++++++ python/private/pyproject_utils.bzl | 14 +- tests/pypi/dependency_groups/BUILD.bazel | 3 + .../dependency_groups_tests.bzl | 86 +++++++++++ 5 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 python/private/pypi/dependency_groups.bzl create mode 100644 tests/pypi/dependency_groups/BUILD.bazel create mode 100644 tests/pypi/dependency_groups/dependency_groups_tests.bzl diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index f94d24594f..af005f706e 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -632,6 +632,15 @@ bzl_library( ], ) +bzl_library( + name = "dependency_groups", + srcs = ["dependency_groups.bzl"], + deps = [ + ":pep508_requirement", + "//python/private:normalize_name", + ], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl new file mode 100644 index 0000000000..2ddadee3dd --- /dev/null +++ b/python/private/pypi/dependency_groups.bzl @@ -0,0 +1,142 @@ +"""Select PyPI packages using pyproject.toml dependency groups.""" + +load("//python/private:normalize_name.bzl", "normalize_name") +load(":pep508_requirement.bzl", "requirement") + +def select_dependency_groups(pyproject, specs, _fail = fail): + """Select the packages named by the given dependency group specifiers. + + Supported specifiers are: + + * `"default"`: the `[project] dependencies`. + * `"optional:"`: a `[project.optional-dependencies]` entry. + * `"group:"`: a PEP 735 `[dependency-groups]` entry. + * `"optional:*"` and `"group:*"`: all entries of the table. + * `"*"`: all of the above. + * `""`: a dependency group, or an optional dependency group if + there is no dependency group with that name. + * `"transitive"`: only supported together with `"*"` and selects all + of the packages, including the transitive dependencies. + + Args: + pyproject: {type}`struct` returned by `read_pyproject`. + specs: {type}`list[str]` the specifiers. + _fail: {type}`callable` the failure function, mainly for testing. + + Returns: + {type}`struct` with the attributes: + * `packages`: {type}`list[str]` the sorted normalized package names. + * `all`: {type}`bool` whether all of the packages including the + transitive dependencies should be selected, i.e. `"*"` and + `"transitive"` were given. + """ + groups = {"default": pyproject.dependencies} + for name, reqs in pyproject.optional_dependencies.items(): + groups["optional:" + normalize_name(name)] = reqs + for name, reqs in _expand_dependency_groups(pyproject.dependency_groups, _fail).items(): + groups["group:" + name] = reqs + + if "transitive" in specs: + if "*" not in specs: + _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format(specs)) + return struct(packages = [], all = True) + + selected = {} + for spec in specs: + keys = _match(spec, groups) + if not keys: + _fail("Unknown dependency group '{}'. Available groups: {}".format( + spec, + ", ".join(sorted(groups)), + )) + for key in keys: + selected[key] = None + + # A self reference like `my-project[test]` selects the optional + # dependencies, but only one level deep. + project = normalize_name(pyproject.name or "") + packages = {} + for key in selected: + for req in groups[key]: + req = requirement(req) + names = [req.name_] + if req.name_ == project: + names = [ + requirement(r).name_ + for extra in req.extras + for r in groups.get("optional:" + extra, []) + ] + for name in names: + if name: + packages[name] = None + + return struct(packages = sorted(packages), all = False) + +def _match(spec, groups): + if spec == "*": + return groups.keys() + if spec in ("optional:*", "group:*"): + prefix = spec[:-1] + return [key for key in groups if key.startswith(prefix)] + + kind, _, name = spec.rpartition(":") + name = normalize_name(name) + if kind: + candidates = [kind + ":" + name] + elif name == "default": + candidates = ["default"] + else: + candidates = ["group:" + name, "optional:" + name] + + for key in candidates: + if key in groups: + return [key] + return [] + +def _expand_dependency_groups(dependency_groups, _fail): + """Expand the PEP 735 `include-group` entries. + + Args: + dependency_groups: {type}`dict[str, list]` the `[dependency-groups]` + table. + _fail: {type}`callable` the failure function. + + Returns: + {type}`dict[str, list[str]]` normalized group names mapped to their + requirement strings. + """ + groups = { + normalize_name(name): items + for name, items in dependency_groups.items() + } + + # Each round expands at least one group unless there is a cycle. + expanded = {} + for _ in range(len(groups)): + for name, items in groups.items(): + if name in expanded: + continue + + reqs = [] + for item in items: + if type(item) == type(""): + reqs.append(item) + continue + + include = normalize_name(item.get("include-group", "")) + if include not in groups: + _fail("Unknown dependency group '{}' included from '{}'".format(include, name)) + return {} + if include not in expanded: + reqs = None + break + reqs.extend(expanded[include]) + + if reqs != None: + expanded[name] = reqs + + if len(expanded) != len(groups): + _fail("Cyclic dependency group includes: {}".format( + ", ".join(sorted([name for name in groups if name not in expanded])), + )) + return expanded diff --git a/python/private/pyproject_utils.bzl b/python/private/pyproject_utils.bzl index e96ffd2023..a9494b8ac2 100644 --- a/python/private/pyproject_utils.bzl +++ b/python/private/pyproject_utils.bzl @@ -18,10 +18,22 @@ def read_pyproject(module_ctx, pyproject): {type}`struct` with the attributes: * `requires_python`: {type}`str | None` the raw `requires-python` value (e.g. `"==3.13.9"`), or `None` if it is not set. + * `name`: {type}`str | None` the `[project] name`. + * `dependencies`: {type}`list[str]` the `[project] dependencies`. + * `optional_dependencies`: {type}`dict[str, list[str]]` the + `[project.optional-dependencies]` table. + * `dependency_groups`: {type}`dict[str, list]` the PEP 735 + `[dependency-groups]` table. Items are either requirement strings + or `{include-group = ""}` tables. """ data = toml.decode(module_ctx.read(module_ctx.path(pyproject), watch = "yes")) + project = data.get("project", {}) return struct( - requires_python = data.get("project", {}).get("requires-python"), + requires_python = project.get("requires-python"), + name = project.get("name"), + dependencies = project.get("dependencies") or [], + optional_dependencies = project.get("optional-dependencies") or {}, + dependency_groups = data.get("dependency-groups") or {}, ) def version_from_requires_python(requires_python): diff --git a/tests/pypi/dependency_groups/BUILD.bazel b/tests/pypi/dependency_groups/BUILD.bazel new file mode 100644 index 0000000000..ba6877e124 --- /dev/null +++ b/tests/pypi/dependency_groups/BUILD.bazel @@ -0,0 +1,3 @@ +load(":dependency_groups_tests.bzl", "dependency_groups_test_suite") + +dependency_groups_test_suite(name = "dependency_groups_tests") diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl new file mode 100644 index 0000000000..ee23aec1fb --- /dev/null +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -0,0 +1,86 @@ +"""dependency_groups tests""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:dependency_groups.bzl", "select_dependency_groups") # buildifier: disable=bzl-visibility + +_tests = [] + +_PYPROJECT = struct( + name = "my-project", + dependencies = ["main-pkg>=1.0"], + optional_dependencies = { + "Docs": ["docs-pkg"], + "all": ["My_Project[docs, grpc]"], + "grpc": ["grpc-pkg"], + }, + dependency_groups = { + "dev": ["dev-pkg", "shared-pkg", {"include-group": "Test"}], + "test": ["test_pkg", "shared-pkg"], + }, +) + +def _select(specs, pyproject = _PYPROJECT): + failures = [] + got = select_dependency_groups(pyproject, specs, _fail = failures.append) + return got, failures + +def _test_select(env): + for specs, want in [ + (["default"], ["main_pkg"]), + (["optional:docs"], ["docs_pkg"]), + (["group:test"], ["shared_pkg", "test_pkg"]), + (["dev"], ["dev_pkg", "shared_pkg", "test_pkg"]), + (["grpc"], ["grpc_pkg"]), + (["all"], ["docs_pkg", "grpc_pkg"]), + (["optional:*"], ["docs_pkg", "grpc_pkg"]), + (["group:*"], ["dev_pkg", "shared_pkg", "test_pkg"]), + (["default", "group:test"], ["main_pkg", "shared_pkg", "test_pkg"]), + (["*"], ["dev_pkg", "docs_pkg", "grpc_pkg", "main_pkg", "shared_pkg", "test_pkg"]), + ]: + got, failures = _select(specs) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) + env.expect.where(specs = specs).that_collection(got.packages).contains_exactly(want).in_order() + env.expect.where(specs = specs).that_bool(got.all).equals(False) + +_tests.append(_test_select) + +def _test_select_all(env): + got, failures = _select(["*", "transitive"]) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_bool(got.all).equals(True) + env.expect.that_collection(got.packages).contains_exactly([]) + +_tests.append(_test_select_all) + +def _test_select_errors(env): + for specs, want in [ + (["missing"], "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test, optional:all, optional:docs, optional:grpc"), + (["default", "transitive"], "The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: [\"default\", \"transitive\"]"), + ]: + _, failures = _select(specs) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([want]) + +_tests.append(_test_select_errors) + +def _test_include_group_errors(env): + for groups, want in [ + ({"a": [{"include-group": "missing"}]}, "Unknown dependency group 'missing' included from 'a'"), + ({"a": [{"include-group": "b"}], "b": [{"include-group": "a"}]}, "Cyclic dependency group includes: a, b"), + ]: + _, failures = _select(["*"], pyproject = struct( + name = None, + dependencies = [], + optional_dependencies = {}, + dependency_groups = groups, + )) + env.expect.that_collection(failures).contains_exactly([want]) + +_tests.append(_test_include_group_errors) + +def dependency_groups_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 755737072827673c74afba95147995086a579543 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:07:32 +0900 Subject: [PATCH 02/20] refactor(pypi): support hiding the unexposed packages of a hub Add the `hide_unexposed_packages` attribute to the hub repository. When set, the aliases of the packages that are not exposed are only visible to the wheel repositories that depend on them. --- python/private/pypi/hub_repository.bzl | 13 +++- python/private/pypi/render_pkg_aliases.bzl | 45 +++++++++++++- .../render_pkg_aliases_test.bzl | 62 +++++++++++++++++++ 3 files changed, 115 insertions(+), 5 deletions(-) diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index f915aa1c77..d421f3cc09 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -26,12 +26,16 @@ exports_files(["requirements.bzl"]) """ def _impl(rctx): - bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + if rctx.attr.hide_unexposed_packages: + bzl_packages = rctx.attr.packages + else: + bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() aliases = render_multiplatform_pkg_aliases( aliases = { key: _whl_config_settings_from_json(values) for key, values in rctx.attr.whl_map.items() }, + exposed_packages = bzl_packages if rctx.attr.hide_unexposed_packages else None, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, @@ -78,10 +82,15 @@ hub_repository = repository_rule( "groups": attr.string_list_dict( mandatory = False, ), + "hide_unexposed_packages": attr.bool( + default = False, + doc = "Make the aliases of the packages that are not in `packages` visible only to the wheel repositories.", + ), "packages": attr.string_list( mandatory = False, doc = """\ -The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys. +The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys +unless `hide_unexposed_packages` is set. """, ), "platform_config_settings": attr.string_list_dict( diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 0a1c328491..5582cc00c9 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -65,7 +65,7 @@ def _repr_actual(aliases): else: return render.dict(aliases, key_repr = _repr_config_setting) -def _render_common_aliases(*, name, aliases, **kwargs): +def _render_common_aliases(*, name, aliases, visibility, **kwargs): pkg_aliases = render.call( "pkg_aliases", name = repr(name), @@ -80,14 +80,21 @@ def _render_common_aliases(*, name, aliases, **kwargs): return """\ load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") {extra_loads} -package(default_visibility = ["//visibility:public"]) +package(default_visibility = {visibility}) {aliases}""".format( aliases = pkg_aliases, extra_loads = extra_loads, + visibility = render.list(visibility), ) -def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, **kwargs): +def render_pkg_aliases( + *, + aliases, + requirement_cycles = None, + extra_hub_aliases = {}, + exposed_packages = None, + **kwargs): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -100,6 +107,9 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases requirement_cycles: any package groups to also add. extra_hub_aliases: The list of extra aliases for each whl to be added in addition to the default ones. + exposed_packages: The packages that should be public. When set, the + aliases of the other packages are only visible to the wheel + repositories. **kwargs: Extra kwargs to pass to the rules. Returns: @@ -124,12 +134,18 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases for whl_name in group_whls } + public = ["//visibility:public"] + if exposed_packages != None: + exposed_packages = {normalize_name(name): None for name in exposed_packages} + internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) + files = { "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( name = normalize_name(name), aliases = pkg_aliases, extra_aliases = extra_hub_aliases.get(normalize_name(name), []), group_name = whl_group_mapping.get(normalize_name(name)), + visibility = public if exposed_packages == None or normalize_name(name) in exposed_packages else internal_visibility, **kwargs ).strip() for name, pkg_aliases in aliases.items() @@ -139,6 +155,29 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) return files +def _internal_visibility(aliases, has_groups = False): + """The visibility of the packages that are not exposed. + + The wheel repositories depend on the aliases of the hub, so the aliases + have to be visible to them. + """ + repo_names = {} + for pkg_aliases in aliases.values(): + if type(pkg_aliases) == type(""): + repo_names[pkg_aliases] = None + continue + + for repo_name in pkg_aliases.values(): + repo_names[repo_name] = None + + visibility = [ + "@{}//:__pkg__".format(repo_name) + for repo_name in sorted(repo_names) + ] + if has_groups: + visibility.append("//_groups:__pkg__") + return visibility + def _major_minor(python_version): major, _, tail = python_version.partition(".") minor, _, _ = tail.partition(".") diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 9114f59279..13bee38ff5 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -58,6 +58,68 @@ pkg_aliases( _tests.append(_test_legacy_aliases) +def _test_hidden_aliases(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "foo": "pypi_foo", + }, + exposed_packages = ["foo"], + ) + + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = [ + "@pypi_bar//:__pkg__", + "@pypi_foo//:__pkg__", +]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", +)""") + env.expect.that_str(actual["foo/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = ["//visibility:public"]) + +pkg_aliases( + name = "foo", + actual = "pypi_foo", +)""") + +_tests.append(_test_hidden_aliases) + +def _test_hidden_aliases_in_groups(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "baz": "pypi_baz", + }, + requirement_cycles = { + "group": ["bar", "baz"], + }, + exposed_packages = ["baz"], + ) + + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = [ + "@pypi_bar//:__pkg__", + "@pypi_baz//:__pkg__", + "//_groups:__pkg__", +]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", + group_name = "group", +)""") + +_tests.append(_test_hidden_aliases_in_groups) + def _test_bzlmod_aliases(env): # Use this function as it is used in pip_repository actual = render_multiplatform_pkg_aliases( From 23127e5b1bea7fd8a1be0f3dfbab80b4134b8399 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:08:09 +0900 Subject: [PATCH 03/20] feat(pypi): add pip.configure to select the exposed packages of a hub Add the `pip.configure` tag class that limits the packages that a hub exposes to the ones listed in the dependency groups of a `pyproject.toml` file. The other packages of the hub are still used by the wheels that need them, but their aliases are not public. --- docs/pypi/download.md | 50 ++++++ news/4395.added.md | 2 + python/private/pypi/BUILD.bazel | 10 ++ python/private/pypi/extension.bzl | 108 +++++++++++++ python/private/pypi/hub_exposure.bzl | 84 ++++++++++ tests/integration/unified_pypi/BUILD.bazel | 16 ++ tests/integration/unified_pypi/MODULE.bazel | 14 ++ .../unified_pypi/pyproject_filtered.toml | 4 + tests/integration/unified_pypi_test.py | 13 ++ tests/pypi/extension/extension_tests.bzl | 92 ++++++++++- tests/pypi/extension/pip_parse.bzl | 12 ++ tests/pypi/hub_exposure/BUILD.bazel | 3 + .../pypi/hub_exposure/hub_exposure_tests.bzl | 153 ++++++++++++++++++ 13 files changed, 559 insertions(+), 2 deletions(-) create mode 100644 news/4395.added.md create mode 100644 python/private/pypi/hub_exposure.bzl create mode 100644 tests/integration/unified_pypi/pyproject_filtered.toml create mode 100644 tests/pypi/hub_exposure/BUILD.bazel create mode 100644 tests/pypi/hub_exposure/hub_exposure_tests.bzl diff --git a/docs/pypi/download.md b/docs/pypi/download.md index cc8b7890aa..5f16606d24 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -134,6 +134,56 @@ rename, and `requirement("numpy")` routes through the active hub like fixed at loading time, before the active hub is known, so load them from a concrete hub instead. Prefer `@pypi//` labels in new code. +(pip-configure)= +### Configuring the Exposed Packages (pip.configure) + +By default a hub exposes all of the packages from its requirements files or +`uv.lock` file, including the transitive dependencies. A `pip.configure` tag +can limit the hub to the packages listed in a `pyproject.toml` file: the +`[project] dependencies`, `[project.optional-dependencies]` and PEP 735 +`[dependency-groups]` tables. + +```toml +[project] +dependencies = ["numpy"] + +[dependency-groups] +dev = ["pytest"] +``` + +```starlark +pip.parse( + hub_name = "pypi_hub", + pyproject_toml = "//:pyproject.toml", + requirements_lock = "//:requirements_lock.txt", +) + +pip.configure( + hub_name = "pypi_hub", + dependency_groups = ["default", "group:dev"], +) +``` + +Here `@pypi_hub//numpy` and `@pypi_hub//pytest` can be used, but the +dependencies of `numpy` are only used through `@pypi_hub//numpy`. The +`all_requirements` lists in the hub's `requirements.bzl` and the +[Unified @pypi Hub](unified-pypi-hub) only contain the exposed packages. + +The {attr}`pip.configure.dependency_groups` attribute defaults to `["*"]`, +which selects every group. Use `["*", "transitive"]` to expose all of the +packages of the hub, including the transitive dependencies. Selecting the +transitive dependencies of only some of the groups is not supported. + +Multiple tags for the same hub are combined. Only the root module and the +module that created the hub can configure it; if both do, the root module +wins. A self reference such as `my-project[test]` in the `pyproject.toml` +selects the optional dependencies of the project. The packages that are not +part of the hub are ignored, so a single `pyproject.toml` can list tools that +are missing from the lock file. + +:::{versionadded} VERSION_NEXT_FEATURE +::: + ### Declaring Abstract Dependencies (pip.dep) :::{versionadded} 2.2.0 diff --git a/news/4395.added.md b/news/4395.added.md new file mode 100644 index 0000000000..62d7f10880 --- /dev/null +++ b/news/4395.added.md @@ -0,0 +1,2 @@ +(pypi) Added {obj}`pip.configure` to limit the packages that a hub exposes to +the ones listed in the dependency groups of a `pyproject.toml` file. diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index af005f706e..caf482b67f 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -129,6 +129,7 @@ bzl_library( deps = [ ":default_platforms", ":hub_builder", + ":hub_exposure", ":hub_repository", ":parse_whl_name", ":pep508_env", @@ -641,6 +642,15 @@ bzl_library( ], ) +bzl_library( + name = "hub_exposure", + srcs = ["hub_exposure.bzl"], + deps = [ + ":dependency_groups", + "//python/private:pyproject_utils", + ], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index f7a1dadaf7..e64e856489 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -24,6 +24,7 @@ load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_req load("//python/private:repo_utils.bzl", "repo_utils") load(":default_platforms.bzl", "default_platforms") load(":hub_builder.bzl", "hub_builder") +load(":hub_exposure.bzl", "hub_exposure") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_whl_name.bzl", "parse_whl_name") load(":pep508_env.bzl", "env") @@ -89,6 +90,7 @@ def build_config( unified_hub_requirements_bzl = False defaults = { "platforms": default_platforms(), + "pyproject_toml": None, "python_version": None, } for mod in module_ctx.modules: @@ -105,6 +107,8 @@ def build_config( unified_hub_requirements_bzl = True pyproject_toml = tag.pyproject_toml if pyproject_toml: + if mod.is_root: + defaults["pyproject_toml"] = pyproject_toml pyproject = read_pyproject(module_ctx, pyproject_toml) if pyproject.requires_python: defaults["python_version"] = version_from_requires_python(pyproject.requires_python) @@ -151,6 +155,7 @@ def build_config( for name, values in defaults["platforms"].items() }, enable_pipstar_extract = enable_pipstar_extract, + pyproject_toml = defaults["pyproject_toml"], toml_decode = toml.decode, unified_hub_requirements_bzl = unified_hub_requirements_bzl, ) @@ -243,6 +248,7 @@ You cannot use both the additive_build_content and additive_build_content_file a # dict[str repo, HubBuilder] # See `hub_builder.bzl%hub_builder()` for `HubBuilder` pip_hub_map = {} + hub_pyproject = {} simpleapi_cache = pypi_cache(mctx = module_ctx) is_pypi_hub_reserved = module_ctx.getenv("RULES_PYTHON_PYPI_HUB_RESERVED", "0") == "1" @@ -319,6 +325,9 @@ You cannot use both the additive_build_content and additive_build_content_file a else: builder = pip_hub_map[hub_name] + if pip_attr.pyproject_toml and hub_name not in hub_pyproject: + hub_pyproject[hub_name] = pip_attr.pyproject_toml + builder.pip_parse( module_ctx, pip_attr = pip_attr, @@ -342,6 +351,7 @@ You cannot use both the additive_build_content and additive_build_content_file a exposed_packages = {} extra_aliases = {} whl_libraries = {} + hubs = {} for hub in pip_hub_map.values(): out = hub.build() @@ -355,8 +365,25 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases[hub.name] = out.extra_aliases hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map + hubs[hub.name] = struct( + module_name = hub.module_name, + pyproject = hub_pyproject.get(hub.name), + packages = out.whl_map.keys(), + exposed_packages = out.exposed_packages, + ) + + configured_hubs = hub_exposure( + module_ctx, + hubs = hubs, + default_pyproject = config.pyproject_toml, + _fail = _fail, + ) + if configured_hubs == None: + return None + exposed_packages.update(configured_hubs) return struct( + configured_hubs = configured_hubs, config = config, declared_deps = declared_deps, default_hub = config.default_hub or renamed_default_hub, @@ -511,6 +538,8 @@ def _pip_impl(module_ctx): packages = mods.exposed_packages.get(hub_name, []), platform_config_settings = mods.platform_config_settings.get(hub_name, {}), groups = mods.hub_group_map.get(hub_name), + # The hubs configured with `pip.configure` hide the other packages. + hide_unexposed_packages = hub_name in mods.configured_hubs, ) _create_unified_hub_repo(mods) @@ -638,6 +667,10 @@ This is designed to work with dependency management tools like Renovate. :::{versionadded} 2.3.0 ::: + +:::{versionchanged} VERSION_NEXT_FEATURE +Also used by {obj}`pip.configure` for the hubs of the root module. +::: """, ), "unified_hub_requirements_bzl": attr.bool( @@ -841,6 +874,10 @@ The version must be specified as `==X.Y.Z` (exact version with full semver). :::{versionadded} 2.3.0 ::: + +:::{versionchanged} VERSION_NEXT_FEATURE +Also used by {obj}`pip.configure` to select the packages that the hub exposes. +::: """, ), "python_version": attr.string( @@ -1026,6 +1063,72 @@ execution-phase error when built. """, ) +_configure_tag = tag_class( + attrs = { + "dependency_groups": attr.string_list( + default = ["*"], + doc = """\ +The `pyproject.toml` dependency groups whose packages the hub exposes. + +Each item is one of: + +* `"default"`: the `[project] dependencies`. +* `"optional:"`: the `[project.optional-dependencies]` entry ``. +* `"group:"`: the PEP 735 `[dependency-groups]` entry ``. +* `"optional:*"` or `"group:*"`: all entries of the table. +* `"*"`: all of the above. +* `""`: the dependency group `` if it exists, otherwise the + optional dependency group ``. +* `"transitive"`: only supported together with `"*"`, exposes all of the + packages of the hub including the transitive dependencies, i.e. all of the + packages from the requirements files or the `uv.lock` file. + +See [Configuring the Exposed Packages](#pip-configure) for examples. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "hub_name": attr.string( + mandatory = True, + doc = """\ +The name of the hub to configure. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "pyproject_toml": attr.label( + mandatory = False, + doc = """\ +The `pyproject.toml` file that defines the dependency groups. Defaults to the +{attr}`pip.parse.pyproject_toml` of the hub, then to +{attr}`pip.default.pyproject_toml` if the hub is defined by the root module. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + }, + doc = """\ +Configure the packages exposed by a hub. + +The packages are selected from a `pyproject.toml` file using the +`[project] dependencies`, `[project.optional-dependencies]` and PEP 735 +`[dependency-groups]` tables. The other packages of the hub, e.g. the +transitive dependencies, are still used by the packages that need them, but +they are not public. + +Only the root module and the module that created the hub can configure it. If +both do, the root module wins. Multiple tags for the same hub are combined. + +See [Configuring the Exposed Packages](#pip-configure) for examples. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", +) + pypi = module_extension( environ = ["RULES_PYTHON_PYPI_HUB_RESERVED"], doc = """\ @@ -1049,12 +1152,17 @@ Environment Variables: - `RULES_PYTHON_PYPI_HUB_RESERVED`: Enable fallback renaming for reserved hub name collisions. See the {envvar}`RULES_PYTHON_PYPI_HUB_RESERVED` documentation for details. +pip.configure: +This tag class configures the packages exposed by a hub using the dependency +groups from a pyproject.toml file. + pip.whl_mods: This tag class is used to help create JSON files to describe modifications to the BUILD files for wheels. """, implementation = _pip_impl, tag_classes = { + "configure": _configure_tag, "default": tag_class( attrs = _default_attrs, doc = """\ diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl new file mode 100644 index 0000000000..6ca9315a81 --- /dev/null +++ b/python/private/pypi/hub_exposure.bzl @@ -0,0 +1,84 @@ +"""Compute the packages exposed by the hubs configured with `pip.configure`.""" + +load("//python/private:pyproject_utils.bzl", "read_pyproject") +load(":dependency_groups.bzl", "select_dependency_groups") + +def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): + """Compute the packages exposed by the hubs configured with `pip.configure`. + + Args: + module_ctx: {type}`module_ctx` the module context. + hubs: {type}`dict[str, struct]` the hubs by name. Each struct has: + * `module_name`: {type}`str` the module that created the hub. + * `pyproject`: {type}`Label | None` from `pip.parse`. + * `packages`: {type}`list[str]` the normalized names of all + packages in the hub. + * `exposed_packages`: {type}`list[str]` the packages that the hub + exposes when it is not configured. + default_pyproject: {type}`Label | None` from `pip.default`, only used + for the hubs of the root module. + _fail: {type}`callable` the failure function, mainly for testing. + + Returns: + {type}`dict[str, list[str]] | None` the configured hubs mapped to the + sorted packages that they expose. The hubs that are not configured + are not included. Returns `None` on failure. + """ + + # The root module wins over the owner of the hub when both configure it. + root_modules = [mod for mod in module_ctx.modules if mod.is_root] + root_name = root_modules[0].name if root_modules else None + root_configured = { + tag.hub_name: None + for mod in root_modules + for tag in mod.tags.configure + } + + configured = {} + for mod in module_ctx.modules: + for tag in mod.tags.configure: + hub = tag.hub_name + if hub not in hubs: + _fail("pip.configure() refers to unknown hub '{}'. Available hubs: {}".format( + hub, + ", ".join(sorted(hubs)), + )) + return None + + info = hubs[hub] + if not mod.is_root and (mod.name != info.module_name or hub in root_configured): + # Only the root module and the module that created the hub can + # configure it. + continue + + pyproject = ( + tag.pyproject_toml or + info.pyproject or + (default_pyproject if info.module_name == root_name else None) + ) + if not pyproject: + _fail("pip.configure() for hub '{}' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`".format(hub)) + return None + + selection = select_dependency_groups( + read_pyproject(module_ctx, pyproject), + tag.dependency_groups, + _fail = _fail, + ) + if selection.all: + packages = info.exposed_packages + else: + packages = selection.packages + + # Multiple tags for the same hub are combined. + exposed = configured.setdefault(hub, {}) + for name in packages: + # The pyproject.toml may list packages that are not in the hub, + # e.g. tools that are not locked. + if name in info.packages: + exposed[name] = None + + return { + hub: sorted(packages) + for hub, packages in configured.items() + } diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index 13d188d087..f3a9513d1f 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -95,3 +95,19 @@ platform( "@platforms//os:windows", ], ) + +# The pypi_filtered hub only makes its pyproject.toml packages public. +py_test( + name = "test_filtered_direct", + srcs = ["test_default.py"], + main = "test_default.py", + deps = ["@pypi_filtered//colorama"], +) + +py_binary( + name = "bin_filtered_six", + srcs = ["bin_six_a.py"], + main = "bin_six_a.py", + tags = ["manual"], + deps = ["@pypi_filtered//six"], +) diff --git a/tests/integration/unified_pypi/MODULE.bazel b/tests/integration/unified_pypi/MODULE.bazel index ee8f622cb4..70e01ccdac 100644 --- a/tests/integration/unified_pypi/MODULE.bazel +++ b/tests/integration/unified_pypi/MODULE.bazel @@ -53,6 +53,20 @@ pip.parse( ) use_repo(pip, "pypi_b") +# `pypi_filtered` only exposes the `[project] dependencies` of the +# `pyproject.toml`, the other packages are not public. +pip.parse( + hub_name = "pypi_filtered", + pyproject_toml = "//:pyproject_filtered.toml", + python_version = "3.11", + requirements_lock = "//:requirements_b.txt", +) +pip.configure( + dependency_groups = ["default"], + hub_name = "pypi_filtered", +) +use_repo(pip, "pypi_filtered") + pip.default(default_hub = "pypi_b") pip.default(unified_hub_requirements_bzl = True) pip.dep( diff --git a/tests/integration/unified_pypi/pyproject_filtered.toml b/tests/integration/unified_pypi/pyproject_filtered.toml new file mode 100644 index 0000000000..632f72fe22 --- /dev/null +++ b/tests/integration/unified_pypi/pyproject_filtered.toml @@ -0,0 +1,4 @@ +[project] +name = "unified-pypi" +version = "0.0.0" +dependencies = ["colorama"] diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index 9ce41fafb7..5ac2f7df8b 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -7,6 +7,19 @@ class UnifiedPypiTest(runner.TestCase): + def test_configure_hides_unexposed_packages(self): + self.run_bazel("test", "//:test_filtered_direct") + result = self.run_bazel("build", "//:bin_filtered_six", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected the alias of the unexposed package to be hidden", + ) + self.assert_result_matches( + result, + r"alias '[^']*//six:six'.*is not visible from", + ) + def test_default_fallback_hub(self): self.run_bazel("test", "//:test_default") diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index ee8e58dfe4..17f8d6741e 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -21,7 +21,7 @@ load("//python/private/pypi:extension.bzl", "build_config", "parse_modules") # load("//python/private/pypi:platform.bzl", _plat = "platform") # buildifier: disable=bzl-visibility load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") -load(":pip_parse.bzl", _parse = "pip_parse") +load(":pip_parse.bzl", _configure = "configure", _parse = "pip_parse") _tests = [] @@ -101,10 +101,11 @@ def _dep(*, name, extra_targets = []): extra_targets = extra_targets, ) -def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], is_root = True): +def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], configure = [], is_root = True): return struct( name = name, tags = struct( + configure = configure, parse = parse, override = override, whl_mods = whl_mods, @@ -118,6 +119,7 @@ def _parse_modules(env, **kwargs): return env.expect.that_struct( parse_modules(**kwargs), attrs = dict( + configured_hubs = subjects.dict, declared_deps = subjects.dict, default_hub = subjects.str, exposed_packages = subjects.dict, @@ -766,6 +768,92 @@ simple==0.0.1 \ _tests.append(_test_default_index_setting) +_CONFIGURE_REQUIREMENTS = """\ +simple==0.0.1 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +other-pkg==0.0.2 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +transitive-pkg==0.0.3 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +""" + +_CONFIGURE_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple"] +""" + +def _configure_parse_modules(env, *modules): + return _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "pyproject.toml": _CONFIGURE_PYPROJECT, + "requirements.txt": _CONFIGURE_REQUIREMENTS, + }, + *modules + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + +def _configure_parse(*, hub_name = "pypi_a", pyproject_toml = "pyproject.toml"): + return _parse( + hub_name = hub_name, + pyproject_toml = pyproject_toml, + python_version = "3.15", + simpleapi_skip = ["simple", "other-pkg", "transitive-pkg"], + requirements_lock = "requirements.txt", + ) + +def _test_configure(env): + for groups, want in [ + (["*"], ["other_pkg", "simple"]), + (["test"], ["simple"]), + (["dev"], ["other_pkg"]), + (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), + ]: + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse()], + configure = [_configure(hub_name = "pypi_a", dependency_groups = groups)], + ), + ) + + pypi.exposed_packages().contains_exactly({"pypi_a": want}) + pypi.configured_hubs().contains_exactly({"pypi_a": want}) + +_tests.append(_test_configure) + +def _test_not_configured_exposes_all(env): + # The pyproject.toml does not filter the hub without `pip.configure`. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse()], + ), + ) + + pypi.exposed_packages().contains_exactly({ + "pypi_a": ["other_pkg", "simple", "transitive_pkg"], + }) + pypi.configured_hubs().contains_exactly({}) + +_tests.append(_test_not_configured_exposes_all) + def extension_test_suite(name): """Create the test suite. diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index 7b5bdfdfd6..505c249169 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -1,5 +1,17 @@ """A simple test helper""" +def configure( + *, + dependency_groups = ["*"], + hub_name, + pyproject_toml = None): + """A simple helper for testing to simulate the PyPI extension configure tag class""" + return struct( + dependency_groups = dependency_groups, + hub_name = hub_name, + pyproject_toml = pyproject_toml, + ) + def pip_parse( *, hub_name, diff --git a/tests/pypi/hub_exposure/BUILD.bazel b/tests/pypi/hub_exposure/BUILD.bazel new file mode 100644 index 0000000000..187361e1ce --- /dev/null +++ b/tests/pypi/hub_exposure/BUILD.bazel @@ -0,0 +1,3 @@ +load(":hub_exposure_tests.bzl", "hub_exposure_test_suite") + +hub_exposure_test_suite(name = "hub_exposure_tests") diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl new file mode 100644 index 0000000000..cafbb229b5 --- /dev/null +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -0,0 +1,153 @@ +"""hub_exposure tests""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:hub_exposure.bzl", "hub_exposure") # buildifier: disable=bzl-visibility +load("//tests/support/mocks:mocks.bzl", "mocks") + +_tests = [] + +_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple", "not-in-hub"] +""" + +def _hub(*, module_name = "root", pyproject = None): + return struct( + module_name = module_name, + pyproject = pyproject, + packages = ["other_pkg", "simple", "transitive_pkg"], + exposed_packages = ["other_pkg", "simple", "transitive_pkg"], + ) + +def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None): + return mocks.tag( + hub_name = hub_name, + dependency_groups = dependency_groups, + pyproject_toml = pyproject_toml, + ) + +def _module(name = "root", is_root = True, configure = []): + return mocks.module(name = name, is_root = is_root, configure = configure) + +def _exposure(*modules, hubs = None, default_pyproject = None): + failures = [] + got = hub_exposure( + mocks.mctx( + modules = list(modules), + mock_files = { + "other.toml": "[project]\ndependencies = [\"other-pkg\"]\n", + "pyproject.toml": _PYPROJECT, + }, + ), + hubs = hubs or {"hub": _hub(pyproject = "pyproject.toml")}, + default_pyproject = default_pyproject, + _fail = failures.append, + ) + return got, failures + +def _test_not_configured(env): + got, failures = _exposure(_module()) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_not_configured) + +def _test_configure(env): + for groups, want in [ + (["*"], ["other_pkg", "simple"]), + (["default"], ["simple"]), + (["dev"], ["other_pkg"]), + # `not-in-hub` is ignored + (["group:test"], ["simple"]), + (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), + ]: + got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) + env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) + env.expect.where(groups = groups).that_dict(got).contains_exactly({"hub": want}) + +_tests.append(_test_configure) + +def _test_configure_combines_tags(env): + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["dev"]), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg", "simple"]}) + +_tests.append(_test_configure_combines_tags) + +def _test_configure_pyproject_precedence(env): + # The tag wins over the `pip.parse` value. + got, _ = _exposure(_module(configure = [ + _configure(pyproject_toml = "other.toml"), + ])) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + + # The `pip.default` value is used for the hubs of the root module. + got, failures = _exposure( + _module(configure = [_configure(dependency_groups = ["default"])]), + hubs = {"hub": _hub()}, + default_pyproject = "pyproject.toml", + ) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + +_tests.append(_test_configure_pyproject_precedence) + +def _test_configure_module_precedence(env): + hubs = {"hub": _hub(module_name = "dep", pyproject = "pyproject.toml")} + + # The owner of the hub can configure it. + got, _ = _exposure( + _module(), + _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + + # The root module wins over the owner. + got, _ = _exposure( + _module(configure = [_configure(dependency_groups = ["default"])]), + _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + + # Other modules are ignored. + got, _ = _exposure( + _module(), + _module(name = "other", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_configure_module_precedence) + +def _test_configure_errors(env): + _, failures = _exposure(_module(configure = [_configure(hub_name = "missing")])) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() refers to unknown hub 'missing'. Available hubs: hub", + ]) + + _, failures = _exposure( + _module(configure = [_configure()]), + hubs = {"hub": _hub()}, + ) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() for hub 'hub' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`", + ]) + +_tests.append(_test_configure_errors) + +def hub_exposure_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 6bb7f8379a68c92c78b12acb4a0fad3b8c99a36a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:11:22 +0900 Subject: [PATCH 04/20] fix(pypi): only add the exposed packages of a configured hub to @pypi Otherwise the unified hub would point to the hidden aliases of the hub and fail with a visibility error instead of the missing package error. --- python/private/pypi/extension.bzl | 4 +++- tests/integration/unified_pypi/BUILD.bazel | 12 ++++++++++++ tests/integration/unified_pypi_test.py | 13 +++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index e64e856489..ce5e95492d 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -416,7 +416,9 @@ def _create_unified_hub_repo(mods): extra_aliases = {} for hub_name in hubs: - for pkg_name in mods.hub_whl_map[hub_name]: + # The hubs configured with `pip.configure` only expose some packages. + hub_packages = mods.configured_hubs.get(hub_name, mods.hub_whl_map[hub_name]) + for pkg_name in hub_packages: norm_pkg = normalize_name(pkg_name) if norm_pkg not in packages: packages[norm_pkg] = [] diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index f3a9513d1f..78f9a857ee 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -104,6 +104,18 @@ py_test( deps = ["@pypi_filtered//colorama"], ) +# six is not exposed by pypi_filtered, so it is missing in the unified hub. +py_binary( + name = "bin_filtered_six_unified", + srcs = ["bin_six_a.py"], + config_settings = { + "@rules_python//python/config_settings:venv": "pypi_filtered", + }, + main = "bin_six_a.py", + tags = ["manual"], + deps = ["@pypi//six"], +) + py_binary( name = "bin_filtered_six", srcs = ["bin_six_a.py"], diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index 5ac2f7df8b..f59a19fb42 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -20,6 +20,19 @@ def test_configure_hides_unexposed_packages(self): r"alias '[^']*//six:six'.*is not visible from", ) + def test_configure_hides_unexposed_packages_in_unified_hub(self): + self.run_bazel("cquery", "//:bin_filtered_six_unified") + result = self.run_bazel("build", "//:bin_filtered_six_unified", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected build to fail during execution phase", + ) + self.assert_result_matches( + result, + 'ERROR: PyPI package "six" is not available when building under PyPI hub "pypi_filtered"\\. Try adding it to the requirements of this hub', + ) + def test_default_fallback_hub(self): self.run_bazel("test", "//:test_default") From fcaac6659712438657768a81294119cb865ffae7 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:22:46 +0900 Subject: [PATCH 05/20] refactor(pypi): support testonly packages in the hub repositories Bazel does not enforce `testonly` on aliases, so the main public targets of a testonly package are testonly wrappers around the aliases. The testonly packages are listed in `all_testonly_requirements` instead of the `all_*requirements` lists of the hub's requirements.bzl. --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/hub_repository.bzl | 19 ++++- python/private/pypi/pkg_aliases.bzl | 68 +++++++++++++++++- python/private/pypi/render_pkg_aliases.bzl | 8 +++ .../private/pypi/requirements.bzl.tmpl.bzlmod | 2 + python/private/pypi/unified_hub_repo.bzl | 10 +++ python/private/pypi/unified_hub_setup.bzl | 5 +- tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 70 +++++++++++++++++++ .../render_pkg_aliases_test.bzl | 31 ++++++++ 9 files changed, 207 insertions(+), 7 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index caf482b67f..c0294975cd 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -354,6 +354,7 @@ bzl_library( deps = [ ":labels", ":whl_library_deps_targets", + "//python:py_library", "//python/private:common_labels", "//python/private:text_util", "@bazel_skylib//lib:selects", diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index d421f3cc09..91f2c2d0e4 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -30,6 +30,7 @@ def _impl(rctx): bzl_packages = rctx.attr.packages else: bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + testonly_packages = rctx.attr.testonly_packages aliases = render_multiplatform_pkg_aliases( aliases = { key: _whl_config_settings_from_json(values) @@ -39,6 +40,7 @@ def _impl(rctx): extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, + testonly_packages = testonly_packages, ) for path, contents in aliases.items(): rctx.file(path, contents) @@ -57,18 +59,25 @@ def _impl(rctx): "%%PACKAGES%%": render.dict(rctx.attr.whl_map, value_repr = lambda x: "None"), }, ) + + # The testonly packages are only listed in `all_testonly_requirements`. + all_packages = [p for p in bzl_packages if p not in testonly_packages] rctx.template("requirements.bzl", rctx.attr._requirements_bzl_template, substitutions = { "%%ALL_DATA_REQUIREMENTS%%": render.list([ macro_tmpl.format(p, "data") - for p in bzl_packages + for p in all_packages ]), "%%ALL_REQUIREMENTS%%": render.list([ macro_tmpl.format(p, "pkg") - for p in bzl_packages + for p in all_packages + ]), + "%%ALL_TESTONLY_REQUIREMENTS%%": render.list([ + macro_tmpl.format(p, "pkg") + for p in testonly_packages ]), "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": render.dict({ p: macro_tmpl.format(p, "whl") - for p in bzl_packages + for p in all_packages }), "%%MACRO_TMPL%%": macro_tmpl, }) @@ -101,6 +110,10 @@ unless `hide_unexposed_packages` is set. mandatory = True, doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", ), + "testonly_packages": attr.string_list( + default = [], + doc = "The packages whose aliases should be testonly.", + ), "whl_map": attr.string_dict( mandatory = True, doc = """\ diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index 227e9bf10f..b838434088 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -32,6 +32,7 @@ setting maps to and their precedence, refer to documentation on that page. """ load("@bazel_skylib//lib:selects.bzl", "selects") +load("//python:py_library.bzl", "py_library") load("//python/private:common_labels.bzl", "labels") load("//python/private:text_util.bzl", "render") load( @@ -86,6 +87,7 @@ def pkg_aliases( extras = [], include = [], group_deps = [], + testonly = False, **kwargs): """Create aliases for an actual package. @@ -107,10 +109,14 @@ def pkg_aliases( dependencies when parsing the requires_dist. Comes from METADATA or a lock file. include: {type}`list[str]` The subset of packages to include. Comes from `//:config.bzl#packages` + testonly: {type}`bool` Whether the public targets should be testonly. + Bazel does not enforce `testonly` on aliases, so the main public + targets are testonly wrappers around the aliases. **kwargs: extra kwargs to pass to {bzl:obj}`get_config_settings`. """ _native = kwargs.pop("native", native) rules = kwargs.pop("rules", struct( + py_library = py_library, whl_library_deps_targets = whl_library_deps_targets, )) select = kwargs.pop("select", selects.with_or) @@ -118,9 +124,13 @@ def pkg_aliases( _native.alias( name = name, actual = ":" + PY_LIBRARY_PUBLIC_LABEL, + **({"testonly": True} if testonly else {}) ) if requires_dist: + if testonly: + fail("testonly is not supported together with requires_dist") + # if it has a group_name set, then it will create the impl label target in the # current macro, we still need to do the aliases to the actual groups package as # per below @@ -210,20 +220,72 @@ def pkg_aliases( elif target_name in (WHEEL_FILE, PY_SRCS_LABEL): kwargs["visibility"] = ["//visibility:private"] - _native.alias( + # The private targets are used by the non-testonly `//_groups` libraries. + _alias( + _native, + rules, name = target_name, actual = _actual, + testonly = testonly and not target_name.startswith("_"), **kwargs ) if group_name: - _native.alias( + _alias( + _native, + rules, name = PY_LIBRARY_PUBLIC_LABEL, actual = "//_groups:{}_pkg".format(group_name), + testonly = testonly, ) - _native.alias( + _alias( + _native, + rules, name = WHEEL_FILE_PUBLIC_LABEL, actual = "//_groups:{}_whl".format(group_name), + testonly = testonly, + ) + +# The public targets that are wrapped in testonly targets, because Bazel does +# not enforce `testonly` on aliases. +_TESTONLY_WRAPPED = [ + PY_LIBRARY_PUBLIC_LABEL, + WHEEL_FILE_PUBLIC_LABEL, + DATA_LABEL, + DIST_INFO_LABEL, + EXTRACTED_WHEEL_FILES, +] + +def _alias(_native, rules, *, name, actual, testonly, **kwargs): + if not testonly: + _native.alias(name = name, actual = actual, **kwargs) + return + + if name not in _TESTONLY_WRAPPED: + # E.g. the extra aliases, which may point to executables. Best effort, + # because Bazel does not enforce `testonly` on aliases. + _native.alias(name = name, actual = actual, testonly = True, **kwargs) + return + + impl = "_testonly_" + name + _native.alias( + name = impl, + actual = actual, + visibility = ["//visibility:private"], + ) + if name == PY_LIBRARY_PUBLIC_LABEL: + rules.py_library( + name = name, + deps = [":" + impl], + testonly = True, + **kwargs + ) + else: + _native.filegroup( + name = name, + srcs = [":" + impl], + testonly = True, + **kwargs ) def multiplatform_whl_aliases( diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 5582cc00c9..8271387418 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -94,6 +94,7 @@ def render_pkg_aliases( requirement_cycles = None, extra_hub_aliases = {}, exposed_packages = None, + testonly_packages = [], **kwargs): """Create alias declarations for each PyPI package. @@ -110,6 +111,7 @@ def render_pkg_aliases( exposed_packages: The packages that should be public. When set, the aliases of the other packages are only visible to the wheel repositories. + testonly_packages: The packages whose aliases should be testonly. **kwargs: Extra kwargs to pass to the rules. Returns: @@ -134,6 +136,11 @@ def render_pkg_aliases( for whl_name in group_whls } + testonly_packages = { + normalize_name(name): None + for name in testonly_packages + } + public = ["//visibility:public"] if exposed_packages != None: exposed_packages = {normalize_name(name): None for name in exposed_packages} @@ -146,6 +153,7 @@ def render_pkg_aliases( extra_aliases = extra_hub_aliases.get(normalize_name(name), []), group_name = whl_group_mapping.get(normalize_name(name)), visibility = public if exposed_packages == None or normalize_name(name) in exposed_packages else internal_visibility, + testonly = normalize_name(name) in testonly_packages, **kwargs ).strip() for name, pkg_aliases in aliases.items() diff --git a/python/private/pypi/requirements.bzl.tmpl.bzlmod b/python/private/pypi/requirements.bzl.tmpl.bzlmod index ba227aeb2d..7904ed5578 100644 --- a/python/private/pypi/requirements.bzl.tmpl.bzlmod +++ b/python/private/pypi/requirements.bzl.tmpl.bzlmod @@ -7,6 +7,8 @@ load("@rules_python//python:pip.bzl", "pip_utils") all_requirements = %%ALL_REQUIREMENTS%% +all_testonly_requirements = %%ALL_TESTONLY_REQUIREMENTS%% + all_whl_requirements_by_package = %%ALL_WHL_REQUIREMENTS_BY_PACKAGE%% all_whl_requirements = all_whl_requirements_by_package.values() diff --git a/python/private/pypi/unified_hub_repo.bzl b/python/private/pypi/unified_hub_repo.bzl index ddd15c0d86..984c3cacbe 100644 --- a/python/private/pypi/unified_hub_repo.bzl +++ b/python/private/pypi/unified_hub_repo.bzl @@ -25,6 +25,7 @@ define_pypi_package_targets( extra_aliases = {extra_aliases}, hubs = {hubs}, pkg_hubs = {pkg_hubs}, + testonly = {testonly}, ) """ @@ -79,6 +80,10 @@ def _unified_hub_repo_impl(rctx): extra_aliases_by_pkg.setdefault(pkg, {})[alias] = alias_hubs # 3. Generate package subpackages + testonly_packages = { + pkg_name: None + for pkg_name in rctx.attr.testonly_packages + } for pkg_name, pkg_hubs in rctx.attr.packages.items(): extra_aliases = extra_aliases_by_pkg.get(pkg_name, {}) rctx.file( @@ -89,6 +94,7 @@ def _unified_hub_repo_impl(rctx): hubs = hubs, pkg_hubs = pkg_hubs, pkg_name = pkg_name, + testonly = pkg_name in testonly_packages, ), ) @@ -130,6 +136,10 @@ unified_hub_repo = repository_rule( default = False, doc = "Whether to generate a requirements.bzl with the per-package `requirement` macros.", ), + "testonly_packages": attr.string_list( + default = [], + doc = "Packages that are testonly in at least one hub.", + ), }, doc = "Private repository rule creating the automatic Unified PyPI Hub.", ) diff --git a/python/private/pypi/unified_hub_setup.bzl b/python/private/pypi/unified_hub_setup.bzl index b19410fd4b..3ee8ae25d3 100644 --- a/python/private/pypi/unified_hub_setup.bzl +++ b/python/private/pypi/unified_hub_setup.bzl @@ -31,7 +31,7 @@ _STANDARD_ALIASES = [ EXTRACTED_WHEEL_FILES, ] -def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub = None): +def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub = None, testonly = False): """Define the targets for a PyPI package in the unified PyPI hub. Args: @@ -40,6 +40,7 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub extra_aliases: dict mapping extra alias names to lists of hubs that support them. hubs: list of all concrete hub names. default_hub: the hub to use by default. + testonly: whether all targets should be marked as testonly. """ pkg_name = name @@ -47,6 +48,7 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub native.alias( name = pkg_name, actual = ":pkg", + testonly = testonly, ) all_aliases = _STANDARD_ALIASES + sorted(extra_aliases.keys()) @@ -100,6 +102,7 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub native.alias( name = alias_name, actual = select(select_map), + testonly = testonly, ) # Generate missing package error targets diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 2d54bd34db..fdec8877fc 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -55,6 +55,76 @@ def _test_legacy_aliases(env): _tests.append(_test_legacy_aliases) +def _testonly_mocks(): + got = {} + + def _add(kind): + return lambda name, **kwargs: got.setdefault(name, (kind, kwargs.get("testonly", False))) + + return got, struct( + native = struct( + alias = _add("alias"), + filegroup = _add("filegroup"), + ), + rules = struct( + py_library = _add("py_library"), + ), + ) + +def _test_testonly_aliases(env): + got, mocks = _testonly_mocks() + pkg_aliases( + name = "foo", + actual = "repo", + testonly = True, + native = mocks.native, + rules = mocks.rules, + ) + + # Bazel does not enforce `testonly` on aliases, so the public targets are + # wrapped. + env.expect.that_dict(got).contains_exactly({ + "data": ("filegroup", True), + "dist_info": ("filegroup", True), + "extracted_whl_files": ("filegroup", True), + "foo": ("alias", True), + "pkg": ("py_library", True), + "srcs": ("alias", True), + "whl": ("filegroup", True), + "whl_file": ("alias", True), + "_testonly_data": ("alias", False), + "_testonly_dist_info": ("alias", False), + "_testonly_extracted_whl_files": ("alias", False), + "_testonly_pkg": ("alias", False), + "_testonly_whl": ("alias", False), + }) + +_tests.append(_test_testonly_aliases) + +def _test_testonly_group_aliases(env): + got, mocks = _testonly_mocks() + pkg_aliases( + name = "foo", + actual = "repo", + group_name = "my_group", + testonly = True, + native = mocks.native, + rules = mocks.rules, + ) + + # The private targets are used by the non-testonly group libraries. + env.expect.that_dict(got).contains_at_least({ + "foo": ("alias", True), + "pkg": ("py_library", True), + "whl": ("filegroup", True), + "_pkg": ("alias", False), + "_testonly_pkg": ("alias", False), + "_testonly_whl": ("alias", False), + "_whl": ("alias", False), + }) + +_tests.append(_test_testonly_group_aliases) + def _test_config_setting_aliases(env): # Use this function as it is used in pip_repository got = {} diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 13bee38ff5..38699f130b 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -58,6 +58,37 @@ pkg_aliases( _tests.append(_test_legacy_aliases) +def _test_testonly_aliases(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "foo": "pypi_foo", + }, + testonly_packages = ["Foo"], + ) + + env.expect.that_str(actual["foo/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = ["//visibility:public"]) + +pkg_aliases( + name = "foo", + actual = "pypi_foo", + testonly = True, +)""") + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = ["//visibility:public"]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", +)""") + +_tests.append(_test_testonly_aliases) + def _test_hidden_aliases(env): actual = render_pkg_aliases( aliases = { From 06306138419466b0d7a0faa8041110a298256efb Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:23:01 +0900 Subject: [PATCH 06/20] feat(pypi): add pip.configure.testonly to make packages testonly The packages selected by a pip.configure tag with `testonly = True` are testonly, unless they are also selected by a tag without `testonly`. --- docs/pypi/download.md | 28 +++++++++++++ news/pip-configure-testonly.added.md | 2 + python/private/pypi/extension.bzl | 29 ++++++++++++- python/private/pypi/hub_exposure.bzl | 20 ++++++--- tests/integration/unified_pypi/BUILD.bazel | 42 +++++++++++++++++++ tests/integration/unified_pypi/MODULE.bazel | 14 +++++++ .../unified_pypi/pyproject_testonly.toml | 6 +++ .../unified_pypi/requirements_testonly.txt | 3 ++ .../unified_pypi/test_iniconfig.py | 3 ++ tests/integration/unified_pypi_test.py | 19 +++++++++ tests/pypi/extension/extension_tests.bzl | 28 ++++++++++++- tests/pypi/extension/pip_parse.bzl | 4 +- .../pypi/hub_exposure/hub_exposure_tests.bzl | 36 ++++++++++++---- 13 files changed, 218 insertions(+), 16 deletions(-) create mode 100644 news/pip-configure-testonly.added.md create mode 100644 tests/integration/unified_pypi/pyproject_testonly.toml create mode 100644 tests/integration/unified_pypi/requirements_testonly.txt create mode 100644 tests/integration/unified_pypi/test_iniconfig.py diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 5f16606d24..52f9ca60a4 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -181,6 +181,34 @@ selects the optional dependencies of the project. The packages that are not part of the hub are ignored, so a single `pyproject.toml` can list tools that are missing from the lock file. +#### Testonly Packages + +Set {attr}`pip.configure.testonly` to make the selected packages testonly, so +that they can only be used by tests and other testonly targets: + +```starlark +pip.configure( + hub_name = "pypi_hub", + dependency_groups = ["default"], +) +pip.configure( + hub_name = "pypi_hub", + dependency_groups = ["group:test"], + testonly = True, +) +``` + +A package that is also selected by a tag without `testonly` is not testonly. +The testonly packages are left out of the `all_requirements`, +`all_whl_requirements` and `all_data_requirements` lists in the hub's +`requirements.bzl` and are listed in `all_testonly_requirements` instead. In +the [Unified @pypi Hub](unified-pypi-hub), a package is testonly if it is +testonly in any of the hubs. + +The other packages of the hub depend on the testonly packages through the same +targets, so a package that a non-testonly package depends on can't be +testonly. + :::{versionadded} VERSION_NEXT_FEATURE ::: diff --git a/news/pip-configure-testonly.added.md b/news/pip-configure-testonly.added.md new file mode 100644 index 0000000000..ebc0594f7e --- /dev/null +++ b/news/pip-configure-testonly.added.md @@ -0,0 +1,2 @@ +(pypi) Added {attr}`pip.configure.testonly` to make the packages selected by +{obj}`pip.configure` testonly. diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index ce5e95492d..7448f6b5c2 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -380,7 +380,8 @@ You cannot use both the additive_build_content and additive_build_content_file a ) if configured_hubs == None: return None - exposed_packages.update(configured_hubs) + for hub_name, info in configured_hubs.items(): + exposed_packages[hub_name] = info.packages return struct( configured_hubs = configured_hubs, @@ -417,7 +418,7 @@ def _create_unified_hub_repo(mods): for hub_name in hubs: # The hubs configured with `pip.configure` only expose some packages. - hub_packages = mods.configured_hubs.get(hub_name, mods.hub_whl_map[hub_name]) + hub_packages = mods.configured_hubs[hub_name].packages if hub_name in mods.configured_hubs else mods.hub_whl_map[hub_name] for pkg_name in hub_packages: norm_pkg = normalize_name(pkg_name) if norm_pkg not in packages: @@ -449,6 +450,13 @@ def _create_unified_hub_repo(mods): hubs = hubs, packages = packages, requirements_bzl = mods.unified_hub_requirements_bzl, + # An alias can't be less testonly than what it points to, so a package + # is testonly if it is testonly in any hub. + testonly_packages = sorted({ + pkg: None + for hub_name in hubs + for pkg in (mods.configured_hubs[hub_name].testonly_packages if hub_name in mods.configured_hubs else []) + }), ) def _pip_impl(module_ctx): @@ -542,6 +550,7 @@ def _pip_impl(module_ctx): groups = mods.hub_group_map.get(hub_name), # The hubs configured with `pip.configure` hide the other packages. hide_unexposed_packages = hub_name in mods.configured_hubs, + testonly_packages = mods.configured_hubs[hub_name].testonly_packages if hub_name in mods.configured_hubs else [], ) _create_unified_hub_repo(mods) @@ -1107,6 +1116,22 @@ The `pyproject.toml` file that defines the dependency groups. Defaults to the {attr}`pip.parse.pyproject_toml` of the hub, then to {attr}`pip.default.pyproject_toml` if the hub is defined by the root module. +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "testonly": attr.bool( + default = False, + doc = """\ +Make the aliases of the selected packages testonly. A package that is also +selected by a `pip.configure` tag without `testonly` is not testonly. + +The testonly packages are left out of the `all_requirements`, +`all_whl_requirements` and `all_data_requirements` lists in the hub's +`requirements.bzl` and listed in `all_testonly_requirements` instead. In the +[Unified @pypi Hub](unified-pypi-hub), a package is testonly if it is testonly +in any of the hubs. + :::{versionadded} VERSION_NEXT_FEATURE ::: """, diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index 6ca9315a81..5d5bfae8e9 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -20,9 +20,16 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): _fail: {type}`callable` the failure function, mainly for testing. Returns: - {type}`dict[str, list[str]] | None` the configured hubs mapped to the - sorted packages that they expose. The hubs that are not configured - are not included. Returns `None` on failure. + {type}`dict[str, struct] | None` the configured hubs mapped to a + struct with: + * `packages`: {type}`list[str]` the sorted packages that the hub + exposes. + * `testonly_packages`: {type}`list[str]` the sorted packages whose + aliases are testonly. A package is testonly only if all of the + tags that select it are testonly. + + The hubs that are not configured are not included. Returns `None` on + failure. """ # The root module wins over the owner of the hub when both configure it. @@ -76,9 +83,12 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): # The pyproject.toml may list packages that are not in the hub, # e.g. tools that are not locked. if name in info.packages: - exposed[name] = None + exposed[name] = exposed.get(name, True) and tag.testonly return { - hub: sorted(packages) + hub: struct( + packages = sorted(packages), + testonly_packages = sorted([name for name, testonly in packages.items() if testonly]), + ) for hub, packages in configured.items() } diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index 78f9a857ee..e7bbca4f9d 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -1,4 +1,5 @@ load("@pypi//:requirements.bzl", "requirement") +load("@pypi_testonly//:requirements.bzl", "all_requirements", "all_testonly_requirements") load("@rules_python//python:py_binary.bzl", "py_binary") load("@rules_python//python:py_test.bzl", "py_test") @@ -123,3 +124,44 @@ py_binary( tags = ["manual"], deps = ["@pypi_filtered//six"], ) + +# The pypi_testonly hub only exposes testonly packages. +py_test( + name = "test_testonly", + srcs = ["test_iniconfig.py"], + main = "test_iniconfig.py", + deps = ["@pypi_testonly//iniconfig"], +) + +py_test( + name = "test_testonly_unified", + srcs = ["test_iniconfig.py"], + config_settings = { + "@rules_python//python/config_settings:venv": "pypi_testonly", + }, + main = "test_iniconfig.py", + deps = ["@pypi//iniconfig"], +) + +py_test( + name = "test_all_testonly_requirements", + srcs = ["test_iniconfig.py"], + main = "test_iniconfig.py", + deps = all_testonly_requirements, +) + +# The testonly packages are not in `all_requirements`. +py_binary( + name = "bin_all_requirements", + srcs = ["test_default.py"], + main = "test_default.py", + deps = ["@pypi//colorama"] + all_requirements, +) + +py_binary( + name = "bin_testonly", + srcs = ["test_iniconfig.py"], + main = "test_iniconfig.py", + tags = ["manual"], + deps = ["@pypi_testonly//iniconfig"], +) diff --git a/tests/integration/unified_pypi/MODULE.bazel b/tests/integration/unified_pypi/MODULE.bazel index 70e01ccdac..dfb19bc749 100644 --- a/tests/integration/unified_pypi/MODULE.bazel +++ b/tests/integration/unified_pypi/MODULE.bazel @@ -67,6 +67,20 @@ pip.configure( ) use_repo(pip, "pypi_filtered") +# `pypi_testonly` only exposes `iniconfig` and it is testonly. +pip.parse( + hub_name = "pypi_testonly", + pyproject_toml = "//:pyproject_testonly.toml", + python_version = "3.11", + requirements_lock = "//:requirements_testonly.txt", +) +pip.configure( + testonly = True, + dependency_groups = ["test"], + hub_name = "pypi_testonly", +) +use_repo(pip, "pypi_testonly") + pip.default(default_hub = "pypi_b") pip.default(unified_hub_requirements_bzl = True) pip.dep( diff --git a/tests/integration/unified_pypi/pyproject_testonly.toml b/tests/integration/unified_pypi/pyproject_testonly.toml new file mode 100644 index 0000000000..0541ccc0b4 --- /dev/null +++ b/tests/integration/unified_pypi/pyproject_testonly.toml @@ -0,0 +1,6 @@ +[project] +name = "unified-pypi" +version = "0.0.0" + +[dependency-groups] +test = ["iniconfig"] diff --git a/tests/integration/unified_pypi/requirements_testonly.txt b/tests/integration/unified_pypi/requirements_testonly.txt new file mode 100644 index 0000000000..862092a324 --- /dev/null +++ b/tests/integration/unified_pypi/requirements_testonly.txt @@ -0,0 +1,3 @@ +iniconfig==2.1.0 \ + --hash=sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7 \ + --hash=sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760 diff --git a/tests/integration/unified_pypi/test_iniconfig.py b/tests/integration/unified_pypi/test_iniconfig.py new file mode 100644 index 0000000000..45bc1a5008 --- /dev/null +++ b/tests/integration/unified_pypi/test_iniconfig.py @@ -0,0 +1,3 @@ +import iniconfig + +assert iniconfig.IniConfig diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index f59a19fb42..ea81f3814d 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -33,6 +33,25 @@ def test_configure_hides_unexposed_packages_in_unified_hub(self): 'ERROR: PyPI package "six" is not available when building under PyPI hub "pypi_filtered"\\. Try adding it to the requirements of this hub', ) + def test_configure_testonly(self): + self.run_bazel( + "test", + "//:test_testonly", + "//:test_testonly_unified", + "//:test_all_testonly_requirements", + ) + self.run_bazel("build", "//:bin_all_requirements") + result = self.run_bazel("build", "//:bin_testonly", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected the testonly package to be rejected in a binary", + ) + self.assert_result_matches( + result, + r"non-test target '//:bin_testonly' depends on testonly", + ) + def test_default_fallback_hub(self): self.run_bazel("test", "//:test_default") diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 17f8d6741e..c9272f4eaf 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -833,10 +833,36 @@ def _test_configure(env): ) pypi.exposed_packages().contains_exactly({"pypi_a": want}) - pypi.configured_hubs().contains_exactly({"pypi_a": want}) + pypi.configured_hubs().contains_exactly({"pypi_a": struct( + packages = want, + testonly_packages = [], + )}) _tests.append(_test_configure) +def _test_configure_testonly(env): + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse()], + configure = [ + _configure(hub_name = "pypi_a", dependency_groups = ["default"]), + _configure(hub_name = "pypi_a", dependency_groups = ["dev", "test"], testonly = True), + ], + ), + ) + + pypi.exposed_packages().contains_exactly({"pypi_a": ["other_pkg", "simple"]}) + + # `simple` is also selected by the non-testonly tag. + pypi.configured_hubs().contains_exactly({"pypi_a": struct( + packages = ["other_pkg", "simple"], + testonly_packages = ["other_pkg"], + )}) + +_tests.append(_test_configure_testonly) + def _test_not_configured_exposes_all(env): # The pyproject.toml does not filter the hub without `pip.configure`. pypi = _configure_parse_modules( diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index 505c249169..e28d20875c 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -4,12 +4,14 @@ def configure( *, dependency_groups = ["*"], hub_name, - pyproject_toml = None): + pyproject_toml = None, + testonly = False): """A simple helper for testing to simulate the PyPI extension configure tag class""" return struct( dependency_groups = dependency_groups, hub_name = hub_name, pyproject_toml = pyproject_toml, + testonly = testonly, ) def pip_parse( diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index cafbb229b5..2a765b4f89 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -23,11 +23,12 @@ def _hub(*, module_name = "root", pyproject = None): exposed_packages = ["other_pkg", "simple", "transitive_pkg"], ) -def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None): +def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None, testonly = False): return mocks.tag( hub_name = hub_name, dependency_groups = dependency_groups, pyproject_toml = pyproject_toml, + testonly = testonly, ) def _module(name = "root", is_root = True, configure = []): @@ -49,6 +50,9 @@ def _exposure(*modules, hubs = None, default_pyproject = None): ) return got, failures +def _packages(got): + return {hub: info.packages for hub, info in got.items()} + def _test_not_configured(env): got, failures = _exposure(_module()) env.expect.that_collection(failures).contains_exactly([]) @@ -67,7 +71,7 @@ def _test_configure(env): ]: got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) - env.expect.where(groups = groups).that_dict(got).contains_exactly({"hub": want}) + env.expect.where(groups = groups).that_dict(_packages(got)).contains_exactly({"hub": want}) _tests.append(_test_configure) @@ -77,7 +81,7 @@ def _test_configure_combines_tags(env): _configure(dependency_groups = ["dev"]), ])) env.expect.that_collection(failures).contains_exactly([]) - env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg", "simple"]}) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["other_pkg", "simple"]}) _tests.append(_test_configure_combines_tags) @@ -86,7 +90,7 @@ def _test_configure_pyproject_precedence(env): got, _ = _exposure(_module(configure = [ _configure(pyproject_toml = "other.toml"), ])) - env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["other_pkg"]}) # The `pip.default` value is used for the hubs of the root module. got, failures = _exposure( @@ -95,7 +99,7 @@ def _test_configure_pyproject_precedence(env): default_pyproject = "pyproject.toml", ) env.expect.that_collection(failures).contains_exactly([]) - env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["simple"]}) _tests.append(_test_configure_pyproject_precedence) @@ -108,7 +112,7 @@ def _test_configure_module_precedence(env): _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), hubs = hubs, ) - env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["other_pkg"]}) # The root module wins over the owner. got, _ = _exposure( @@ -116,7 +120,7 @@ def _test_configure_module_precedence(env): _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), hubs = hubs, ) - env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["simple"]}) # Other modules are ignored. got, _ = _exposure( @@ -144,6 +148,24 @@ def _test_configure_errors(env): _tests.append(_test_configure_errors) +def _test_configure_testonly(env): + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["dev", "test"], testonly = True), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["other_pkg", "simple"]}) + + # `simple` is also selected by the non-testonly tag. + env.expect.that_collection(got["hub"].testonly_packages).contains_exactly(["other_pkg"]) + + got, _ = _exposure(_module(configure = [ + _configure(dependency_groups = ["*"], testonly = True), + ])) + env.expect.that_collection(got["hub"].testonly_packages).contains_exactly(["other_pkg", "simple"]) + +_tests.append(_test_configure_testonly) + def hub_exposure_test_suite(name): """Create the test suite. From b31a734fd273538849adba68184df7b801212fce Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:05:33 +0900 Subject: [PATCH 07/20] fix(pypi): treat exposing all packages as not configuring the hub pip.configure with ["*", "transitive"] used the packages that are available on all platforms, which made the platform-specific packages private. Treat it as if the hub was not configured instead, which keeps these packages public and out of all_requirements as before. --- docs/pypi/download.md | 5 ++-- python/private/pypi/extension.bzl | 1 - python/private/pypi/hub_exposure.bzl | 24 +++++++++---------- .../pypi/hub_exposure/hub_exposure_tests.bzl | 14 +++++++++-- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 5f16606d24..654e156921 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -171,8 +171,9 @@ dependencies of `numpy` are only used through `@pypi_hub//numpy`. The The {attr}`pip.configure.dependency_groups` attribute defaults to `["*"]`, which selects every group. Use `["*", "transitive"]` to expose all of the -packages of the hub, including the transitive dependencies. Selecting the -transitive dependencies of only some of the groups is not supported. +packages of the hub, including the transitive dependencies, which is the same +as not configuring the hub. Selecting the transitive dependencies of only some +of the groups is not supported. Multiple tags for the same hub are combined. Only the root module and the module that created the hub can configure it; if both do, the root module diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index ce5e95492d..10e0c456a2 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -369,7 +369,6 @@ You cannot use both the additive_build_content and additive_build_content_file a module_name = hub.module_name, pyproject = hub_pyproject.get(hub.name), packages = out.whl_map.keys(), - exposed_packages = out.exposed_packages, ) configured_hubs = hub_exposure( diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index 6ca9315a81..fe9e586153 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -13,28 +13,27 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): * `pyproject`: {type}`Label | None` from `pip.parse`. * `packages`: {type}`list[str]` the normalized names of all packages in the hub. - * `exposed_packages`: {type}`list[str]` the packages that the hub - exposes when it is not configured. default_pyproject: {type}`Label | None` from `pip.default`, only used for the hubs of the root module. _fail: {type}`callable` the failure function, mainly for testing. Returns: {type}`dict[str, list[str]] | None` the configured hubs mapped to the - sorted packages that they expose. The hubs that are not configured - are not included. Returns `None` on failure. + sorted packages that they expose. The hubs that are not configured or + that expose all of their packages are not included, so that they + behave the same. Returns `None` on failure. """ # The root module wins over the owner of the hub when both configure it. root_modules = [mod for mod in module_ctx.modules if mod.is_root] root_name = root_modules[0].name if root_modules else None - root_configured = { - tag.hub_name: None - for mod in root_modules - for tag in mod.tags.configure - } + root_configured = {} + for mod in root_modules: + for tag in mod.tags.configure: + root_configured[tag.hub_name] = None configured = {} + exposes_all = {} for mod in module_ctx.modules: for tag in mod.tags.configure: hub = tag.hub_name @@ -66,13 +65,11 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): _fail = _fail, ) if selection.all: - packages = info.exposed_packages - else: - packages = selection.packages + exposes_all[hub] = None # Multiple tags for the same hub are combined. exposed = configured.setdefault(hub, {}) - for name in packages: + for name in selection.packages: # The pyproject.toml may list packages that are not in the hub, # e.g. tools that are not locked. if name in info.packages: @@ -81,4 +78,5 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): return { hub: sorted(packages) for hub, packages in configured.items() + if hub not in exposes_all } diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index cafbb229b5..5d84a351b5 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -20,7 +20,6 @@ def _hub(*, module_name = "root", pyproject = None): module_name = module_name, pyproject = pyproject, packages = ["other_pkg", "simple", "transitive_pkg"], - exposed_packages = ["other_pkg", "simple", "transitive_pkg"], ) def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None): @@ -63,7 +62,6 @@ def _test_configure(env): (["dev"], ["other_pkg"]), # `not-in-hub` is ignored (["group:test"], ["simple"]), - (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), ]: got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) @@ -71,6 +69,18 @@ def _test_configure(env): _tests.append(_test_configure) +def _test_configure_all_packages(env): + # Exposing all of the packages is the same as not configuring the hub, so + # that e.g. the platform-specific packages stay public. + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["*", "transitive"]), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_configure_all_packages) + def _test_configure_combines_tags(env): got, failures = _exposure(_module(configure = [ _configure(dependency_groups = ["default"]), From 9d97360242b44f6a73472793d17a17287cabb1f3 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:05:56 +0900 Subject: [PATCH 08/20] refactor(pypi): use explicit loops and simpler conditions in pip.configure Replace the nested comprehensions and the long inline conditions with explicit loops and use types.is_string. --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/dependency_groups.bzl | 19 ++++++++----------- python/private/pypi/hub_repository.bzl | 4 +++- python/private/pypi/render_pkg_aliases.bzl | 19 +++++++++++-------- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index caf482b67f..fe28b17413 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -639,6 +639,7 @@ bzl_library( deps = [ ":pep508_requirement", "//python/private:normalize_name", + "@bazel_skylib//lib:types", ], ) diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index 2ddadee3dd..4e40c7e856 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -1,5 +1,6 @@ """Select PyPI packages using pyproject.toml dependency groups.""" +load("@bazel_skylib//lib:types.bzl", "types") load("//python/private:normalize_name.bzl", "normalize_name") load(":pep508_requirement.bzl", "requirement") @@ -59,16 +60,12 @@ def select_dependency_groups(pyproject, specs, _fail = fail): for key in selected: for req in groups[key]: req = requirement(req) - names = [req.name_] - if req.name_ == project: - names = [ - requirement(r).name_ - for extra in req.extras - for r in groups.get("optional:" + extra, []) - ] - for name in names: - if name: - packages[name] = None + if req.name_ != project: + packages[req.name_] = None + continue + for extra in req.extras: + for r in groups.get("optional:" + extra, []): + packages[requirement(r).name_] = None return struct(packages = sorted(packages), all = False) @@ -119,7 +116,7 @@ def _expand_dependency_groups(dependency_groups, _fail): reqs = [] for item in items: - if type(item) == type(""): + if types.is_string(item): reqs.append(item) continue diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index d421f3cc09..ff5311c6ea 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -28,14 +28,16 @@ exports_files(["requirements.bzl"]) def _impl(rctx): if rctx.attr.hide_unexposed_packages: bzl_packages = rctx.attr.packages + exposed_packages = bzl_packages else: bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + exposed_packages = None aliases = render_multiplatform_pkg_aliases( aliases = { key: _whl_config_settings_from_json(values) for key, values in rctx.attr.whl_map.items() }, - exposed_packages = bzl_packages if rctx.attr.hide_unexposed_packages else None, + exposed_packages = exposed_packages, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 5582cc00c9..2c86155876 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -139,17 +139,20 @@ def render_pkg_aliases( exposed_packages = {normalize_name(name): None for name in exposed_packages} internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) - files = { - "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( - name = normalize_name(name), + files = {} + for name, pkg_aliases in aliases.items(): + name = normalize_name(name) + visibility = public + if exposed_packages != None and name not in exposed_packages: + visibility = internal_visibility + files["{}/BUILD.bazel".format(name)] = _render_common_aliases( + name = name, aliases = pkg_aliases, - extra_aliases = extra_hub_aliases.get(normalize_name(name), []), - group_name = whl_group_mapping.get(normalize_name(name)), - visibility = public if exposed_packages == None or normalize_name(name) in exposed_packages else internal_visibility, + extra_aliases = extra_hub_aliases.get(name, []), + group_name = whl_group_mapping.get(name), + visibility = visibility, **kwargs ).strip() - for name, pkg_aliases in aliases.items() - } if requirement_cycles: files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) From 35c066052cb0bea531aba78784149420d8056aaa Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:06:00 +0900 Subject: [PATCH 09/20] test(pypi): test the pip.default pyproject_toml with pip.configure Keep a single wiring test for pip.configure in the extension tests, as the selection is tested in hub_exposure_tests, and test that the pip.default pyproject_toml is used. Also link the issue in the news entry. --- news/4395.added.md | 3 +- tests/pypi/extension/extension_tests.bzl | 49 +++++++++++++++--------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/news/4395.added.md b/news/4395.added.md index 62d7f10880..3e633cc997 100644 --- a/news/4395.added.md +++ b/news/4395.added.md @@ -1,2 +1,3 @@ (pypi) Added {obj}`pip.configure` to limit the packages that a hub exposes to -the ones listed in the dependency groups of a `pyproject.toml` file. +the ones listed in the dependency groups of a `pyproject.toml` file +([#3413](https://github.com/bazel-contrib/rules_python/issues/3413)). diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 17f8d6741e..680853a0d0 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -807,36 +807,47 @@ def _configure_parse_modules(env, *modules): minor_mapping = {"3.15": "3.15.19"}, ) -def _configure_parse(*, hub_name = "pypi_a", pyproject_toml = "pyproject.toml"): +def _configure_parse(**kwargs): return _parse( - hub_name = hub_name, - pyproject_toml = pyproject_toml, + hub_name = "pypi_a", python_version = "3.15", simpleapi_skip = ["simple", "other-pkg", "transitive-pkg"], requirements_lock = "requirements.txt", + **kwargs ) def _test_configure(env): - for groups, want in [ - (["*"], ["other_pkg", "simple"]), - (["test"], ["simple"]), - (["dev"], ["other_pkg"]), - (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), - ]: - pypi = _configure_parse_modules( - env, - _mod( - name = "rules_python", - parse = [_configure_parse()], - configure = [_configure(hub_name = "pypi_a", dependency_groups = groups)], - ), - ) + # The selection itself is tested in `hub_exposure_tests`. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse(pyproject_toml = "pyproject.toml")], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["test"])], + ), + ) - pypi.exposed_packages().contains_exactly({"pypi_a": want}) - pypi.configured_hubs().contains_exactly({"pypi_a": want}) + pypi.exposed_packages().contains_exactly({"pypi_a": ["simple"]}) + pypi.configured_hubs().contains_exactly({"pypi_a": ["simple"]}) _tests.append(_test_configure) +def _test_configure_default_pyproject(env): + # The `pip.default` pyproject.toml is used for the hubs of the root module. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + default = _default_tags_default + [_default(pyproject_toml = "pyproject.toml")], + parse = [_configure_parse()], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["dev"])], + ), + ) + + pypi.configured_hubs().contains_exactly({"pypi_a": ["other_pkg"]}) + +_tests.append(_test_configure_default_pyproject) + def _test_not_configured_exposes_all(env): # The pyproject.toml does not filter the hub without `pip.configure`. pypi = _configure_parse_modules( From 003655c8ab6329907b1953ece8094c24241bed2e Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:08:42 +0900 Subject: [PATCH 10/20] refactor(pypi): drop the unreachable testonly check in pkg_aliases --- news/{pip-configure-testonly.added.md => 4428.added.md} | 0 python/private/pypi/pkg_aliases.bzl | 5 +---- 2 files changed, 1 insertion(+), 4 deletions(-) rename news/{pip-configure-testonly.added.md => 4428.added.md} (100%) diff --git a/news/pip-configure-testonly.added.md b/news/4428.added.md similarity index 100% rename from news/pip-configure-testonly.added.md rename to news/4428.added.md diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index b838434088..f165e5dffe 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -124,13 +124,10 @@ def pkg_aliases( _native.alias( name = name, actual = ":" + PY_LIBRARY_PUBLIC_LABEL, - **({"testonly": True} if testonly else {}) + testonly = testonly, ) if requires_dist: - if testonly: - fail("testonly is not supported together with requires_dist") - # if it has a group_name set, then it will create the impl label target in the # current macro, we still need to do the aliases to the actual groups package as # per below From 5e4c8098726042175cd265302bff4df5e4c6697d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:08:46 +0900 Subject: [PATCH 11/20] refactor(pypi): compute the testonly packages once in parse_modules Compute the testonly packages of each hub and of the unified hub in parse_modules instead of using long inline conditions, and test that a package is testonly in the unified hub if it is testonly in any hub. --- python/private/pypi/extension.bzl | 24 ++++++++++++-------- tests/pypi/extension/extension_tests.bzl | 29 ++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index ddbb43ebc4..9479d08eaf 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -379,8 +379,16 @@ You cannot use both the additive_build_content and additive_build_content_file a ) if configured_hubs == None: return None + testonly_packages = {} + unified_testonly_packages = {} for hub_name, info in configured_hubs.items(): exposed_packages[hub_name] = info.packages + testonly_packages[hub_name] = info.testonly_packages + + # An alias can't be less testonly than what it points to, so a package + # is testonly in the unified hub if it is testonly in any hub. + for pkg in info.testonly_packages: + unified_testonly_packages[pkg] = None return struct( configured_hubs = configured_hubs, @@ -392,7 +400,9 @@ You cannot use both the additive_build_content and additive_build_content_file a facts = simpleapi_cache.get_facts(), hub_group_map = hub_group_map, hub_whl_map = hub_whl_map, + testonly_packages = testonly_packages, unified_hub_requirements_bzl = config.unified_hub_requirements_bzl, + unified_testonly_packages = sorted(unified_testonly_packages), whl_libraries = whl_libraries, whl_mods = whl_mods, platform_config_settings = { @@ -417,7 +427,9 @@ def _create_unified_hub_repo(mods): for hub_name in hubs: # The hubs configured with `pip.configure` only expose some packages. - hub_packages = mods.configured_hubs[hub_name].packages if hub_name in mods.configured_hubs else mods.hub_whl_map[hub_name] + hub_packages = mods.hub_whl_map[hub_name] + if hub_name in mods.configured_hubs: + hub_packages = mods.configured_hubs[hub_name].packages for pkg_name in hub_packages: norm_pkg = normalize_name(pkg_name) if norm_pkg not in packages: @@ -449,13 +461,7 @@ def _create_unified_hub_repo(mods): hubs = hubs, packages = packages, requirements_bzl = mods.unified_hub_requirements_bzl, - # An alias can't be less testonly than what it points to, so a package - # is testonly if it is testonly in any hub. - testonly_packages = sorted({ - pkg: None - for hub_name in hubs - for pkg in (mods.configured_hubs[hub_name].testonly_packages if hub_name in mods.configured_hubs else []) - }), + testonly_packages = mods.unified_testonly_packages, ) def _pip_impl(module_ctx): @@ -549,7 +555,7 @@ def _pip_impl(module_ctx): groups = mods.hub_group_map.get(hub_name), # The hubs configured with `pip.configure` hide the other packages. hide_unexposed_packages = hub_name in mods.configured_hubs, - testonly_packages = mods.configured_hubs[hub_name].testonly_packages if hub_name in mods.configured_hubs else [], + testonly_packages = mods.testonly_packages.get(hub_name, []), ) _create_unified_hub_repo(mods) diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 93651937df..590be1cbb2 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -125,6 +125,8 @@ def _parse_modules(env, **kwargs): exposed_packages = subjects.dict, hub_group_map = subjects.dict, hub_whl_map = subjects.dict, + testonly_packages = subjects.dict, + unified_testonly_packages = subjects.collection, unified_hub_requirements_bzl = subjects.bool, whl_libraries = subjects.dict, whl_mods = subjects.dict, @@ -807,9 +809,9 @@ def _configure_parse_modules(env, *modules): minor_mapping = {"3.15": "3.15.19"}, ) -def _configure_parse(**kwargs): +def _configure_parse(hub_name = "pypi_a", **kwargs): return _parse( - hub_name = "pypi_a", + hub_name = hub_name, python_version = "3.15", simpleapi_skip = ["simple", "other-pkg", "transitive-pkg"], requirements_lock = "requirements.txt", @@ -858,6 +860,29 @@ def _test_configure_testonly(env): _tests.append(_test_configure_testonly) +def _test_configure_testonly_unified(env): + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [ + _configure_parse(hub_name = "pypi_a", pyproject_toml = "pyproject.toml"), + _configure_parse(hub_name = "pypi_b", pyproject_toml = "pyproject.toml"), + ], + configure = [ + _configure(hub_name = "pypi_a", dependency_groups = ["default"], testonly = True), + _configure(hub_name = "pypi_b", dependency_groups = ["default"]), + ], + ), + ) + + pypi.testonly_packages().contains_exactly({"pypi_a": ["simple"], "pypi_b": []}) + + # A package is testonly in the unified hub if it is testonly in any hub. + pypi.unified_testonly_packages().contains_exactly(["simple"]) + +_tests.append(_test_configure_testonly_unified) + def _test_configure_default_pyproject(env): # The `pip.default` pyproject.toml is used for the hubs of the root module. pypi = _configure_parse_modules( From 41a0664a8e26d19089cdc0269861e1432c46b7f4 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:08:48 +0900 Subject: [PATCH 12/20] news: name the pip.configure.testonly entry after the PR --- news/4428.added.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/news/4428.added.md b/news/4428.added.md index ebc0594f7e..179e302535 100644 --- a/news/4428.added.md +++ b/news/4428.added.md @@ -1,2 +1,3 @@ (pypi) Added {attr}`pip.configure.testonly` to make the packages selected by -{obj}`pip.configure` testonly. +{obj}`pip.configure` testonly +([#3413](https://github.com/bazel-contrib/rules_python/issues/3413)). From 71be0c06b8f42f8e68b0d6f2f3c2c5fbb52765de Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:12:30 +0900 Subject: [PATCH 13/20] fix(pypi): only pass testonly to the package alias when it is set The existing tests mock the alias without a testonly argument. --- python/private/pypi/pkg_aliases.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index f165e5dffe..d4fac5b269 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -124,7 +124,7 @@ def pkg_aliases( _native.alias( name = name, actual = ":" + PY_LIBRARY_PUBLIC_LABEL, - testonly = testonly, + **({"testonly": True} if testonly else {}) ) if requires_dist: From 1ffebb84dee0f508ea8b237bf7eea252fd4e18b0 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 18 Nov 2026 19:45:58 -0800 Subject: [PATCH 14/20] refactor(pypi): polish pip.configure docs, docstrings, and formatting Align the new `pip.configure` documentation, docstrings, and Starlark files with repository style and documentation conventions. Add `requires-python` to the `docs/pypi/download.md` example, move the `pip.configure` section to top-level `##` heading order, clarify the `pyproject_toml` attribute docstrings and `{ref}` links, document helper arguments/returns and returned `struct` fields, flatten `hub_exposure` guards, move module-level test constants before functions, and wrap Starlark lines and `doc` strings to 80 columns. --- docs/pypi/download.md | 87 ++++++++++--------- python/private/pypi/dependency_groups.bzl | 39 +++++++-- python/private/pypi/extension.bzl | 52 +++++++---- python/private/pypi/hub_exposure.bzl | 16 ++-- python/private/pypi/hub_repository.bzl | 11 ++- python/private/pypi/render_pkg_aliases.bzl | 19 +++- python/private/pyproject_utils.bzl | 5 ++ .../dependency_groups_tests.bzl | 49 +++++++++-- tests/pypi/extension/extension_tests.bzl | 65 ++++++++------ tests/pypi/extension/pip_parse.bzl | 2 +- .../pypi/hub_exposure/hub_exposure_tests.bzl | 48 +++++++--- 11 files changed, 266 insertions(+), 127 deletions(-) diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 654e156921..ef069da5b5 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -134,8 +134,50 @@ rename, and `requirement("numpy")` routes through the active hub like fixed at loading time, before the active hub is known, so load them from a concrete hub instead. Prefer `@pypi//` labels in new code. +### Declaring Abstract Dependencies (pip.dep) + +:::{versionadded} 2.2.0 +Declaring abstract PyPI dependencies via `pip.dep` tags. +::: + +Sometimes a shared library target or a ruleset needs to depend on a PyPI +package (e.g., `@pypi//numpy`), but does not want to force a specific package +version or a concrete `requirements.txt` lock file on its consumers. + +Instead of calling `pip.parse()`, the module can declare its dependency using +the `pip.dep` tag: + +```starlark +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") + +# Declare an abstract dependency on 'numpy' and specify extra targets that +# are expected to be available in the package. +pip.dep( + name = "numpy", + extra_targets = ["extra-alias"], +) +``` + +This ensures that the target structure `@pypi//numpy` (and +`@pypi//numpy:extra-alias`) exists in the unified `@pypi` hub repository, so the +declaring module can compile and analyze successfully without needing any local +requirements file. + +The actual concrete implementation and version of the package must be provided +by a downstream module calling `pip.parse`. + +If a downstream module attempts to build a target that depends on an abstract +dependency, but has not provided a concrete implementation for it via any +`pip.parse` call, the build will fail at execution time. + + +As with any repository rule or extension, if you would like to ensure that `pip_parse` is +re-executed to pick up a non-hermetic change to your environment (e.g., updating your system +`python` interpreter), you can force it to re-execute by running `bazel sync --only [pip_parse +name]`. + (pip-configure)= -### Configuring the Exposed Packages (pip.configure) +## Configuring the Exposed Packages (pip.configure) By default a hub exposes all of the packages from its requirements files or `uv.lock` file, including the transitive dependencies. A `pip.configure` tag @@ -145,6 +187,7 @@ can limit the hub to the packages listed in a `pyproject.toml` file: the ```toml [project] +requires-python = "==3.13.0" dependencies = ["numpy"] [dependency-groups] @@ -185,48 +228,6 @@ are missing from the lock file. :::{versionadded} VERSION_NEXT_FEATURE ::: -### Declaring Abstract Dependencies (pip.dep) - -:::{versionadded} 2.2.0 -Declaring abstract PyPI dependencies via `pip.dep` tags. -::: - -Sometimes a shared library target or a ruleset needs to depend on a PyPI -package (e.g., `@pypi//numpy`), but does not want to force a specific package -version or a concrete `requirements.txt` lock file on its consumers. - -Instead of calling `pip.parse()`, the module can declare its dependency using -the `pip.dep` tag: - -```starlark -pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") - -# Declare an abstract dependency on 'numpy' and specify extra targets that -# are expected to be available in the package. -pip.dep( - name = "numpy", - extra_targets = ["extra-alias"], -) -``` - -This ensures that the target structure `@pypi//numpy` (and -`@pypi//numpy:extra-alias`) exists in the unified `@pypi` hub repository, so the -declaring module can compile and analyze successfully without needing any local -requirements file. - -The actual concrete implementation and version of the package must be provided -by a downstream module calling `pip.parse`. - -If a downstream module attempts to build a target that depends on an abstract -dependency, but has not provided a concrete implementation for it via any -`pip.parse` call, the build will fail at execution time. - - -As with any repository rule or extension, if you would like to ensure that `pip_parse` is -re-executed to pick up a non-hermetic change to your environment (e.g., updating your system -`python` interpreter), you can force it to re-execute by running `bazel sync --only [pip_parse -name]`. - (per-os-arch-requirements)= ## Requirements for a specific OS/Architecture diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index 4e40c7e856..e40e67a4c3 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -34,13 +34,21 @@ def select_dependency_groups(pyproject, specs, _fail = fail): groups = {"default": pyproject.dependencies} for name, reqs in pyproject.optional_dependencies.items(): groups["optional:" + normalize_name(name)] = reqs - for name, reqs in _expand_dependency_groups(pyproject.dependency_groups, _fail).items(): + expanded = _expand_dependency_groups(pyproject.dependency_groups, _fail) + for name, reqs in expanded.items(): groups["group:" + name] = reqs if "transitive" in specs: if "*" not in specs: - _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format(specs)) - return struct(packages = [], all = True) + _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format( + specs, + )) + return struct( + # list[str]: sorted normalized package names selected by `specs`. + packages = [], + # bool: True when all hub packages are selected. + all = True, + ) selected = {} for spec in specs: @@ -67,9 +75,24 @@ def select_dependency_groups(pyproject, specs, _fail = fail): for r in groups.get("optional:" + extra, []): packages[requirement(r).name_] = None - return struct(packages = sorted(packages), all = False) + return struct( + # list[str]: sorted normalized package names selected by `specs`. + packages = sorted(packages), + # bool: True when all packages including transitive ones are selected. + all = False, + ) def _match(spec, groups): + """Match a dependency group specifier against normalized group keys. + + Args: + spec: {type}`str` the dependency group specifier. + groups: {type}`dict[str, list[str]]` available groups keyed by + prefixed group name. + + Returns: + {type}`list[str]` matching keys in `groups`. + """ if spec == "*": return groups.keys() if spec in ("optional:*", "group:*"): @@ -122,7 +145,10 @@ def _expand_dependency_groups(dependency_groups, _fail): include = normalize_name(item.get("include-group", "")) if include not in groups: - _fail("Unknown dependency group '{}' included from '{}'".format(include, name)) + _fail("Unknown dependency group '{}' included from '{}'".format( + include, + name, + )) return {} if include not in expanded: reqs = None @@ -133,7 +159,8 @@ def _expand_dependency_groups(dependency_groups, _fail): expanded[name] = reqs if len(expanded) != len(groups): + unexpanded = [name for name in groups if name not in expanded] _fail("Cyclic dependency group includes: {}".format( - ", ".join(sorted([name for name in groups if name not in expanded])), + ", ".join(sorted(unexpanded)), )) return expanded diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index dcc7aa1a94..78119835af 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -157,6 +157,7 @@ def build_config( for name, values in defaults["platforms"].items() }, enable_pipstar_extract = enable_pipstar_extract, + # Label | None: default `pyproject.toml` from `pip.default`. pyproject_toml = defaults["pyproject_toml"], toml_decode = toml.decode, unified_hub_requirements_bzl = unified_hub_requirements_bzl, @@ -369,8 +370,11 @@ You cannot use both the additive_build_content and additive_build_content_file a hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map hubs[hub.name] = struct( + # str: module that created the hub. module_name = hub.module_name, + # Label | None: `pyproject_toml` set on `pip.parse`. pyproject = hub_pyproject.get(hub.name), + # list[str]: normalized names of all packages in the hub. packages = out.whl_map.keys(), ) @@ -385,6 +389,7 @@ You cannot use both the additive_build_content and additive_build_content_file a exposed_packages.update(configured_hubs) return struct( + # dict[str, list[str]]: configured hubs and exposed packages. configured_hubs = configured_hubs, config = config, declared_deps = declared_deps, @@ -419,7 +424,10 @@ def _create_unified_hub_repo(mods): for hub_name in hubs: # The hubs configured with `pip.configure` only expose some packages. - hub_packages = mods.configured_hubs.get(hub_name, mods.hub_whl_map[hub_name]) + hub_packages = mods.configured_hubs.get( + hub_name, + mods.hub_whl_map[hub_name], + ) for pkg_name in hub_packages: norm_pkg = normalize_name(pkg_name) if norm_pkg not in packages: @@ -673,15 +681,18 @@ If you are defining custom platforms in your project and don't want things to cl ), "pyproject_toml": attr.label( mandatory = False, - doc = """\ -Label pointing to pyproject.toml file to read the default Python version from. -When specified, reads the `requires-python` field from pyproject.toml and uses -it as the default python_version for all `pip.parse()` calls that don't -explicitly specify one. + doc = """ +Label pointing to a `pyproject.toml` file used to read the default Python +version and, for hubs of the root module, to select exposed packages via +{obj}`pip.configure`. + +When `[project] requires-python` is present, it is used as the default +`python_version` for `pip.parse()` calls that do not specify one. :::{note} -The version must be specified as `==X.Y.Z` (exact version with full semver). -This is designed to work with dependency management tools like Renovate. +If `requires-python` is set, the version must be specified as `==X.Y.Z` (exact +version with full semver). This is designed to work with dependency management +tools like Renovate. ::: :::{versionadded} 2.3.0 @@ -898,13 +909,16 @@ find in case extra indexes are specified. ), "pyproject_toml": attr.label( mandatory = False, - doc = """\ -Label pointing to a pyproject.toml file to read the Python version from. -When specified, the `requires-python` field is used as the `python_version` -for this `pip.parse()` call, unless `python_version` is set explicitly. + doc = """ +Label pointing to a `pyproject.toml` file used to read the Python version and +to select exposed packages via {obj}`pip.configure`. + +When `python_version` is not set explicitly, the `[project] requires-python` +field is used as the `python_version` for this `pip.parse()` call. :::{note} -The version must be specified as `==X.Y.Z` (exact version with full semver). +When used to derive `python_version`, `requires-python` must be specified as +`==X.Y.Z` (exact version with full semver). ::: :::{versionadded} 2.3.0 @@ -1116,7 +1130,7 @@ _configure_tag = tag_class( attrs = { "dependency_groups": attr.string_list( default = ["*"], - doc = """\ + doc = """ The `pyproject.toml` dependency groups whose packages the hub exposes. Each item is one of: @@ -1132,7 +1146,7 @@ Each item is one of: packages of the hub including the transitive dependencies, i.e. all of the packages from the requirements files or the `uv.lock` file. -See [Configuring the Exposed Packages](#pip-configure) for examples. +See {ref}`Configuring the Exposed Packages ` for examples. :::{versionadded} VERSION_NEXT_FEATURE ::: @@ -1140,7 +1154,7 @@ See [Configuring the Exposed Packages](#pip-configure) for examples. ), "hub_name": attr.string( mandatory = True, - doc = """\ + doc = """ The name of the hub to configure. :::{versionadded} VERSION_NEXT_FEATURE @@ -1149,7 +1163,7 @@ The name of the hub to configure. ), "pyproject_toml": attr.label( mandatory = False, - doc = """\ + doc = """ The `pyproject.toml` file that defines the dependency groups. Defaults to the {attr}`pip.parse.pyproject_toml` of the hub, then to {attr}`pip.default.pyproject_toml` if the hub is defined by the root module. @@ -1159,7 +1173,7 @@ The `pyproject.toml` file that defines the dependency groups. Defaults to the """, ), }, - doc = """\ + doc = """ Configure the packages exposed by a hub. The packages are selected from a `pyproject.toml` file using the @@ -1171,7 +1185,7 @@ they are not public. Only the root module and the module that created the hub can configure it. If both do, the root module wins. Multiple tags for the same hub are combined. -See [Configuring the Exposed Packages](#pip-configure) for examples. +See {ref}`Configuring the Exposed Packages ` for examples. :::{versionadded} VERSION_NEXT_FEATURE ::: diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index fe9e586153..ff9f24694d 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -45,18 +45,20 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): return None info = hubs[hub] - if not mod.is_root and (mod.name != info.module_name or hub in root_configured): + if not mod.is_root and mod.name != info.module_name: # Only the root module and the module that created the hub can # configure it. continue + if not mod.is_root and hub in root_configured: + continue - pyproject = ( - tag.pyproject_toml or - info.pyproject or - (default_pyproject if info.module_name == root_name else None) - ) + pyproject = tag.pyproject_toml or info.pyproject + if not pyproject and info.module_name == root_name: + pyproject = default_pyproject if not pyproject: - _fail("pip.configure() for hub '{}' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`".format(hub)) + _fail("pip.configure() for hub '{}' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`".format( + hub, + )) return None selection = select_dependency_groups( diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index ff5311c6ea..99f659ba31 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -86,13 +86,16 @@ hub_repository = repository_rule( ), "hide_unexposed_packages": attr.bool( default = False, - doc = "Make the aliases of the packages that are not in `packages` visible only to the wheel repositories.", + doc = """ +Make the aliases of the packages that are not in `packages` visible only to +the wheel repositories. +""", ), "packages": attr.string_list( mandatory = False, - doc = """\ -The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys -unless `hide_unexposed_packages` is set. + doc = """ +The list of packages that will be exposed via all_*requirements macros. +Defaults to whl_map keys unless `hide_unexposed_packages` is set. """, ), "platform_config_settings": attr.string_list_dict( diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 2c86155876..ba0ad2cc75 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -136,8 +136,14 @@ def render_pkg_aliases( public = ["//visibility:public"] if exposed_packages != None: - exposed_packages = {normalize_name(name): None for name in exposed_packages} - internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) + exposed_packages = { + normalize_name(name): None + for name in exposed_packages + } + internal_visibility = _internal_visibility( + aliases, + has_groups = bool(requirement_cycles), + ) files = {} for name, pkg_aliases in aliases.items(): @@ -159,10 +165,17 @@ def render_pkg_aliases( return files def _internal_visibility(aliases, has_groups = False): - """The visibility of the packages that are not exposed. + """Return the visibility list for packages that are not exposed. The wheel repositories depend on the aliases of the hub, so the aliases have to be visible to them. + + Args: + aliases: {type}`dict[str, str | dict]` package-to-repo alias mapping. + has_groups: {type}`bool` whether requirement cycle groups exist. + + Returns: + {type}`list[str]` visibility package specifications. """ repo_names = {} for pkg_aliases in aliases.values(): diff --git a/python/private/pyproject_utils.bzl b/python/private/pyproject_utils.bzl index a9494b8ac2..ceb5c803aa 100644 --- a/python/private/pyproject_utils.bzl +++ b/python/private/pyproject_utils.bzl @@ -29,10 +29,15 @@ def read_pyproject(module_ctx, pyproject): data = toml.decode(module_ctx.read(module_ctx.path(pyproject), watch = "yes")) project = data.get("project", {}) return struct( + # str | None: raw `[project] requires-python` value. requires_python = project.get("requires-python"), + # str | None: `[project] name`. name = project.get("name"), + # list[str]: `[project] dependencies` requirement strings. dependencies = project.get("dependencies") or [], + # dict[str, list[str]]: `[project.optional-dependencies]` table. optional_dependencies = project.get("optional-dependencies") or {}, + # dict[str, list]: PEP 735 `[dependency-groups]` table. dependency_groups = data.get("dependency-groups") or {}, ) diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl index ee23aec1fb..d494cae9d7 100644 --- a/tests/pypi/dependency_groups/dependency_groups_tests.bzl +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -1,7 +1,10 @@ """dependency_groups tests""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:dependency_groups.bzl", "select_dependency_groups") # buildifier: disable=bzl-visibility +load( + "//python/private/pypi:dependency_groups.bzl", + "select_dependency_groups", +) # buildifier: disable=bzl-visibility _tests = [] @@ -35,11 +38,25 @@ def _test_select(env): (["optional:*"], ["docs_pkg", "grpc_pkg"]), (["group:*"], ["dev_pkg", "shared_pkg", "test_pkg"]), (["default", "group:test"], ["main_pkg", "shared_pkg", "test_pkg"]), - (["*"], ["dev_pkg", "docs_pkg", "grpc_pkg", "main_pkg", "shared_pkg", "test_pkg"]), + ( + ["*"], + [ + "dev_pkg", + "docs_pkg", + "grpc_pkg", + "main_pkg", + "shared_pkg", + "test_pkg", + ], + ), ]: got, failures = _select(specs) - env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) - env.expect.where(specs = specs).that_collection(got.packages).contains_exactly(want).in_order() + env.expect.where(specs = specs).that_collection( + failures, + ).contains_exactly([]) + env.expect.where(specs = specs).that_collection( + got.packages, + ).contains_exactly(want).in_order() env.expect.where(specs = specs).that_bool(got.all).equals(False) _tests.append(_test_select) @@ -54,18 +71,32 @@ _tests.append(_test_select_all) def _test_select_errors(env): for specs, want in [ - (["missing"], "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test, optional:all, optional:docs, optional:grpc"), - (["default", "transitive"], "The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: [\"default\", \"transitive\"]"), + ( + ["missing"], + "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test, optional:all, optional:docs, optional:grpc", + ), + ( + ["default", "transitive"], + "The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: [\"default\", \"transitive\"]", + ), ]: _, failures = _select(specs) - env.expect.where(specs = specs).that_collection(failures).contains_exactly([want]) + env.expect.where(specs = specs).that_collection( + failures, + ).contains_exactly([want]) _tests.append(_test_select_errors) def _test_include_group_errors(env): for groups, want in [ - ({"a": [{"include-group": "missing"}]}, "Unknown dependency group 'missing' included from 'a'"), - ({"a": [{"include-group": "b"}], "b": [{"include-group": "a"}]}, "Cyclic dependency group includes: a, b"), + ( + {"a": [{"include-group": "missing"}]}, + "Unknown dependency group 'missing' included from 'a'", + ), + ( + {"a": [{"include-group": "b"}], "b": [{"include-group": "a"}]}, + "Cyclic dependency group includes: a, b", + ), ]: _, failures = _select(["*"], pyproject = struct( name = None, diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index bc80ed679c..b3a6e7837c 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -25,6 +25,27 @@ load(":pip_parse.bzl", _configure = "configure", _parse = "pip_parse") _tests = [] +_CONFIGURE_REQUIREMENTS = """\ +simple==0.0.1 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +other-pkg==0.0.2 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +transitive-pkg==0.0.3 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +""" + +_CONFIGURE_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple"] +""" + def _pypi_mock_mctx(*modules, os_name = "unittest", arch_name = "exotic", environ = {}, read = None, mock_files = {}): _ = read # @unused return mocks.mctx( @@ -107,7 +128,16 @@ def _dep(*, name, extra_targets = []): extra_targets = extra_targets, ) -def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], configure = [], is_root = True): +def _mod( + *, + name, + default = _default_tags_default, + parse = [], + override = [], + whl_mods = [], + dep = [], + configure = [], + is_root = True): return struct( name = name, tags = struct( @@ -821,27 +851,6 @@ simple==0.0.1 \ _tests.append(_test_default_index_setting) -_CONFIGURE_REQUIREMENTS = """\ -simple==0.0.1 \ - --hash=sha256:deadbeef \ - --hash=sha256:deadbaaf -other-pkg==0.0.2 \ - --hash=sha256:deadbeef \ - --hash=sha256:deadbaaf -transitive-pkg==0.0.3 \ - --hash=sha256:deadbeef \ - --hash=sha256:deadbaaf -""" - -_CONFIGURE_PYPROJECT = """\ -[project] -dependencies = ["simple"] - -[dependency-groups] -dev = ["other-pkg"] -test = ["simple"] -""" - def _configure_parse_modules(env, *modules): return _parse_modules( env, @@ -876,7 +885,9 @@ def _test_configure(env): _mod( name = "rules_python", parse = [_configure_parse(pyproject_toml = "pyproject.toml")], - configure = [_configure(hub_name = "pypi_a", dependency_groups = ["test"])], + configure = [ + _configure(hub_name = "pypi_a", dependency_groups = ["test"]), + ], ), ) @@ -891,9 +902,13 @@ def _test_configure_default_pyproject(env): env, _mod( name = "rules_python", - default = _default_tags_default + [_default(pyproject_toml = "pyproject.toml")], + default = _default_tags_default + [ + _default(pyproject_toml = "pyproject.toml"), + ], parse = [_configure_parse()], - configure = [_configure(hub_name = "pypi_a", dependency_groups = ["dev"])], + configure = [ + _configure(hub_name = "pypi_a", dependency_groups = ["dev"]), + ], ), ) diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index 3670282a92..db3008d431 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -5,7 +5,7 @@ def configure( dependency_groups = ["*"], hub_name, pyproject_toml = None): - """A simple helper for testing to simulate the PyPI extension configure tag class""" + """Simulate the PyPI extension configure tag class for testing.""" return struct( dependency_groups = dependency_groups, hub_name = hub_name, diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index 5d84a351b5..a51deb9c56 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -1,7 +1,10 @@ """hub_exposure tests""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:hub_exposure.bzl", "hub_exposure") # buildifier: disable=bzl-visibility +load( + "//python/private/pypi:hub_exposure.bzl", + "hub_exposure", +) # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") _tests = [] @@ -22,7 +25,10 @@ def _hub(*, module_name = "root", pyproject = None): packages = ["other_pkg", "simple", "transitive_pkg"], ) -def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None): +def _configure( + hub_name = "hub", + dependency_groups = ["*"], + pyproject_toml = None): return mocks.tag( hub_name = hub_name, dependency_groups = dependency_groups, @@ -63,9 +69,15 @@ def _test_configure(env): # `not-in-hub` is ignored (["group:test"], ["simple"]), ]: - got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) - env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) - env.expect.where(groups = groups).that_dict(got).contains_exactly({"hub": want}) + got, failures = _exposure( + _module(configure = [_configure(dependency_groups = groups)]), + ) + env.expect.where(groups = groups).that_collection( + failures, + ).contains_exactly([]) + env.expect.where(groups = groups).that_dict(got).contains_exactly( + {"hub": want}, + ) _tests.append(_test_configure) @@ -87,7 +99,9 @@ def _test_configure_combines_tags(env): _configure(dependency_groups = ["dev"]), ])) env.expect.that_collection(failures).contains_exactly([]) - env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg", "simple"]}) + env.expect.that_dict(got).contains_exactly( + {"hub": ["other_pkg", "simple"]}, + ) _tests.append(_test_configure_combines_tags) @@ -115,7 +129,11 @@ def _test_configure_module_precedence(env): # The owner of the hub can configure it. got, _ = _exposure( _module(), - _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + _module( + name = "dep", + is_root = False, + configure = [_configure(dependency_groups = ["dev"])], + ), hubs = hubs, ) env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) @@ -123,7 +141,11 @@ def _test_configure_module_precedence(env): # The root module wins over the owner. got, _ = _exposure( _module(configure = [_configure(dependency_groups = ["default"])]), - _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + _module( + name = "dep", + is_root = False, + configure = [_configure(dependency_groups = ["dev"])], + ), hubs = hubs, ) env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) @@ -131,7 +153,11 @@ def _test_configure_module_precedence(env): # Other modules are ignored. got, _ = _exposure( _module(), - _module(name = "other", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + _module( + name = "other", + is_root = False, + configure = [_configure(dependency_groups = ["dev"])], + ), hubs = hubs, ) env.expect.that_dict(got).contains_exactly({}) @@ -139,7 +165,9 @@ def _test_configure_module_precedence(env): _tests.append(_test_configure_module_precedence) def _test_configure_errors(env): - _, failures = _exposure(_module(configure = [_configure(hub_name = "missing")])) + _, failures = _exposure( + _module(configure = [_configure(hub_name = "missing")]), + ) env.expect.that_collection(failures).contains_exactly([ "pip.configure() refers to unknown hub 'missing'. Available hubs: hub", ]) From a40d1d5e00aee45af7f6724304cdf369a7c3aebc Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 18 Nov 2026 19:49:15 -0800 Subject: [PATCH 15/20] refactor(pypi): restore pyproject_toml attribute docstrings Restore the `pip.default.pyproject_toml` and `pip.parse.pyproject_toml` attribute docstrings to their original text per review decision. --- python/private/pypi/extension.bzl | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 78119835af..7814246e62 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -681,18 +681,15 @@ If you are defining custom platforms in your project and don't want things to cl ), "pyproject_toml": attr.label( mandatory = False, - doc = """ -Label pointing to a `pyproject.toml` file used to read the default Python -version and, for hubs of the root module, to select exposed packages via -{obj}`pip.configure`. - -When `[project] requires-python` is present, it is used as the default -`python_version` for `pip.parse()` calls that do not specify one. + doc = """\ +Label pointing to pyproject.toml file to read the default Python version from. +When specified, reads the `requires-python` field from pyproject.toml and uses +it as the default python_version for all `pip.parse()` calls that don't +explicitly specify one. :::{note} -If `requires-python` is set, the version must be specified as `==X.Y.Z` (exact -version with full semver). This is designed to work with dependency management -tools like Renovate. +The version must be specified as `==X.Y.Z` (exact version with full semver). +This is designed to work with dependency management tools like Renovate. ::: :::{versionadded} 2.3.0 @@ -909,16 +906,13 @@ find in case extra indexes are specified. ), "pyproject_toml": attr.label( mandatory = False, - doc = """ -Label pointing to a `pyproject.toml` file used to read the Python version and -to select exposed packages via {obj}`pip.configure`. - -When `python_version` is not set explicitly, the `[project] requires-python` -field is used as the `python_version` for this `pip.parse()` call. + doc = """\ +Label pointing to a pyproject.toml file to read the Python version from. +When specified, the `requires-python` field is used as the `python_version` +for this `pip.parse()` call, unless `python_version` is set explicitly. :::{note} -When used to derive `python_version`, `requires-python` must be specified as -`==X.Y.Z` (exact version with full semver). +The version must be specified as `==X.Y.Z` (exact version with full semver). ::: :::{versionadded} 2.3.0 From 1291919372fc4283939655250438ee646f9e6ba3 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 18 Nov 2026 20:20:05 -0800 Subject: [PATCH 16/20] refactor(pypi): short-circuit _fail paths with None in group selection When a non-aborting `_fail` callback is injected in tests, `select_dependency_groups` and `_expand_dependency_groups` previously continued executing after `_fail(...)`, causing `hub_exposure` to fail on `None` attribute access or return partial results instead of `None`. Return `None` immediately after each `_fail(...)` call in `dependency_groups.bzl`, guard `selection == None` in `hub_exposure.bzl`, and assert `None` return values in error tests. --- python/private/pypi/dependency_groups.bzl | 14 ++++++++++---- python/private/pypi/hub_exposure.bzl | 2 ++ .../dependency_groups/dependency_groups_tests.bzl | 6 ++++-- tests/pypi/hub_exposure/hub_exposure_tests.bzl | 14 ++++++++++++-- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index e40e67a4c3..511041417d 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -25,16 +25,19 @@ def select_dependency_groups(pyproject, specs, _fail = fail): _fail: {type}`callable` the failure function, mainly for testing. Returns: - {type}`struct` with the attributes: + {type}`struct | None` with the attributes: * `packages`: {type}`list[str]` the sorted normalized package names. * `all`: {type}`bool` whether all of the packages including the transitive dependencies should be selected, i.e. `"*"` and `"transitive"` were given. + Returns `None` on failure. """ groups = {"default": pyproject.dependencies} for name, reqs in pyproject.optional_dependencies.items(): groups["optional:" + normalize_name(name)] = reqs expanded = _expand_dependency_groups(pyproject.dependency_groups, _fail) + if expanded == None: + return None for name, reqs in expanded.items(): groups["group:" + name] = reqs @@ -43,6 +46,7 @@ def select_dependency_groups(pyproject, specs, _fail = fail): _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format( specs, )) + return None return struct( # list[str]: sorted normalized package names selected by `specs`. packages = [], @@ -58,6 +62,7 @@ def select_dependency_groups(pyproject, specs, _fail = fail): spec, ", ".join(sorted(groups)), )) + return None for key in keys: selected[key] = None @@ -122,8 +127,8 @@ def _expand_dependency_groups(dependency_groups, _fail): _fail: {type}`callable` the failure function. Returns: - {type}`dict[str, list[str]]` normalized group names mapped to their - requirement strings. + {type}`dict[str, list[str]] | None` normalized group names mapped to + their requirement strings, or `None` on failure. """ groups = { normalize_name(name): items @@ -149,7 +154,7 @@ def _expand_dependency_groups(dependency_groups, _fail): include, name, )) - return {} + return None if include not in expanded: reqs = None break @@ -163,4 +168,5 @@ def _expand_dependency_groups(dependency_groups, _fail): _fail("Cyclic dependency group includes: {}".format( ", ".join(sorted(unexpanded)), )) + return None return expanded diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index ff9f24694d..ed103fd2d6 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -66,6 +66,8 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): tag.dependency_groups, _fail = _fail, ) + if selection == None: + return None if selection.all: exposes_all[hub] = None diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl index d494cae9d7..208b30639f 100644 --- a/tests/pypi/dependency_groups/dependency_groups_tests.bzl +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -80,7 +80,8 @@ def _test_select_errors(env): "The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: [\"default\", \"transitive\"]", ), ]: - _, failures = _select(specs) + got, failures = _select(specs) + env.expect.where(specs = specs).that_str(got).equals(None) env.expect.where(specs = specs).that_collection( failures, ).contains_exactly([want]) @@ -98,12 +99,13 @@ def _test_include_group_errors(env): "Cyclic dependency group includes: a, b", ), ]: - _, failures = _select(["*"], pyproject = struct( + got, failures = _select(["a"], pyproject = struct( name = None, dependencies = [], optional_dependencies = {}, dependency_groups = groups, )) + env.expect.that_str(got).equals(None) env.expect.that_collection(failures).contains_exactly([want]) _tests.append(_test_include_group_errors) diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index a51deb9c56..2713b9355b 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -165,21 +165,31 @@ def _test_configure_module_precedence(env): _tests.append(_test_configure_module_precedence) def _test_configure_errors(env): - _, failures = _exposure( + got, failures = _exposure( _module(configure = [_configure(hub_name = "missing")]), ) + env.expect.that_str(got).equals(None) env.expect.that_collection(failures).contains_exactly([ "pip.configure() refers to unknown hub 'missing'. Available hubs: hub", ]) - _, failures = _exposure( + got, failures = _exposure( _module(configure = [_configure()]), hubs = {"hub": _hub()}, ) + env.expect.that_str(got).equals(None) env.expect.that_collection(failures).contains_exactly([ "pip.configure() for hub 'hub' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`", ]) + got, failures = _exposure( + _module(configure = [_configure(dependency_groups = ["missing"])]), + ) + env.expect.that_str(got).equals(None) + env.expect.that_collection(failures).contains_exactly([ + "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test", + ]) + _tests.append(_test_configure_errors) def hub_exposure_test_suite(name): From 28b48d5ba2771d3c019e03bcffbd3a0993364162 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 18 Nov 2026 20:30:56 -0800 Subject: [PATCH 17/20] refactor(pypi): expand nested self-extras and refine type comments Single-level self-reference expansion dropped nested extras from composite extras such as `my-project[all]` and added the project name itself to `packages`, and `dependency_groups` type comments omitted list element types. Expand self-referenced optional groups iteratively with a bounded queue in `dependency_groups.bzl` (with unit tests covering nested, bare, and cyclic self-references), and specify parameterized element types in `pyproject_utils.bzl`, `dependency_groups.bzl`, and `render_pkg_aliases.bzl`. --- python/private/pypi/dependency_groups.bzl | 19 +++++++----- python/private/pypi/render_pkg_aliases.bzl | 3 +- python/private/pyproject_utils.bzl | 12 ++++--- .../dependency_groups_tests.bzl | 31 +++++++++++++++++++ 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index 511041417d..c3b435a956 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -67,18 +67,23 @@ def select_dependency_groups(pyproject, specs, _fail = fail): selected[key] = None # A self reference like `my-project[test]` selects the optional - # dependencies, but only one level deep. + # dependencies, including nested composite extras. project = normalize_name(pyproject.name or "") packages = {} - for key in selected: - for req in groups[key]: + queue = list(selected) + for _ in range(len(groups)): + if not queue: + break + for req in groups[queue.pop(0)]: req = requirement(req) if req.name_ != project: packages[req.name_] = None continue for extra in req.extras: - for r in groups.get("optional:" + extra, []): - packages[requirement(r).name_] = None + opt_key = "optional:" + extra + if opt_key in groups and opt_key not in selected: + selected[opt_key] = None + queue.append(opt_key) return struct( # list[str]: sorted normalized package names selected by `specs`. @@ -122,8 +127,8 @@ def _expand_dependency_groups(dependency_groups, _fail): """Expand the PEP 735 `include-group` entries. Args: - dependency_groups: {type}`dict[str, list]` the `[dependency-groups]` - table. + dependency_groups: {type}`dict[str, list[str | dict[str, str]]]` the + `[dependency-groups]` table. _fail: {type}`callable` the failure function. Returns: diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index ba0ad2cc75..f03c33355e 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -171,7 +171,8 @@ def _internal_visibility(aliases, has_groups = False): have to be visible to them. Args: - aliases: {type}`dict[str, str | dict]` package-to-repo alias mapping. + aliases: {type}`dict[str, str | dict[struct, str]]` package-to-repo + alias mapping. has_groups: {type}`bool` whether requirement cycle groups exist. Returns: diff --git a/python/private/pyproject_utils.bzl b/python/private/pyproject_utils.bzl index ceb5c803aa..c65157df83 100644 --- a/python/private/pyproject_utils.bzl +++ b/python/private/pyproject_utils.bzl @@ -22,11 +22,12 @@ def read_pyproject(module_ctx, pyproject): * `dependencies`: {type}`list[str]` the `[project] dependencies`. * `optional_dependencies`: {type}`dict[str, list[str]]` the `[project.optional-dependencies]` table. - * `dependency_groups`: {type}`dict[str, list]` the PEP 735 - `[dependency-groups]` table. Items are either requirement strings - or `{include-group = ""}` tables. + * `dependency_groups`: {type}`dict[str, list[str | dict[str, str]]]` + the PEP 735 `[dependency-groups]` table. Items are either + requirement strings or `{include-group = ""}` tables. """ - data = toml.decode(module_ctx.read(module_ctx.path(pyproject), watch = "yes")) + raw = module_ctx.read(module_ctx.path(pyproject), watch = "yes") + data = toml.decode(raw) project = data.get("project", {}) return struct( # str | None: raw `[project] requires-python` value. @@ -37,7 +38,8 @@ def read_pyproject(module_ctx, pyproject): dependencies = project.get("dependencies") or [], # dict[str, list[str]]: `[project.optional-dependencies]` table. optional_dependencies = project.get("optional-dependencies") or {}, - # dict[str, list]: PEP 735 `[dependency-groups]` table. + # dict[str, list[str | dict[str, str]]]: PEP 735 `[dependency-groups]` + # table of requirement strings and `{"include-group": str}` dicts. dependency_groups = data.get("dependency-groups") or {}, ) diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl index 208b30639f..ab56109c7c 100644 --- a/tests/pypi/dependency_groups/dependency_groups_tests.bzl +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -61,6 +61,37 @@ def _test_select(env): _tests.append(_test_select) +def _test_self_reference_extras(env): + pyproject = struct( + name = "my-project", + dependencies = [], + optional_dependencies = { + "a": ["a-pkg", "my-project[b]"], + "all": ["My_Project[docs, grpc]"], + "b": ["b-pkg", "my-project[a]"], + "docs": ["docs-pkg"], + "everything": ["my-project[all]", "my-project"], + "grpc": ["grpc-pkg"], + }, + dependency_groups = { + "ci": ["ci-pkg", "my-project[all]"], + }, + ) + for specs, want in [ + (["everything"], ["docs_pkg", "grpc_pkg"]), + (["ci"], ["ci_pkg", "docs_pkg", "grpc_pkg"]), + (["a"], ["a_pkg", "b_pkg"]), + ]: + got, failures = _select(specs, pyproject = pyproject) + env.expect.where(specs = specs).that_collection( + failures, + ).contains_exactly([]) + env.expect.where(specs = specs).that_collection( + got.packages, + ).contains_exactly(want).in_order() + +_tests.append(_test_self_reference_extras) + def _test_select_all(env): got, failures = _select(["*", "transitive"]) env.expect.that_collection(failures).contains_exactly([]) From 0f35bd1b060054334ec1b1c0ff0c9eb000b21f92 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:32:52 +0900 Subject: [PATCH 18/20] feat(pypi): make pip.configure a builder that does not require a pyproject.toml The dependency_groups default is now empty, so a pip.configure tag only needs a pyproject.toml when it selects dependency groups. A pyproject_toml set on a tag is used by the later tags of the same module and hub. --- docs/pypi/download.md | 36 ++++++++--- python/private/pypi/extension.bzl | 23 +++++-- python/private/pypi/hub_exposure.bzl | 17 +++++- tests/pypi/extension/pip_parse.bzl | 2 +- .../pypi/hub_exposure/hub_exposure_tests.bzl | 61 +++++++++++++++++-- 5 files changed, 116 insertions(+), 23 deletions(-) diff --git a/docs/pypi/download.md b/docs/pypi/download.md index ef069da5b5..931b64cf57 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -212,15 +212,33 @@ dependencies of `numpy` are only used through `@pypi_hub//numpy`. The `all_requirements` lists in the hub's `requirements.bzl` and the [Unified @pypi Hub](unified-pypi-hub) only contain the exposed packages. -The {attr}`pip.configure.dependency_groups` attribute defaults to `["*"]`, -which selects every group. Use `["*", "transitive"]` to expose all of the -packages of the hub, including the transitive dependencies, which is the same -as not configuring the hub. Selecting the transitive dependencies of only some -of the groups is not supported. - -Multiple tags for the same hub are combined. Only the root module and the -module that created the hub can configure it; if both do, the root module -wins. A self reference such as `my-project[test]` in the `pyproject.toml` +Use `["*"]` to select every group, or `["*", "transitive"]` to expose all of +the packages of the hub, including the transitive dependencies, which is the +same as not configuring the hub. Selecting the transitive dependencies of only +some of the groups is not supported. A tag without +{attr}`pip.configure.dependency_groups` does not limit the hub and does not +need a `pyproject.toml`. + +The `pip.configure` tags work like a builder: they are applied in order and +the tags for the same hub are combined. A `pyproject_toml` set on a tag is used +by the later tags of the same module and hub, so it only needs to be set once: + +```starlark +pip.configure( + hub_name = "pypi_hub", + pyproject_toml = "//tools:pyproject.toml", + dependency_groups = ["default"], +) + +# Uses //tools:pyproject.toml +pip.configure( + hub_name = "pypi_hub", + dependency_groups = ["group:dev"], +) +``` + +Only the root module and the module that created the hub can configure it; if +both do, the root module wins. A self reference such as `my-project[test]` in the `pyproject.toml` selects the optional dependencies of the project. The packages that are not part of the hub are ignored, so a single `pyproject.toml` can list tools that are missing from the lock file. diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 7814246e62..38ec1d0e09 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -1123,9 +1123,11 @@ execution-phase error when built. _configure_tag = tag_class( attrs = { "dependency_groups": attr.string_list( - default = ["*"], + default = [], doc = """ -The `pyproject.toml` dependency groups whose packages the hub exposes. +The `pyproject.toml` dependency groups whose packages the hub exposes. The +default, `[]`, does not select any packages, so the tag does not limit the hub +and it does not need a `pyproject.toml`. Each item is one of: @@ -1158,9 +1160,12 @@ The name of the hub to configure. "pyproject_toml": attr.label( mandatory = False, doc = """ -The `pyproject.toml` file that defines the dependency groups. Defaults to the -{attr}`pip.parse.pyproject_toml` of the hub, then to -{attr}`pip.default.pyproject_toml` if the hub is defined by the root module. +The `pyproject.toml` file that defines the dependency groups. It is also used +by the later `pip.configure` tags of the same module and hub that don't set it. +Defaults to the last `pyproject_toml` set by an earlier `pip.configure` tag of +the same module and hub, then to the {attr}`pip.parse.pyproject_toml` of the +hub, then to {attr}`pip.default.pyproject_toml` if the hub is defined by the +root module. It is only needed when {attr}`dependency_groups` is set. :::{versionadded} VERSION_NEXT_FEATURE ::: @@ -1176,8 +1181,14 @@ The packages are selected from a `pyproject.toml` file using the transitive dependencies, are still used by the packages that need them, but they are not public. +The tags work like a builder: they are applied in order and multiple tags for +the same hub are combined, so each tag can configure a different part of the +hub. A `pyproject_toml` set on a tag is used by the later tags of the same +module and hub. A tag that does not select any {attr}`dependency_groups` does +not limit the hub. + Only the root module and the module that created the hub can configure it. If -both do, the root module wins. Multiple tags for the same hub are combined. +both do, the root module wins. See {ref}`Configuring the Exposed Packages ` for examples. diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index ed103fd2d6..7393c28fbe 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -30,11 +30,15 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): root_configured = {} for mod in root_modules: for tag in mod.tags.configure: - root_configured[tag.hub_name] = None + if tag.dependency_groups: + root_configured[tag.hub_name] = None configured = {} exposes_all = {} for mod in module_ctx.modules: + # The tags of a module are applied in order like a builder: a tag + # without a `pyproject_toml` uses the last one set for the hub. + last_pyproject = {} for tag in mod.tags.configure: hub = tag.hub_name if hub not in hubs: @@ -52,12 +56,19 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): if not mod.is_root and hub in root_configured: continue - pyproject = tag.pyproject_toml or info.pyproject + if tag.pyproject_toml: + last_pyproject[hub] = tag.pyproject_toml + if not tag.dependency_groups: + # The tag does not select any packages. + continue + + pyproject = last_pyproject.get(hub) or info.pyproject if not pyproject and info.module_name == root_name: pyproject = default_pyproject if not pyproject: - _fail("pip.configure() for hub '{}' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`".format( + _fail("pip.configure() for hub '{}' selects the dependency groups {}, which need a pyproject.toml, set `pyproject_toml` on this or an earlier `pip.configure` or on `pip.parse`".format( hub, + tag.dependency_groups, )) return None diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index db3008d431..d048f99bd0 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -2,7 +2,7 @@ def configure( *, - dependency_groups = ["*"], + dependency_groups = [], hub_name, pyproject_toml = None): """Simulate the PyPI extension configure tag class for testing.""" diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index 2713b9355b..1f2b13622c 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -27,7 +27,7 @@ def _hub(*, module_name = "root", pyproject = None): def _configure( hub_name = "hub", - dependency_groups = ["*"], + dependency_groups = [], pyproject_toml = None): return mocks.tag( hub_name = hub_name, @@ -108,7 +108,7 @@ _tests.append(_test_configure_combines_tags) def _test_configure_pyproject_precedence(env): # The tag wins over the `pip.parse` value. got, _ = _exposure(_module(configure = [ - _configure(pyproject_toml = "other.toml"), + _configure(pyproject_toml = "other.toml", dependency_groups = ["*"]), ])) env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) @@ -123,6 +123,59 @@ def _test_configure_pyproject_precedence(env): _tests.append(_test_configure_pyproject_precedence) +def _test_configure_without_pyproject(env): + # A tag without the dependency groups does not need a pyproject.toml and + # does not limit the hub. + got, failures = _exposure( + _module(configure = [_configure()]), + hubs = {"hub": _hub()}, + ) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_configure_without_pyproject) + +def _test_configure_builder(env): + # The `pyproject_toml` of a tag is used by the later tags of the hub. + got, failures = _exposure( + _module(configure = [ + _configure(pyproject_toml = "other.toml"), + _configure(dependency_groups = ["default"]), + ]), + hubs = {"hub": _hub(pyproject = "pyproject.toml")}, + ) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + + # The tags before it use the `pip.parse` value. + got, failures = _exposure( + _module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(pyproject_toml = "other.toml", dependency_groups = ["default"]), + ]), + hubs = {"hub": _hub(pyproject = "pyproject.toml")}, + ) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg", "simple"]}) + + # It is not used by the other modules. + hubs = {"hub": _hub(module_name = "dep")} + got, failures = _exposure( + _module(configure = [_configure(hub_name = "hub", pyproject_toml = "other.toml")]), + _module( + name = "dep", + is_root = False, + configure = [_configure(hub_name = "hub", dependency_groups = ["default"])], + ), + hubs = hubs, + ) + env.expect.that_str(got).equals(None) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() for hub 'hub' selects the dependency groups [\"default\"], which need a pyproject.toml, set `pyproject_toml` on this or an earlier `pip.configure` or on `pip.parse`", + ]) + +_tests.append(_test_configure_builder) + def _test_configure_module_precedence(env): hubs = {"hub": _hub(module_name = "dep", pyproject = "pyproject.toml")} @@ -174,12 +227,12 @@ def _test_configure_errors(env): ]) got, failures = _exposure( - _module(configure = [_configure()]), + _module(configure = [_configure(dependency_groups = ["*"])]), hubs = {"hub": _hub()}, ) env.expect.that_str(got).equals(None) env.expect.that_collection(failures).contains_exactly([ - "pip.configure() for hub 'hub' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`", + "pip.configure() for hub 'hub' selects the dependency groups [\"*\"], which need a pyproject.toml, set `pyproject_toml` on this or an earlier `pip.configure` or on `pip.parse`", ]) got, failures = _exposure( From 591ccf19a8dc096e2d876a456835c5abfe846ed4 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:36:22 +0900 Subject: [PATCH 19/20] fix(pypi): fail on pip.configure.testonly without the dependency groups --- python/private/pypi/hub_exposure.bzl | 3 +++ tests/pypi/hub_exposure/hub_exposure_tests.bzl | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index a73293e535..3098af040e 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -67,6 +67,9 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): last_pyproject[hub] = tag.pyproject_toml if not tag.dependency_groups: # The tag does not select any packages. + if tag.testonly: + _fail("pip.configure() for hub '{}' uses `testonly`, which needs the `dependency_groups` of the testonly packages".format(hub)) + return None continue pyproject = last_pyproject.get(hub) or info.pyproject diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index 73f015fae2..d4ea3de496 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -263,6 +263,13 @@ def _test_configure_testonly(env): "pip.configure() for hub 'hub' cannot use `testonly` when exposing all of the packages with [\"*\", \"transitive\"]", ]) + # The testonly packages need to be selected. + got, failures = _exposure(_module(configure = [_configure(testonly = True)])) + env.expect.that_str(got).equals(None) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() for hub 'hub' uses `testonly`, which needs the `dependency_groups` of the testonly packages", + ]) + _tests.append(_test_configure_testonly) def hub_exposure_test_suite(name): From 7c58d7b20a47efb5a0f33d9db412d439ad97191d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:36:34 +0900 Subject: [PATCH 20/20] docs(pypi): rewrap the pip.configure docs --- docs/pypi/download.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 931b64cf57..ad8b40e70d 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -238,10 +238,10 @@ pip.configure( ``` Only the root module and the module that created the hub can configure it; if -both do, the root module wins. A self reference such as `my-project[test]` in the `pyproject.toml` -selects the optional dependencies of the project. The packages that are not -part of the hub are ignored, so a single `pyproject.toml` can list tools that -are missing from the lock file. +both do, the root module wins. A self reference such as `my-project[test]` in +the `pyproject.toml` selects the optional dependencies of the project. The +packages that are not part of the hub are ignored, so a single `pyproject.toml` +can list tools that are missing from the lock file. :::{versionadded} VERSION_NEXT_FEATURE :::