From 3ef53b2098194d6754702dffa6defc24b0fd5566 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:07:15 +0900 Subject: [PATCH 01/15] refactor(pypi): select packages from the pyproject.toml dependency groups Read the dependencies, the optional dependencies and the PEP 735 dependency groups from the pyproject.toml and add a helper to select the packages using dependency group specifiers. --- python/private/pypi/BUILD.bazel | 9 ++ python/private/pypi/dependency_groups.bzl | 142 ++++++++++++++++++ python/private/pyproject_utils.bzl | 14 +- tests/pypi/dependency_groups/BUILD.bazel | 3 + .../dependency_groups_tests.bzl | 86 +++++++++++ 5 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 python/private/pypi/dependency_groups.bzl create mode 100644 tests/pypi/dependency_groups/BUILD.bazel create mode 100644 tests/pypi/dependency_groups/dependency_groups_tests.bzl diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index f94d24594f..af005f706e 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -632,6 +632,15 @@ bzl_library( ], ) +bzl_library( + name = "dependency_groups", + srcs = ["dependency_groups.bzl"], + deps = [ + ":pep508_requirement", + "//python/private:normalize_name", + ], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl new file mode 100644 index 0000000000..2ddadee3dd --- /dev/null +++ b/python/private/pypi/dependency_groups.bzl @@ -0,0 +1,142 @@ +"""Select PyPI packages using pyproject.toml dependency groups.""" + +load("//python/private:normalize_name.bzl", "normalize_name") +load(":pep508_requirement.bzl", "requirement") + +def select_dependency_groups(pyproject, specs, _fail = fail): + """Select the packages named by the given dependency group specifiers. + + Supported specifiers are: + + * `"default"`: the `[project] dependencies`. + * `"optional:"`: a `[project.optional-dependencies]` entry. + * `"group:"`: a PEP 735 `[dependency-groups]` entry. + * `"optional:*"` and `"group:*"`: all entries of the table. + * `"*"`: all of the above. + * `""`: a dependency group, or an optional dependency group if + there is no dependency group with that name. + * `"transitive"`: only supported together with `"*"` and selects all + of the packages, including the transitive dependencies. + + Args: + pyproject: {type}`struct` returned by `read_pyproject`. + specs: {type}`list[str]` the specifiers. + _fail: {type}`callable` the failure function, mainly for testing. + + Returns: + {type}`struct` with the attributes: + * `packages`: {type}`list[str]` the sorted normalized package names. + * `all`: {type}`bool` whether all of the packages including the + transitive dependencies should be selected, i.e. `"*"` and + `"transitive"` were given. + """ + groups = {"default": pyproject.dependencies} + for name, reqs in pyproject.optional_dependencies.items(): + groups["optional:" + normalize_name(name)] = reqs + for name, reqs in _expand_dependency_groups(pyproject.dependency_groups, _fail).items(): + groups["group:" + name] = reqs + + if "transitive" in specs: + if "*" not in specs: + _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format(specs)) + return struct(packages = [], all = True) + + selected = {} + for spec in specs: + keys = _match(spec, groups) + if not keys: + _fail("Unknown dependency group '{}'. Available groups: {}".format( + spec, + ", ".join(sorted(groups)), + )) + for key in keys: + selected[key] = None + + # A self reference like `my-project[test]` selects the optional + # dependencies, but only one level deep. + project = normalize_name(pyproject.name or "") + packages = {} + for key in selected: + for req in groups[key]: + req = requirement(req) + names = [req.name_] + if req.name_ == project: + names = [ + requirement(r).name_ + for extra in req.extras + for r in groups.get("optional:" + extra, []) + ] + for name in names: + if name: + packages[name] = None + + return struct(packages = sorted(packages), all = False) + +def _match(spec, groups): + if spec == "*": + return groups.keys() + if spec in ("optional:*", "group:*"): + prefix = spec[:-1] + return [key for key in groups if key.startswith(prefix)] + + kind, _, name = spec.rpartition(":") + name = normalize_name(name) + if kind: + candidates = [kind + ":" + name] + elif name == "default": + candidates = ["default"] + else: + candidates = ["group:" + name, "optional:" + name] + + for key in candidates: + if key in groups: + return [key] + return [] + +def _expand_dependency_groups(dependency_groups, _fail): + """Expand the PEP 735 `include-group` entries. + + Args: + dependency_groups: {type}`dict[str, list]` the `[dependency-groups]` + table. + _fail: {type}`callable` the failure function. + + Returns: + {type}`dict[str, list[str]]` normalized group names mapped to their + requirement strings. + """ + groups = { + normalize_name(name): items + for name, items in dependency_groups.items() + } + + # Each round expands at least one group unless there is a cycle. + expanded = {} + for _ in range(len(groups)): + for name, items in groups.items(): + if name in expanded: + continue + + reqs = [] + for item in items: + if type(item) == type(""): + reqs.append(item) + continue + + include = normalize_name(item.get("include-group", "")) + if include not in groups: + _fail("Unknown dependency group '{}' included from '{}'".format(include, name)) + return {} + if include not in expanded: + reqs = None + break + reqs.extend(expanded[include]) + + if reqs != None: + expanded[name] = reqs + + if len(expanded) != len(groups): + _fail("Cyclic dependency group includes: {}".format( + ", ".join(sorted([name for name in groups if name not in expanded])), + )) + return expanded diff --git a/python/private/pyproject_utils.bzl b/python/private/pyproject_utils.bzl index e96ffd2023..a9494b8ac2 100644 --- a/python/private/pyproject_utils.bzl +++ b/python/private/pyproject_utils.bzl @@ -18,10 +18,22 @@ def read_pyproject(module_ctx, pyproject): {type}`struct` with the attributes: * `requires_python`: {type}`str | None` the raw `requires-python` value (e.g. `"==3.13.9"`), or `None` if it is not set. + * `name`: {type}`str | None` the `[project] name`. + * `dependencies`: {type}`list[str]` the `[project] dependencies`. + * `optional_dependencies`: {type}`dict[str, list[str]]` the + `[project.optional-dependencies]` table. + * `dependency_groups`: {type}`dict[str, list]` the PEP 735 + `[dependency-groups]` table. Items are either requirement strings + or `{include-group = ""}` tables. """ data = toml.decode(module_ctx.read(module_ctx.path(pyproject), watch = "yes")) + project = data.get("project", {}) return struct( - requires_python = data.get("project", {}).get("requires-python"), + requires_python = project.get("requires-python"), + name = project.get("name"), + dependencies = project.get("dependencies") or [], + optional_dependencies = project.get("optional-dependencies") or {}, + dependency_groups = data.get("dependency-groups") or {}, ) def version_from_requires_python(requires_python): diff --git a/tests/pypi/dependency_groups/BUILD.bazel b/tests/pypi/dependency_groups/BUILD.bazel new file mode 100644 index 0000000000..ba6877e124 --- /dev/null +++ b/tests/pypi/dependency_groups/BUILD.bazel @@ -0,0 +1,3 @@ +load(":dependency_groups_tests.bzl", "dependency_groups_test_suite") + +dependency_groups_test_suite(name = "dependency_groups_tests") diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl new file mode 100644 index 0000000000..ee23aec1fb --- /dev/null +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -0,0 +1,86 @@ +"""dependency_groups tests""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:dependency_groups.bzl", "select_dependency_groups") # buildifier: disable=bzl-visibility + +_tests = [] + +_PYPROJECT = struct( + name = "my-project", + dependencies = ["main-pkg>=1.0"], + optional_dependencies = { + "Docs": ["docs-pkg"], + "all": ["My_Project[docs, grpc]"], + "grpc": ["grpc-pkg"], + }, + dependency_groups = { + "dev": ["dev-pkg", "shared-pkg", {"include-group": "Test"}], + "test": ["test_pkg", "shared-pkg"], + }, +) + +def _select(specs, pyproject = _PYPROJECT): + failures = [] + got = select_dependency_groups(pyproject, specs, _fail = failures.append) + return got, failures + +def _test_select(env): + for specs, want in [ + (["default"], ["main_pkg"]), + (["optional:docs"], ["docs_pkg"]), + (["group:test"], ["shared_pkg", "test_pkg"]), + (["dev"], ["dev_pkg", "shared_pkg", "test_pkg"]), + (["grpc"], ["grpc_pkg"]), + (["all"], ["docs_pkg", "grpc_pkg"]), + (["optional:*"], ["docs_pkg", "grpc_pkg"]), + (["group:*"], ["dev_pkg", "shared_pkg", "test_pkg"]), + (["default", "group:test"], ["main_pkg", "shared_pkg", "test_pkg"]), + (["*"], ["dev_pkg", "docs_pkg", "grpc_pkg", "main_pkg", "shared_pkg", "test_pkg"]), + ]: + got, failures = _select(specs) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) + env.expect.where(specs = specs).that_collection(got.packages).contains_exactly(want).in_order() + env.expect.where(specs = specs).that_bool(got.all).equals(False) + +_tests.append(_test_select) + +def _test_select_all(env): + got, failures = _select(["*", "transitive"]) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_bool(got.all).equals(True) + env.expect.that_collection(got.packages).contains_exactly([]) + +_tests.append(_test_select_all) + +def _test_select_errors(env): + for specs, want in [ + (["missing"], "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test, optional:all, optional:docs, optional:grpc"), + (["default", "transitive"], "The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: [\"default\", \"transitive\"]"), + ]: + _, failures = _select(specs) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([want]) + +_tests.append(_test_select_errors) + +def _test_include_group_errors(env): + for groups, want in [ + ({"a": [{"include-group": "missing"}]}, "Unknown dependency group 'missing' included from 'a'"), + ({"a": [{"include-group": "b"}], "b": [{"include-group": "a"}]}, "Cyclic dependency group includes: a, b"), + ]: + _, failures = _select(["*"], pyproject = struct( + name = None, + dependencies = [], + optional_dependencies = {}, + dependency_groups = groups, + )) + env.expect.that_collection(failures).contains_exactly([want]) + +_tests.append(_test_include_group_errors) + +def dependency_groups_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 755737072827673c74afba95147995086a579543 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:07:32 +0900 Subject: [PATCH 02/15] refactor(pypi): support hiding the unexposed packages of a hub Add the `hide_unexposed_packages` attribute to the hub repository. When set, the aliases of the packages that are not exposed are only visible to the wheel repositories that depend on them. --- python/private/pypi/hub_repository.bzl | 13 +++- python/private/pypi/render_pkg_aliases.bzl | 45 +++++++++++++- .../render_pkg_aliases_test.bzl | 62 +++++++++++++++++++ 3 files changed, 115 insertions(+), 5 deletions(-) diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index f915aa1c77..d421f3cc09 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -26,12 +26,16 @@ exports_files(["requirements.bzl"]) """ def _impl(rctx): - bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + if rctx.attr.hide_unexposed_packages: + bzl_packages = rctx.attr.packages + else: + bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() aliases = render_multiplatform_pkg_aliases( aliases = { key: _whl_config_settings_from_json(values) for key, values in rctx.attr.whl_map.items() }, + exposed_packages = bzl_packages if rctx.attr.hide_unexposed_packages else None, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, @@ -78,10 +82,15 @@ hub_repository = repository_rule( "groups": attr.string_list_dict( mandatory = False, ), + "hide_unexposed_packages": attr.bool( + default = False, + doc = "Make the aliases of the packages that are not in `packages` visible only to the wheel repositories.", + ), "packages": attr.string_list( mandatory = False, doc = """\ -The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys. +The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys +unless `hide_unexposed_packages` is set. """, ), "platform_config_settings": attr.string_list_dict( diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 0a1c328491..5582cc00c9 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -65,7 +65,7 @@ def _repr_actual(aliases): else: return render.dict(aliases, key_repr = _repr_config_setting) -def _render_common_aliases(*, name, aliases, **kwargs): +def _render_common_aliases(*, name, aliases, visibility, **kwargs): pkg_aliases = render.call( "pkg_aliases", name = repr(name), @@ -80,14 +80,21 @@ def _render_common_aliases(*, name, aliases, **kwargs): return """\ load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") {extra_loads} -package(default_visibility = ["//visibility:public"]) +package(default_visibility = {visibility}) {aliases}""".format( aliases = pkg_aliases, extra_loads = extra_loads, + visibility = render.list(visibility), ) -def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, **kwargs): +def render_pkg_aliases( + *, + aliases, + requirement_cycles = None, + extra_hub_aliases = {}, + exposed_packages = None, + **kwargs): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -100,6 +107,9 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases requirement_cycles: any package groups to also add. extra_hub_aliases: The list of extra aliases for each whl to be added in addition to the default ones. + exposed_packages: The packages that should be public. When set, the + aliases of the other packages are only visible to the wheel + repositories. **kwargs: Extra kwargs to pass to the rules. Returns: @@ -124,12 +134,18 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases for whl_name in group_whls } + public = ["//visibility:public"] + if exposed_packages != None: + exposed_packages = {normalize_name(name): None for name in exposed_packages} + internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) + files = { "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( name = normalize_name(name), aliases = pkg_aliases, extra_aliases = extra_hub_aliases.get(normalize_name(name), []), group_name = whl_group_mapping.get(normalize_name(name)), + visibility = public if exposed_packages == None or normalize_name(name) in exposed_packages else internal_visibility, **kwargs ).strip() for name, pkg_aliases in aliases.items() @@ -139,6 +155,29 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) return files +def _internal_visibility(aliases, has_groups = False): + """The visibility of the packages that are not exposed. + + The wheel repositories depend on the aliases of the hub, so the aliases + have to be visible to them. + """ + repo_names = {} + for pkg_aliases in aliases.values(): + if type(pkg_aliases) == type(""): + repo_names[pkg_aliases] = None + continue + + for repo_name in pkg_aliases.values(): + repo_names[repo_name] = None + + visibility = [ + "@{}//:__pkg__".format(repo_name) + for repo_name in sorted(repo_names) + ] + if has_groups: + visibility.append("//_groups:__pkg__") + return visibility + def _major_minor(python_version): major, _, tail = python_version.partition(".") minor, _, _ = tail.partition(".") diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 9114f59279..13bee38ff5 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -58,6 +58,68 @@ pkg_aliases( _tests.append(_test_legacy_aliases) +def _test_hidden_aliases(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "foo": "pypi_foo", + }, + exposed_packages = ["foo"], + ) + + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = [ + "@pypi_bar//:__pkg__", + "@pypi_foo//:__pkg__", +]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", +)""") + env.expect.that_str(actual["foo/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = ["//visibility:public"]) + +pkg_aliases( + name = "foo", + actual = "pypi_foo", +)""") + +_tests.append(_test_hidden_aliases) + +def _test_hidden_aliases_in_groups(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "baz": "pypi_baz", + }, + requirement_cycles = { + "group": ["bar", "baz"], + }, + exposed_packages = ["baz"], + ) + + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = [ + "@pypi_bar//:__pkg__", + "@pypi_baz//:__pkg__", + "//_groups:__pkg__", +]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", + group_name = "group", +)""") + +_tests.append(_test_hidden_aliases_in_groups) + def _test_bzlmod_aliases(env): # Use this function as it is used in pip_repository actual = render_multiplatform_pkg_aliases( From 23127e5b1bea7fd8a1be0f3dfbab80b4134b8399 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:08:09 +0900 Subject: [PATCH 03/15] feat(pypi): add pip.configure to select the exposed packages of a hub Add the `pip.configure` tag class that limits the packages that a hub exposes to the ones listed in the dependency groups of a `pyproject.toml` file. The other packages of the hub are still used by the wheels that need them, but their aliases are not public. --- docs/pypi/download.md | 50 ++++++ news/4395.added.md | 2 + python/private/pypi/BUILD.bazel | 10 ++ python/private/pypi/extension.bzl | 108 +++++++++++++ python/private/pypi/hub_exposure.bzl | 84 ++++++++++ tests/integration/unified_pypi/BUILD.bazel | 16 ++ tests/integration/unified_pypi/MODULE.bazel | 14 ++ .../unified_pypi/pyproject_filtered.toml | 4 + tests/integration/unified_pypi_test.py | 13 ++ tests/pypi/extension/extension_tests.bzl | 92 ++++++++++- tests/pypi/extension/pip_parse.bzl | 12 ++ tests/pypi/hub_exposure/BUILD.bazel | 3 + .../pypi/hub_exposure/hub_exposure_tests.bzl | 153 ++++++++++++++++++ 13 files changed, 559 insertions(+), 2 deletions(-) create mode 100644 news/4395.added.md create mode 100644 python/private/pypi/hub_exposure.bzl create mode 100644 tests/integration/unified_pypi/pyproject_filtered.toml create mode 100644 tests/pypi/hub_exposure/BUILD.bazel create mode 100644 tests/pypi/hub_exposure/hub_exposure_tests.bzl diff --git a/docs/pypi/download.md b/docs/pypi/download.md index cc8b7890aa..5f16606d24 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -134,6 +134,56 @@ rename, and `requirement("numpy")` routes through the active hub like fixed at loading time, before the active hub is known, so load them from a concrete hub instead. Prefer `@pypi//` labels in new code. +(pip-configure)= +### Configuring the Exposed Packages (pip.configure) + +By default a hub exposes all of the packages from its requirements files or +`uv.lock` file, including the transitive dependencies. A `pip.configure` tag +can limit the hub to the packages listed in a `pyproject.toml` file: the +`[project] dependencies`, `[project.optional-dependencies]` and PEP 735 +`[dependency-groups]` tables. + +```toml +[project] +dependencies = ["numpy"] + +[dependency-groups] +dev = ["pytest"] +``` + +```starlark +pip.parse( + hub_name = "pypi_hub", + pyproject_toml = "//:pyproject.toml", + requirements_lock = "//:requirements_lock.txt", +) + +pip.configure( + hub_name = "pypi_hub", + dependency_groups = ["default", "group:dev"], +) +``` + +Here `@pypi_hub//numpy` and `@pypi_hub//pytest` can be used, but the +dependencies of `numpy` are only used through `@pypi_hub//numpy`. The +`all_requirements` lists in the hub's `requirements.bzl` and the +[Unified @pypi Hub](unified-pypi-hub) only contain the exposed packages. + +The {attr}`pip.configure.dependency_groups` attribute defaults to `["*"]`, +which selects every group. Use `["*", "transitive"]` to expose all of the +packages of the hub, including the transitive dependencies. Selecting the +transitive dependencies of only some of the groups is not supported. + +Multiple tags for the same hub are combined. Only the root module and the +module that created the hub can configure it; if both do, the root module +wins. A self reference such as `my-project[test]` in the `pyproject.toml` +selects the optional dependencies of the project. The packages that are not +part of the hub are ignored, so a single `pyproject.toml` can list tools that +are missing from the lock file. + +:::{versionadded} VERSION_NEXT_FEATURE +::: + ### Declaring Abstract Dependencies (pip.dep) :::{versionadded} 2.2.0 diff --git a/news/4395.added.md b/news/4395.added.md new file mode 100644 index 0000000000..62d7f10880 --- /dev/null +++ b/news/4395.added.md @@ -0,0 +1,2 @@ +(pypi) Added {obj}`pip.configure` to limit the packages that a hub exposes to +the ones listed in the dependency groups of a `pyproject.toml` file. diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index af005f706e..caf482b67f 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -129,6 +129,7 @@ bzl_library( deps = [ ":default_platforms", ":hub_builder", + ":hub_exposure", ":hub_repository", ":parse_whl_name", ":pep508_env", @@ -641,6 +642,15 @@ bzl_library( ], ) +bzl_library( + name = "hub_exposure", + srcs = ["hub_exposure.bzl"], + deps = [ + ":dependency_groups", + "//python/private:pyproject_utils", + ], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index f7a1dadaf7..e64e856489 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -24,6 +24,7 @@ load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_req load("//python/private:repo_utils.bzl", "repo_utils") load(":default_platforms.bzl", "default_platforms") load(":hub_builder.bzl", "hub_builder") +load(":hub_exposure.bzl", "hub_exposure") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_whl_name.bzl", "parse_whl_name") load(":pep508_env.bzl", "env") @@ -89,6 +90,7 @@ def build_config( unified_hub_requirements_bzl = False defaults = { "platforms": default_platforms(), + "pyproject_toml": None, "python_version": None, } for mod in module_ctx.modules: @@ -105,6 +107,8 @@ def build_config( unified_hub_requirements_bzl = True pyproject_toml = tag.pyproject_toml if pyproject_toml: + if mod.is_root: + defaults["pyproject_toml"] = pyproject_toml pyproject = read_pyproject(module_ctx, pyproject_toml) if pyproject.requires_python: defaults["python_version"] = version_from_requires_python(pyproject.requires_python) @@ -151,6 +155,7 @@ def build_config( for name, values in defaults["platforms"].items() }, enable_pipstar_extract = enable_pipstar_extract, + pyproject_toml = defaults["pyproject_toml"], toml_decode = toml.decode, unified_hub_requirements_bzl = unified_hub_requirements_bzl, ) @@ -243,6 +248,7 @@ You cannot use both the additive_build_content and additive_build_content_file a # dict[str repo, HubBuilder] # See `hub_builder.bzl%hub_builder()` for `HubBuilder` pip_hub_map = {} + hub_pyproject = {} simpleapi_cache = pypi_cache(mctx = module_ctx) is_pypi_hub_reserved = module_ctx.getenv("RULES_PYTHON_PYPI_HUB_RESERVED", "0") == "1" @@ -319,6 +325,9 @@ You cannot use both the additive_build_content and additive_build_content_file a else: builder = pip_hub_map[hub_name] + if pip_attr.pyproject_toml and hub_name not in hub_pyproject: + hub_pyproject[hub_name] = pip_attr.pyproject_toml + builder.pip_parse( module_ctx, pip_attr = pip_attr, @@ -342,6 +351,7 @@ You cannot use both the additive_build_content and additive_build_content_file a exposed_packages = {} extra_aliases = {} whl_libraries = {} + hubs = {} for hub in pip_hub_map.values(): out = hub.build() @@ -355,8 +365,25 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases[hub.name] = out.extra_aliases hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map + hubs[hub.name] = struct( + module_name = hub.module_name, + pyproject = hub_pyproject.get(hub.name), + packages = out.whl_map.keys(), + exposed_packages = out.exposed_packages, + ) + + configured_hubs = hub_exposure( + module_ctx, + hubs = hubs, + default_pyproject = config.pyproject_toml, + _fail = _fail, + ) + if configured_hubs == None: + return None + exposed_packages.update(configured_hubs) return struct( + configured_hubs = configured_hubs, config = config, declared_deps = declared_deps, default_hub = config.default_hub or renamed_default_hub, @@ -511,6 +538,8 @@ def _pip_impl(module_ctx): packages = mods.exposed_packages.get(hub_name, []), platform_config_settings = mods.platform_config_settings.get(hub_name, {}), groups = mods.hub_group_map.get(hub_name), + # The hubs configured with `pip.configure` hide the other packages. + hide_unexposed_packages = hub_name in mods.configured_hubs, ) _create_unified_hub_repo(mods) @@ -638,6 +667,10 @@ This is designed to work with dependency management tools like Renovate. :::{versionadded} 2.3.0 ::: + +:::{versionchanged} VERSION_NEXT_FEATURE +Also used by {obj}`pip.configure` for the hubs of the root module. +::: """, ), "unified_hub_requirements_bzl": attr.bool( @@ -841,6 +874,10 @@ The version must be specified as `==X.Y.Z` (exact version with full semver). :::{versionadded} 2.3.0 ::: + +:::{versionchanged} VERSION_NEXT_FEATURE +Also used by {obj}`pip.configure` to select the packages that the hub exposes. +::: """, ), "python_version": attr.string( @@ -1026,6 +1063,72 @@ execution-phase error when built. """, ) +_configure_tag = tag_class( + attrs = { + "dependency_groups": attr.string_list( + default = ["*"], + doc = """\ +The `pyproject.toml` dependency groups whose packages the hub exposes. + +Each item is one of: + +* `"default"`: the `[project] dependencies`. +* `"optional:"`: the `[project.optional-dependencies]` entry ``. +* `"group:"`: the PEP 735 `[dependency-groups]` entry ``. +* `"optional:*"` or `"group:*"`: all entries of the table. +* `"*"`: all of the above. +* `""`: the dependency group `` if it exists, otherwise the + optional dependency group ``. +* `"transitive"`: only supported together with `"*"`, exposes all of the + packages of the hub including the transitive dependencies, i.e. all of the + packages from the requirements files or the `uv.lock` file. + +See [Configuring the Exposed Packages](#pip-configure) for examples. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "hub_name": attr.string( + mandatory = True, + doc = """\ +The name of the hub to configure. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "pyproject_toml": attr.label( + mandatory = False, + doc = """\ +The `pyproject.toml` file that defines the dependency groups. Defaults to the +{attr}`pip.parse.pyproject_toml` of the hub, then to +{attr}`pip.default.pyproject_toml` if the hub is defined by the root module. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + }, + doc = """\ +Configure the packages exposed by a hub. + +The packages are selected from a `pyproject.toml` file using the +`[project] dependencies`, `[project.optional-dependencies]` and PEP 735 +`[dependency-groups]` tables. The other packages of the hub, e.g. the +transitive dependencies, are still used by the packages that need them, but +they are not public. + +Only the root module and the module that created the hub can configure it. If +both do, the root module wins. Multiple tags for the same hub are combined. + +See [Configuring the Exposed Packages](#pip-configure) for examples. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", +) + pypi = module_extension( environ = ["RULES_PYTHON_PYPI_HUB_RESERVED"], doc = """\ @@ -1049,12 +1152,17 @@ Environment Variables: - `RULES_PYTHON_PYPI_HUB_RESERVED`: Enable fallback renaming for reserved hub name collisions. See the {envvar}`RULES_PYTHON_PYPI_HUB_RESERVED` documentation for details. +pip.configure: +This tag class configures the packages exposed by a hub using the dependency +groups from a pyproject.toml file. + pip.whl_mods: This tag class is used to help create JSON files to describe modifications to the BUILD files for wheels. """, implementation = _pip_impl, tag_classes = { + "configure": _configure_tag, "default": tag_class( attrs = _default_attrs, doc = """\ diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl new file mode 100644 index 0000000000..6ca9315a81 --- /dev/null +++ b/python/private/pypi/hub_exposure.bzl @@ -0,0 +1,84 @@ +"""Compute the packages exposed by the hubs configured with `pip.configure`.""" + +load("//python/private:pyproject_utils.bzl", "read_pyproject") +load(":dependency_groups.bzl", "select_dependency_groups") + +def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): + """Compute the packages exposed by the hubs configured with `pip.configure`. + + Args: + module_ctx: {type}`module_ctx` the module context. + hubs: {type}`dict[str, struct]` the hubs by name. Each struct has: + * `module_name`: {type}`str` the module that created the hub. + * `pyproject`: {type}`Label | None` from `pip.parse`. + * `packages`: {type}`list[str]` the normalized names of all + packages in the hub. + * `exposed_packages`: {type}`list[str]` the packages that the hub + exposes when it is not configured. + default_pyproject: {type}`Label | None` from `pip.default`, only used + for the hubs of the root module. + _fail: {type}`callable` the failure function, mainly for testing. + + Returns: + {type}`dict[str, list[str]] | None` the configured hubs mapped to the + sorted packages that they expose. The hubs that are not configured + are not included. Returns `None` on failure. + """ + + # The root module wins over the owner of the hub when both configure it. + root_modules = [mod for mod in module_ctx.modules if mod.is_root] + root_name = root_modules[0].name if root_modules else None + root_configured = { + tag.hub_name: None + for mod in root_modules + for tag in mod.tags.configure + } + + configured = {} + for mod in module_ctx.modules: + for tag in mod.tags.configure: + hub = tag.hub_name + if hub not in hubs: + _fail("pip.configure() refers to unknown hub '{}'. Available hubs: {}".format( + hub, + ", ".join(sorted(hubs)), + )) + return None + + info = hubs[hub] + if not mod.is_root and (mod.name != info.module_name or hub in root_configured): + # Only the root module and the module that created the hub can + # configure it. + continue + + pyproject = ( + tag.pyproject_toml or + info.pyproject or + (default_pyproject if info.module_name == root_name else None) + ) + if not pyproject: + _fail("pip.configure() for hub '{}' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`".format(hub)) + return None + + selection = select_dependency_groups( + read_pyproject(module_ctx, pyproject), + tag.dependency_groups, + _fail = _fail, + ) + if selection.all: + packages = info.exposed_packages + else: + packages = selection.packages + + # Multiple tags for the same hub are combined. + exposed = configured.setdefault(hub, {}) + for name in packages: + # The pyproject.toml may list packages that are not in the hub, + # e.g. tools that are not locked. + if name in info.packages: + exposed[name] = None + + return { + hub: sorted(packages) + for hub, packages in configured.items() + } diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index 13d188d087..f3a9513d1f 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -95,3 +95,19 @@ platform( "@platforms//os:windows", ], ) + +# The pypi_filtered hub only makes its pyproject.toml packages public. +py_test( + name = "test_filtered_direct", + srcs = ["test_default.py"], + main = "test_default.py", + deps = ["@pypi_filtered//colorama"], +) + +py_binary( + name = "bin_filtered_six", + srcs = ["bin_six_a.py"], + main = "bin_six_a.py", + tags = ["manual"], + deps = ["@pypi_filtered//six"], +) diff --git a/tests/integration/unified_pypi/MODULE.bazel b/tests/integration/unified_pypi/MODULE.bazel index ee8f622cb4..70e01ccdac 100644 --- a/tests/integration/unified_pypi/MODULE.bazel +++ b/tests/integration/unified_pypi/MODULE.bazel @@ -53,6 +53,20 @@ pip.parse( ) use_repo(pip, "pypi_b") +# `pypi_filtered` only exposes the `[project] dependencies` of the +# `pyproject.toml`, the other packages are not public. +pip.parse( + hub_name = "pypi_filtered", + pyproject_toml = "//:pyproject_filtered.toml", + python_version = "3.11", + requirements_lock = "//:requirements_b.txt", +) +pip.configure( + dependency_groups = ["default"], + hub_name = "pypi_filtered", +) +use_repo(pip, "pypi_filtered") + pip.default(default_hub = "pypi_b") pip.default(unified_hub_requirements_bzl = True) pip.dep( diff --git a/tests/integration/unified_pypi/pyproject_filtered.toml b/tests/integration/unified_pypi/pyproject_filtered.toml new file mode 100644 index 0000000000..632f72fe22 --- /dev/null +++ b/tests/integration/unified_pypi/pyproject_filtered.toml @@ -0,0 +1,4 @@ +[project] +name = "unified-pypi" +version = "0.0.0" +dependencies = ["colorama"] diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index 9ce41fafb7..5ac2f7df8b 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -7,6 +7,19 @@ class UnifiedPypiTest(runner.TestCase): + def test_configure_hides_unexposed_packages(self): + self.run_bazel("test", "//:test_filtered_direct") + result = self.run_bazel("build", "//:bin_filtered_six", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected the alias of the unexposed package to be hidden", + ) + self.assert_result_matches( + result, + r"alias '[^']*//six:six'.*is not visible from", + ) + def test_default_fallback_hub(self): self.run_bazel("test", "//:test_default") diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index ee8e58dfe4..17f8d6741e 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -21,7 +21,7 @@ load("//python/private/pypi:extension.bzl", "build_config", "parse_modules") # load("//python/private/pypi:platform.bzl", _plat = "platform") # buildifier: disable=bzl-visibility load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") -load(":pip_parse.bzl", _parse = "pip_parse") +load(":pip_parse.bzl", _configure = "configure", _parse = "pip_parse") _tests = [] @@ -101,10 +101,11 @@ def _dep(*, name, extra_targets = []): extra_targets = extra_targets, ) -def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], is_root = True): +def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], configure = [], is_root = True): return struct( name = name, tags = struct( + configure = configure, parse = parse, override = override, whl_mods = whl_mods, @@ -118,6 +119,7 @@ def _parse_modules(env, **kwargs): return env.expect.that_struct( parse_modules(**kwargs), attrs = dict( + configured_hubs = subjects.dict, declared_deps = subjects.dict, default_hub = subjects.str, exposed_packages = subjects.dict, @@ -766,6 +768,92 @@ simple==0.0.1 \ _tests.append(_test_default_index_setting) +_CONFIGURE_REQUIREMENTS = """\ +simple==0.0.1 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +other-pkg==0.0.2 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +transitive-pkg==0.0.3 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +""" + +_CONFIGURE_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple"] +""" + +def _configure_parse_modules(env, *modules): + return _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "pyproject.toml": _CONFIGURE_PYPROJECT, + "requirements.txt": _CONFIGURE_REQUIREMENTS, + }, + *modules + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + +def _configure_parse(*, hub_name = "pypi_a", pyproject_toml = "pyproject.toml"): + return _parse( + hub_name = hub_name, + pyproject_toml = pyproject_toml, + python_version = "3.15", + simpleapi_skip = ["simple", "other-pkg", "transitive-pkg"], + requirements_lock = "requirements.txt", + ) + +def _test_configure(env): + for groups, want in [ + (["*"], ["other_pkg", "simple"]), + (["test"], ["simple"]), + (["dev"], ["other_pkg"]), + (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), + ]: + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse()], + configure = [_configure(hub_name = "pypi_a", dependency_groups = groups)], + ), + ) + + pypi.exposed_packages().contains_exactly({"pypi_a": want}) + pypi.configured_hubs().contains_exactly({"pypi_a": want}) + +_tests.append(_test_configure) + +def _test_not_configured_exposes_all(env): + # The pyproject.toml does not filter the hub without `pip.configure`. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse()], + ), + ) + + pypi.exposed_packages().contains_exactly({ + "pypi_a": ["other_pkg", "simple", "transitive_pkg"], + }) + pypi.configured_hubs().contains_exactly({}) + +_tests.append(_test_not_configured_exposes_all) + def extension_test_suite(name): """Create the test suite. diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index 7b5bdfdfd6..505c249169 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -1,5 +1,17 @@ """A simple test helper""" +def configure( + *, + dependency_groups = ["*"], + hub_name, + pyproject_toml = None): + """A simple helper for testing to simulate the PyPI extension configure tag class""" + return struct( + dependency_groups = dependency_groups, + hub_name = hub_name, + pyproject_toml = pyproject_toml, + ) + def pip_parse( *, hub_name, diff --git a/tests/pypi/hub_exposure/BUILD.bazel b/tests/pypi/hub_exposure/BUILD.bazel new file mode 100644 index 0000000000..187361e1ce --- /dev/null +++ b/tests/pypi/hub_exposure/BUILD.bazel @@ -0,0 +1,3 @@ +load(":hub_exposure_tests.bzl", "hub_exposure_test_suite") + +hub_exposure_test_suite(name = "hub_exposure_tests") diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl new file mode 100644 index 0000000000..cafbb229b5 --- /dev/null +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -0,0 +1,153 @@ +"""hub_exposure tests""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:hub_exposure.bzl", "hub_exposure") # buildifier: disable=bzl-visibility +load("//tests/support/mocks:mocks.bzl", "mocks") + +_tests = [] + +_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple", "not-in-hub"] +""" + +def _hub(*, module_name = "root", pyproject = None): + return struct( + module_name = module_name, + pyproject = pyproject, + packages = ["other_pkg", "simple", "transitive_pkg"], + exposed_packages = ["other_pkg", "simple", "transitive_pkg"], + ) + +def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None): + return mocks.tag( + hub_name = hub_name, + dependency_groups = dependency_groups, + pyproject_toml = pyproject_toml, + ) + +def _module(name = "root", is_root = True, configure = []): + return mocks.module(name = name, is_root = is_root, configure = configure) + +def _exposure(*modules, hubs = None, default_pyproject = None): + failures = [] + got = hub_exposure( + mocks.mctx( + modules = list(modules), + mock_files = { + "other.toml": "[project]\ndependencies = [\"other-pkg\"]\n", + "pyproject.toml": _PYPROJECT, + }, + ), + hubs = hubs or {"hub": _hub(pyproject = "pyproject.toml")}, + default_pyproject = default_pyproject, + _fail = failures.append, + ) + return got, failures + +def _test_not_configured(env): + got, failures = _exposure(_module()) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_not_configured) + +def _test_configure(env): + for groups, want in [ + (["*"], ["other_pkg", "simple"]), + (["default"], ["simple"]), + (["dev"], ["other_pkg"]), + # `not-in-hub` is ignored + (["group:test"], ["simple"]), + (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), + ]: + got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) + env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) + env.expect.where(groups = groups).that_dict(got).contains_exactly({"hub": want}) + +_tests.append(_test_configure) + +def _test_configure_combines_tags(env): + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["dev"]), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg", "simple"]}) + +_tests.append(_test_configure_combines_tags) + +def _test_configure_pyproject_precedence(env): + # The tag wins over the `pip.parse` value. + got, _ = _exposure(_module(configure = [ + _configure(pyproject_toml = "other.toml"), + ])) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + + # The `pip.default` value is used for the hubs of the root module. + got, failures = _exposure( + _module(configure = [_configure(dependency_groups = ["default"])]), + hubs = {"hub": _hub()}, + default_pyproject = "pyproject.toml", + ) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + +_tests.append(_test_configure_pyproject_precedence) + +def _test_configure_module_precedence(env): + hubs = {"hub": _hub(module_name = "dep", pyproject = "pyproject.toml")} + + # The owner of the hub can configure it. + got, _ = _exposure( + _module(), + _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + + # The root module wins over the owner. + got, _ = _exposure( + _module(configure = [_configure(dependency_groups = ["default"])]), + _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + + # Other modules are ignored. + got, _ = _exposure( + _module(), + _module(name = "other", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_configure_module_precedence) + +def _test_configure_errors(env): + _, failures = _exposure(_module(configure = [_configure(hub_name = "missing")])) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() refers to unknown hub 'missing'. Available hubs: hub", + ]) + + _, failures = _exposure( + _module(configure = [_configure()]), + hubs = {"hub": _hub()}, + ) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() for hub 'hub' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`", + ]) + +_tests.append(_test_configure_errors) + +def hub_exposure_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 6bb7f8379a68c92c78b12acb4a0fad3b8c99a36a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:11:22 +0900 Subject: [PATCH 04/15] fix(pypi): only add the exposed packages of a configured hub to @pypi Otherwise the unified hub would point to the hidden aliases of the hub and fail with a visibility error instead of the missing package error. --- python/private/pypi/extension.bzl | 4 +++- tests/integration/unified_pypi/BUILD.bazel | 12 ++++++++++++ tests/integration/unified_pypi_test.py | 13 +++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index e64e856489..ce5e95492d 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -416,7 +416,9 @@ def _create_unified_hub_repo(mods): extra_aliases = {} for hub_name in hubs: - for pkg_name in mods.hub_whl_map[hub_name]: + # The hubs configured with `pip.configure` only expose some packages. + hub_packages = mods.configured_hubs.get(hub_name, mods.hub_whl_map[hub_name]) + for pkg_name in hub_packages: norm_pkg = normalize_name(pkg_name) if norm_pkg not in packages: packages[norm_pkg] = [] diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index f3a9513d1f..78f9a857ee 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -104,6 +104,18 @@ py_test( deps = ["@pypi_filtered//colorama"], ) +# six is not exposed by pypi_filtered, so it is missing in the unified hub. +py_binary( + name = "bin_filtered_six_unified", + srcs = ["bin_six_a.py"], + config_settings = { + "@rules_python//python/config_settings:venv": "pypi_filtered", + }, + main = "bin_six_a.py", + tags = ["manual"], + deps = ["@pypi//six"], +) + py_binary( name = "bin_filtered_six", srcs = ["bin_six_a.py"], diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index 5ac2f7df8b..f59a19fb42 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -20,6 +20,19 @@ def test_configure_hides_unexposed_packages(self): r"alias '[^']*//six:six'.*is not visible from", ) + def test_configure_hides_unexposed_packages_in_unified_hub(self): + self.run_bazel("cquery", "//:bin_filtered_six_unified") + result = self.run_bazel("build", "//:bin_filtered_six_unified", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected build to fail during execution phase", + ) + self.assert_result_matches( + result, + 'ERROR: PyPI package "six" is not available when building under PyPI hub "pypi_filtered"\\. Try adding it to the requirements of this hub', + ) + def test_default_fallback_hub(self): self.run_bazel("test", "//:test_default") From b31a734fd273538849adba68184df7b801212fce Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:05:33 +0900 Subject: [PATCH 05/15] fix(pypi): treat exposing all packages as not configuring the hub pip.configure with ["*", "transitive"] used the packages that are available on all platforms, which made the platform-specific packages private. Treat it as if the hub was not configured instead, which keeps these packages public and out of all_requirements as before. --- docs/pypi/download.md | 5 ++-- python/private/pypi/extension.bzl | 1 - python/private/pypi/hub_exposure.bzl | 24 +++++++++---------- .../pypi/hub_exposure/hub_exposure_tests.bzl | 14 +++++++++-- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 5f16606d24..654e156921 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -171,8 +171,9 @@ dependencies of `numpy` are only used through `@pypi_hub//numpy`. The The {attr}`pip.configure.dependency_groups` attribute defaults to `["*"]`, which selects every group. Use `["*", "transitive"]` to expose all of the -packages of the hub, including the transitive dependencies. Selecting the -transitive dependencies of only some of the groups is not supported. +packages of the hub, including the transitive dependencies, which is the same +as not configuring the hub. Selecting the transitive dependencies of only some +of the groups is not supported. Multiple tags for the same hub are combined. Only the root module and the module that created the hub can configure it; if both do, the root module diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index ce5e95492d..10e0c456a2 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -369,7 +369,6 @@ You cannot use both the additive_build_content and additive_build_content_file a module_name = hub.module_name, pyproject = hub_pyproject.get(hub.name), packages = out.whl_map.keys(), - exposed_packages = out.exposed_packages, ) configured_hubs = hub_exposure( diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index 6ca9315a81..fe9e586153 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -13,28 +13,27 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): * `pyproject`: {type}`Label | None` from `pip.parse`. * `packages`: {type}`list[str]` the normalized names of all packages in the hub. - * `exposed_packages`: {type}`list[str]` the packages that the hub - exposes when it is not configured. default_pyproject: {type}`Label | None` from `pip.default`, only used for the hubs of the root module. _fail: {type}`callable` the failure function, mainly for testing. Returns: {type}`dict[str, list[str]] | None` the configured hubs mapped to the - sorted packages that they expose. The hubs that are not configured - are not included. Returns `None` on failure. + sorted packages that they expose. The hubs that are not configured or + that expose all of their packages are not included, so that they + behave the same. Returns `None` on failure. """ # The root module wins over the owner of the hub when both configure it. root_modules = [mod for mod in module_ctx.modules if mod.is_root] root_name = root_modules[0].name if root_modules else None - root_configured = { - tag.hub_name: None - for mod in root_modules - for tag in mod.tags.configure - } + root_configured = {} + for mod in root_modules: + for tag in mod.tags.configure: + root_configured[tag.hub_name] = None configured = {} + exposes_all = {} for mod in module_ctx.modules: for tag in mod.tags.configure: hub = tag.hub_name @@ -66,13 +65,11 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): _fail = _fail, ) if selection.all: - packages = info.exposed_packages - else: - packages = selection.packages + exposes_all[hub] = None # Multiple tags for the same hub are combined. exposed = configured.setdefault(hub, {}) - for name in packages: + for name in selection.packages: # The pyproject.toml may list packages that are not in the hub, # e.g. tools that are not locked. if name in info.packages: @@ -81,4 +78,5 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): return { hub: sorted(packages) for hub, packages in configured.items() + if hub not in exposes_all } diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index cafbb229b5..5d84a351b5 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -20,7 +20,6 @@ def _hub(*, module_name = "root", pyproject = None): module_name = module_name, pyproject = pyproject, packages = ["other_pkg", "simple", "transitive_pkg"], - exposed_packages = ["other_pkg", "simple", "transitive_pkg"], ) def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None): @@ -63,7 +62,6 @@ def _test_configure(env): (["dev"], ["other_pkg"]), # `not-in-hub` is ignored (["group:test"], ["simple"]), - (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), ]: got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) @@ -71,6 +69,18 @@ def _test_configure(env): _tests.append(_test_configure) +def _test_configure_all_packages(env): + # Exposing all of the packages is the same as not configuring the hub, so + # that e.g. the platform-specific packages stay public. + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["*", "transitive"]), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_configure_all_packages) + def _test_configure_combines_tags(env): got, failures = _exposure(_module(configure = [ _configure(dependency_groups = ["default"]), From 9d97360242b44f6a73472793d17a17287cabb1f3 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:05:56 +0900 Subject: [PATCH 06/15] refactor(pypi): use explicit loops and simpler conditions in pip.configure Replace the nested comprehensions and the long inline conditions with explicit loops and use types.is_string. --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/dependency_groups.bzl | 19 ++++++++----------- python/private/pypi/hub_repository.bzl | 4 +++- python/private/pypi/render_pkg_aliases.bzl | 19 +++++++++++-------- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index caf482b67f..fe28b17413 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -639,6 +639,7 @@ bzl_library( deps = [ ":pep508_requirement", "//python/private:normalize_name", + "@bazel_skylib//lib:types", ], ) diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index 2ddadee3dd..4e40c7e856 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -1,5 +1,6 @@ """Select PyPI packages using pyproject.toml dependency groups.""" +load("@bazel_skylib//lib:types.bzl", "types") load("//python/private:normalize_name.bzl", "normalize_name") load(":pep508_requirement.bzl", "requirement") @@ -59,16 +60,12 @@ def select_dependency_groups(pyproject, specs, _fail = fail): for key in selected: for req in groups[key]: req = requirement(req) - names = [req.name_] - if req.name_ == project: - names = [ - requirement(r).name_ - for extra in req.extras - for r in groups.get("optional:" + extra, []) - ] - for name in names: - if name: - packages[name] = None + if req.name_ != project: + packages[req.name_] = None + continue + for extra in req.extras: + for r in groups.get("optional:" + extra, []): + packages[requirement(r).name_] = None return struct(packages = sorted(packages), all = False) @@ -119,7 +116,7 @@ def _expand_dependency_groups(dependency_groups, _fail): reqs = [] for item in items: - if type(item) == type(""): + if types.is_string(item): reqs.append(item) continue diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index d421f3cc09..ff5311c6ea 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -28,14 +28,16 @@ exports_files(["requirements.bzl"]) def _impl(rctx): if rctx.attr.hide_unexposed_packages: bzl_packages = rctx.attr.packages + exposed_packages = bzl_packages else: bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + exposed_packages = None aliases = render_multiplatform_pkg_aliases( aliases = { key: _whl_config_settings_from_json(values) for key, values in rctx.attr.whl_map.items() }, - exposed_packages = bzl_packages if rctx.attr.hide_unexposed_packages else None, + exposed_packages = exposed_packages, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 5582cc00c9..2c86155876 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -139,17 +139,20 @@ def render_pkg_aliases( exposed_packages = {normalize_name(name): None for name in exposed_packages} internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) - files = { - "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( - name = normalize_name(name), + files = {} + for name, pkg_aliases in aliases.items(): + name = normalize_name(name) + visibility = public + if exposed_packages != None and name not in exposed_packages: + visibility = internal_visibility + files["{}/BUILD.bazel".format(name)] = _render_common_aliases( + name = name, aliases = pkg_aliases, - extra_aliases = extra_hub_aliases.get(normalize_name(name), []), - group_name = whl_group_mapping.get(normalize_name(name)), - visibility = public if exposed_packages == None or normalize_name(name) in exposed_packages else internal_visibility, + extra_aliases = extra_hub_aliases.get(name, []), + group_name = whl_group_mapping.get(name), + visibility = visibility, **kwargs ).strip() - for name, pkg_aliases in aliases.items() - } if requirement_cycles: files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) From 35c066052cb0bea531aba78784149420d8056aaa Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:06:00 +0900 Subject: [PATCH 07/15] test(pypi): test the pip.default pyproject_toml with pip.configure Keep a single wiring test for pip.configure in the extension tests, as the selection is tested in hub_exposure_tests, and test that the pip.default pyproject_toml is used. Also link the issue in the news entry. --- news/4395.added.md | 3 +- tests/pypi/extension/extension_tests.bzl | 49 +++++++++++++++--------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/news/4395.added.md b/news/4395.added.md index 62d7f10880..3e633cc997 100644 --- a/news/4395.added.md +++ b/news/4395.added.md @@ -1,2 +1,3 @@ (pypi) Added {obj}`pip.configure` to limit the packages that a hub exposes to -the ones listed in the dependency groups of a `pyproject.toml` file. +the ones listed in the dependency groups of a `pyproject.toml` file +([#3413](https://github.com/bazel-contrib/rules_python/issues/3413)). diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 17f8d6741e..680853a0d0 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -807,36 +807,47 @@ def _configure_parse_modules(env, *modules): minor_mapping = {"3.15": "3.15.19"}, ) -def _configure_parse(*, hub_name = "pypi_a", pyproject_toml = "pyproject.toml"): +def _configure_parse(**kwargs): return _parse( - hub_name = hub_name, - pyproject_toml = pyproject_toml, + hub_name = "pypi_a", python_version = "3.15", simpleapi_skip = ["simple", "other-pkg", "transitive-pkg"], requirements_lock = "requirements.txt", + **kwargs ) def _test_configure(env): - for groups, want in [ - (["*"], ["other_pkg", "simple"]), - (["test"], ["simple"]), - (["dev"], ["other_pkg"]), - (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), - ]: - pypi = _configure_parse_modules( - env, - _mod( - name = "rules_python", - parse = [_configure_parse()], - configure = [_configure(hub_name = "pypi_a", dependency_groups = groups)], - ), - ) + # The selection itself is tested in `hub_exposure_tests`. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse(pyproject_toml = "pyproject.toml")], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["test"])], + ), + ) - pypi.exposed_packages().contains_exactly({"pypi_a": want}) - pypi.configured_hubs().contains_exactly({"pypi_a": want}) + pypi.exposed_packages().contains_exactly({"pypi_a": ["simple"]}) + pypi.configured_hubs().contains_exactly({"pypi_a": ["simple"]}) _tests.append(_test_configure) +def _test_configure_default_pyproject(env): + # The `pip.default` pyproject.toml is used for the hubs of the root module. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + default = _default_tags_default + [_default(pyproject_toml = "pyproject.toml")], + parse = [_configure_parse()], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["dev"])], + ), + ) + + pypi.configured_hubs().contains_exactly({"pypi_a": ["other_pkg"]}) + +_tests.append(_test_configure_default_pyproject) + def _test_not_configured_exposes_all(env): # The pyproject.toml does not filter the hub without `pip.configure`. pypi = _configure_parse_modules( From 1ffebb84dee0f508ea8b237bf7eea252fd4e18b0 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 18 Nov 2026 19:45:58 -0800 Subject: [PATCH 08/15] refactor(pypi): polish pip.configure docs, docstrings, and formatting Align the new `pip.configure` documentation, docstrings, and Starlark files with repository style and documentation conventions. Add `requires-python` to the `docs/pypi/download.md` example, move the `pip.configure` section to top-level `##` heading order, clarify the `pyproject_toml` attribute docstrings and `{ref}` links, document helper arguments/returns and returned `struct` fields, flatten `hub_exposure` guards, move module-level test constants before functions, and wrap Starlark lines and `doc` strings to 80 columns. --- docs/pypi/download.md | 87 ++++++++++--------- python/private/pypi/dependency_groups.bzl | 39 +++++++-- python/private/pypi/extension.bzl | 52 +++++++---- python/private/pypi/hub_exposure.bzl | 16 ++-- python/private/pypi/hub_repository.bzl | 11 ++- python/private/pypi/render_pkg_aliases.bzl | 19 +++- python/private/pyproject_utils.bzl | 5 ++ .../dependency_groups_tests.bzl | 49 +++++++++-- tests/pypi/extension/extension_tests.bzl | 65 ++++++++------ tests/pypi/extension/pip_parse.bzl | 2 +- .../pypi/hub_exposure/hub_exposure_tests.bzl | 48 +++++++--- 11 files changed, 266 insertions(+), 127 deletions(-) diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 654e156921..ef069da5b5 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -134,8 +134,50 @@ rename, and `requirement("numpy")` routes through the active hub like fixed at loading time, before the active hub is known, so load them from a concrete hub instead. Prefer `@pypi//` labels in new code. +### Declaring Abstract Dependencies (pip.dep) + +:::{versionadded} 2.2.0 +Declaring abstract PyPI dependencies via `pip.dep` tags. +::: + +Sometimes a shared library target or a ruleset needs to depend on a PyPI +package (e.g., `@pypi//numpy`), but does not want to force a specific package +version or a concrete `requirements.txt` lock file on its consumers. + +Instead of calling `pip.parse()`, the module can declare its dependency using +the `pip.dep` tag: + +```starlark +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") + +# Declare an abstract dependency on 'numpy' and specify extra targets that +# are expected to be available in the package. +pip.dep( + name = "numpy", + extra_targets = ["extra-alias"], +) +``` + +This ensures that the target structure `@pypi//numpy` (and +`@pypi//numpy:extra-alias`) exists in the unified `@pypi` hub repository, so the +declaring module can compile and analyze successfully without needing any local +requirements file. + +The actual concrete implementation and version of the package must be provided +by a downstream module calling `pip.parse`. + +If a downstream module attempts to build a target that depends on an abstract +dependency, but has not provided a concrete implementation for it via any +`pip.parse` call, the build will fail at execution time. + + +As with any repository rule or extension, if you would like to ensure that `pip_parse` is +re-executed to pick up a non-hermetic change to your environment (e.g., updating your system +`python` interpreter), you can force it to re-execute by running `bazel sync --only [pip_parse +name]`. + (pip-configure)= -### Configuring the Exposed Packages (pip.configure) +## Configuring the Exposed Packages (pip.configure) By default a hub exposes all of the packages from its requirements files or `uv.lock` file, including the transitive dependencies. A `pip.configure` tag @@ -145,6 +187,7 @@ can limit the hub to the packages listed in a `pyproject.toml` file: the ```toml [project] +requires-python = "==3.13.0" dependencies = ["numpy"] [dependency-groups] @@ -185,48 +228,6 @@ are missing from the lock file. :::{versionadded} VERSION_NEXT_FEATURE ::: -### Declaring Abstract Dependencies (pip.dep) - -:::{versionadded} 2.2.0 -Declaring abstract PyPI dependencies via `pip.dep` tags. -::: - -Sometimes a shared library target or a ruleset needs to depend on a PyPI -package (e.g., `@pypi//numpy`), but does not want to force a specific package -version or a concrete `requirements.txt` lock file on its consumers. - -Instead of calling `pip.parse()`, the module can declare its dependency using -the `pip.dep` tag: - -```starlark -pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") - -# Declare an abstract dependency on 'numpy' and specify extra targets that -# are expected to be available in the package. -pip.dep( - name = "numpy", - extra_targets = ["extra-alias"], -) -``` - -This ensures that the target structure `@pypi//numpy` (and -`@pypi//numpy:extra-alias`) exists in the unified `@pypi` hub repository, so the -declaring module can compile and analyze successfully without needing any local -requirements file. - -The actual concrete implementation and version of the package must be provided -by a downstream module calling `pip.parse`. - -If a downstream module attempts to build a target that depends on an abstract -dependency, but has not provided a concrete implementation for it via any -`pip.parse` call, the build will fail at execution time. - - -As with any repository rule or extension, if you would like to ensure that `pip_parse` is -re-executed to pick up a non-hermetic change to your environment (e.g., updating your system -`python` interpreter), you can force it to re-execute by running `bazel sync --only [pip_parse -name]`. - (per-os-arch-requirements)= ## Requirements for a specific OS/Architecture diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index 4e40c7e856..e40e67a4c3 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -34,13 +34,21 @@ def select_dependency_groups(pyproject, specs, _fail = fail): groups = {"default": pyproject.dependencies} for name, reqs in pyproject.optional_dependencies.items(): groups["optional:" + normalize_name(name)] = reqs - for name, reqs in _expand_dependency_groups(pyproject.dependency_groups, _fail).items(): + expanded = _expand_dependency_groups(pyproject.dependency_groups, _fail) + for name, reqs in expanded.items(): groups["group:" + name] = reqs if "transitive" in specs: if "*" not in specs: - _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format(specs)) - return struct(packages = [], all = True) + _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format( + specs, + )) + return struct( + # list[str]: sorted normalized package names selected by `specs`. + packages = [], + # bool: True when all hub packages are selected. + all = True, + ) selected = {} for spec in specs: @@ -67,9 +75,24 @@ def select_dependency_groups(pyproject, specs, _fail = fail): for r in groups.get("optional:" + extra, []): packages[requirement(r).name_] = None - return struct(packages = sorted(packages), all = False) + return struct( + # list[str]: sorted normalized package names selected by `specs`. + packages = sorted(packages), + # bool: True when all packages including transitive ones are selected. + all = False, + ) def _match(spec, groups): + """Match a dependency group specifier against normalized group keys. + + Args: + spec: {type}`str` the dependency group specifier. + groups: {type}`dict[str, list[str]]` available groups keyed by + prefixed group name. + + Returns: + {type}`list[str]` matching keys in `groups`. + """ if spec == "*": return groups.keys() if spec in ("optional:*", "group:*"): @@ -122,7 +145,10 @@ def _expand_dependency_groups(dependency_groups, _fail): include = normalize_name(item.get("include-group", "")) if include not in groups: - _fail("Unknown dependency group '{}' included from '{}'".format(include, name)) + _fail("Unknown dependency group '{}' included from '{}'".format( + include, + name, + )) return {} if include not in expanded: reqs = None @@ -133,7 +159,8 @@ def _expand_dependency_groups(dependency_groups, _fail): expanded[name] = reqs if len(expanded) != len(groups): + unexpanded = [name for name in groups if name not in expanded] _fail("Cyclic dependency group includes: {}".format( - ", ".join(sorted([name for name in groups if name not in expanded])), + ", ".join(sorted(unexpanded)), )) return expanded diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index dcc7aa1a94..78119835af 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -157,6 +157,7 @@ def build_config( for name, values in defaults["platforms"].items() }, enable_pipstar_extract = enable_pipstar_extract, + # Label | None: default `pyproject.toml` from `pip.default`. pyproject_toml = defaults["pyproject_toml"], toml_decode = toml.decode, unified_hub_requirements_bzl = unified_hub_requirements_bzl, @@ -369,8 +370,11 @@ You cannot use both the additive_build_content and additive_build_content_file a hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map hubs[hub.name] = struct( + # str: module that created the hub. module_name = hub.module_name, + # Label | None: `pyproject_toml` set on `pip.parse`. pyproject = hub_pyproject.get(hub.name), + # list[str]: normalized names of all packages in the hub. packages = out.whl_map.keys(), ) @@ -385,6 +389,7 @@ You cannot use both the additive_build_content and additive_build_content_file a exposed_packages.update(configured_hubs) return struct( + # dict[str, list[str]]: configured hubs and exposed packages. configured_hubs = configured_hubs, config = config, declared_deps = declared_deps, @@ -419,7 +424,10 @@ def _create_unified_hub_repo(mods): for hub_name in hubs: # The hubs configured with `pip.configure` only expose some packages. - hub_packages = mods.configured_hubs.get(hub_name, mods.hub_whl_map[hub_name]) + hub_packages = mods.configured_hubs.get( + hub_name, + mods.hub_whl_map[hub_name], + ) for pkg_name in hub_packages: norm_pkg = normalize_name(pkg_name) if norm_pkg not in packages: @@ -673,15 +681,18 @@ If you are defining custom platforms in your project and don't want things to cl ), "pyproject_toml": attr.label( mandatory = False, - doc = """\ -Label pointing to pyproject.toml file to read the default Python version from. -When specified, reads the `requires-python` field from pyproject.toml and uses -it as the default python_version for all `pip.parse()` calls that don't -explicitly specify one. + doc = """ +Label pointing to a `pyproject.toml` file used to read the default Python +version and, for hubs of the root module, to select exposed packages via +{obj}`pip.configure`. + +When `[project] requires-python` is present, it is used as the default +`python_version` for `pip.parse()` calls that do not specify one. :::{note} -The version must be specified as `==X.Y.Z` (exact version with full semver). -This is designed to work with dependency management tools like Renovate. +If `requires-python` is set, the version must be specified as `==X.Y.Z` (exact +version with full semver). This is designed to work with dependency management +tools like Renovate. ::: :::{versionadded} 2.3.0 @@ -898,13 +909,16 @@ find in case extra indexes are specified. ), "pyproject_toml": attr.label( mandatory = False, - doc = """\ -Label pointing to a pyproject.toml file to read the Python version from. -When specified, the `requires-python` field is used as the `python_version` -for this `pip.parse()` call, unless `python_version` is set explicitly. + doc = """ +Label pointing to a `pyproject.toml` file used to read the Python version and +to select exposed packages via {obj}`pip.configure`. + +When `python_version` is not set explicitly, the `[project] requires-python` +field is used as the `python_version` for this `pip.parse()` call. :::{note} -The version must be specified as `==X.Y.Z` (exact version with full semver). +When used to derive `python_version`, `requires-python` must be specified as +`==X.Y.Z` (exact version with full semver). ::: :::{versionadded} 2.3.0 @@ -1116,7 +1130,7 @@ _configure_tag = tag_class( attrs = { "dependency_groups": attr.string_list( default = ["*"], - doc = """\ + doc = """ The `pyproject.toml` dependency groups whose packages the hub exposes. Each item is one of: @@ -1132,7 +1146,7 @@ Each item is one of: packages of the hub including the transitive dependencies, i.e. all of the packages from the requirements files or the `uv.lock` file. -See [Configuring the Exposed Packages](#pip-configure) for examples. +See {ref}`Configuring the Exposed Packages ` for examples. :::{versionadded} VERSION_NEXT_FEATURE ::: @@ -1140,7 +1154,7 @@ See [Configuring the Exposed Packages](#pip-configure) for examples. ), "hub_name": attr.string( mandatory = True, - doc = """\ + doc = """ The name of the hub to configure. :::{versionadded} VERSION_NEXT_FEATURE @@ -1149,7 +1163,7 @@ The name of the hub to configure. ), "pyproject_toml": attr.label( mandatory = False, - doc = """\ + doc = """ The `pyproject.toml` file that defines the dependency groups. Defaults to the {attr}`pip.parse.pyproject_toml` of the hub, then to {attr}`pip.default.pyproject_toml` if the hub is defined by the root module. @@ -1159,7 +1173,7 @@ The `pyproject.toml` file that defines the dependency groups. Defaults to the """, ), }, - doc = """\ + doc = """ Configure the packages exposed by a hub. The packages are selected from a `pyproject.toml` file using the @@ -1171,7 +1185,7 @@ they are not public. Only the root module and the module that created the hub can configure it. If both do, the root module wins. Multiple tags for the same hub are combined. -See [Configuring the Exposed Packages](#pip-configure) for examples. +See {ref}`Configuring the Exposed Packages ` for examples. :::{versionadded} VERSION_NEXT_FEATURE ::: diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index fe9e586153..ff9f24694d 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -45,18 +45,20 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): return None info = hubs[hub] - if not mod.is_root and (mod.name != info.module_name or hub in root_configured): + if not mod.is_root and mod.name != info.module_name: # Only the root module and the module that created the hub can # configure it. continue + if not mod.is_root and hub in root_configured: + continue - pyproject = ( - tag.pyproject_toml or - info.pyproject or - (default_pyproject if info.module_name == root_name else None) - ) + pyproject = tag.pyproject_toml or info.pyproject + if not pyproject and info.module_name == root_name: + pyproject = default_pyproject if not pyproject: - _fail("pip.configure() for hub '{}' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`".format(hub)) + _fail("pip.configure() for hub '{}' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`".format( + hub, + )) return None selection = select_dependency_groups( diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index ff5311c6ea..99f659ba31 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -86,13 +86,16 @@ hub_repository = repository_rule( ), "hide_unexposed_packages": attr.bool( default = False, - doc = "Make the aliases of the packages that are not in `packages` visible only to the wheel repositories.", + doc = """ +Make the aliases of the packages that are not in `packages` visible only to +the wheel repositories. +""", ), "packages": attr.string_list( mandatory = False, - doc = """\ -The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys -unless `hide_unexposed_packages` is set. + doc = """ +The list of packages that will be exposed via all_*requirements macros. +Defaults to whl_map keys unless `hide_unexposed_packages` is set. """, ), "platform_config_settings": attr.string_list_dict( diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 2c86155876..ba0ad2cc75 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -136,8 +136,14 @@ def render_pkg_aliases( public = ["//visibility:public"] if exposed_packages != None: - exposed_packages = {normalize_name(name): None for name in exposed_packages} - internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) + exposed_packages = { + normalize_name(name): None + for name in exposed_packages + } + internal_visibility = _internal_visibility( + aliases, + has_groups = bool(requirement_cycles), + ) files = {} for name, pkg_aliases in aliases.items(): @@ -159,10 +165,17 @@ def render_pkg_aliases( return files def _internal_visibility(aliases, has_groups = False): - """The visibility of the packages that are not exposed. + """Return the visibility list for packages that are not exposed. The wheel repositories depend on the aliases of the hub, so the aliases have to be visible to them. + + Args: + aliases: {type}`dict[str, str | dict]` package-to-repo alias mapping. + has_groups: {type}`bool` whether requirement cycle groups exist. + + Returns: + {type}`list[str]` visibility package specifications. """ repo_names = {} for pkg_aliases in aliases.values(): diff --git a/python/private/pyproject_utils.bzl b/python/private/pyproject_utils.bzl index a9494b8ac2..ceb5c803aa 100644 --- a/python/private/pyproject_utils.bzl +++ b/python/private/pyproject_utils.bzl @@ -29,10 +29,15 @@ def read_pyproject(module_ctx, pyproject): data = toml.decode(module_ctx.read(module_ctx.path(pyproject), watch = "yes")) project = data.get("project", {}) return struct( + # str | None: raw `[project] requires-python` value. requires_python = project.get("requires-python"), + # str | None: `[project] name`. name = project.get("name"), + # list[str]: `[project] dependencies` requirement strings. dependencies = project.get("dependencies") or [], + # dict[str, list[str]]: `[project.optional-dependencies]` table. optional_dependencies = project.get("optional-dependencies") or {}, + # dict[str, list]: PEP 735 `[dependency-groups]` table. dependency_groups = data.get("dependency-groups") or {}, ) diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl index ee23aec1fb..d494cae9d7 100644 --- a/tests/pypi/dependency_groups/dependency_groups_tests.bzl +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -1,7 +1,10 @@ """dependency_groups tests""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:dependency_groups.bzl", "select_dependency_groups") # buildifier: disable=bzl-visibility +load( + "//python/private/pypi:dependency_groups.bzl", + "select_dependency_groups", +) # buildifier: disable=bzl-visibility _tests = [] @@ -35,11 +38,25 @@ def _test_select(env): (["optional:*"], ["docs_pkg", "grpc_pkg"]), (["group:*"], ["dev_pkg", "shared_pkg", "test_pkg"]), (["default", "group:test"], ["main_pkg", "shared_pkg", "test_pkg"]), - (["*"], ["dev_pkg", "docs_pkg", "grpc_pkg", "main_pkg", "shared_pkg", "test_pkg"]), + ( + ["*"], + [ + "dev_pkg", + "docs_pkg", + "grpc_pkg", + "main_pkg", + "shared_pkg", + "test_pkg", + ], + ), ]: got, failures = _select(specs) - env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) - env.expect.where(specs = specs).that_collection(got.packages).contains_exactly(want).in_order() + env.expect.where(specs = specs).that_collection( + failures, + ).contains_exactly([]) + env.expect.where(specs = specs).that_collection( + got.packages, + ).contains_exactly(want).in_order() env.expect.where(specs = specs).that_bool(got.all).equals(False) _tests.append(_test_select) @@ -54,18 +71,32 @@ _tests.append(_test_select_all) def _test_select_errors(env): for specs, want in [ - (["missing"], "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test, optional:all, optional:docs, optional:grpc"), - (["default", "transitive"], "The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: [\"default\", \"transitive\"]"), + ( + ["missing"], + "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test, optional:all, optional:docs, optional:grpc", + ), + ( + ["default", "transitive"], + "The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: [\"default\", \"transitive\"]", + ), ]: _, failures = _select(specs) - env.expect.where(specs = specs).that_collection(failures).contains_exactly([want]) + env.expect.where(specs = specs).that_collection( + failures, + ).contains_exactly([want]) _tests.append(_test_select_errors) def _test_include_group_errors(env): for groups, want in [ - ({"a": [{"include-group": "missing"}]}, "Unknown dependency group 'missing' included from 'a'"), - ({"a": [{"include-group": "b"}], "b": [{"include-group": "a"}]}, "Cyclic dependency group includes: a, b"), + ( + {"a": [{"include-group": "missing"}]}, + "Unknown dependency group 'missing' included from 'a'", + ), + ( + {"a": [{"include-group": "b"}], "b": [{"include-group": "a"}]}, + "Cyclic dependency group includes: a, b", + ), ]: _, failures = _select(["*"], pyproject = struct( name = None, diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index bc80ed679c..b3a6e7837c 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -25,6 +25,27 @@ load(":pip_parse.bzl", _configure = "configure", _parse = "pip_parse") _tests = [] +_CONFIGURE_REQUIREMENTS = """\ +simple==0.0.1 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +other-pkg==0.0.2 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +transitive-pkg==0.0.3 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +""" + +_CONFIGURE_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple"] +""" + def _pypi_mock_mctx(*modules, os_name = "unittest", arch_name = "exotic", environ = {}, read = None, mock_files = {}): _ = read # @unused return mocks.mctx( @@ -107,7 +128,16 @@ def _dep(*, name, extra_targets = []): extra_targets = extra_targets, ) -def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], configure = [], is_root = True): +def _mod( + *, + name, + default = _default_tags_default, + parse = [], + override = [], + whl_mods = [], + dep = [], + configure = [], + is_root = True): return struct( name = name, tags = struct( @@ -821,27 +851,6 @@ simple==0.0.1 \ _tests.append(_test_default_index_setting) -_CONFIGURE_REQUIREMENTS = """\ -simple==0.0.1 \ - --hash=sha256:deadbeef \ - --hash=sha256:deadbaaf -other-pkg==0.0.2 \ - --hash=sha256:deadbeef \ - --hash=sha256:deadbaaf -transitive-pkg==0.0.3 \ - --hash=sha256:deadbeef \ - --hash=sha256:deadbaaf -""" - -_CONFIGURE_PYPROJECT = """\ -[project] -dependencies = ["simple"] - -[dependency-groups] -dev = ["other-pkg"] -test = ["simple"] -""" - def _configure_parse_modules(env, *modules): return _parse_modules( env, @@ -876,7 +885,9 @@ def _test_configure(env): _mod( name = "rules_python", parse = [_configure_parse(pyproject_toml = "pyproject.toml")], - configure = [_configure(hub_name = "pypi_a", dependency_groups = ["test"])], + configure = [ + _configure(hub_name = "pypi_a", dependency_groups = ["test"]), + ], ), ) @@ -891,9 +902,13 @@ def _test_configure_default_pyproject(env): env, _mod( name = "rules_python", - default = _default_tags_default + [_default(pyproject_toml = "pyproject.toml")], + default = _default_tags_default + [ + _default(pyproject_toml = "pyproject.toml"), + ], parse = [_configure_parse()], - configure = [_configure(hub_name = "pypi_a", dependency_groups = ["dev"])], + configure = [ + _configure(hub_name = "pypi_a", dependency_groups = ["dev"]), + ], ), ) diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index 3670282a92..db3008d431 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -5,7 +5,7 @@ def configure( dependency_groups = ["*"], hub_name, pyproject_toml = None): - """A simple helper for testing to simulate the PyPI extension configure tag class""" + """Simulate the PyPI extension configure tag class for testing.""" return struct( dependency_groups = dependency_groups, hub_name = hub_name, diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index 5d84a351b5..a51deb9c56 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -1,7 +1,10 @@ """hub_exposure tests""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:hub_exposure.bzl", "hub_exposure") # buildifier: disable=bzl-visibility +load( + "//python/private/pypi:hub_exposure.bzl", + "hub_exposure", +) # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") _tests = [] @@ -22,7 +25,10 @@ def _hub(*, module_name = "root", pyproject = None): packages = ["other_pkg", "simple", "transitive_pkg"], ) -def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None): +def _configure( + hub_name = "hub", + dependency_groups = ["*"], + pyproject_toml = None): return mocks.tag( hub_name = hub_name, dependency_groups = dependency_groups, @@ -63,9 +69,15 @@ def _test_configure(env): # `not-in-hub` is ignored (["group:test"], ["simple"]), ]: - got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) - env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) - env.expect.where(groups = groups).that_dict(got).contains_exactly({"hub": want}) + got, failures = _exposure( + _module(configure = [_configure(dependency_groups = groups)]), + ) + env.expect.where(groups = groups).that_collection( + failures, + ).contains_exactly([]) + env.expect.where(groups = groups).that_dict(got).contains_exactly( + {"hub": want}, + ) _tests.append(_test_configure) @@ -87,7 +99,9 @@ def _test_configure_combines_tags(env): _configure(dependency_groups = ["dev"]), ])) env.expect.that_collection(failures).contains_exactly([]) - env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg", "simple"]}) + env.expect.that_dict(got).contains_exactly( + {"hub": ["other_pkg", "simple"]}, + ) _tests.append(_test_configure_combines_tags) @@ -115,7 +129,11 @@ def _test_configure_module_precedence(env): # The owner of the hub can configure it. got, _ = _exposure( _module(), - _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + _module( + name = "dep", + is_root = False, + configure = [_configure(dependency_groups = ["dev"])], + ), hubs = hubs, ) env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) @@ -123,7 +141,11 @@ def _test_configure_module_precedence(env): # The root module wins over the owner. got, _ = _exposure( _module(configure = [_configure(dependency_groups = ["default"])]), - _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + _module( + name = "dep", + is_root = False, + configure = [_configure(dependency_groups = ["dev"])], + ), hubs = hubs, ) env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) @@ -131,7 +153,11 @@ def _test_configure_module_precedence(env): # Other modules are ignored. got, _ = _exposure( _module(), - _module(name = "other", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + _module( + name = "other", + is_root = False, + configure = [_configure(dependency_groups = ["dev"])], + ), hubs = hubs, ) env.expect.that_dict(got).contains_exactly({}) @@ -139,7 +165,9 @@ def _test_configure_module_precedence(env): _tests.append(_test_configure_module_precedence) def _test_configure_errors(env): - _, failures = _exposure(_module(configure = [_configure(hub_name = "missing")])) + _, failures = _exposure( + _module(configure = [_configure(hub_name = "missing")]), + ) env.expect.that_collection(failures).contains_exactly([ "pip.configure() refers to unknown hub 'missing'. Available hubs: hub", ]) From a40d1d5e00aee45af7f6724304cdf369a7c3aebc Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 18 Nov 2026 19:49:15 -0800 Subject: [PATCH 09/15] refactor(pypi): restore pyproject_toml attribute docstrings Restore the `pip.default.pyproject_toml` and `pip.parse.pyproject_toml` attribute docstrings to their original text per review decision. --- python/private/pypi/extension.bzl | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 78119835af..7814246e62 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -681,18 +681,15 @@ If you are defining custom platforms in your project and don't want things to cl ), "pyproject_toml": attr.label( mandatory = False, - doc = """ -Label pointing to a `pyproject.toml` file used to read the default Python -version and, for hubs of the root module, to select exposed packages via -{obj}`pip.configure`. - -When `[project] requires-python` is present, it is used as the default -`python_version` for `pip.parse()` calls that do not specify one. + doc = """\ +Label pointing to pyproject.toml file to read the default Python version from. +When specified, reads the `requires-python` field from pyproject.toml and uses +it as the default python_version for all `pip.parse()` calls that don't +explicitly specify one. :::{note} -If `requires-python` is set, the version must be specified as `==X.Y.Z` (exact -version with full semver). This is designed to work with dependency management -tools like Renovate. +The version must be specified as `==X.Y.Z` (exact version with full semver). +This is designed to work with dependency management tools like Renovate. ::: :::{versionadded} 2.3.0 @@ -909,16 +906,13 @@ find in case extra indexes are specified. ), "pyproject_toml": attr.label( mandatory = False, - doc = """ -Label pointing to a `pyproject.toml` file used to read the Python version and -to select exposed packages via {obj}`pip.configure`. - -When `python_version` is not set explicitly, the `[project] requires-python` -field is used as the `python_version` for this `pip.parse()` call. + doc = """\ +Label pointing to a pyproject.toml file to read the Python version from. +When specified, the `requires-python` field is used as the `python_version` +for this `pip.parse()` call, unless `python_version` is set explicitly. :::{note} -When used to derive `python_version`, `requires-python` must be specified as -`==X.Y.Z` (exact version with full semver). +The version must be specified as `==X.Y.Z` (exact version with full semver). ::: :::{versionadded} 2.3.0 From 1291919372fc4283939655250438ee646f9e6ba3 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 18 Nov 2026 20:20:05 -0800 Subject: [PATCH 10/15] refactor(pypi): short-circuit _fail paths with None in group selection When a non-aborting `_fail` callback is injected in tests, `select_dependency_groups` and `_expand_dependency_groups` previously continued executing after `_fail(...)`, causing `hub_exposure` to fail on `None` attribute access or return partial results instead of `None`. Return `None` immediately after each `_fail(...)` call in `dependency_groups.bzl`, guard `selection == None` in `hub_exposure.bzl`, and assert `None` return values in error tests. --- python/private/pypi/dependency_groups.bzl | 14 ++++++++++---- python/private/pypi/hub_exposure.bzl | 2 ++ .../dependency_groups/dependency_groups_tests.bzl | 6 ++++-- tests/pypi/hub_exposure/hub_exposure_tests.bzl | 14 ++++++++++++-- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index e40e67a4c3..511041417d 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -25,16 +25,19 @@ def select_dependency_groups(pyproject, specs, _fail = fail): _fail: {type}`callable` the failure function, mainly for testing. Returns: - {type}`struct` with the attributes: + {type}`struct | None` with the attributes: * `packages`: {type}`list[str]` the sorted normalized package names. * `all`: {type}`bool` whether all of the packages including the transitive dependencies should be selected, i.e. `"*"` and `"transitive"` were given. + Returns `None` on failure. """ groups = {"default": pyproject.dependencies} for name, reqs in pyproject.optional_dependencies.items(): groups["optional:" + normalize_name(name)] = reqs expanded = _expand_dependency_groups(pyproject.dependency_groups, _fail) + if expanded == None: + return None for name, reqs in expanded.items(): groups["group:" + name] = reqs @@ -43,6 +46,7 @@ def select_dependency_groups(pyproject, specs, _fail = fail): _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format( specs, )) + return None return struct( # list[str]: sorted normalized package names selected by `specs`. packages = [], @@ -58,6 +62,7 @@ def select_dependency_groups(pyproject, specs, _fail = fail): spec, ", ".join(sorted(groups)), )) + return None for key in keys: selected[key] = None @@ -122,8 +127,8 @@ def _expand_dependency_groups(dependency_groups, _fail): _fail: {type}`callable` the failure function. Returns: - {type}`dict[str, list[str]]` normalized group names mapped to their - requirement strings. + {type}`dict[str, list[str]] | None` normalized group names mapped to + their requirement strings, or `None` on failure. """ groups = { normalize_name(name): items @@ -149,7 +154,7 @@ def _expand_dependency_groups(dependency_groups, _fail): include, name, )) - return {} + return None if include not in expanded: reqs = None break @@ -163,4 +168,5 @@ def _expand_dependency_groups(dependency_groups, _fail): _fail("Cyclic dependency group includes: {}".format( ", ".join(sorted(unexpanded)), )) + return None return expanded diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index ff9f24694d..ed103fd2d6 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -66,6 +66,8 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): tag.dependency_groups, _fail = _fail, ) + if selection == None: + return None if selection.all: exposes_all[hub] = None diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl index d494cae9d7..208b30639f 100644 --- a/tests/pypi/dependency_groups/dependency_groups_tests.bzl +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -80,7 +80,8 @@ def _test_select_errors(env): "The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: [\"default\", \"transitive\"]", ), ]: - _, failures = _select(specs) + got, failures = _select(specs) + env.expect.where(specs = specs).that_str(got).equals(None) env.expect.where(specs = specs).that_collection( failures, ).contains_exactly([want]) @@ -98,12 +99,13 @@ def _test_include_group_errors(env): "Cyclic dependency group includes: a, b", ), ]: - _, failures = _select(["*"], pyproject = struct( + got, failures = _select(["a"], pyproject = struct( name = None, dependencies = [], optional_dependencies = {}, dependency_groups = groups, )) + env.expect.that_str(got).equals(None) env.expect.that_collection(failures).contains_exactly([want]) _tests.append(_test_include_group_errors) diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index a51deb9c56..2713b9355b 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -165,21 +165,31 @@ def _test_configure_module_precedence(env): _tests.append(_test_configure_module_precedence) def _test_configure_errors(env): - _, failures = _exposure( + got, failures = _exposure( _module(configure = [_configure(hub_name = "missing")]), ) + env.expect.that_str(got).equals(None) env.expect.that_collection(failures).contains_exactly([ "pip.configure() refers to unknown hub 'missing'. Available hubs: hub", ]) - _, failures = _exposure( + got, failures = _exposure( _module(configure = [_configure()]), hubs = {"hub": _hub()}, ) + env.expect.that_str(got).equals(None) env.expect.that_collection(failures).contains_exactly([ "pip.configure() for hub 'hub' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`", ]) + got, failures = _exposure( + _module(configure = [_configure(dependency_groups = ["missing"])]), + ) + env.expect.that_str(got).equals(None) + env.expect.that_collection(failures).contains_exactly([ + "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test", + ]) + _tests.append(_test_configure_errors) def hub_exposure_test_suite(name): From 28b48d5ba2771d3c019e03bcffbd3a0993364162 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 18 Nov 2026 20:30:56 -0800 Subject: [PATCH 11/15] refactor(pypi): expand nested self-extras and refine type comments Single-level self-reference expansion dropped nested extras from composite extras such as `my-project[all]` and added the project name itself to `packages`, and `dependency_groups` type comments omitted list element types. Expand self-referenced optional groups iteratively with a bounded queue in `dependency_groups.bzl` (with unit tests covering nested, bare, and cyclic self-references), and specify parameterized element types in `pyproject_utils.bzl`, `dependency_groups.bzl`, and `render_pkg_aliases.bzl`. --- python/private/pypi/dependency_groups.bzl | 19 +++++++----- python/private/pypi/render_pkg_aliases.bzl | 3 +- python/private/pyproject_utils.bzl | 12 ++++--- .../dependency_groups_tests.bzl | 31 +++++++++++++++++++ 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index 511041417d..c3b435a956 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -67,18 +67,23 @@ def select_dependency_groups(pyproject, specs, _fail = fail): selected[key] = None # A self reference like `my-project[test]` selects the optional - # dependencies, but only one level deep. + # dependencies, including nested composite extras. project = normalize_name(pyproject.name or "") packages = {} - for key in selected: - for req in groups[key]: + queue = list(selected) + for _ in range(len(groups)): + if not queue: + break + for req in groups[queue.pop(0)]: req = requirement(req) if req.name_ != project: packages[req.name_] = None continue for extra in req.extras: - for r in groups.get("optional:" + extra, []): - packages[requirement(r).name_] = None + opt_key = "optional:" + extra + if opt_key in groups and opt_key not in selected: + selected[opt_key] = None + queue.append(opt_key) return struct( # list[str]: sorted normalized package names selected by `specs`. @@ -122,8 +127,8 @@ def _expand_dependency_groups(dependency_groups, _fail): """Expand the PEP 735 `include-group` entries. Args: - dependency_groups: {type}`dict[str, list]` the `[dependency-groups]` - table. + dependency_groups: {type}`dict[str, list[str | dict[str, str]]]` the + `[dependency-groups]` table. _fail: {type}`callable` the failure function. Returns: diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index ba0ad2cc75..f03c33355e 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -171,7 +171,8 @@ def _internal_visibility(aliases, has_groups = False): have to be visible to them. Args: - aliases: {type}`dict[str, str | dict]` package-to-repo alias mapping. + aliases: {type}`dict[str, str | dict[struct, str]]` package-to-repo + alias mapping. has_groups: {type}`bool` whether requirement cycle groups exist. Returns: diff --git a/python/private/pyproject_utils.bzl b/python/private/pyproject_utils.bzl index ceb5c803aa..c65157df83 100644 --- a/python/private/pyproject_utils.bzl +++ b/python/private/pyproject_utils.bzl @@ -22,11 +22,12 @@ def read_pyproject(module_ctx, pyproject): * `dependencies`: {type}`list[str]` the `[project] dependencies`. * `optional_dependencies`: {type}`dict[str, list[str]]` the `[project.optional-dependencies]` table. - * `dependency_groups`: {type}`dict[str, list]` the PEP 735 - `[dependency-groups]` table. Items are either requirement strings - or `{include-group = ""}` tables. + * `dependency_groups`: {type}`dict[str, list[str | dict[str, str]]]` + the PEP 735 `[dependency-groups]` table. Items are either + requirement strings or `{include-group = ""}` tables. """ - data = toml.decode(module_ctx.read(module_ctx.path(pyproject), watch = "yes")) + raw = module_ctx.read(module_ctx.path(pyproject), watch = "yes") + data = toml.decode(raw) project = data.get("project", {}) return struct( # str | None: raw `[project] requires-python` value. @@ -37,7 +38,8 @@ def read_pyproject(module_ctx, pyproject): dependencies = project.get("dependencies") or [], # dict[str, list[str]]: `[project.optional-dependencies]` table. optional_dependencies = project.get("optional-dependencies") or {}, - # dict[str, list]: PEP 735 `[dependency-groups]` table. + # dict[str, list[str | dict[str, str]]]: PEP 735 `[dependency-groups]` + # table of requirement strings and `{"include-group": str}` dicts. dependency_groups = data.get("dependency-groups") or {}, ) diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl index 208b30639f..ab56109c7c 100644 --- a/tests/pypi/dependency_groups/dependency_groups_tests.bzl +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -61,6 +61,37 @@ def _test_select(env): _tests.append(_test_select) +def _test_self_reference_extras(env): + pyproject = struct( + name = "my-project", + dependencies = [], + optional_dependencies = { + "a": ["a-pkg", "my-project[b]"], + "all": ["My_Project[docs, grpc]"], + "b": ["b-pkg", "my-project[a]"], + "docs": ["docs-pkg"], + "everything": ["my-project[all]", "my-project"], + "grpc": ["grpc-pkg"], + }, + dependency_groups = { + "ci": ["ci-pkg", "my-project[all]"], + }, + ) + for specs, want in [ + (["everything"], ["docs_pkg", "grpc_pkg"]), + (["ci"], ["ci_pkg", "docs_pkg", "grpc_pkg"]), + (["a"], ["a_pkg", "b_pkg"]), + ]: + got, failures = _select(specs, pyproject = pyproject) + env.expect.where(specs = specs).that_collection( + failures, + ).contains_exactly([]) + env.expect.where(specs = specs).that_collection( + got.packages, + ).contains_exactly(want).in_order() + +_tests.append(_test_self_reference_extras) + def _test_select_all(env): got, failures = _select(["*", "transitive"]) env.expect.that_collection(failures).contains_exactly([]) From 0f35bd1b060054334ec1b1c0ff0c9eb000b21f92 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:32:52 +0900 Subject: [PATCH 12/15] feat(pypi): make pip.configure a builder that does not require a pyproject.toml The dependency_groups default is now empty, so a pip.configure tag only needs a pyproject.toml when it selects dependency groups. A pyproject_toml set on a tag is used by the later tags of the same module and hub. --- docs/pypi/download.md | 36 ++++++++--- python/private/pypi/extension.bzl | 23 +++++-- python/private/pypi/hub_exposure.bzl | 17 +++++- tests/pypi/extension/pip_parse.bzl | 2 +- .../pypi/hub_exposure/hub_exposure_tests.bzl | 61 +++++++++++++++++-- 5 files changed, 116 insertions(+), 23 deletions(-) diff --git a/docs/pypi/download.md b/docs/pypi/download.md index ef069da5b5..931b64cf57 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -212,15 +212,33 @@ dependencies of `numpy` are only used through `@pypi_hub//numpy`. The `all_requirements` lists in the hub's `requirements.bzl` and the [Unified @pypi Hub](unified-pypi-hub) only contain the exposed packages. -The {attr}`pip.configure.dependency_groups` attribute defaults to `["*"]`, -which selects every group. Use `["*", "transitive"]` to expose all of the -packages of the hub, including the transitive dependencies, which is the same -as not configuring the hub. Selecting the transitive dependencies of only some -of the groups is not supported. - -Multiple tags for the same hub are combined. Only the root module and the -module that created the hub can configure it; if both do, the root module -wins. A self reference such as `my-project[test]` in the `pyproject.toml` +Use `["*"]` to select every group, or `["*", "transitive"]` to expose all of +the packages of the hub, including the transitive dependencies, which is the +same as not configuring the hub. Selecting the transitive dependencies of only +some of the groups is not supported. A tag without +{attr}`pip.configure.dependency_groups` does not limit the hub and does not +need a `pyproject.toml`. + +The `pip.configure` tags work like a builder: they are applied in order and +the tags for the same hub are combined. A `pyproject_toml` set on a tag is used +by the later tags of the same module and hub, so it only needs to be set once: + +```starlark +pip.configure( + hub_name = "pypi_hub", + pyproject_toml = "//tools:pyproject.toml", + dependency_groups = ["default"], +) + +# Uses //tools:pyproject.toml +pip.configure( + hub_name = "pypi_hub", + dependency_groups = ["group:dev"], +) +``` + +Only the root module and the module that created the hub can configure it; if +both do, the root module wins. A self reference such as `my-project[test]` in the `pyproject.toml` selects the optional dependencies of the project. The packages that are not part of the hub are ignored, so a single `pyproject.toml` can list tools that are missing from the lock file. diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 7814246e62..38ec1d0e09 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -1123,9 +1123,11 @@ execution-phase error when built. _configure_tag = tag_class( attrs = { "dependency_groups": attr.string_list( - default = ["*"], + default = [], doc = """ -The `pyproject.toml` dependency groups whose packages the hub exposes. +The `pyproject.toml` dependency groups whose packages the hub exposes. The +default, `[]`, does not select any packages, so the tag does not limit the hub +and it does not need a `pyproject.toml`. Each item is one of: @@ -1158,9 +1160,12 @@ The name of the hub to configure. "pyproject_toml": attr.label( mandatory = False, doc = """ -The `pyproject.toml` file that defines the dependency groups. Defaults to the -{attr}`pip.parse.pyproject_toml` of the hub, then to -{attr}`pip.default.pyproject_toml` if the hub is defined by the root module. +The `pyproject.toml` file that defines the dependency groups. It is also used +by the later `pip.configure` tags of the same module and hub that don't set it. +Defaults to the last `pyproject_toml` set by an earlier `pip.configure` tag of +the same module and hub, then to the {attr}`pip.parse.pyproject_toml` of the +hub, then to {attr}`pip.default.pyproject_toml` if the hub is defined by the +root module. It is only needed when {attr}`dependency_groups` is set. :::{versionadded} VERSION_NEXT_FEATURE ::: @@ -1176,8 +1181,14 @@ The packages are selected from a `pyproject.toml` file using the transitive dependencies, are still used by the packages that need them, but they are not public. +The tags work like a builder: they are applied in order and multiple tags for +the same hub are combined, so each tag can configure a different part of the +hub. A `pyproject_toml` set on a tag is used by the later tags of the same +module and hub. A tag that does not select any {attr}`dependency_groups` does +not limit the hub. + Only the root module and the module that created the hub can configure it. If -both do, the root module wins. Multiple tags for the same hub are combined. +both do, the root module wins. See {ref}`Configuring the Exposed Packages ` for examples. diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index ed103fd2d6..7393c28fbe 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -30,11 +30,15 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): root_configured = {} for mod in root_modules: for tag in mod.tags.configure: - root_configured[tag.hub_name] = None + if tag.dependency_groups: + root_configured[tag.hub_name] = None configured = {} exposes_all = {} for mod in module_ctx.modules: + # The tags of a module are applied in order like a builder: a tag + # without a `pyproject_toml` uses the last one set for the hub. + last_pyproject = {} for tag in mod.tags.configure: hub = tag.hub_name if hub not in hubs: @@ -52,12 +56,19 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): if not mod.is_root and hub in root_configured: continue - pyproject = tag.pyproject_toml or info.pyproject + if tag.pyproject_toml: + last_pyproject[hub] = tag.pyproject_toml + if not tag.dependency_groups: + # The tag does not select any packages. + continue + + pyproject = last_pyproject.get(hub) or info.pyproject if not pyproject and info.module_name == root_name: pyproject = default_pyproject if not pyproject: - _fail("pip.configure() for hub '{}' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`".format( + _fail("pip.configure() for hub '{}' selects the dependency groups {}, which need a pyproject.toml, set `pyproject_toml` on this or an earlier `pip.configure` or on `pip.parse`".format( hub, + tag.dependency_groups, )) return None diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index db3008d431..d048f99bd0 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -2,7 +2,7 @@ def configure( *, - dependency_groups = ["*"], + dependency_groups = [], hub_name, pyproject_toml = None): """Simulate the PyPI extension configure tag class for testing.""" diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index 2713b9355b..1f2b13622c 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -27,7 +27,7 @@ def _hub(*, module_name = "root", pyproject = None): def _configure( hub_name = "hub", - dependency_groups = ["*"], + dependency_groups = [], pyproject_toml = None): return mocks.tag( hub_name = hub_name, @@ -108,7 +108,7 @@ _tests.append(_test_configure_combines_tags) def _test_configure_pyproject_precedence(env): # The tag wins over the `pip.parse` value. got, _ = _exposure(_module(configure = [ - _configure(pyproject_toml = "other.toml"), + _configure(pyproject_toml = "other.toml", dependency_groups = ["*"]), ])) env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) @@ -123,6 +123,59 @@ def _test_configure_pyproject_precedence(env): _tests.append(_test_configure_pyproject_precedence) +def _test_configure_without_pyproject(env): + # A tag without the dependency groups does not need a pyproject.toml and + # does not limit the hub. + got, failures = _exposure( + _module(configure = [_configure()]), + hubs = {"hub": _hub()}, + ) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_configure_without_pyproject) + +def _test_configure_builder(env): + # The `pyproject_toml` of a tag is used by the later tags of the hub. + got, failures = _exposure( + _module(configure = [ + _configure(pyproject_toml = "other.toml"), + _configure(dependency_groups = ["default"]), + ]), + hubs = {"hub": _hub(pyproject = "pyproject.toml")}, + ) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + + # The tags before it use the `pip.parse` value. + got, failures = _exposure( + _module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(pyproject_toml = "other.toml", dependency_groups = ["default"]), + ]), + hubs = {"hub": _hub(pyproject = "pyproject.toml")}, + ) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg", "simple"]}) + + # It is not used by the other modules. + hubs = {"hub": _hub(module_name = "dep")} + got, failures = _exposure( + _module(configure = [_configure(hub_name = "hub", pyproject_toml = "other.toml")]), + _module( + name = "dep", + is_root = False, + configure = [_configure(hub_name = "hub", dependency_groups = ["default"])], + ), + hubs = hubs, + ) + env.expect.that_str(got).equals(None) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() for hub 'hub' selects the dependency groups [\"default\"], which need a pyproject.toml, set `pyproject_toml` on this or an earlier `pip.configure` or on `pip.parse`", + ]) + +_tests.append(_test_configure_builder) + def _test_configure_module_precedence(env): hubs = {"hub": _hub(module_name = "dep", pyproject = "pyproject.toml")} @@ -174,12 +227,12 @@ def _test_configure_errors(env): ]) got, failures = _exposure( - _module(configure = [_configure()]), + _module(configure = [_configure(dependency_groups = ["*"])]), hubs = {"hub": _hub()}, ) env.expect.that_str(got).equals(None) env.expect.that_collection(failures).contains_exactly([ - "pip.configure() for hub 'hub' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`", + "pip.configure() for hub 'hub' selects the dependency groups [\"*\"], which need a pyproject.toml, set `pyproject_toml` on this or an earlier `pip.configure` or on `pip.parse`", ]) got, failures = _exposure( From 7c58d7b20a47efb5a0f33d9db412d439ad97191d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:36:34 +0900 Subject: [PATCH 13/15] docs(pypi): rewrap the pip.configure docs --- docs/pypi/download.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 931b64cf57..ad8b40e70d 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -238,10 +238,10 @@ pip.configure( ``` Only the root module and the module that created the hub can configure it; if -both do, the root module wins. A self reference such as `my-project[test]` in the `pyproject.toml` -selects the optional dependencies of the project. The packages that are not -part of the hub are ignored, so a single `pyproject.toml` can list tools that -are missing from the lock file. +both do, the root module wins. A self reference such as `my-project[test]` in +the `pyproject.toml` selects the optional dependencies of the project. The +packages that are not part of the hub are ignored, so a single `pyproject.toml` +can list tools that are missing from the lock file. :::{versionadded} VERSION_NEXT_FEATURE ::: From a49b6c8ee942a15dbe0cf631ea473ebd4c95cfa4 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 21 Nov 2026 00:35:41 +0900 Subject: [PATCH 14/15] fix(pypi): keep the platform specific packages out of all_requirements The pip.configure selection replaced the packages that the hub lists in the all_*requirements, so a selected package that has no wheel on some of the platforms broke the builds there. Only list the selected packages that are available on all of the platforms and keep all of the selected packages public. --- python/private/pypi/extension.bzl | 13 ++++++++- python/private/pypi/hub_repository.bzl | 14 +++++++-- tests/pypi/extension/extension_tests.bzl | 37 ++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 38ec1d0e09..753e34e541 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -386,7 +386,17 @@ You cannot use both the additive_build_content and additive_build_content_file a ) if configured_hubs == None: return None - exposed_packages.update(configured_hubs) + + # The `all_*requirements` only list the selected packages that are + # available on all of the platforms, while all of the selected packages + # stay public. + for hub_name, packages in configured_hubs.items(): + selected = {name: None for name in packages} + exposed_packages[hub_name] = [ + name + for name in exposed_packages[hub_name] + if name in selected + ] return struct( # dict[str, list[str]]: configured hubs and exposed packages. @@ -552,6 +562,7 @@ def _pip_impl(module_ctx): groups = mods.hub_group_map.get(hub_name), # The hubs configured with `pip.configure` hide the other packages. hide_unexposed_packages = hub_name in mods.configured_hubs, + public_packages = mods.configured_hubs.get(hub_name, []), ) _create_unified_hub_repo(mods) diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 99f659ba31..c401168873 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -28,7 +28,7 @@ exports_files(["requirements.bzl"]) def _impl(rctx): if rctx.attr.hide_unexposed_packages: bzl_packages = rctx.attr.packages - exposed_packages = bzl_packages + exposed_packages = rctx.attr.public_packages else: bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() exposed_packages = None @@ -87,8 +87,8 @@ hub_repository = repository_rule( "hide_unexposed_packages": attr.bool( default = False, doc = """ -Make the aliases of the packages that are not in `packages` visible only to -the wheel repositories. +Make the aliases of the packages that are not in `public_packages` visible only +to the wheel repositories. """, ), "packages": attr.string_list( @@ -102,6 +102,14 @@ Defaults to whl_map keys unless `hide_unexposed_packages` is set. doc = "The constraint values for each platform name. The values are string canonical string Label representations", mandatory = False, ), + "public_packages": attr.string_list( + mandatory = False, + doc = """ +The packages whose aliases are public when `hide_unexposed_packages` is set. +Unlike `packages`, it includes the packages that are not available on all of +the platforms. +""", + ), "repo_name": attr.string( mandatory = True, doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index b3a6e7837c..f1974cf88c 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -896,6 +896,43 @@ def _test_configure(env): _tests.append(_test_configure) +def _test_configure_platform_specific(env): + # A selected package that is not available on all of the platforms is + # public, but it is not in the `all_*requirements`. + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + _mod( + name = "rules_python", + parse = [_configure_parse( + pyproject_toml = "pyproject.toml", + target_platforms = ["linux_x86_64", "windows_aarch64"], + )], + configure = [ + _configure(hub_name = "pypi_a", dependency_groups = ["default", "dev"]), + ], + ), + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "pyproject.toml": _CONFIGURE_PYPROJECT, + "requirements.txt": _CONFIGURE_REQUIREMENTS.replace( + "other-pkg==0.0.2", + "other-pkg==0.0.2 ; sys_platform == 'win32'", + ), + }, + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + + pypi.exposed_packages().contains_exactly({"pypi_a": ["simple"]}) + pypi.configured_hubs().contains_exactly({"pypi_a": ["other_pkg", "simple"]}) + +_tests.append(_test_configure_platform_specific) + def _test_configure_default_pyproject(env): # The `pip.default` pyproject.toml is used for the hubs of the root module. pypi = _configure_parse_modules( From c29740e819e1e088d9c9e8c71e4f5b6178df3287 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 21 Nov 2026 00:35:56 +0900 Subject: [PATCH 15/15] fix(pypi): select nothing with a wildcard for an empty pyproject.toml table --- python/private/pypi/dependency_groups.bzl | 4 +++- .../dependency_groups_tests.bzl | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index c3b435a956..13b164c2b4 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -57,7 +57,9 @@ def select_dependency_groups(pyproject, specs, _fail = fail): selected = {} for spec in specs: keys = _match(spec, groups) - if not keys: + + # A wildcard matches nothing when its table is empty or missing. + if not keys and spec not in ("*", "optional:*", "group:*"): _fail("Unknown dependency group '{}'. Available groups: {}".format( spec, ", ".join(sorted(groups)), diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl index ab56109c7c..a53905bb04 100644 --- a/tests/pypi/dependency_groups/dependency_groups_tests.bzl +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -100,6 +100,24 @@ def _test_select_all(env): _tests.append(_test_select_all) +def _test_select_empty_wildcards(env): + pyproject = struct( + name = "my-project", + dependencies = ["main-pkg"], + optional_dependencies = {}, + dependency_groups = {}, + ) + for specs, want in [ + (["optional:*"], []), + (["group:*"], []), + (["*"], ["main_pkg"]), + ]: + got, failures = _select(specs, pyproject = pyproject) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) + env.expect.where(specs = specs).that_collection(got.packages).contains_exactly(want) + +_tests.append(_test_select_empty_wildcards) + def _test_select_errors(env): for specs, want in [ (