Implement a --group flag for dependency-group parsing for pip-compile

piptools/dependency_groups.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import sys
+from typing import Any, Iterable
+
+import click
+from dependency_groups import DependencyGroupResolver
+from pip._internal.req import InstallRequirement
+from pip._vendor.packaging.requirements import Requirement
+
+from .utils import ParsedDependencyGroupParam
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+
+
+def parse_dependency_groups(
+    params: tuple[ParsedDependencyGroupParam, ...],
+) -> list[InstallRequirement]:
+    resolvers = _build_resolvers(param.path for param in params)
+    reqs: list[InstallRequirement] = []
+    for param in params:
+        resolver = resolvers[param.path]
+        try:
+            reqs.extend(
+                InstallRequirement(
+                    Requirement(str(req)),
+                    comes_from=f"--group '{param}'",
+                )
+                for req in resolver.resolve(param.group)
+            )
+        except (ValueError, TypeError, LookupError) as e:
+            raise click.UsageError(
+                f"[dependency-groups] resolution failed for '{param.group}' "
+                f"from '{param.path}': {e}"
+            ) from e
+    return reqs
+
+
+def _build_resolvers(paths: Iterable[str]) -> dict[str, Any]:
+    resolvers = {}
+    for path in paths:
+        if path in resolvers:
+            continue
+
+        pyproject = _load_pyproject(path)
+        if "dependency-groups" not in pyproject:
+            raise click.UsageError(
+                f"[dependency-groups] table was missing from '{path}'. "
+                "Cannot resolve '--group' option."
+            )
+        raw_dependency_groups = pyproject["dependency-groups"]
+        if not isinstance(raw_dependency_groups, dict):
+            raise click.UsageError(
+                f"[dependency-groups] table was malformed in {path}. "
+                "Cannot resolve '--group' option."
+            )
+
+        resolvers[path] = DependencyGroupResolver(raw_dependency_groups)
+    return resolvers
+
+
+def _load_pyproject(path: str) -> dict[str, Any]:
+    try:
+        with open(path, "rb") as fp:
+            return tomllib.load(fp)
+    except FileNotFoundError:
+        raise click.UsageError(f"{path} not found. Cannot resolve '--group' option.")
+    except tomllib.TOMLDecodeError as e:
+        raise click.UsageError(f"Error parsing {path}: {e}") from e
+    except OSError as e:
+        raise click.UsageError(f"Error reading {path}: {e}") from e

piptools/scripts/compile.py

@@ -17,12 +17,14 @@
 from .._compat import parse_requirements
 from ..build import ProjectMetadata, build_project_metadata
 from ..cache import DependencyCache
+from ..dependency_groups import parse_dependency_groups
 from ..exceptions import NoCandidateFound, PipToolsError
 from ..logging import log
 from ..repositories import LocalRequirementsRepository, PyPIRepository
 from ..repositories.base import BaseRepository
 from ..resolver import BacktrackingResolver, LegacyResolver
 from ..utils import (
+    ParsedDependencyGroupParam,
     dedup,
     drop_extras,
     install_req_from_line,
@@ -109,6 +111,7 @@ def _determine_linesep(
 @options.reuse_hashes
 @options.max_rounds
 @options.src_files
+@options.group
 @options.build_isolation
 @options.emit_find_links
 @options.cache_dir
@@ -153,6 +156,7 @@ def cli(
     generate_hashes: bool,
     reuse_hashes: bool,
     src_files: tuple[str, ...],
+    groups: tuple[ParsedDependencyGroupParam, ...],
     max_rounds: int,
     build_isolation: bool,
     emit_find_links: bool,
@@ -201,7 +205,7 @@ def cli(
             "--only-build-deps cannot be used with any of --extra, --all-extras"
         )
 
-    if len(src_files) == 0:
+    if len(src_files) == 0 and len(groups) == 0:
         for file_path in DEFAULT_REQUIREMENTS_FILES:
             if os.path.exists(file_path):
                 src_files = (file_path,)
@@ -227,7 +231,7 @@ def cli(
                 os.path.dirname(src_files[0]), DEFAULT_REQUIREMENTS_OUTPUT_FILE
             )
         # An output file must be provided if there are multiple source files
-        elif len(src_files) > 1:
+        elif len(src_files) + len(groups) > 1:
             raise click.BadParameter(
                 "--output-file is required if two or more input files are given."
             )
@@ -400,6 +404,9 @@ def cli(
                 )
             )
 
+    # Parse `--group` dependency-groups and add them to constraints
+    constraints.extend(parse_dependency_groups(groups))
+
     # Parse all constraints from `--constraint` files
     for filename in constraint:
         constraints.extend(

piptools/scripts/options.py

@@ -7,7 +7,11 @@
 from pip._internal.utils.misc import redact_auth_from_url
 
 from piptools.locations import CACHE_DIR, DEFAULT_CONFIG_FILE_NAMES
-from piptools.utils import UNSAFE_PACKAGES, override_defaults_from_config_file
+from piptools.utils import (
+    UNSAFE_PACKAGES,
+    DependencyGroupParamType,
+    override_defaults_from_config_file,
+)
 
 BuildTargetT = Literal["sdist", "wheel", "editable"]
 ALL_BUILD_TARGETS: tuple[BuildTargetT, ...] = (
@@ -248,6 +252,18 @@ def _get_default_option(option_name: str) -> Any:
     type=click.Path(exists=True, allow_dash=True),
 )
 
+group = click.option(
+    "--group",
+    "groups",
+    type=DependencyGroupParamType(),
+    multiple=True,
+    help=(
+        'Specify a named dependency-group from a "pyproject.toml" file. '
+        'If a path is given, the name of the file must be "pyproject.toml". '
+        'Defaults to using "pyproject.toml" in the current directory.'
+    ),
+)
+
 build_isolation = click.option(
     "--build-isolation/--no-build-isolation",
     is_flag=True,

piptools/utils.py

@@ -6,20 +6,21 @@
 import itertools
 import json
 import os
+import pathlib
 import re
 import shlex
 import sys
 from pathlib import Path
 from typing import Any, Callable, Iterable, Iterator, TypeVar, cast
 
+import click
 from click.core import ParameterSource
 
 if sys.version_info >= (3, 11):
     import tomllib
 else:
     import tomli as tomllib
 
-import click
 import pip
 from click.utils import LazyFile
 from pip._internal.req import InstallRequirement
@@ -415,6 +416,10 @@ def get_compile_command(click_ctx: click.Context) -> str:
             value = [value]
 
         for val in value:
+            # use the input value for --group params
+            if isinstance(val, ParsedDependencyGroupParam):
+                val = val.input_arg
+
             # Flags don't have a value, thus add to args true or false option long name
             if option.is_flag:
                 # If there are false-options, choose an option name depending on a value
@@ -768,3 +773,55 @@ def is_path_relative_to(path1: Path, path2: Path) -> bool:
     except ValueError:
         return False
     return True
+
+
+class DependencyGroupParamType(click.ParamType):
+    def get_metavar(self, param: click.Parameter) -> str:
+        return "[pyproject-path::]groupname"
+
+    def convert(
+        self, value: str, param: click.Parameter | None, ctx: click.Context | None
+    ) -> ParsedDependencyGroupParam:
+        """Parse a ``[dependency-groups]`` group reference."""
+        return ParsedDependencyGroupParam(value)
+
+
+class ParsedDependencyGroupParam:
+    """
+    Parse a dependency group input, but retain the input value.
+
+    Splits on the rightmost ":", and validates that the path (if present) ends
+    in ``pyproject.toml``. Defaults the path to ``pyproject.toml`` when one is not given.
+
+    ``:`` cannot appear in dependency group names, so this is a safe and simple parse.
+
+    If the path portion ends in ":", then the ":" is removed, effectively resulting in
+    a split on "::" when that is used.
+
+    The following conversions are expected::
+
+        'foo' -> ('pyproject.toml', 'foo')
+        'foo/pyproject.toml:bar' -> ('foo/pyproject.toml', 'bar')
+        'foo/pyproject.toml::bar' -> ('foo/pyproject.toml', 'bar')
+    """
+
+    def __init__(self, value: str) -> None:
+        self.input_arg = value
+
+        path, sep, groupname = value.rpartition(":")
+        if not sep:
+            path = "pyproject.toml"
+        else:
+            # strip a rightmost ":" if one was present
+            if path.endswith(":"):
+                path = path[:-1]
+            # check for 'pyproject.toml' filenames using pathlib
+            if pathlib.PurePath(path).name != "pyproject.toml":
+                msg = "group paths use 'pyproject.toml' filenames"
+                raise click.UsageError(msg)
+
+        self.path = path
+        self.group = groupname
+
+    def __str__(self) -> str:
+        return self.input_arg

pyproject.toml

@@ -39,6 +39,7 @@ dependencies = [
   "build >= 1.0.0",
   "click >= 8",
   "pip >= 22.2",
+  "dependency-groups >= 1.3.0",
   "pyproject_hooks",
   "tomli; python_version < '3.11'",
   # indirect dependencies

tests/test_cli_compile.py

@@ -1458,6 +1458,134 @@ def test_stdin(pip_conf, runner):
     )
 
 
+@pytest.mark.parametrize(
+    "groupspec",
+    ("mygroup", "pyproject.toml:mygroup", "pyproject.toml::mygroup"),
+)
+def test_dependency_group_resolution(pip_conf, runner, tmpdir_cwd, groupspec):
+    """
+    Test compile requirements from pyproject.toml [dependency-groups].
+    """
+    pyproj = tmpdir_cwd / "pyproject.toml"
+    pyproj.write_text(
+        dedent(
+            """\
+            [project]
+            name = "foo"
+            version = "1.0"
+
+            [dependency-groups]
+            mygroup = ["small-fake-a==0.1"]
+            """
+        )
+    )
+
+    out = runner.invoke(
+        cli,
+        [
+            "--group",
+            groupspec,
+            "--output-file",
+            "-",
+            "--quiet",
+            "--no-emit-options",
+            "--no-header",
+        ],
+    )
+    assert out.exit_code == 0, out.stderr
+
+    assert out.stdout == dedent(
+        f"""\
+        small-fake-a==0.1
+            # via --group '{groupspec}'
+        """
+    )
+
+
+@pytest.mark.parametrize(
+    ("pyproject_content", "group_arg", "expect_error"),
+    (
+        pytest.param(None, "mygroup", "pyproject.toml not found", id="missing-file"),
+        pytest.param(
+            """\
+            [project]
+            name = "foo"
+            version = "1.0"
+            """,
+            "mygroup",
+            "[dependency-groups] table was missing from 'pyproject.toml'",
+            id="missing-table",
+        ),
+        pytest.param(
+            # the malformed-toml test case needs to be in a non-cwd directory
+            # so that it properly tests malformed file parsing in this context
+            # if it's in the cwd, pip-compile will attempt to parse `./pyproject.toml` first
+            # (which will fail in a separate location)
+            """\
+            [project
+            name = "foo"
+            version = "1.0"
+            """,
+            "./foo/pyproject.toml:mygroup",
+            "Error parsing ./foo/pyproject.toml",
+            id="malformed-toml",
+        ),
+        pytest.param(
+            """\
+            [project]
+            name = "foo"
+            version = "1.0"
+            [dependency-groups]
+            mygroup = [{include-group = "mygroup"}]
+            """,
+            "mygroup",
+            "[dependency-groups] resolution failed for 'mygroup' from 'pyproject.toml'",
+            id="cyclic",
+        ),
+        pytest.param(
+            """\
+            [project]
+            name = "foo"
+            version = "1.0"
+            [[dependency-groups]]
+            mygroup = "foo"
+            """,
+            "mygroup",
+            (
+                "[dependency-groups] table was malformed in pyproject.toml. "
+                "Cannot resolve '--group' option."
+            ),
+            id="improper-table",
+        ),
+    ),
+)
+def test_dependency_group_resolution_fails_due_to_bad_data(
+    pip_conf, runner, tmpdir_cwd, pyproject_content, group_arg, expect_error
+):
+    if ":" in group_arg:
+        path, _, _ = group_arg.partition(":")
+        (tmpdir_cwd / path).parent.mkdir(parents=True)
+    else:
+        path = "pyproject.toml"
+    pyproj = tmpdir_cwd / path
+    if pyproject_content is not None:
+        pyproj.write_text(dedent(pyproject_content))
+    out = runner.invoke(
+        cli,
+        [
+            "--group",
+            group_arg,
+            "--output-file",
+            "-",
+            "--quiet",
+            "--no-emit-options",
+            "--no-header",
+        ],
+    )
+    assert out.exit_code == 2
+    assert expect_error in out.stderr
+
+
 def test_multiple_input_files_without_output_file(runner):
     """
     The --output-file option is required for multiple requirement input files.