Merge pull request #695 from BrianPugh/n-token

Re-introduce `Paramter.n_tokens` to control number of tokens consumed/provided to custom converters.

Author: BrianPugh

cyclopts/_convert.py

@@ -1,4 +1,5 @@
 import collections.abc
+import inspect
 import json
 import operator
 import re
@@ -525,7 +526,20 @@ def _convert(
 
             def converter_with_token(t_, value):
                 assert cparam.converter
-                return cparam.converter(t_, (value,))
+
+                # Resolve string converters to methods on the type
+                resolved_converter = cparam.converter
+                if isinstance(resolved_converter, str):
+                    resolved_converter = getattr(t_, resolved_converter)
+
+                # Detect bound methods (classmethods/instance methods)
+                # Bound methods already have their first parameter bound
+                if inspect.ismethod(resolved_converter):
+                    # Call with just tokens - cls/self already bound
+                    return resolved_converter((value,))
+                else:
+                    # Regular function - pass type and tokens
+                    return resolved_converter(t_, (value,))
 
             converter = converter_with_token
 
@@ -813,13 +827,16 @@ def name_transform(s: str) -> str:
             raise NotImplementedError("Unreachable?")
 
 
-def token_count(type_: Any) -> tuple[int, bool]:
+def token_count(type_: Any, skip_converter_params: bool = False) -> tuple[int, bool]:
     """The number of tokens after a keyword the parameter should consume.
 
     Parameters
     ----------
     type_: Type
         A type hint/annotation to infer token_count from if not explicitly specified.
+    skip_converter_params: bool
+        If True, don't extract converter parameters from __cyclopts__.
+        Used to prevent infinite recursion when determining consume_all behavior.
 
     Returns
     -------
@@ -829,7 +846,30 @@ def token_count(type_: Any) -> tuple[int, bool]:
         If this is ``True`` and positional, consume all remaining tokens.
         The returned number of tokens constitutes a single element of the iterable-to-be-parsed.
     """
-    type_ = resolve(type_)
+    # Check for explicit n_tokens in Parameter annotation before resolving
+    # This handles nested cases like tuple[Annotated[str, Parameter(n_tokens=2)], int]
+    from cyclopts.parameter import get_parameters
+
+    resolved_type, parameters = get_parameters(type_, skip_converter_params=skip_converter_params)
+    for param in parameters:
+        if param.n_tokens is not None:
+            if param.n_tokens == -1:
+                return 1, True
+            else:
+                # Recursively determine consume_all from the type's natural structure.
+                # Only recurse if the type has changed (e.g., Annotated wrapper was removed).
+                # If resolved_type is the same as type_, recursing would cause infinite loop.
+                if resolved_type is not type_:
+                    # Skip converter params to avoid infinite recursion when converter is decorated
+                    # with @Parameter(n_tokens=...) and attached to a class via @Parameter(converter=...).
+                    _, consume_all_from_type = token_count(resolved_type, skip_converter_params=True)
+                else:
+                    # Type didn't change (e.g., class decorated with @Parameter(n_tokens=...))
+                    # Can't determine natural consume_all by recursing on same type
+                    consume_all_from_type = False
+                return param.n_tokens, consume_all_from_type
+
+    type_ = resolved_type
     origin_type = get_origin(type_)
     # Normalize abstract origin types to concrete types early
     if origin_type in _abstract_to_concrete_type_mapping:

cyclopts/argument/_argument.py

@@ -1,5 +1,6 @@
 """Argument class and related functionality."""
 
+import inspect
 import json
 import operator
 import sys
@@ -556,10 +557,16 @@ def _convert(self, converter: Callable | None = None):
         from cyclopts.argument._collection import update_argument_collection
 
         if self.parameter.converter:
-            converter = self.parameter.converter
+            # Resolve string converters to methods on the type
+            if isinstance(self.parameter.converter, str):
+                converter = getattr(self.hint, self.parameter.converter)
+            else:
+                converter = self.parameter.converter
         elif converter is None:
             converter = partial(convert, name_transform=self.parameter.name_transform)
 
+        assert converter is not None  # Ensure converter is set at this point
+
         def safe_converter(hint, tokens):
             if isinstance(tokens, dict):
                 try:
@@ -568,7 +575,13 @@ def safe_converter(hint, tokens):
                     raise CoercionError(msg=e.args[0] if e.args else None, argument=self, target_type=hint) from e
             else:
                 try:
-                    return converter(hint, tokens)
+                    # Detect bound methods (classmethods/instance methods)
+                    if inspect.ismethod(converter):
+                        # Call with just tokens - cls/self already bound
+                        return converter(tokens)  # pyright: ignore[reportCallIssue]
+                    else:
+                        # Regular function - pass type and tokens
+                        return converter(hint, tokens)  # pyright: ignore[reportCallIssue]
                 except (AssertionError, ValueError, TypeError) as e:
                     token = tokens[0] if len(tokens) == 1 else None
                     raise CoercionError(
@@ -831,6 +844,27 @@ def token_count(self, keys: tuple[str, ...] = ()):
         if self.parameter.count:
             return 0, False
 
+        # Check for explicit n_tokens override
+        # This applies to values at any level: root values (keys=()) or nested values (keys=(...))
+        # For example, **kwargs: Annotated[str, Parameter(n_tokens=2)] means each kwarg value needs 2 tokens
+        if self.parameter.n_tokens is not None:
+            if self.parameter.n_tokens == -1:
+                return 1, True
+            else:
+                # Determine consume_all based on the hint at the requested level
+                # by recursively calling token_count on the hint
+                if len(keys) > 1:
+                    hint = self._default
+                elif len(keys) == 1:
+                    hint = self._type_hint_for_key(keys[0])
+                else:
+                    hint = self.hint
+
+                # Recursively call token_count to get the consume_all behavior
+                # We ignore the token count from the recursive call and use our explicit n_tokens
+                _, consume_all_from_type = token_count(hint)
+                return self.parameter.n_tokens, consume_all_from_type
+
         if len(keys) > 1:
             hint = self._default
         elif len(keys) == 1:

cyclopts/parameter.py

@@ -34,7 +34,6 @@
 )
 from cyclopts.field_info import get_field_infos, signature_parameters
 from cyclopts.group import Group
-from cyclopts.token import Token
 from cyclopts.utils import (
     default_name_transform,
     frozen,
@@ -117,7 +116,8 @@ def main(foo: Annotated[int, Parameter(name="bar")]):
         converter=lambda x: cast(tuple[str, ...], to_tuple_converter(x)),
     )
 
-    converter: Callable[[type, Sequence[Token]], Any] | None = field(
+    # Accepts regular converters (type, tokens) -> Any, bound methods (tokens) -> Any, or string references
+    converter: Callable[..., Any] | str | None = field(
         default=None,
         kw_only=True,
     )
@@ -255,6 +255,11 @@ def main(foo: Annotated[int, Parameter(name="bar")]):
         kw_only=True,
     )
 
+    n_tokens: int | None = field(
+        default=None,
+        kw_only=True,
+    )
+
     # Populated by the record_attrs_init_args decorator.
     _provided_args: tuple[str, ...] = field(factory=tuple, init=False, eq=False)
 
@@ -462,26 +467,71 @@ def validate_command(f: Callable):
                 )
 
 
-def get_parameters(hint: T) -> tuple[T, list[Parameter]]:
+def get_parameters(hint: T, skip_converter_params: bool = False) -> tuple[T, list[Parameter]]:
     """At root level, checks for cyclopts.Parameter annotations.
 
-    Includes checking the ``__cyclopts__`` attribute.
+    Includes checking the ``__cyclopts__`` attribute on both the type and any converter functions.
+
+    Parameters
+    ----------
+    hint
+        Type hint to extract parameters from.
+    skip_converter_params
+        If True, skip extracting parameters from converter's __cyclopts__.
+        Used to prevent infinite recursion in token_count.
 
     Returns
     -------
     hint
         Annotation hint with :obj:`Annotated` and :obj:`Optional` resolved.
     list[Parameter]
-        List of parameters discovered.
+        List of parameters discovered, ordered by priority (lowest to highest):
+        converter-decoration < type-decoration < annotation.
     """
-    parameters = []
     hint = resolve_optional(hint)
+
+    # Extract parameters from type's __cyclopts__ attribute
+    type_cyclopts_config_params = []
     if cyclopts_config := getattr(hint, "__cyclopts__", None):
-        parameters.extend(cyclopts_config.parameters)
+        type_cyclopts_config_params.extend(cyclopts_config.parameters)
+
+    # Extract parameters from Annotated metadata
+    annotated_params = []
     if is_annotated(hint):
         inner = get_args(hint)
         hint = inner[0]
-        parameters.extend(x for x in inner[1:] if isinstance(x, Parameter))
+        annotated_params.extend(x for x in inner[1:] if isinstance(x, Parameter))
+
+    # Check if any parameter has a converter with __cyclopts__ and extract its parameters
+    converter_params = []
+    if not skip_converter_params:
+        for param in annotated_params + type_cyclopts_config_params:
+            if param.converter:
+                converter = param.converter
+
+                # Resolve string converters to methods on the type
+                if isinstance(converter, str):
+                    converter = getattr(hint, converter)
+
+                # Check for __cyclopts__ on the converter
+                if hasattr(converter, "__cyclopts__"):
+                    converter_params.extend(converter.__cyclopts__.parameters)
+                    break
+                # For bound methods from classmethods/staticmethods, access the descriptor via __self__
+                elif (
+                    hasattr(converter, "__self__")
+                    and hasattr(converter, "__name__")
+                    and hasattr(converter.__self__, "__dict__")
+                ):
+                    # Get the descriptor from the class's __dict__
+                    descriptor = converter.__self__.__dict__.get(converter.__name__)
+                    if descriptor and hasattr(descriptor, "__cyclopts__"):
+                        converter_params.extend(descriptor.__cyclopts__.parameters)
+                        break
+
+    # Return parameters in priority order (lowest to highest)
+    # This allows Parameter.combine() to correctly prioritize later parameters
+    parameters = converter_params + type_cyclopts_config_params + annotated_params
 
     return hint, parameters
 

docs/source/api.rst

@@ -917,9 +917,45 @@ API
            }
 
       If not provided, defaults to Cyclopts's internal coercion engine.
-      If a pydantic type-hint is provided, Cyclopts will disable it's internal coercion
+      If a pydantic type-hint is provided, Cyclopts will disable its internal coercion
       engine (including this `converter` argument) and leave the coercion to pydantic.
 
+      The number of tokens passed to the converter is inferred from the type hint by default,
+      but can be explicitly controlled with :attr:`~.Parameter.n_tokens`. This is useful when
+      the type signature doesn't match the desired CLI token consumption. When loading complex
+      objects with multiple fields, it may also be useful to combine with :attr:`~.Parameter.accepts_keys`.
+
+      **Decorating Converters:** Converter functions can be decorated with :class:`.Parameter` to define
+      reusable conversion behavior:
+
+      .. code-block:: python
+
+         @Parameter(n_tokens=1, accepts_keys=False)
+         def load_from_id(type_, tokens):
+             """Load object from database by ID."""
+             return fetch_from_db(tokens[0].value)
+
+         @app.default
+         def main(obj: Annotated[MyType, Parameter(converter=load_from_id)]):
+             # Automatically inherits n_tokens=1 and accepts_keys=False
+             pass
+
+      **Classmethod Support:** Converters can be classmethods. Use string references for class decoration
+      or direct references in annotations. Classmethod signature should be ``(cls, tokens)`` instead of
+      ``(type_, tokens)``:
+
+      .. code-block:: python
+
+         @Parameter(converter="from_env")
+         class Config:
+             @Parameter(n_tokens=1, accepts_keys=False)
+             @classmethod
+             def from_env(cls, tokens):
+                 env = tokens[0].value
+                 configs = {"dev": ("localhost", 8080), "prod": ("api.example.com", 443)}
+                 return cls(*configs[env])
+
+
    .. attribute:: validator
       :type: Union[None, Callable, Iterable[Callable]]
       :value: None
@@ -1225,6 +1261,9 @@ API
          $ my-program --image foo.jpg nature
          image=Image(path='foo.jpg', label='nature')
 
+      The ``accepts_keys=False`` option is commonly used with :attr:`~.Parameter.converter` and
+      :attr:`~.Parameter.n_tokens`.
+
    .. attribute:: consume_multiple
       :type: Optional[bool]
       :value: None
@@ -1355,6 +1394,63 @@ API
 
       See :ref:`Coercion Rules` for more details.
 
+   .. attribute:: n_tokens
+      :type: Optional[int]
+      :value: None
+
+      Explicitly override the number of CLI tokens this parameter consumes.
+
+      By default, Cyclopts infers the token count from the parameter's type hint
+      (e.g., :obj:`int` consumes 1 token, ``tuple[int, int]`` consumes 2, :obj:`list` consumes all remaining).
+      This attribute allows you to override that inference, which is particularly useful when:
+
+      * Using custom converters that need a different token count than the type suggests.
+      * Loading complex types from a single token (e.g., loading from a file path).
+      * Implementing selection/lookup patterns where one token identifies an object.
+
+      Values:
+
+      * ``None`` (default): Infer token count from the type hint.
+      * non-negative integer: Consume exactly that many tokens.
+      * ``-1``: Consume all remaining tokens (similar to iterables).
+
+      For ``*args`` parameters, ``n_tokens`` specifies tokens **per element**.
+      For example, ``n_tokens=2`` with 6 tokens creates 3 elements.
+
+      .. code-block:: python
+
+         from cyclopts import App, Parameter
+         from typing import Annotated
+
+         class Config:
+             def __init__(self, host: str, port: int):
+                 self.host = host
+                 self.port = port
+
+         def load_config(type_, tokens):
+             # Load config from a file path (single token)
+             filepath = tokens[0].value
+             # ... load from file ...
+             return Config("example.com", 8080)
+
+         app = App()
+
+         @app.default
+         def main(
+             config: Annotated[
+                 Config,
+                 Parameter(n_tokens=1, converter=load_config, accepts_keys=False)
+             ]
+         ):
+             print(f"Connecting to {config.host}:{config.port}")
+
+         app()
+
+      .. code-block:: console
+
+         $ my-script --config prod.conf
+         Connecting to example.com:8080
+
    .. automethod:: combine
 
    .. automethod:: default