Add request partitioning to waterdata_client. (#325)

* specify chunk size

* update templates so that generate query returns a list of queries

* regenerate clients and models

* add and test method for grouping iterable of strings

* add and test iterable field type

* add basic sequence validation for monitoring location id

* update build scripts with new validated type

* pass client chunk size into query generation

* fallback to default location id set by user if not list

* tweaks to streamline code

* bump version to 0.9.0b0

Author: jarq6c

python/waterdata_client/pyproject.toml

@@ -12,7 +12,7 @@ readme = "README.md"
 requires-python = ">=3.11"
 license = "Apache-2.0"
 classifiers = [
-    "Development Status :: 3 - Alpha",
+    "Development Status :: 4 - Beta",
     "Intended Audience :: Education",
     "Intended Audience :: Science/Research",
     "Programming Language :: Python :: 3.11",

python/waterdata_client/scripts/build_clients.py

@@ -26,14 +26,16 @@
 from hydrotools.waterdata_client._version import __version__
 
 TYPE_OVERRIDES: dict[str, str] = {
-    "datetime": "str | DateTimeInput"
+    "datetime": "str | DateTimeInput",
+    "monitoring_location_id": "str | StringList"
 }
 """Mapping from specific arguments to special type hints. This facilitates
 various query interventions.
 """
 
 EXTRA_IMPORTS: dict[str, str] = {
-    "datetime": "from hydrotools.waterdata_client.datetime_utilities import DateTimeInput"
+    "datetime": "from hydrotools.waterdata_client.datetime_utilities import DateTimeInput",
+    "monitoring_location_id": "from hydrotools.waterdata_client.request_partitioning import StringList"
 }
 """Mapping from specific arguments to extra required imports."""
 

python/waterdata_client/scripts/build_request_models.py

@@ -26,14 +26,16 @@
 from hydrotools.waterdata_client._version import __version__
 
 TYPE_OVERRIDES: dict[str, str] = {
-    "datetime": "DateTimeField"
+    "datetime": "DateTimeField",
+    "monitoring_location_id": "StringList"
 }
 """Mapping from specific arguments to special type hints. This facilitates
 various query interventions.
 """
 
 EXTRA_IMPORTS: dict[str, str] = {
-    "datetime": "from hydrotools.waterdata_client.datetime_utilities import DateTimeField"
+    "datetime": "from hydrotools.waterdata_client.datetime_utilities import DateTimeField",
+    "monitoring_location_id": "from hydrotools.waterdata_client.request_partitioning import StringList"
 }
 """Mapping from specific arguments to extra required imports."""
 

python/waterdata_client/src/hydrotools/waterdata_client/_version.py

@@ -1,2 +1,2 @@
 """Track dynamic package versions here."""
-__version__ = "0.8.3a0"
+__version__ = "0.9.0b0"

python/waterdata_client/src/hydrotools/waterdata_client/base_client.py

@@ -49,13 +49,15 @@ def __init__(
         max_retries: int = SETTINGS.default_retries,
         timeout_seconds: int = SETTINGS.timeout_seconds,
         ssl_context: Optional[ssl.SSLContext] = None,
-        transformer: Optional[ResponseTransformer[TransformedResponseT_co]] = check_features
+        transformer: Optional[ResponseTransformer[TransformedResponseT_co]] = check_features,
+        chunk_size: int = SETTINGS.chunk_size
     ) -> None:
         self.concurrency_limit = concurrency_limit
         self.max_retries = max_retries
         self.timeout_seconds = timeout_seconds
         self.ssl_context = ssl_context
         self.transformer = transformer
+        self.chunk_size = chunk_size
 
         # Setup request builder
         self._builder = partial(

python/waterdata_client/src/hydrotools/waterdata_client/client_config.py

@@ -73,6 +73,7 @@ class EnvironmentKey(StrEnum):
     TIMEOUT = f"{_KEY_START}TIMEOUT"
     API_KEY = f"{_KEY_START}USGS_API_KEY"
     MAX_PAGES = f"{_KEY_START}MAX_PAGES"
+    CHUNK_SIZE = f"{_KEY_START}CHUNK_SIZE"
 
     @classmethod
     def describe_keys(cls) -> str:
@@ -115,6 +116,7 @@ class _Settings:
             cache_expires.
         max_pages: Maximum number of times to follow paginated 'next' links
             in JSON responses.
+        chunk_size: Size of request chunks for arguments that accept Sequences.
     """
     usgs_base_url: URL = URL("https://api.waterdata.usgs.gov/ogcapi/v0")
     schema_path: str = "openapi"
@@ -129,6 +131,7 @@ class _Settings:
     timeout_seconds: int = 900
     usgs_api_key: Optional[str] = None
     max_pages: int = 20
+    chunk_size: int = 5
 
     @classmethod
     def from_env(cls) -> Self:
@@ -154,7 +157,8 @@ def from_env(cls) -> Self:
             default_retries=int(os.getenv(EnvironmentKey.RETRIES, cls.default_retries)),
             timeout_seconds=int(os.getenv(EnvironmentKey.TIMEOUT, cls.timeout_seconds)),
             usgs_api_key=os.getenv(EnvironmentKey.API_KEY, cls.usgs_api_key),
-            max_pages=int(os.getenv(EnvironmentKey.MAX_PAGES, cls.max_pages))
+            max_pages=int(os.getenv(EnvironmentKey.MAX_PAGES, cls.max_pages)),
+            chunk_size=int(os.getenv(EnvironmentKey.CHUNK_SIZE, cls.chunk_size))
             )
 
     @cached_property

python/waterdata_client/src/hydrotools/waterdata_client/clients/__init__.py

@@ -3,9 +3,9 @@
 client classes for USGS OGC API items endpoints. These classes are generated
 by inspecting the USGS OGC API JSON schema and identifying all "items" endpoints.
 
-Package version: 0.8.0a0
+Package version: 0.9.0b0
 Generation script: build_clients.py
-Generated: 2026-05-20 18:31:11 Z
+Generated: 2026-05-29 15:52:35 Z
 JSON Schema source: https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=json
 JSON Schema version: 0.49.2
 OpenAPI version: 3.0.2

python/waterdata_client/src/hydrotools/waterdata_client/clients/agency_codes.py

@@ -3,9 +3,9 @@
 client classes for USGS OGC API items endpoints. These classes are generated
 by inspecting the USGS OGC API JSON schema and identifying all "items" endpoints.
 
-Package version: 0.8.0a0
+Package version: 0.9.0b0
 Generation script: build_clients.py
-Generated: 2026-05-20 18:31:11 Z
+Generated: 2026-05-29 15:52:35 Z
 JSON Schema source: https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=json
 JSON Schema version: 0.49.2
 OpenAPI version: 3.0.2
@@ -96,7 +96,7 @@ def get(
                 our APIs. You can obtain an API key at https://api.waterdata.usgs.gov/signup
         """
         # Validate query
-        query = AgencyCodesRequest(
+        queries = AgencyCodesRequest(
             agency_name=agency_name,
             bbox=bbox,
             bbox_crs=bbox_crs,
@@ -111,10 +111,10 @@ def get(
             skipgeometry=skipgeometry,
             sortby=sortby,
             api_key=api_key if api_key else SETTINGS.usgs_api_key
-        ).generate_query()
+        ).generate_queries()
 
         # Get responses
-        data = self._get_json_responses(queries=[query])
+        data = self._get_json_responses(queries=queries)
 
         # Transform
         return self._handle_response(data)

python/waterdata_client/src/hydrotools/waterdata_client/clients/altitude_datums.py

@@ -3,9 +3,9 @@
 client classes for USGS OGC API items endpoints. These classes are generated
 by inspecting the USGS OGC API JSON schema and identifying all "items" endpoints.
 
-Package version: 0.8.0a0
+Package version: 0.9.0b0
 Generation script: build_clients.py
-Generated: 2026-05-20 18:31:11 Z
+Generated: 2026-05-29 15:52:35 Z
 JSON Schema source: https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=json
 JSON Schema version: 0.49.2
 OpenAPI version: 3.0.2
@@ -100,7 +100,7 @@ def get(
                 our APIs. You can obtain an API key at https://api.waterdata.usgs.gov/signup
         """
         # Validate query
-        query = AltitudeDatumsRequest(
+        queries = AltitudeDatumsRequest(
             altitude_datum_description=altitude_datum_description,
             bbox=bbox,
             bbox_crs=bbox_crs,
@@ -115,10 +115,10 @@ def get(
             skipgeometry=skipgeometry,
             sortby=sortby,
             api_key=api_key if api_key else SETTINGS.usgs_api_key
-        ).generate_query()
+        ).generate_queries()
 
         # Get responses
-        data = self._get_json_responses(queries=[query])
+        data = self._get_json_responses(queries=queries)
 
         # Transform
         return self._handle_response(data)

python/waterdata_client/src/hydrotools/waterdata_client/clients/aquifer_codes.py

@@ -3,9 +3,9 @@
 client classes for USGS OGC API items endpoints. These classes are generated
 by inspecting the USGS OGC API JSON schema and identifying all "items" endpoints.
 
-Package version: 0.8.0a0
+Package version: 0.9.0b0
 Generation script: build_clients.py
-Generated: 2026-05-20 18:31:11 Z
+Generated: 2026-05-29 15:52:35 Z
 JSON Schema source: https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=json
 JSON Schema version: 0.49.2
 OpenAPI version: 3.0.2
@@ -110,7 +110,7 @@ def get(
                 our APIs. You can obtain an API key at https://api.waterdata.usgs.gov/signup
         """
         # Validate query
-        query = AquiferCodesRequest(
+        queries = AquiferCodesRequest(
             aquifer_name=aquifer_name,
             bbox=bbox,
             bbox_crs=bbox_crs,
@@ -125,10 +125,10 @@ def get(
             skipgeometry=skipgeometry,
             sortby=sortby,
             api_key=api_key if api_key else SETTINGS.usgs_api_key
-        ).generate_query()
+        ).generate_queries()
 
         # Get responses
-        data = self._get_json_responses(queries=[query])
+        data = self._get_json_responses(queries=queries)
 
         # Transform
         return self._handle_response(data)