Task exceptions break the schedule loop #4
Description
I have observed that every time a task raises an exception the schedule loop gets broken and the task is never scheduled again for a new execution as per the defined interval.
The behaviour is kind of unexpected as the task/function should be invoked regardless of the last execution finish status.
I schedule this task:
scheduler.register_task(druid_historical_data, target_dl="historical-minute").schedule(interval=datetime.timedelta(seconds=1200))That eventually trows an HTTP timeout:
1744367900976 Task exception was never retrieved
1744367900977 future: <Task finished name='Task-3' coro=<Scheduler._run_with_interval() done, defined at /usr/local/lib/python3.12/site-packages/aioskd/scheduler.py:121> exception=ReadTimeout('')>
1744367900977 Traceback (most recent call last):
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpx/_transports/default.py", line 101, in map_httpcore_exceptions
1744367900977 yield
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpx/_transports/default.py", line 394, in handle_async_request
1744367900977 resp = await self._pool.handle_async_request(req)
1744367900977 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpcore/_async/connection_pool.py", line 256, in handle_async_request
1744367900977 raise exc from None
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpcore/_async/connection_pool.py", line 236, in handle_async_request
1744367900977 response = await connection.handle_async_request(
1744367900977 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpcore/_async/connection.py", line 103, in handle_async_request
1744367900977 return await self._connection.handle_async_request(request)
1744367900977 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpcore/_async/http11.py", line 136, in handle_async_request
1744367900977 raise exc
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpcore/_async/http11.py", line 106, in handle_async_request
1744367900977 ) = await self._receive_response_headers(**kwargs)
1744367900977 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpcore/_async/http11.py", line 177, in _receive_response_headers
1744367900977 event = await self._receive_event(timeout=timeout)
1744367900977 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpcore/_async/http11.py", line 217, in _receive_event
1744367900977 data = await self._network_stream.read(
1744367900977 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpcore/_backends/anyio.py", line 32, in read
1744367900977 with map_exceptions(exc_map):
1744367900977 ^^^^^^^^^^^^^^^^^^^^^^^
1744367900977 File "/usr/local/lib/python3.12/contextlib.py", line 158, in __exit__
1744367900977 self.gen.throw(value)
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpcore/_exceptions.py", line 14, in map_exceptions
1744367900977 raise to_exc(exc) from exc
1744367900977 httpcore.ReadTimeout
1744367900977
1744367900977 The above exception was the direct cause of the following exception:
1744367900977
1744367900977 Traceback (most recent call last):
1744367900977 File "/usr/local/lib/python3.12/site-packages/aioskd/scheduler.py", line 127, in _run_with_interval
1744367900977 await asyncio.gather(*tasks)
1744367900977 File "/usr/local/lib/python3.12/site-packages/aioskd/scheduler.py", line 108, in _execute_task
1744367900977 await task.task()
1744367900977 File "/app/dremel_bot/checks/druid.py", line 307, in __call__
1744367900977 source_result = await self.druid_api.query(SqlQuery(query=source_query))
1744367900977 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1744367900977 File "/app/dremel_bot/utils/druid.py", line 143, in query
1744367900977 resp = await self.client.post(
1744367900977 ^^^^^^^^^^^^^^^^^^^^^^^
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpx/_client.py", line 1859, in post
1744367900977 return await self.request(
1744367900977 ^^^^^^^^^^^^^^^^^^^
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpx/_client.py", line 1540, in request
1744367900977 return await self.send(request, auth=auth, follow_redirects=follow_redirects)
1744367900977 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpx/_client.py", line 1629, in send
1744367900977 response = await self._send_handling_auth(
1744367900977 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpx/_client.py", line 1657, in _send_handling_auth
1744367900977 response = await self._send_handling_redirects(
1744367900977 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpx/_client.py", line 1694, in _send_handling_redirects
1744367900977 response = await self._send_single_request(request)
1744367900977 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpx/_client.py", line 1730, in _send_single_request
1744367900977 response = await transport.handle_async_request(request)
1744367900977 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpx/_transports/default.py", line 393, in handle_async_request
1744367900977 with map_httpcore_exceptions():
1744367900977 ^^^^^^^^^^^^^^^^^^^^^^^^^
1744367900977 File "/usr/local/lib/python3.12/contextlib.py", line 158, in __exit__
1744367900977 self.gen.throw(value)
1744367900977 File "/usr/local/lib/python3.12/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions
1744367900977 raise mapped_exc(message) from exc
1744367900977 httpx.ReadTimeout