feat(configuration): Mark 4xx calls as erroneous spans and adapt current instrumentations.

Signed-off-by: Cagri Yonca <cagri@ibm.com>

Author: CagriYonca

.gitignore

@@ -106,3 +106,4 @@ uv.lock
 # Sandbox
 sandbox/
 .bob/
+.serena/

src/instana/instrumentation/aiohttp/client.py

@@ -2,21 +2,22 @@
 # (c) Copyright Instana Inc. 2019
 
 
-from types import SimpleNamespace
-from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Tuple
-
-import wrapt
-from opentelemetry.semconv.trace import SpanAttributes
-
-from instana.log import logger
-from instana.propagators.format import Format
-from instana.singletons import agent
-from instana.util.secrets import strip_secrets_from_query
-from instana.util.traceutils import extract_custom_headers, get_tracer_tuple
-
 try:
+    from collections.abc import Awaitable, Callable
+    from types import SimpleNamespace
+    from typing import TYPE_CHECKING, Any
+
     import aiohttp
+    import wrapt
     from opentelemetry.context import get_current
+    from opentelemetry.semconv.trace import SpanAttributes
+
+    from instana.log import logger
+    from instana.propagators.format import Format
+    from instana.singletons import agent
+    from instana.util.http import should_mark_http_exit_as_error
+    from instana.util.secrets import strip_secrets_from_query
+    from instana.util.traceutils import extract_custom_headers, get_tracer_tuple
 
     if TYPE_CHECKING:
         from aiohttp.client import ClientSession
@@ -65,8 +66,9 @@ async def stan_request_end(
 
                 extract_custom_headers(span, params.response.headers)
 
-                if params.response.status >= 500:
-                    span.mark_as_errored({"http.error": params.response.reason})
+                if should_mark_http_exit_as_error(params.response.status, agent.options):
+                    error_msg = f"{params.response.status} {params.response.reason}"
+                    span.mark_as_errored({"http.error": error_msg})
 
                 if span.is_recording():
                     span.end()
@@ -92,8 +94,8 @@ async def stan_request_exception(
     def init_with_instana(
         wrapped: Callable[..., Awaitable["ClientSession"]],
         instance: aiohttp.client.ClientSession,
-        args: Tuple[int, str, Tuple[object, ...]],
-        kwargs: Dict[str, Any],
+        args: tuple[int, str, tuple[object, ...]],
+        kwargs: dict[str, Any],
     ) -> object:
         instana_trace_config = aiohttp.TraceConfig()
         instana_trace_config.on_request_start.append(stan_request_start)

src/instana/instrumentation/httpx.py

@@ -1,7 +1,7 @@
 # (c) Copyright IBM Corp. 2025
 
 try:
-    from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple
+    from typing import TYPE_CHECKING, Any, Callable, Optional
 
     import httpx
     import wrapt
@@ -12,6 +12,7 @@
     from instana.log import logger
     from instana.propagators.format import Format
     from instana.singletons import agent
+    from instana.util.http import should_mark_http_exit_as_error
     from instana.util.secrets import strip_secrets_from_query
     from instana.util.traceutils import extract_custom_headers, get_tracer_tuple
 
@@ -50,25 +51,26 @@ def _set_request_span_attributes(
 
     def _set_response_span_attributes(
         span: "InstanaSpan",
-        response: Optional[httpx.Response] = None,
+        response: "Optional[httpx.Response]" = None,
     ) -> None:
         try:
             if response.headers:
                 extract_custom_headers(span, response.headers)
 
             status_code = response.status_code
             span.set_attribute(SpanAttributes.HTTP_STATUS_CODE, status_code)
-            if status_code >= 500:
-                span.mark_as_errored()
+            if should_mark_http_exit_as_error(status_code, agent.options):
+                error_msg = f"{status_code} {response.reason_phrase}"
+                span.mark_as_errored({"http.error": error_msg})
         except Exception:
             logger.debug("httpx _set_request_span_attributes error: ", exc_info=True)
 
     @wrapt.patch_function_wrapper("httpx", "HTTPTransport.handle_request")
     def handle_request_with_instana(
         wrapped: Callable[..., "httpx.HTTPTransport.handle_request"],
         instance: httpx.HTTPTransport,
-        args: Tuple[int, str, Tuple[Any, ...]],
-        kwargs: Dict[str, Any],
+        args: tuple[int, str, tuple[Any, ...]],
+        kwargs: dict[str, Any],
     ) -> httpx.Response:
         tracer, _, _ = get_tracer_tuple()
         # If we're not tracing, just return
@@ -102,8 +104,8 @@ def handle_request_with_instana(
     async def handle_async_request_with_instana(
         wrapped: Callable[..., "httpx.AsyncHTTPTransport.handle_async_request"],
         instance: httpx.AsyncHTTPTransport,
-        args: Tuple[int, str, Tuple[Any, ...]],
-        kwargs: Dict[str, Any],
+        args: tuple[int, str, tuple[Any, ...]],
+        kwargs: dict[str, Any],
     ) -> httpx.Response:
         tracer, _, _ = get_tracer_tuple()
         # If we're not tracing, just return

src/instana/instrumentation/tornado/client.py

@@ -4,7 +4,7 @@
 
 try:
     import functools
-    from typing import TYPE_CHECKING, Any, Callable, Dict, Tuple
+    from typing import TYPE_CHECKING, Any, Callable
 
     import tornado
     import wrapt
@@ -23,15 +23,16 @@
     from instana.propagators.format import Format
     from instana.singletons import agent, get_tracer
     from instana.span.span import get_current_span
+    from instana.util.http import should_mark_http_exit_as_error
     from instana.util.secrets import strip_secrets_from_query
     from instana.util.traceutils import extract_custom_headers
 
     @wrapt.patch_function_wrapper("tornado.httpclient", "AsyncHTTPClient.fetch")
     def fetch_with_instana(
         wrapped: Callable[..., object],
         instance: "AsyncHTTPClient",
-        argv: Tuple[object, ...],
-        kwargs: Dict[str, Any],
+        argv: tuple[object, ...],
+        kwargs: dict[str, Any],
     ) -> "Future":
         try:
             parent_span = get_current_span()
@@ -47,13 +48,12 @@ def fetch_with_instana(
             # To modify request headers, we have to preemptively create an HTTPRequest object if a
             # URL string was passed.
             if not isinstance(request, tornado.httpclient.HTTPRequest):
+                # "callback" and "raise_error" are fetch()-level kwargs, not
+                # HTTPRequest constructor params — extract them first so they
+                # are not forwarded to HTTPRequest.__init__.
+                fetch_only_params = ("callback", "raise_error")
+                new_kwargs = {p: kwargs.pop(p) for p in fetch_only_params if p in kwargs}
                 request = tornado.httpclient.HTTPRequest(url=request, **kwargs)
-
-                new_kwargs = {}
-                for param in ("callback", "raise_error"):
-                    # if not in instead and pop
-                    if param in kwargs:
-                        new_kwargs[param] = kwargs.pop(param)
                 kwargs = new_kwargs
 
             parent_context = get_current()
@@ -89,8 +89,10 @@ def finish_tracing(future: "Future", span: "InstanaSpan") -> None:
         try:
             response = future.result()
             span.set_attribute(SpanAttributes.HTTP_STATUS_CODE, response.code)
-
             extract_custom_headers(span, response.headers)
+            if should_mark_http_exit_as_error(response.code, agent.options):
+                error_msg = f"{response.code} {response.reason}"
+                span.mark_as_errored({"http.error": error_msg})
         except tornado.httpclient.HTTPClientError as e:
             span.set_attribute(SpanAttributes.HTTP_STATUS_CODE, e.code)
             span.record_exception(e)

src/instana/instrumentation/tornado/server.py

@@ -77,6 +77,7 @@ def set_default_headers_with_instana(
         span = instance.request._instana
         tracer = get_tracer()
         tracer.inject(span.context, Format.HTTP_HEADERS, instance._headers)
+        return wrapped(*argv, **kwargs)
 
     @wrapt.patch_function_wrapper("tornado.web", "RequestHandler.on_finish")
     def on_finish_with_instana(

src/instana/instrumentation/twisted/client.py

@@ -19,6 +19,7 @@
     from instana.propagators.format import Format
     from instana.singletons import agent, get_tracer
     from instana.span.span import get_current_span
+    from instana.util.http import should_mark_http_exit_as_error
     from instana.util.secrets import strip_secrets_from_query
     from instana.util.traceutils import extract_custom_headers
 
@@ -133,9 +134,10 @@ def finish_tracing(
                 }
                 extract_custom_headers(span, headers_dict)
 
-                if status_code >= 500:
+                if should_mark_http_exit_as_error(status_code, agent.options):
+                    phrase = result.phrase.decode("latin-1")
                     span.mark_as_errored({
-                        "http.error": result.phrase.decode("latin-1")
+                        "http.error": f"{status_code} {phrase}"
                     })
         except Exception:
             logger.debug("twisted client finish_tracing", exc_info=True)

src/instana/instrumentation/twisted/server.py

@@ -62,10 +62,20 @@ def render_with_instana(
             span = tracer.start_span(
                 "twisted-server", context=parent_context)
 
-            # Set span as current so downstream code
-            # (e.g. twisted-client) can find it during the synchronous
-            # wrapped() call.  We detach unconditionally in the finally
-            # block below once wrapped() has returned.
+            # Set span as current so that any async work started during
+            # wrapped() (e.g. outgoing Agent.request Deferreds) can find
+            # this span as their parent after the event loop resumes.
+            #
+            # IMPORTANT: we do NOT detach the token here in the `finally`
+            # block.  Twisted's render() returns NOT_DONE_YET for async
+            # handlers, and the event loop only fires pending Deferreds
+            # after render() has returned — by which point a `finally`
+            # detach would have already removed the context, leaving
+            # downstream spans (e.g. twisted-client) with no active parent.
+            #
+            # Instead the token is stored on the request object and detached
+            # inside finish_tracing(), which is called by notifyFinish() only
+            # after the full async response lifecycle has completed.
             ctx = trace.set_span_in_context(span)
             token = context.attach(ctx)
 
@@ -119,21 +129,24 @@ def render_with_instana(
             for key, value in response_headers.items():
                 request.setHeader(key.encode("latin-1"), value.encode("utf-8"))
 
-            # Store span on request for later retrieval
+            # Store span and context token on the request so finish_tracing
+            # can detach the token after the full async lifecycle completes.
             request._instana = span
+            request._instana_token = token
             request._instana_finished = False
 
             finish_deferred = request.notifyFinish()
             finish_deferred.addBoth(finish_tracing, request)
 
             return wrapped(*argv, **kwargs)
         except Exception:
+            # On instrumentation error detach immediately (we never reach
+            # finish_tracing in this path) and fall through to the bare call.
+            if token is not None:
+                context.detach(token)
             if span is not None and span.is_recording():
                 span.end()
             logger.debug("twisted server render_with_instana", exc_info=True)
-        finally:
-            if token is not None:
-                context.detach(token)
 
         return wrapped(*argv, **kwargs)
 
@@ -146,6 +159,7 @@ def finish_tracing(
 
         request._instana_finished = True
         span = request._instana
+        token = getattr(request, "_instana_token", None)
         try:
             status_code = request.code
             if isinstance(status_code, int):
@@ -159,12 +173,19 @@ def finish_tracing(
             extract_custom_headers(span, response_hdrs)
 
             if isinstance(status_code, int) and status_code >= 500:
+                phrase = request.code_message.decode("latin-1")
                 span.mark_as_errored({
-                    "http.error": request.code_message.decode("latin-1")
+                    "http.error": f"{status_code} {phrase}"
                 })
         except Exception:
             logger.debug("twisted server finish_tracing", exc_info=True)
         finally:
+            # Detach the OTel context token here — after the full async
+            # response lifecycle — instead of in render_with_instana's
+            # finally block.  This ensures the server span remains the
+            # active context for any Deferreds started during render().
+            if token is not None:
+                context.detach(token)
             if span.is_recording():
                 span.end()
 

src/instana/instrumentation/urllib3.py

@@ -2,32 +2,33 @@
 # (c) Copyright Instana Inc. 2017
 
 
-from typing import TYPE_CHECKING, Any, Callable, Dict, Tuple, Union
+try:
+    from typing import TYPE_CHECKING, Any, Callable, Union
 
-import wrapt
-from opentelemetry.context import get_current
-from opentelemetry.semconv.trace import SpanAttributes
+    import wrapt
+    from opentelemetry.context import get_current
+    from opentelemetry.semconv.trace import SpanAttributes
 
-from instana.log import logger
-from instana.propagators.format import Format
-from instana.singletons import agent
-from instana.util.secrets import strip_secrets_from_query
-from instana.util.traceutils import extract_custom_headers, get_tracer_tuple
+    from instana.log import logger
+    from instana.propagators.format import Format
+    from instana.singletons import agent
+    from instana.util.http import should_mark_http_exit_as_error
+    from instana.util.secrets import strip_secrets_from_query
+    from instana.util.traceutils import extract_custom_headers, get_tracer_tuple
 
-if TYPE_CHECKING:
-    from instana.span.span import InstanaSpan
+    if TYPE_CHECKING:
+        from instana.span.span import InstanaSpan
 
-try:
     import urllib3
 
     def _collect_kvs(
         instance: Union[
             urllib3.connectionpool.HTTPConnectionPool,
             urllib3.connectionpool.HTTPSConnectionPool,
         ],
-        args: Tuple[int, str, Tuple[Any, ...]],
-        kwargs: Dict[str, Any],
-    ) -> Dict[str, Any]:
+        args: tuple[int, str, tuple[Any, ...]],
+        kwargs: dict[str, Any],
+    ) -> dict[str, Any]:
         kvs = dict()
         try:
             kvs["host"] = instance.host
@@ -74,8 +75,9 @@ def collect_response(
 
             extract_custom_headers(span, response.headers)
 
-            if response.status >= 500:
-                span.mark_as_errored()
+            if should_mark_http_exit_as_error(response.status, agent.options):
+                error_msg = f"{response.status} {response.reason}"
+                span.mark_as_errored({"http.error": error_msg})
         except Exception:
             logger.debug("urllib3 collect_response error: ", exc_info=True)
 
@@ -88,8 +90,8 @@ def urlopen_with_instana(
             urllib3.connectionpool.HTTPConnectionPool,
             urllib3.connectionpool.HTTPSConnectionPool,
         ],
-        args: Tuple[int, str, Tuple[Any, ...]],
-        kwargs: Dict[str, Any],
+        args: tuple[int, str, tuple[Any, ...]],
+        kwargs: dict[str, Any],
     ) -> urllib3.response.HTTPResponse:
         tracer, _, span_name = get_tracer_tuple()