feat: heal and track urllib.request calls

CONTRIBUTING.md

@@ -56,10 +56,11 @@ The SDK works with:
 - `httpx` and `httpx2`, sync and async
 - `requests`
 - `aiohttp` 3.13 and later
+- `urllib.request`
 - Python 3.10 through 3.14
 - `mnfst run <command>` or a `manifest()` call at startup
 
-Custom transports, `urllib` and other clients are not covered. See
+Custom transports and other clients are not covered. See
 [the coverage details](docs/guide.md).
 
 ## License

docs/guide.md

@@ -74,7 +74,7 @@ Then send a JSON or `application/x-www-form-urlencoded` request that your test A
 
 ## Behavior and limits
 
-- Standard httpx and httpx2 transports, requests adapters and aiohttp 3.13+ sessions are instrumented process-wide, including existing clients. aiohttp is covered through its client middleware chain; the SDK's middleware runs innermost, so your own middlewares and `raise_for_status` see the result of the heal. Custom transports, `urllib`, aiohttp before 3.13, websocket handshakes, browsers and other languages are not covered.
+- Standard httpx and httpx2 transports, requests adapters, aiohttp 3.13+ sessions and `urllib.request` openers are instrumented process-wide, including existing clients. aiohttp is covered through its client middleware chain; the SDK's middleware runs innermost, so your own middlewares and `raise_for_status` see the result of the heal. A `urllib.request` failure is captured before urllib raises its `HTTPError`, so a healed call returns normally. Custom transports and handlers, aiohttp before 3.13, websocket handshakes, browsers and other languages are not covered.
 - Any 4xx is sent to Manifest for healing except 401, 402, 403 and 429. Successful calls, those four, and every 5xx are never healed: authentication, billing, rate limiting and server faults are not repaired by editing the request. They are tracked as metadata only (see "Data sent to Manifest"). A network failure before an HTTP response passes through untracked.
 - Manifest selects repairs. The SDK retries at most once per captured failure. The original error response is returned if healing is unavailable, no repair can be applied, or the retry has a transport error.
 - A successful streaming retry remains streamed. Error capture reads a bounded prefix and preserves the original response bytes for the caller. Error reads use the caller's read timeout; healing adds up to 60 seconds, and the retry uses the caller's timeout. aiohttp's total timeout covers the whole call, so there healing takes at most half of the time left and the rest is kept for the retry.

src/mnfst/outbound.py

@@ -1,4 +1,5 @@
-"""Auto-instrumentation of outbound HTTP clients (httpx, httpx2, requests and aiohttp).
+"""Auto-instrumentation of outbound HTTP clients (httpx, httpx2, requests,
+aiohttp and urllib.request).
 Patches at TRANSPORT level: one hook per client library, so every client —
 including ones created before manifest() ran — is covered, and redirects,
 retries and streaming stay the client's business. The internal_call guard
@@ -10,6 +11,7 @@
 from __future__ import annotations
 
 import atexit
+import copy
 import platform
 import time
 import uuid
@@ -26,7 +28,8 @@
 from .gate import should_capture
 from .heal_api import NOT_ATTEMPTED, NOT_SENT, AsyncHealApi, HealApi, HealEvent, internal_call
 from .merge import merge_healed_body
-from .response_capture import capture_aiohttp, capture_httpx, capture_httpx_async, capture_requests
+from .response_capture import (capture_aiohttp, capture_httpx, capture_httpx_async,
+                               capture_requests, capture_urllib)
 from .tracking import CallBuffer
 from .wire import capped_response_body, heal_payload, safe_error_text, tracked_url, traveling_body
 
@@ -261,6 +264,7 @@ def install_outbound(config: Config, heal_api: Optional[HealApi] = None,
         _install_httpx(mod, config, sync_api, async_api)
     install_requests(config, sync_api)
     install_aiohttp(config, async_api)
+    install_urllib(config, sync_api)
     # Announce the install, so that silence stops being ambiguous: a healthy
     # app and a broken one are otherwise the same nothing on the dashboard.
     # Once per process, fire-and-forget — it must never delay startup, and a
@@ -382,6 +386,7 @@ def uninstall_outbound() -> None:
             mod.AsyncHTTPTransport.handle_async_request = _originals.pop(async_key)
     uninstall_requests()
     uninstall_aiohttp()
+    uninstall_urllib()
     _installed = False
     _installed_config = None
 
@@ -611,3 +616,91 @@ async def mnfst_middleware(request, handler):
             return response  # nothing in here may reach the caller
 
     return mnfst_middleware
+
+
+# --- urllib.request ------------------------------------------------------------
+# AbstractHTTPHandler.do_open is urllib's one hop on the wire, shared by the
+# http and https handlers. It returns the raw response for any status, before
+# urllib's processors turn a 4xx into an HTTPError and follow redirects.
+
+def install_urllib(config: Config, heal_api: HealApi) -> None:
+    import urllib.request
+    if "urllib_do_open" in _originals:
+        return
+
+    _originals["urllib_do_open"] = urllib.request.AbstractHTTPHandler.do_open
+
+    def patched_do_open(self, http_class, req, **http_conn_args):
+        original = _originals["urllib_do_open"]
+        started_at = time.time()
+        started = time.monotonic()
+        response = original(self, http_class, req, **http_conn_args)
+        elapsed_ms = int((time.monotonic() - started) * 1000)
+        if internal_call.get() or _excluded(config, req.full_url):
+            return response
+        method = req.get_method()
+        try:
+            if not should_capture(response.status) or not heal_api.healing_enabled():
+                _track(method, req.full_url, response.status, started_at, elapsed_ms)
+                return response
+            try:
+                response, raw, incomplete = capture_urllib(response)
+            except Exception:
+                return response
+            # A file or iterable body is not bytes: capture without it, so an
+            # upload is never consumed.
+            data = req.data
+            content = data if isinstance(data, bytes) else (b"" if data is None else None)
+            capture = _Capture(method, req.full_url, dict(req.header_items()), content,
+                               response.status, raw, elapsed_ms, incomplete)
+            result = heal_api.heal(capture.payload)
+            if result is NOT_SENT:  # every heal slot busy: track it, never lose it
+                _track(method, req.full_url, response.status, started_at, elapsed_ms)
+                return response
+            retry = _decide(config, heal_api, capture, result)
+            if retry is None:
+                return response
+            try:
+                retried = original(self, http_class, _retarget_urllib(req, retry), **http_conn_args)
+                retry_body, truncated = None, False
+                if retried.status >= 400:
+                    retried, raw, incomplete = capture_urllib(retried)
+                    retry_body, truncated = capped_response_body(raw)
+                    truncated = truncated or incomplete
+            except Exception as exc:
+                _report(heal_api, result, 0, safe_error_text(exc))
+                _emit(config, capture, result, None)
+                return response
+            _report(heal_api, result, retried.status, retry_body, truncated)
+            _emit(config, capture, result, retried.status)
+            try:
+                response.close()
+            except Exception:
+                pass
+            return retried
+        except Exception:
+            return response  # nothing in here may reach the caller
+
+    urllib.request.AbstractHTTPHandler.do_open = patched_do_open
+
+
+def uninstall_urllib() -> None:
+    if "urllib_do_open" not in _originals:
+        return
+    import urllib.request
+    urllib.request.AbstractHTTPHandler.do_open = _originals.pop("urllib_do_open")
+
+
+def _retarget_urllib(req, retry: _Retry):
+    """A copy of the request with the retry applied. The processors that set
+    Content-Length and Host already ran on the original; http.client sizes a
+    bytes body itself, and Host rides in the captured headers."""
+    rebuilt = copy.copy(req)
+    rebuilt.full_url = retry.url  # same origin: _apply refuses anything else
+    rebuilt.host = req.host  # the proxy, when there is one
+    if req.has_proxy():
+        rebuilt.selector = rebuilt.full_url
+    rebuilt.headers, rebuilt.unredirected_hdrs = dict(retry.headers), {}
+    rebuilt.method = req.get_method()  # else urllib infers POST from any body
+    rebuilt.data = retry.content
+    return rebuilt

src/mnfst/response_capture.py

@@ -8,6 +8,7 @@
 import io
 import warnings
 import zlib
+from itertools import chain
 from types import ModuleType
 from typing import Iterator
 
@@ -151,7 +152,6 @@ def close(self):
 
 
 def capture_requests(response):
-    from itertools import chain
     from urllib3.response import HTTPResponse
     if response._content is not False:
         return response, response.content[:RESPONSE_BODY_CAP + 1]
@@ -212,3 +212,40 @@ async def capture_aiohttp(response):
     if getattr(stream, 'total_compressed_bytes', None) is None:
         raw = _decode(raw, response.headers.get('content-encoding', ''))
     return raw, incomplete
+
+
+def capture_urllib(response):
+    """Read up to the cap from a stdlib http.client failure, then give the
+    same response object a body of those bytes followed by the unread rest.
+    It stays an HTTPResponse, so an HTTPError built on it, or a caller
+    without urllib's error processing, reads the whole body. Returns
+    (response, decoded prefix, incomplete)."""
+    rest = _clone(response)  # reads the connection from here on
+    chunks, size = [], 0
+    incomplete = True
+    iterator = iter(lambda: rest.read(65536), b'')
+    try:
+        for chunk in iterator:
+            chunks.append(chunk)
+            size += len(chunk)
+            if size > RESPONSE_BODY_CAP:
+                break
+        else:
+            incomplete = False
+    except Exception as exc:
+        error = exc
+        def failed():
+            raise error
+            yield  # pragma: no cover
+        iterator = failed()
+    # The replacement body is already de-chunked and has no known length.
+    response.fp = io.BufferedReader(_Reader(chain(chunks, iterator), rest))
+    response.chunked, response.length = False, None
+    raw = b''.join(chunks)[:RESPONSE_BODY_CAP + 1]
+    return response, _decode(raw, response.headers.get('content-encoding', '')), incomplete
+
+
+def _clone(response):
+    twin = type(response).__new__(type(response))
+    twin.__dict__.update(response.__dict__)
+    return twin