Serve the workspace UI over HTTP/2 (TLS + h2 at the mngr forward proxy)

apps/minds/changelog/gabriel-http-2-forward-spec.md

@@ -0,0 +1,5 @@
+Serve the workspace UI over HTTP/2 to fix the hang that occurred once enough streaming tabs were open in one workspace.
+
+The whole workspace origin was previously served over plain HTTP/1.1, which Chromium caps at ~6 held-open connections per origin. Once a workspace had that many long-lived streams (one SSE per open chat tab, plus any `/service/` app's own streams), the pool was exhausted and every further request -- including the plain GETs that bootstrap a new terminal or app iframe -- queued indefinitely, hanging the UI even though the backend was healthy.
+
+Minds now runs its single `mngr forward` proxy with `--use-http2`, so the workspace origin terminates TLS and negotiates HTTP/2 (which multiplexes many streams over one connection). The Python readiness/recovery probes and the `/goto/` redirect URLs flip to `https` in lockstep, and the Electron shell builds `https`/`wss` workspace URLs, marks the `mngr_forward_session` cookie `Secure`, and silently trusts the proxy's ephemeral self-signed cert for its loopback origins (`localhost`, `*.localhost`, `127.0.0.1`) in the workspace-content session only -- every real https origin still gets Chromium's normal verification. The minds bare-origin backend (Home/Create/chrome/sidebar and its `minds_session` cookie) stays plain HTTP; the app deliberately runs a mixed http/https transport, which is not mixed-content-blocked.

apps/minds/electron/main.js

@@ -2981,24 +2981,53 @@ function handleNotification(event) {
 //
 // Mirrors the cookie into the workspace content partition so any
 // chrome/iframe / WebContentsView using that partition is authenticated too.
+// Loopback hosts the `mngr forward` proxy serves under its self-signed cert:
+// the bare `localhost` origin, the `agent-<id>.localhost` workspace subdomains
+// (`*.localhost`), and the `127.0.0.1` IP host. Every other host must fall
+// through to Chromium's normal verification.
+function isLoopbackHostname(hostname) {
+  return hostname === 'localhost' || hostname.endsWith('.localhost') || hostname === '127.0.0.1';
+}
+
+// Trust the proxy's ephemeral self-signed cert for its loopback origins, in the
+// workspace-content partition only. The proxy regenerates the cert every
+// startup and only minds' own loopback origins use it, so there is no OS trust
+// store or CA to install; every real https origin still gets Chromium's default
+// verification (cb(-3)). Registered only when the proxy actually serves TLS.
+function trustLoopbackCertsForWorkspaceContent() {
+  const contentSession = session.fromPartition(CONTENT_PARTITION);
+  contentSession.setCertificateVerifyProc((request, callback) => {
+    if (isLoopbackHostname(request.hostname)) {
+      callback(0); // trust
+    } else {
+      callback(-3); // defer to Chromium's default verification
+    }
+  });
+  console.log('[startup] Trusting self-signed loopback certs for workspace content (HTTP/2 on)');
+}
+
 async function handleMngrForwardStarted(event) {
   const port = event.mngr_forward_port;
   const preauth = event.preauth_cookie;
   if (!port || !preauth) {
     console.warn('[startup] mngr_forward_started missing port or preauth_cookie:', event);
     return;
   }
-  const url = `http://localhost:${port}`;
+  // The proxy serves TLS + HTTP/2, so the origin is https and the session
+  // cookie must be Secure (a Secure cookie is only sent over https).
+  const url = `https://localhost:${port}`;
   // Cache the plugin origin so workspaceUrlForAgent() can build /goto/ URLs
   // against the correct port (the plugin, not minds).
   mngrForwardBaseUrl = url;
+  trustLoopbackCertsForWorkspaceContent();
   const baseSpec = {
     url,
     name: 'mngr_forward_session',
     value: preauth,
     httpOnly: true,
     sameSite: 'lax',
     path: '/',
+    secure: true,
   };
   try {
     await session.defaultSession.cookies.set(baseSpec);

apps/minds/imbue/minds/cli/run.py

@@ -460,7 +460,7 @@ def run(
             f"{_MNGR_FORWARD_LISTEN_TIMEOUT_SECONDS:.0f}s; the plugin likely failed to start. "
             "Check the logs above for its stderr and retry."
         )
-    logger.info("  mngr forward: http://127.0.0.1:{}", mngr_forward_port)
+    logger.info("  mngr forward: https://127.0.0.1:{}", mngr_forward_port)
 
     # AgentCreator is constructed *after* ``start_mngr_forward`` so the
     # readiness probe can use the same preauth cookie the plugin accepts and

apps/minds/imbue/minds/desktop_client/agent_creator.py

@@ -94,18 +94,29 @@
 # assumption about which app that is or which routes it implements.
 _WORKSPACE_PROBE_PATH: Final[str] = "/"
 
+# Scheme of the `mngr forward` proxy origin. minds always runs the proxy with
+# `--use-http2`, so it terminates TLS and the probe/redirect URLs the Python
+# side builds are always `https`.
+_MNGR_FORWARD_SCHEME: Final[str] = "https"
+
 
 def make_workspace_probe_client(preauth_cookie: str, probe_timeout_seconds: float) -> httpx.Client:
     """Construct a reusable httpx.Client preconfigured for workspace probes.
 
     Callers that probe in a tight poll loop should construct one of these and
     pass it to ``probe_workspace_through_plugin`` on each iteration, instead
     of letting the helper construct a one-shot client per call.
+
+    The proxy serves TLS (HTTP/2), so cert verification is disabled: these
+    probes dial ``127.0.0.1`` with a ``Host: agent-<hex>.localhost`` header, so
+    hostname verification could never pass, and the cert is a self-signed
+    ephemeral one the probe is not positioned to validate anyway. Loopback-only.
     """
     return httpx.Client(
         timeout=probe_timeout_seconds,
         follow_redirects=False,
         cookies={_MNGR_FORWARD_SESSION_COOKIE_NAME: preauth_cookie},
+        verify=False,
     )
 
 
@@ -151,7 +162,7 @@ def probe_workspace_through_plugin(
     one-shot client is constructed for this single probe -- fine for
     one-off / sporadic callers but wasteful in a loop.
     """
-    probe_url = f"http://127.0.0.1:{mngr_forward_port}{_WORKSPACE_PROBE_PATH}"
+    probe_url = f"{_MNGR_FORWARD_SCHEME}://127.0.0.1:{mngr_forward_port}{_WORKSPACE_PROBE_PATH}"
     host_header = f"{agent_id}.localhost"
     if client is not None:
         return _probe_once(client, probe_url, host_header)
@@ -2005,7 +2016,7 @@ def _build_redirect_url(self, agent_id: AgentId) -> str:
         """
         if self.mngr_forward_port == 0:
             return f"/goto/{agent_id}/"
-        return f"http://localhost:{self.mngr_forward_port}/goto/{agent_id}/"
+        return f"{_MNGR_FORWARD_SCHEME}://localhost:{self.mngr_forward_port}/goto/{agent_id}/"
 
     def _wait_for_workspace_ready(self, agent_id: AgentId, log_queue: queue.Queue[str]) -> None:
         """Poll the agent's system_interface through the plugin until it responds 200.

apps/minds/imbue/minds/desktop_client/agent_creator_test.py

@@ -60,6 +60,8 @@
 from imbue.mngr.primitives import AgentId
 from imbue.mngr.primitives import HostName
 from imbue.mngr.utils.git_utils import GIT_MIRROR_PUSH_REFSPECS
+from imbue.mngr_forward.tls import build_server_ssl_context
+from imbue.mngr_forward.tls import generate_self_signed_cert
 from imbue.mngr_latchkey.agent_setup import SECRET_LATCHKEY_ENV_VAR_NAMES
 
 
@@ -938,6 +940,13 @@ def _start_scripted_server(not_ready_count: int) -> tuple[HTTPServer, threading.
         {"not_ready_count": not_ready_count, "request_count": 0, "lock": threading.Lock()},
     )
     server = HTTPServer(("127.0.0.1", 0), handler_cls)
+    # The readiness probe dials the proxy over https (minds always runs it with
+    # HTTP/2), so the stand-in server must speak TLS to match -- otherwise the
+    # probe's TLS handshake fails against a plain-HTTP socket. Reuse the proxy's
+    # own self-signed cert helpers so the test exercises the real https path.
+    cert_pem, key_pem = generate_self_signed_cert()
+    ssl_context = build_server_ssl_context(cert_pem, key_pem)
+    server.socket = ssl_context.wrap_socket(server.socket, server_side=True)
     thread = threading.Thread(target=server.serve_forever, daemon=True)
     thread.start()
     port = server.server_address[1]
@@ -1121,6 +1130,40 @@ def _capture(request: httpx.Request) -> httpx.Response:
     assert status == 503
 
 
+def test_probe_workspace_uses_https_scheme() -> None:
+    """The loopback probe dials https, matching the TLS + HTTP/2 proxy.
+
+    The probe must hit the same transport the proxy speaks; a mismatch would
+    make every readiness probe fail the TLS handshake (or hit a closed http
+    port).
+    """
+    captured: list[httpx.Request] = []
+
+    def _capture(request: httpx.Request) -> httpx.Response:
+        captured.append(request)
+        return httpx.Response(200, text="ok")
+
+    with httpx.Client(transport=httpx.MockTransport(_capture)) as client:
+        probe_workspace_through_plugin(
+            mngr_forward_port=18999,
+            preauth_cookie="any-preauth",
+            agent_id=AgentId.generate(),
+            probe_timeout_seconds=0.5,
+            client=client,
+        )
+
+    assert captured[0].url.scheme == "https"
+    assert captured[0].url.host == "127.0.0.1"
+
+
+def test_build_redirect_url_uses_https_scheme(tmp_path) -> None:
+    """The /goto redirect URL the UI navigates to uses the proxy's https scheme."""
+    creator = _make_test_creator(tmp_path, mngr_forward_port=8421)
+    aid = AgentId.generate()
+    url = creator._build_redirect_url(aid)
+    assert url == f"https://localhost:8421/goto/{aid}/"
+
+
 def test_wait_for_workspace_ready_publishes_anyway_on_timeout(tmp_path) -> None:
     """If the probe times out, we still return so the caller can publish the redirect."""
     server, _thread, port = _start_scripted_server(not_ready_count=10**6)

apps/minds/imbue/minds/desktop_client/app.py

@@ -208,10 +208,13 @@ def _get_mngr_forward_origin() -> str:
     """Build the bare-origin URL of the ``mngr forward`` plugin.
 
     Used by templates to construct ``/goto/<agent>/`` URLs that target the
-    plugin (which owns subdomain forwarding) rather than minds.
+    plugin (which owns subdomain forwarding) rather than minds. minds always
+    runs the proxy with TLS + HTTP/2, so the scheme is ``https`` and the
+    rendered links reach it rather than failing a plaintext request against
+    the TLS listener.
     """
     port = get_state().mngr_forward_port or 8421
-    return f"http://localhost:{port}"
+    return f"https://localhost:{port}"
 
 
 def _get_is_mac() -> bool:
@@ -2435,9 +2438,9 @@ def create_desktop_client(
     The agent-subdomain forwarding lives in the ``mngr_forward`` plugin
     (``libs/mngr_forward``) now; this app only serves minds-specific routes
     on the bare origin (login, landing, accounts, workspace settings,
-    sharing, agent create / destroy). Workspace links go to
-    ``http://localhost:<mngr_forward_port>/goto/<agent>/`` instead of being
-    routed in-process.
+    sharing, agent create / destroy). Workspace links go to the proxy's
+    ``localhost:<mngr_forward_port>/goto/<agent>/`` route (``https`` when the
+    proxy serves HTTP/2, else ``http``) instead of being routed in-process.
 
     ``envelope_stream_consumer`` feeds discovery events into
     ``backend_resolver`` and is also the bounce target for ``SIGHUP``-style

apps/minds/imbue/minds/desktop_client/e2e_workspace_runner.py

@@ -69,8 +69,11 @@
 _INBOX_PATH_PATTERN: Final[re.Pattern[str]] = re.compile(r"^http://localhost:\d+/inbox(?:/|$|\?)")
 # The agent subdomain URL the create flow redirects to once the workspace's
 # ``system_interface`` is reachable. The desktop client wraps that origin in
-# the mngr_forward plugin, so the port may differ from the bare backend.
-_AGENT_SUBDOMAIN_PATTERN: Final[re.Pattern[str]] = re.compile(r"^http://agent-[a-f0-9]+\.localhost:\d+(?:/|$)")
+# the mngr_forward plugin, so the port may differ from the bare backend. The
+# scheme is ``https`` when the proxy serves TLS + HTTP/2 (the default) and
+# ``http`` otherwise, so accept both. (The bare minds backend origin stays
+# plain ``http`` -- see ``_BACKEND_ORIGIN_PATTERN``.)
+_AGENT_SUBDOMAIN_PATTERN: Final[re.Pattern[str]] = re.compile(r"^https?://agent-[a-f0-9]+\.localhost:\d+(?:/|$)")
 
 # Default env tier when nothing is activated. Staging's ``client.toml`` is
 # committed under apps/minds/imbue/minds/config/envs/staging/ so callers

apps/minds/imbue/minds/desktop_client/e2e_workspace_runner_test.py

@@ -70,6 +70,17 @@ def test_wait_returns_when_workspace_url_reached() -> None:
     _wait_for_workspace_ready_or_failure(cast(Page, page), timeout_seconds=5)
 
 
+def test_wait_returns_for_https_workspace_url() -> None:
+    """The workspace origin is https when the proxy serves TLS + HTTP/2 (the default).
+
+    The ready-check must recognize that scheme, not just http -- otherwise the
+    waiter never sees the workspace as ready and times out even though it loaded.
+    """
+    https_ready_url = "https://agent-deadbeef.localhost:8421/"
+    page = _FakePage(urls=[https_ready_url], is_visible_results=[False])
+    _wait_for_workspace_ready_or_failure(cast(Page, page), timeout_seconds=5)
+
+
 def test_wait_raises_with_surfaced_error_on_failure_view() -> None:
     page = _FakePage(
         urls=[_PENDING_URL],

apps/minds/imbue/minds/desktop_client/forward_cli.py

@@ -622,6 +622,28 @@ def start_mngr_forward(
     dropped.
     """
     preauth_cookie = secrets.token_urlsafe(_PREAUTH_TOKEN_LENGTH)
+    command = _build_forward_command(config, preauth_cookie)
+    env = dict(os.environ)
+    env["MNGR_HOST_DIR"] = str(config.mngr_host_dir)
+    logger.info("Spawning `mngr forward` subprocess: {}", " ".join(_redact_secrets(command)))
+    # noqa: S603 — command is fully controlled (mngr binary + the args we
+    # build above), no untrusted input reaches the argv list.
+    process = subprocess.Popen(  # noqa: S603
+        command,
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        bufsize=0,
+        env=env,
+        cwd=str(Path.home()),
+    )
+    consumer = EnvelopeStreamConsumer(resolver=resolver)
+    consumer.attach(process)
+    return consumer, preauth_cookie
+
+
+def _build_forward_command(config: ForwardSubprocessConfig, preauth_cookie: str) -> list[str]:
+    """Build the ``mngr forward`` argv for the subprocess minds spawns."""
     command: list[str] = [
         config.mngr_binary,
         "forward",
@@ -637,27 +659,15 @@ def start_mngr_forward(
         "--format",
         "jsonl",
     ]
+    # TLS + HTTP/2 so the workspace origin is not capped by Chromium's
+    # per-origin HTTP/1.1 connection limit. The Electron shell trusts the
+    # proxy's self-signed cert for its loopback origins.
+    command.append("--use-http2")
     for include in config.agent_include:
         command.extend(["--agent-include", include])
     for spec in config.reverse_specs:
         command.extend(["--reverse", spec])
-    env = dict(os.environ)
-    env["MNGR_HOST_DIR"] = str(config.mngr_host_dir)
-    logger.info("Spawning `mngr forward` subprocess: {}", " ".join(_redact_secrets(command)))
-    # noqa: S603 — command is fully controlled (mngr binary + the args we
-    # build above), no untrusted input reaches the argv list.
-    process = subprocess.Popen(  # noqa: S603
-        command,
-        stdin=subprocess.DEVNULL,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        bufsize=0,
-        env=env,
-        cwd=str(Path.home()),
-    )
-    consumer = EnvelopeStreamConsumer(resolver=resolver)
-    consumer.attach(process)
-    return consumer, preauth_cookie
+    return command
 
 
 def _redact_secrets(command: list[str]) -> list[str]:

apps/minds/imbue/minds/desktop_client/forward_cli_test.py

@@ -25,6 +25,8 @@
 from imbue.imbue_common.event_envelope import IsoTimestamp
 from imbue.minds.desktop_client.backend_resolver import MngrCliBackendResolver
 from imbue.minds.desktop_client.forward_cli import EnvelopeStreamConsumer
+from imbue.minds.desktop_client.forward_cli import ForwardSubprocessConfig
+from imbue.minds.desktop_client.forward_cli import _build_forward_command
 from imbue.minds.desktop_client.forward_cli import _redact_secrets
 from imbue.minds.primitives import ServiceName
 from imbue.mngr.api.discovery_events import AgentDestroyedEvent
@@ -679,6 +681,38 @@ def test_start_before_attach_raises(consumer: EnvelopeStreamConsumer) -> None:
         consumer.start(cg)
 
 
+# --- _build_forward_command ----------------------------------------------
+
+
+def test_build_forward_command_includes_use_http2_flag() -> None:
+    """The spawned argv always carries --use-http2 so the proxy serves TLS.
+
+    minds always runs the proxy with TLS + HTTP/2, matching the https/wss URLs
+    the rest of minds builds, so a client that expects https reaches an https
+    proxy.
+    """
+    config = ForwardSubprocessConfig(service="system_interface")
+    command = _build_forward_command(config, preauth_cookie="a-secret")
+    assert "--use-http2" in command
+    # Core flags are always present alongside the TLS flag.
+    assert command[:2] == [config.mngr_binary, "forward"]
+    assert "--observe-via-file" in command
+    assert command[command.index("--service") + 1] == "system_interface"
+    assert command[command.index("--preauth-cookie") + 1] == "a-secret"
+
+
+def test_build_forward_command_threads_includes_and_reverse_specs() -> None:
+    """Agent-include and reverse specs are expanded into repeated flags."""
+    config = ForwardSubprocessConfig(
+        agent_include=("has(agent.labels.is_primary)", "agent.name == 'x'"),
+        reverse_specs=("8420:8420",),
+    )
+    command = _build_forward_command(config, preauth_cookie="s")
+    includes = [command[i + 1] for i, tok in enumerate(command) if tok == "--agent-include"]
+    assert includes == ["has(agent.labels.is_primary)", "agent.name == 'x'"]
+    assert command[command.index("--reverse") + 1] == "8420:8420"
+
+
 # --- _redact_secrets ------------------------------------------------------