Disable staleness watchdog for frozen Moonbeam chain (#27)

Moonbeam stopped producing blocks at 16,796,699 (0x1004c1b). Because
the chain will never produce a new block, the WSConn staleness
watchdog was closing every connection within 120s and the NodeProxy
was spawning a replacement — both pointing at the same healthy
provider, which never got a newHeads frame to satisfy the freshness
check. The repeated :DOWN messages and the new connections' open_loop
handshakes swamped the Moonbeam NodeProxy mailbox (observed at
>500k pending messages on as1), which in turn made every
eth_getCode from FleetValidation time out at 25s and crash the EdgeV2
ticket handler.

Make the staleness opt-in per chain:
- Chains.Moonbeam.frozen?/0 and final_block_number/0 declare the
  chain's permanent head.
- RemoteChain.frozen?/1 and final_block_number/1 default to
  false/nil for any chain that doesn't opt in, so Base, Diode, Oasis
  and Anvil are unaffected.
- RemoteChain.WSConn.stale_at?/3 and the :ping handler short-circuit
  to false for frozen chains. The subscription_id guard is preserved
  so a real disconnect still tears the WSConn down.
- RemoteChain.ChainList.block_current?/2 accepts the final block on a
  frozen chain regardless of timestamp age, so endpoint re-probes on
  the 5-minute TTL don't reject the provider as 'stopped advancing'.

Regression tests cover the new opt-in behaviour, the staleness
short-circuit, and the block_current? branch for frozen chains.

Author: dominicletz

lib/chains/chains.ex

@@ -132,6 +132,14 @@ end
 defmodule Chains.Moonbeam do
   alias DiodeClient.{Base16, Hash}
 
+  @doc """
+  Last block Moonbeam produced before the chain halted. Treated as the
+  permanent head by `RemoteChain` so the staleness watchdog does not keep
+  evicting healthy connections to a chain that will never produce another
+  block. Update this if/when the chain resumes or hard-forks.
+  """
+  @final_block_number 16_796_699
+
   def chain_id(), do: 1284
   def expected_block_intervall(), do: 6
   def epoch(n), do: Chains.epoch(__MODULE__, n)
@@ -142,6 +150,17 @@ defmodule Chains.Moonbeam do
   def epoch_duration(), do: 2_592_000
   def chain_prefix(), do: "glmr"
 
+  @doc """
+  Whether this chain has stopped producing blocks permanently. When true,
+  the WSConn staleness watchdog is disabled (the chain will never satisfy
+  it) and the provider's `latest` block must match `final_block_number/0`
+  for endpoint health checks.
+  """
+  def frozen?, do: true
+
+  @doc "The final block number of this chain. Only meaningful when `frozen?/0` is true."
+  def final_block_number, do: @final_block_number
+
   def additional_endpoints() do
     ~w(
       https://moonbeam.api.onfinality.io/public

lib/remote_chain/chain_list.ex

@@ -213,8 +213,30 @@ defmodule RemoteChain.ChainList do
   skew is bounded separately by `@max_future_skew_seconds` because the
   staleness predicate only checks how stale a `lastblock_at` is — never how
   far in the future it sits.
+
+  Frozen chains (see `RemoteChain.frozen?/1`) have a fixed head; any
+  provider that returns a block matching `RemoteChain.final_block_number/1`
+  is considered current regardless of timestamp age, because age and
+  staleness are not meaningful when the chain will never produce a newer
+  block.
   """
-  def block_current?(chain, %{"timestamp" => timestamp}) when is_binary(timestamp) do
+  def block_current?(chain, block) when is_map(block) do
+    cond do
+      RemoteChain.frozen?(chain) ->
+        final = RemoteChain.final_block_number(chain)
+        is_integer(final) and current_by_block_number?(block, final)
+
+      is_binary(block["timestamp"]) ->
+        block_current_by_timestamp(chain, block["timestamp"])
+
+      true ->
+        false
+    end
+  end
+
+  def block_current?(_chain, _block), do: false
+
+  defp block_current_by_timestamp(chain, timestamp) do
     block_ts = Base16.decode_int(timestamp)
     now = System.os_time(:second)
     age = now - block_ts
@@ -223,7 +245,11 @@ defmodule RemoteChain.ChainList do
       not RemoteChain.WSConn.stale_at?(block_age_to_lastblock_at(now, age), chain)
   end
 
-  def block_current?(_chain, _block), do: false
+  defp current_by_block_number?(%{"number" => number}, final) when is_binary(number) do
+    Base16.decode_int(number) == final
+  end
+
+  defp current_by_block_number?(_block, _final), do: false
 
   @doc false
   def timestamp_current?(max_age_seconds, block_timestamp)

lib/remote_chain/remote_chain.ex

@@ -96,6 +96,33 @@ defmodule RemoteChain do
     chainimpl(chain) != Chains.Moonbeam
   end
 
+  @doc """
+  Whether the chain has stopped producing blocks permanently.
+
+  Opt-in per chain via `Chain.frozen?/0`; defaults to `false` for chains
+  that do not declare the function. Used to disable the WSConn staleness
+  watchdog and the endpoint freshness check, both of which would otherwise
+  reject healthy connections to a chain whose head is fixed.
+  """
+  def frozen?(chain) do
+    impl = chainimpl(chain)
+    Code.ensure_loaded?(impl) and function_exported?(impl, :frozen?, 0) and impl.frozen?()
+  end
+
+  @doc """
+  The final block number of a frozen chain, or `nil` for chains that are
+  still producing blocks (or have not declared `Chain.final_block_number/0`).
+  """
+  def final_block_number(chain) do
+    impl = chainimpl(chain)
+
+    cond do
+      not Code.ensure_loaded?(impl) -> nil
+      not function_exported?(impl, :final_block_number, 0) -> nil
+      true -> impl.final_block_number()
+    end
+  end
+
   @doc false
   def execution_reverted_rpc_error do
     %{"code" => -32000, "message" => "execution reverted"}

lib/remote_chain/ws_conn.ex

@@ -48,16 +48,25 @@ defmodule RemoteChain.WSConn do
   overrides the multiplier — used by `NodeProxy` for its eviction pass (two
   ping cycles, `@stale_eviction_intervals`).
 
+  Frozen chains (see `RemoteChain.frozen?/1`) never satisfy this predicate:
+  the chain is not producing blocks, so `lastblock_at` stays at connect
+  time and would otherwise trigger constant evictions of healthy
+  connections. Provider health is still enforced by TCP/WS disconnects
+  and the subscription handshake (`handle_info(:ping, ...)`).
+
   This is the single threshold shared by `stale?/2` (pid-based), the
   `:ping` close handler, `NodeProxy`'s consensus and eviction logic, and
   `ChainList.block_current?/2`.
   """
   def stale_at?(lastblock_at, chain, intervals \\ @stale_threshold_intervals) do
-    case lastblock_at do
-      nil ->
+    cond do
+      RemoteChain.frozen?(chain) ->
+        false
+
+      is_nil(lastblock_at) ->
         false
 
-      %DateTime{} ->
+      true ->
         age = DateTime.diff(DateTime.utc_now(), lastblock_at, :second)
         age > chain.expected_block_intervall() * intervals
     end
@@ -276,16 +285,24 @@ defmodule RemoteChain.WSConn do
       raise "No subscription id received, aborting connection with #{ws_url}"
     end
 
-    if stale_at?(state.lastblock_at, chain) do
-      {:message_queue_len, len} = Process.info(self(), :message_queue_len)
+    # Frozen chains: the staleness predicate can never be satisfied, so
+    # skip it. The subscription_id check above still guards against a
+    # WSConn that lost its `eth_subscribe("newHeads")` confirmation.
+    cond do
+      RemoteChain.frozen?(chain) ->
+        {:ok, state}
 
-      Logger.warning(
-        "WSConn #{inspect({self(), len})} block timeout #{chain} (#{ws_url}). Restarting..."
-      )
+      stale_at?(state.lastblock_at, chain) ->
+        {:message_queue_len, len} = Process.info(self(), :message_queue_len)
 
-      {:close, state}
-    else
-      {:ok, state}
+        Logger.warning(
+          "WSConn #{inspect({self(), len})} block timeout #{chain} (#{ws_url}). Restarting..."
+        )
+
+        {:close, state}
+
+      true ->
+        {:ok, state}
     end
   end
 

test/remote_chain/chain_list_test.exs

@@ -181,6 +181,50 @@ defmodule RemoteChain.ChainListTest do
     end
   end
 
+  describe "block_current?/2 for frozen chains" do
+    # Regression: Moonbeam stopped producing blocks at 16_796_699. A
+    # healthy provider's `eth_getBlockByNumber("latest")` returns that
+    # block with an ancient timestamp; the age-based check rejects it,
+    # which causes the endpoint probe to fail on every TTL refresh.
+    test "accepts the final block on a frozen chain regardless of timestamp" do
+      final = RemoteChain.final_block_number(Chains.Moonbeam)
+      ancient = System.os_time(:second) - 365 * 24 * 3600
+
+      assert RemoteChain.ChainList.block_current?(Chains.Moonbeam, %{
+               "number" => "0x" <> Integer.to_string(final, 16),
+               "timestamp" => hex_timestamp(ancient)
+             })
+    end
+
+    test "rejects a block whose number does not match the final block on a frozen chain" do
+      # Provider reports a block newer than the freeze → must be wrong
+      # (or the chain has resumed and the final_block_number is stale).
+      final = RemoteChain.final_block_number(Chains.Moonbeam)
+      wrong = "0x" <> Integer.to_string(final + 1, 16)
+
+      refute RemoteChain.ChainList.block_current?(Chains.Moonbeam, %{
+               "number" => wrong,
+               "timestamp" => hex_timestamp(System.os_time(:second))
+             })
+    end
+
+    test "rejects a frozen-chain block without a decodable number" do
+      refute RemoteChain.ChainList.block_current?(Chains.Moonbeam, %{
+               "timestamp" => hex_timestamp(System.os_time(:second))
+             })
+    end
+
+    test "still applies the timestamp check to non-frozen chains" do
+      # Sanity check: a 24h-old block on Anvil (15s cadence) is rejected.
+      stale = System.os_time(:second) - 24 * 3600
+
+      refute RemoteChain.ChainList.block_current?(Chains.Anvil, %{
+               "number" => "0x1",
+               "timestamp" => hex_timestamp(stale)
+             })
+    end
+  end
+
   describe "timestamp_current?/2" do
     test "accepts blocks within max age, rejects older ones" do
       now = System.os_time(:second)

test/remote_chain/remote_chain_test.exs

@@ -73,4 +73,36 @@ defmodule RemoteChainTest do
       assert RemoteChain.rpc_endpoints(Chains.Diode) == ["http://local-override.example:3834"]
     end
   end
+
+  describe "frozen?/1" do
+    test "returns true for chains that opt in via frozen?/0 (Moonbeam)" do
+      assert RemoteChain.frozen?(Chains.Moonbeam)
+    end
+
+    test "returns false for chains that do not declare frozen?/0" do
+      refute RemoteChain.frozen?(Chains.Diode)
+      refute RemoteChain.frozen?(Chains.OasisSapphire)
+      refute RemoteChain.frozen?(Chains.Base)
+    end
+
+    test "accepts chain_id and chain prefix dispatch" do
+      assert RemoteChain.frozen?(Chains.Moonbeam.chain_id())
+      assert RemoteChain.frozen?("glmr")
+    end
+  end
+
+  describe "final_block_number/1" do
+    test "returns the chain's declared final block number (Moonbeam)" do
+      assert is_integer(RemoteChain.final_block_number(Chains.Moonbeam))
+
+      assert RemoteChain.final_block_number(Chains.Moonbeam) ==
+               Chains.Moonbeam.final_block_number()
+    end
+
+    test "returns nil for chains that have not declared final_block_number/0" do
+      assert RemoteChain.final_block_number(Chains.Diode) == nil
+      assert RemoteChain.final_block_number(Chains.OasisSapphire) == nil
+      assert RemoteChain.final_block_number(Chains.Base) == nil
+    end
+  end
 end

test/remote_chain/ws_conn_test.exs

@@ -89,4 +89,40 @@ defmodule RemoteChain.WSConnTest do
       assert WSConn.stale_threshold_intervals() == 10
     end
   end
+
+  describe "stale_at?/3 for frozen chains" do
+    # Regression: Moonbeam stopped producing blocks at 16_796_699. Without a
+    # short-circuit, every WSConn's `lastblock_at` is "ancient" relative to
+    # the staleness window, and the watchdog closes/restarts them
+    # continuously, swamping the NodeProxy mailbox and starving RPCs.
+    test "returns false for a frozen chain regardless of how old lastblock_at is" do
+      ancient = DateTime.utc_now() |> DateTime.add(-365 * 24 * 3600, :second)
+      refute WSConn.stale_at?(ancient, Chains.Moonbeam)
+      refute WSConn.stale_at?(ancient, Chains.Moonbeam, 1)
+    end
+
+    test "returns false for a frozen chain even with intervals=0" do
+      ancient = DateTime.utc_now() |> DateTime.add(-3600, :second)
+      refute WSConn.stale_at?(ancient, Chains.Moonbeam, 0)
+    end
+
+    test "returns false for a frozen chain when lastblock_at is nil" do
+      refute WSConn.stale_at?(nil, Chains.Moonbeam)
+    end
+
+    test "still returns true for non-frozen chains with old lastblock_at" do
+      # Sanity check: the short-circuit only affects frozen chains. A 1-day
+      # old timestamp on Anvil (15s cadence) is still stale.
+      ancient = DateTime.utc_now() |> DateTime.add(-24 * 3600, :second)
+      assert WSConn.stale_at?(ancient, Chains.Anvil)
+    end
+  end
+
+  describe "stale?/2 for frozen chains" do
+    test "returns false for a frozen chain with a never-updated lastblock_at" do
+      ancient = DateTime.utc_now() |> DateTime.add(-3600, :second)
+      {:ok, pid} = WSConnStateStub.start(%WSConn{lastblock_at: ancient})
+      refute WSConn.stale?(pid, Chains.Moonbeam)
+    end
+  end
 end