diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/handlers/JsonRpcObjectExecutor.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/handlers/JsonRpcObjectExecutor.java index 70c78e9ab3e..ca78d020881 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/handlers/JsonRpcObjectExecutor.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/handlers/JsonRpcObjectExecutor.java @@ -23,6 +23,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.StreamingJsonRpcSuccessResponse; import org.hyperledger.besu.plugin.services.rpc.RpcResponseType; import java.io.IOException; @@ -72,6 +73,22 @@ private void handleJsonObjectResponse( response.setStatusCode(status(jsonRpcResponse).code()); if (jsonRpcResponse.getType() == RpcResponseType.NONE) { response.end(); + } else if (jsonRpcResponse instanceof StreamingJsonRpcSuccessResponse streamingResponse) { + try (final JsonResponseStreamer streamer = + new JsonResponseStreamer(response, ctx.request().remoteAddress())) { + final JsonGenerator generator = + getJsonObjectMapper().getFactory().createGenerator(streamer); + try { + streamingResponse.writeTo(generator); + } finally { + try { + generator.close(); + } catch (final IOException ignored) { + // Generator close flushes the buffer, which may fail if the connection was + // reset mid-stream. Swallow it — the primary exception is already propagating. + } + } + } } else { try (final JsonResponseStreamer streamer = new JsonResponseStreamer(response, ctx.request().remoteAddress())) { diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AbstractDebugTraceBlock.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AbstractDebugTraceBlock.java index eade86613d0..1364a12cfb5 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AbstractDebugTraceBlock.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/AbstractDebugTraceBlock.java @@ -14,36 +14,38 @@ */ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods; -import static org.hyperledger.besu.services.pipeline.PipelineBuilder.createPipelineFrom; +import static org.hyperledger.besu.ethereum.mainnet.feemarket.ExcessBlobGasCalculator.calculateExcessBlobGasForParent; +import org.hyperledger.besu.datatypes.BlobGas; +import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.TransactionTraceParams; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.DebugTraceTransactionResult; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.StreamingJsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.debug.TraceOptions; -import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; +import org.hyperledger.besu.ethereum.mainnet.ImmutableTransactionValidationParams; import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; -import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; -import org.hyperledger.besu.metrics.BesuMetricCategory; +import org.hyperledger.besu.ethereum.mainnet.block.access.list.AccessLocationTracker; +import org.hyperledger.besu.ethereum.mainnet.block.access.list.BlockAccessList; +import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; +import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; +import org.hyperledger.besu.ethereum.vm.StreamingDebugOperationTracer; +import org.hyperledger.besu.evm.blockhash.BlockHashLookup; import org.hyperledger.besu.metrics.ObservableMetricsSystem; -import org.hyperledger.besu.plugin.services.metrics.Counter; -import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; -import org.hyperledger.besu.services.pipeline.Pipeline; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; + +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Optional; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import com.google.common.base.Suppliers; @@ -51,9 +53,7 @@ public abstract class AbstractDebugTraceBlock implements JsonRpcMethod { private final ProtocolSchedule protocolSchedule; - private final LabelledMetric outputCounter; private final Supplier blockchainQueriesSupplier; - private final EthScheduler ethScheduler; public AbstractDebugTraceBlock( final ProtocolSchedule protocolSchedule, @@ -62,14 +62,6 @@ public AbstractDebugTraceBlock( final EthScheduler ethScheduler) { this.blockchainQueriesSupplier = Suppliers.ofInstance(blockchainQueries); this.protocolSchedule = protocolSchedule; - this.outputCounter = - metricsSystem.createLabelledCounter( - BesuMetricCategory.BLOCKCHAIN, - "transactions_debugTraceblock_pipeline_processed_total", - "Number of transactions processed for each block", - "step", - "action"); - this.ethScheduler = ethScheduler; } protected BlockchainQueries getBlockchainQueries() { @@ -97,61 +89,103 @@ protected TraceOptions getTraceOptions(final JsonRpcRequestContext requestContex return traceOptions; } - protected Collection getTraces( - final JsonRpcRequestContext requestContext, - final TraceOptions traceOptions, - final Optional maybeBlock) { - return maybeBlock - .flatMap( - block -> - Tracer.processTracing( - getBlockchainQueries(), - Optional.of(block.getHeader()), - traceableState -> { - List tracesList = - Collections.synchronizedList(new ArrayList<>()); - final ProtocolSpec protocolSpec = - protocolSchedule.getByBlockHeader(block.getHeader()); - final MainnetTransactionProcessor transactionProcessor = - protocolSpec.getTransactionProcessor(); - final TraceBlock.ChainUpdater chainUpdater = - new TraceBlock.ChainUpdater(traceableState); - - TransactionSource transactionSource = new TransactionSource(block); - DebugOperationTracer debugOperationTracer = - new DebugOperationTracer(traceOptions.opCodeTracerConfig(), true); - ExecuteTransactionStep executeTransactionStep = - new ExecuteTransactionStep( - chainUpdater, - transactionProcessor, - getBlockchainQueries().getBlockchain(), - debugOperationTracer, - protocolSpec, - block); - - Pipeline traceBlockPipeline = - createPipelineFrom( - "getTransactions", - transactionSource, - 4, - outputCounter, - false, - "debug_trace_block") - .thenProcess("executeTransaction", executeTransactionStep) - .thenProcessAsyncOrdered( - "debugTraceTransactionStep", - DebugTraceTransactionStepFactory.createAsync( - traceOptions, protocolSpec), - 4) - .andFinishWith("collect_results", tracesList::add); - - try { - ethScheduler.startPipeline(traceBlockPipeline).get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } - return Optional.of(tracesList); - })) - .orElse(null); + /** + * Creates a streaming response that writes each transaction trace directly to the JSON output. + * Uses StreamingDebugOperationTracer which writes each struct log inline during EVM execution, + * achieving O(1) frame memory. Memory is only captured for the 24 opcodes that touch it. + */ + protected StreamingJsonRpcSuccessResponse.ResultWriter getStreamingTraces( + final TraceOptions traceOptions, final Optional maybeBlock) { + return generator -> { + if (maybeBlock.isEmpty()) { + generator.writeNull(); + return; + } + final Block block = maybeBlock.get(); + + generator.writeStartArray(); + final AtomicBoolean lambdaEntered = new AtomicBoolean(false); + + final Optional tracingResult = Tracer.processTracing( + getBlockchainQueries(), + Optional.of(block.getHeader()), + traceableState -> { + lambdaEntered.set(true); + final ProtocolSpec protocolSpec = + protocolSchedule.getByBlockHeader(block.getHeader()); + final MainnetTransactionProcessor transactionProcessor = + protocolSpec.getTransactionProcessor(); + final TraceBlock.ChainUpdater chainUpdater = + new TraceBlock.ChainUpdater(traceableState); + final BlockHeader header = block.getHeader(); + final Optional maybeParentHeader = + getBlockchainQueries().getBlockchain().getBlockHeader(header.getParentHash()); + final Wei blobGasPrice = + protocolSpec + .getFeeMarket() + .blobGasPricePerGas( + maybeParentHeader + .map(parent -> calculateExcessBlobGasForParent(protocolSpec, parent)) + .orElse(BlobGas.ZERO)); + final BlockHashLookup blockHashLookup = + protocolSpec.getPreExecutionProcessor().createBlockHashLookup( + getBlockchainQueries().getBlockchain(), header); + + try { + for (final Transaction tx : block.getBody().getTransactions()) { + generator.writeStartObject(); + generator.writeStringField("txHash", tx.getHash().toHexString()); + generator.writeFieldName("result"); + generator.writeStartObject(); + + generator.writeFieldName("structLogs"); + generator.writeStartArray(); + + final StreamingDebugOperationTracer tracer = + new StreamingDebugOperationTracer( + traceOptions.opCodeTracerConfig(), true, generator); + + final AccessLocationTracker accessListTracker = + BlockAccessList.BlockAccessListBuilder + .createTransactionAccessLocationTracker(0); + final TransactionProcessingResult result = + transactionProcessor.processTransaction( + chainUpdater.getNextUpdater(), + header, + tx, + header.getCoinbase(), + tracer, + blockHashLookup, + ImmutableTransactionValidationParams.builder().build(), + blobGasPrice, + Optional.of(accessListTracker)); + + generator.writeEndArray(); // structLogs + + final long gas = tx.getGasLimit() - result.getGasRemaining(); + final String returnValue = result.getOutput().toString().substring(2); + generator.writeNumberField("gas", gas); + generator.writeBooleanField("failed", !result.isSuccessful()); + generator.writeStringField("returnValue", returnValue); + + generator.writeEndObject(); // result + generator.writeEndObject(); // tx + generator.flush(); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return Optional.of(Boolean.TRUE); + }); + + if (tracingResult.isEmpty() && lambdaEntered.get()) { + // Lambda ran but failed (e.g. connection reset mid-trace). The generator is in an + // unknown nested context — abandon the response rather than producing broken JSON. + throw new IOException("debug_traceBlock failed mid-stream: tracing aborted"); + } + // Either tracing succeeded (all tx objects closed cleanly) or worldstate was unavailable + // (lambda never ran, generator is at top-level array context → produces []). + generator.writeEndArray(); + }; } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java index 9111d3c6ebe..2fc080b9ed8 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlock.java @@ -20,9 +20,8 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcParameter.JsonRpcParameterException; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.DebugTraceTransactionResult; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.StreamingJsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.core.BlockHeaderFunctions; @@ -34,7 +33,6 @@ import org.hyperledger.besu.ethereum.rlp.RLPException; import org.hyperledger.besu.metrics.ObservableMetricsSystem; -import java.util.Collection; import java.util.Optional; import org.apache.tuweni.bytes.Bytes; @@ -80,9 +78,9 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { .getBlockchain() .getBlockByHash(block.getHeader().getParentHash()) .isPresent()) { - final Collection results = - getTraces(requestContext, traceOptions, Optional.ofNullable(block)); - return new JsonRpcSuccessResponse(requestContext.getRequest().getId(), results); + return new StreamingJsonRpcSuccessResponse( + requestContext.getRequest().getId(), + getStreamingTraces(traceOptions, Optional.ofNullable(block))); } else { return new JsonRpcErrorResponse( requestContext.getRequest().getId(), RpcErrorType.PARENT_BLOCK_NOT_FOUND); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByHash.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByHash.java index 0a1cb902aaf..c9d9a4a4bc5 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByHash.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByHash.java @@ -20,9 +20,8 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcParameter.JsonRpcParameterException; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.DebugTraceTransactionResult; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.StreamingJsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.core.Block; import org.hyperledger.besu.ethereum.debug.TraceOptions; @@ -30,7 +29,6 @@ import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.metrics.ObservableMetricsSystem; -import java.util.Collection; import java.util.Optional; public class DebugTraceBlockByHash extends AbstractDebugTraceBlock { @@ -61,8 +59,8 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { TraceOptions traceOptions = getTraceOptions(requestContext); Optional maybeBlock = getBlockchainQueries().getBlockchain().getBlockByHash(blockHash); - final Collection results = - getTraces(requestContext, traceOptions, maybeBlock); - return new JsonRpcSuccessResponse(requestContext.getRequest().getId(), results); + return new StreamingJsonRpcSuccessResponse( + requestContext.getRequest().getId(), + getStreamingTraces(traceOptions, maybeBlock)); } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByNumber.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByNumber.java index edde80c9cb7..c0ae9c3fe5f 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByNumber.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugTraceBlockByNumber.java @@ -14,8 +14,10 @@ */ package org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods; -import static org.hyperledger.besu.services.pipeline.PipelineBuilder.createPipelineFrom; +import static org.hyperledger.besu.ethereum.mainnet.feemarket.ExcessBlobGasCalculator.calculateExcessBlobGasForParent; +import org.hyperledger.besu.datatypes.BlobGas; +import org.hyperledger.besu.datatypes.Wei; import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters; @@ -23,34 +25,37 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonRpcParameter.JsonRpcParameterException; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.TransactionTraceParams; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.Tracer; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.processor.TransactionTrace; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.RpcErrorType; -import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.DebugTraceTransactionResult; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.StreamingJsonRpcSuccessResponse; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.StructLog; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.StructLogWithError; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.core.Block; +import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.debug.TraceOptions; import org.hyperledger.besu.ethereum.eth.manager.EthScheduler; +import org.hyperledger.besu.ethereum.mainnet.ImmutableTransactionValidationParams; import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; +import org.hyperledger.besu.ethereum.mainnet.block.access.list.AccessLocationTracker; +import org.hyperledger.besu.ethereum.mainnet.block.access.list.BlockAccessList; +import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; -import org.hyperledger.besu.metrics.BesuMetricCategory; +import org.hyperledger.besu.evm.blockhash.BlockHashLookup; +import org.hyperledger.besu.evm.tracing.TraceFrame; import org.hyperledger.besu.metrics.ObservableMetricsSystem; -import org.hyperledger.besu.plugin.services.metrics.Counter; -import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; -import org.hyperledger.besu.services.pipeline.Pipeline; -import java.util.ArrayList; -import java.util.Collections; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.List; import java.util.Optional; -import java.util.concurrent.ExecutionException; public class DebugTraceBlockByNumber extends AbstractBlockParameterMethod { - protected final ProtocolSchedule protocolSchedule; - private final LabelledMetric outputCounter; - private final EthScheduler ethScheduler; + private final ProtocolSchedule protocolSchedule; public DebugTraceBlockByNumber( final ProtocolSchedule protocolSchedule, @@ -59,14 +64,6 @@ public DebugTraceBlockByNumber( final EthScheduler ethScheduler) { super(blockchainQueries); this.protocolSchedule = protocolSchedule; - this.outputCounter = - metricsSystem.createLabelledCounter( - BesuMetricCategory.BLOCKCHAIN, - "transactions_debugtraceblock_pipeline_processed_total", - "Number of transactions processed for each block", - "step", - "action"); - this.ethScheduler = ethScheduler; } @Override @@ -85,13 +82,29 @@ protected BlockParameter blockParameter(final JsonRpcRequestContext request) { } @Override - protected Object resultByBlockNumber( - final JsonRpcRequestContext request, final long blockNumber) { + public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { + final BlockParameter blockParam = blockParameter(requestContext); + final Optional blockNumber = blockParam.getNumber(); + final long resolvedBlockNumber; + + if (blockNumber.isPresent()) { + resolvedBlockNumber = blockNumber.get(); + } else if (blockParam.isLatest()) { + resolvedBlockNumber = getBlockchainQueries().headBlockNumber(); + } else if (blockParam.isFinalized()) { + resolvedBlockNumber = + getBlockchainQueries().finalizedBlockHeader().map(h -> h.getNumber()).orElse(-1L); + } else if (blockParam.isSafe()) { + resolvedBlockNumber = + getBlockchainQueries().safeBlockHeader().map(h -> h.getNumber()).orElse(-1L); + } else { + resolvedBlockNumber = getBlockchainQueries().headBlockNumber(); + } final TraceOptions traceOptions; try { traceOptions = - request + requestContext .getOptionalParameter(1, TransactionTraceParams.class) .map(TransactionTraceParams::traceOptions) .orElse(TraceOptions.DEFAULT); @@ -101,64 +114,112 @@ protected Object resultByBlockNumber( RpcErrorType.INVALID_TRANSACTION_TRACE_PARAMS, e); } catch (IllegalArgumentException e) { - // Handle invalid tracer type from TracerType.fromString() throw new InvalidJsonRpcParameters( e.getMessage(), RpcErrorType.INVALID_TRANSACTION_TRACE_PARAMS, e); } + Optional maybeBlock = - getBlockchainQueries().getBlockchain().getBlockByNumber(blockNumber); - - return maybeBlock - .flatMap( - block -> - Tracer.processTracing( - getBlockchainQueries(), - Optional.of(block.getHeader()), - traceableState -> { - List tracesList = - Collections.synchronizedList(new ArrayList<>()); - final ProtocolSpec protocolSpec = - protocolSchedule.getByBlockHeader(block.getHeader()); - final MainnetTransactionProcessor transactionProcessor = - protocolSpec.getTransactionProcessor(); - final TraceBlock.ChainUpdater chainUpdater = - new TraceBlock.ChainUpdater(traceableState); - - TransactionSource transactionSource = new TransactionSource(block); - DebugOperationTracer debugOperationTracer = - new DebugOperationTracer(traceOptions.opCodeTracerConfig(), true); - ExecuteTransactionStep executeTransactionStep = - new ExecuteTransactionStep( - chainUpdater, - transactionProcessor, - getBlockchainQueries().getBlockchain(), - debugOperationTracer, - protocolSpec, - block); - - Pipeline traceBlockPipeline = - createPipelineFrom( - "getTransactions", - transactionSource, - 4, - outputCounter, - false, - "debug_trace_block_by_number") - .thenProcess("executeTransaction", executeTransactionStep) - .thenProcessAsyncOrdered( - "debugTraceTransactionStep", - DebugTraceTransactionStepFactory.createAsync( - traceOptions, protocolSpec), - 4) - .andFinishWith("collect_results", tracesList::add); - - try { - ethScheduler.startPipeline(traceBlockPipeline).get(); - } catch (InterruptedException | ExecutionException e) { - throw new RuntimeException(e); - } - return Optional.of(tracesList); - })) - .orElse(null); + getBlockchainQueries().getBlockchain().getBlockByNumber(resolvedBlockNumber); + + StreamingJsonRpcSuccessResponse.ResultWriter resultWriter = + generator -> { + if (maybeBlock.isEmpty()) { + generator.writeNull(); + return; + } + final Block block = maybeBlock.get(); + + Tracer.processTracing( + getBlockchainQueries(), + Optional.of(block.getHeader()), + traceableState -> { + final ProtocolSpec protocolSpec = + protocolSchedule.getByBlockHeader(block.getHeader()); + final MainnetTransactionProcessor transactionProcessor = + protocolSpec.getTransactionProcessor(); + final TraceBlock.ChainUpdater chainUpdater = + new TraceBlock.ChainUpdater(traceableState); + final BlockHeader header = block.getHeader(); + final Optional maybeParentHeader = + getBlockchainQueries().getBlockchain().getBlockHeader(header.getParentHash()); + final Wei blobGasPrice = + protocolSpec + .getFeeMarket() + .blobGasPricePerGas( + maybeParentHeader + .map(parent -> + calculateExcessBlobGasForParent(protocolSpec, parent)) + .orElse(BlobGas.ZERO)); + final BlockHashLookup blockHashLookup = + protocolSpec.getPreExecutionProcessor().createBlockHashLookup( + getBlockchainQueries().getBlockchain(), header); + + final DebugOperationTracer tracer = + new DebugOperationTracer(traceOptions.opCodeTracerConfig(), true); + + try { + generator.writeStartArray(); + + for (final Transaction tx : block.getBody().getTransactions()) { + final AccessLocationTracker accessListTracker = + BlockAccessList.BlockAccessListBuilder + .createTransactionAccessLocationTracker(0); + final TransactionProcessingResult result = + transactionProcessor.processTransaction( + chainUpdater.getNextUpdater(), + header, + tx, + header.getCoinbase(), + tracer, + blockHashLookup, + ImmutableTransactionValidationParams.builder().build(), + blobGasPrice, + Optional.of(accessListTracker)); + + final List frames = tracer.getTraceFrames(); + final long gas = tx.getGasLimit() - result.getGasRemaining(); + final String returnValue = result.getOutput().toString().substring(2); + final boolean failed = !result.isSuccessful(); + + generator.writeStartObject(); + generator.writeStringField("txHash", tx.getHash().toHexString()); + generator.writeFieldName("result"); + generator.writeStartObject(); + generator.writeNumberField("gas", gas); + generator.writeBooleanField("failed", failed); + generator.writeStringField("returnValue", returnValue); + generator.writeFieldName("structLogs"); + generator.writeStartArray(); + for (final TraceFrame frame : frames) { + final StructLog structLog = frame.getExceptionalHaltReason().isPresent() + ? new StructLogWithError(frame) + : new StructLog(frame); + generator.writeObject(structLog); + } + generator.writeEndArray(); + generator.writeEndObject(); + generator.writeEndObject(); + generator.flush(); + + tracer.reset(); + } + + generator.writeEndArray(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return Optional.empty(); + }); + }; + + return new StreamingJsonRpcSuccessResponse( + requestContext.getRequest().getId(), resultWriter); + } + + @Override + protected Object resultByBlockNumber( + final JsonRpcRequestContext request, final long blockNumber) { + // Not used — response() is overridden to use streaming + throw new UnsupportedOperationException("Use response() directly"); } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/response/StreamingJsonRpcSuccessResponse.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/response/StreamingJsonRpcSuccessResponse.java new file mode 100644 index 00000000000..0a3ba249ace --- /dev/null +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/response/StreamingJsonRpcSuccessResponse.java @@ -0,0 +1,61 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.api.jsonrpc.internal.response; + +import org.hyperledger.besu.plugin.services.rpc.RpcResponseType; + +import java.io.IOException; + +import com.fasterxml.jackson.core.JsonGenerator; + +/** + * A JSON-RPC success response that streams the result field directly to a JsonGenerator, avoiding + * materializing the entire result in memory. Used for large responses like debug_traceBlock. + */ +public class StreamingJsonRpcSuccessResponse implements JsonRpcResponse { + + private final Object id; + private final ResultWriter resultWriter; + + @FunctionalInterface + public interface ResultWriter { + void writeResult(JsonGenerator generator) throws IOException; + } + + public StreamingJsonRpcSuccessResponse(final Object id, final ResultWriter resultWriter) { + this.id = id; + this.resultWriter = resultWriter; + } + + public Object getId() { + return id; + } + + public void writeTo(final JsonGenerator generator) throws IOException { + generator.writeStartObject(); + generator.writeStringField("jsonrpc", "2.0"); + generator.writeFieldName("id"); + generator.writeObject(id); + generator.writeFieldName("result"); + resultWriter.writeResult(generator); + generator.writeEndObject(); + generator.flush(); + } + + @Override + public RpcResponseType getType() { + return RpcResponseType.SUCCESS; + } +} diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/StructLogWithError.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/StructLogWithError.java index 30b9132f253..a90716a5d72 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/StructLogWithError.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/results/StructLogWithError.java @@ -22,7 +22,7 @@ public class StructLogWithError extends StructLog { private final String[] error; - StructLogWithError(final TraceFrame traceFrame) { + public StructLogWithError(final TraceFrame traceFrame) { super(traceFrame); error = traceFrame.getExceptionalHaltReason().map(ehr -> new String[] {ehr.name()}).orElse(null); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/vm/StreamingDebugOperationTracer.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/vm/StreamingDebugOperationTracer.java new file mode 100644 index 00000000000..33019d2833d --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/vm/StreamingDebugOperationTracer.java @@ -0,0 +1,300 @@ +/* + * Copyright contributors to Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.vm; + +import org.hyperledger.besu.evm.ModificationNotAllowedException; +import org.hyperledger.besu.evm.frame.ExceptionalHaltReason; +import org.hyperledger.besu.evm.frame.MessageFrame; +import org.hyperledger.besu.evm.operation.AbstractCallOperation; +import org.hyperledger.besu.evm.operation.Operation; +import org.hyperledger.besu.evm.operation.Operation.OperationResult; +import org.hyperledger.besu.evm.tracing.OpCodeTracerConfigBuilder.OpCodeTracerConfig; +import org.hyperledger.besu.evm.tracing.OperationTracer; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; + +import com.fasterxml.jackson.core.JsonGenerator; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; + +/** + * An operation tracer that streams StructLog entries directly to a JsonGenerator during EVM + * execution. Unlike DebugOperationTracer which accumulates all TraceFrames in an ArrayList, this + * tracer writes each struct log inline and immediately discards it, achieving O(1) frame memory. + * + *

Additionally implements lazy memory capture: EVM memory is only captured for opcodes that + * actually read or write memory (24 out of ~140 opcodes). For all other opcodes, memory is omitted, + * reducing per-frame size from ~256KB to ~1-2KB for the majority of opcode steps. + */ +public class StreamingDebugOperationTracer implements OperationTracer { + + private static final char[] HEX_CHARS = "0123456789abcdef".toCharArray(); + + /** + * Set of opcode numbers that read from or write to EVM memory. Only these opcodes trigger a full + * memory capture in tracePostExecution. All other opcodes emit null/omitted memory field. + */ + private static final Set MEMORY_OPCODES = + Set.of( + 0x20, // KECCAK256 (reads memory to hash) + 0x37, // CALLDATACOPY (writes calldata into memory) + 0x39, // CODECOPY (writes code into memory) + 0x3C, // EXTCODECOPY (writes external code into memory) + 0x3E, // RETURNDATACOPY (writes return data into memory) + 0x51, // MLOAD (reads 32 bytes from memory) + 0x52, // MSTORE (writes 32 bytes to memory) + 0x53, // MSTORE8 (writes 1 byte to memory) + 0x5E, // MCOPY (copies memory region) + 0xA0, // LOG0 (reads memory for log data) + 0xA1, // LOG1 + 0xA2, // LOG2 + 0xA3, // LOG3 + 0xA4, // LOG4 + 0xF0, // CREATE (reads memory for init code) + 0xF1, // CALL (reads input from memory, writes output to memory) + 0xF2, // CALLCODE (reads/writes memory) + 0xF3, // RETURN (reads memory for return data) + 0xF4, // DELEGATECALL (reads/writes memory) + 0xF5, // CREATE2 (reads memory for init code) + 0xFA, // STATICCALL (reads/writes memory) + 0xFD // REVERT (reads memory for revert reason) + ); + + private final OpCodeTracerConfig options; + private final boolean recordChildCallGas; + private final JsonGenerator generator; + + // Pre-execution state captured in tracePreExecution, consumed in tracePostExecution + private Optional preExecutionStack; + private long gasRemaining; + private int pc; + private int depth; + + // traceOpcodes filtering + private boolean traceOpcode; + private Operation previousOpcode = null; + + public StreamingDebugOperationTracer( + final OpCodeTracerConfig options, + final boolean recordChildCallGas, + final JsonGenerator generator) { + this.options = options; + this.recordChildCallGas = recordChildCallGas; + this.generator = generator; + } + + @Override + public void tracePreExecution(final MessageFrame frame) { + final Operation currentOperation = frame.getCurrentOperation(); + if (!(traceOpcode = traceOpcode(currentOperation))) { + return; + } + preExecutionStack = captureStack(frame); + gasRemaining = frame.getRemainingGas(); + pc = frame.getPC(); + depth = frame.getDepth(); + } + + @Override + public void tracePostExecution(final MessageFrame frame, final OperationResult operationResult) { + final Operation currentOperation = frame.getCurrentOperation(); + final String opcode = currentOperation.getName(); + if (!traceOpcode) { + return; + } + final int opcodeNumber = (opcode != null) ? currentOperation.getOpcode() : Integer.MAX_VALUE; + + long thisGasCost = operationResult.getGasCost(); + if (recordChildCallGas && currentOperation instanceof AbstractCallOperation) { + thisGasCost += frame.getMessageFrameStack().getFirst().getRemainingGas(); + } + + final Optional haltReason = + Optional.ofNullable(operationResult.getHaltReason()).or(frame::getExceptionalHaltReason); + + try { + generator.writeStartObject(); + + // pc + generator.writeNumberField("pc", pc); + + // op + generator.writeStringField("op", opcode != null ? opcode : "INVALID"); + + // gas + generator.writeNumberField("gas", gasRemaining); + + // gasCost + generator.writeNumberField("gasCost", thisGasCost); + + // depth (besu uses 0-based internally, geth uses 1-based) + generator.writeNumberField("depth", depth + 1); + + // stack (pre-execution) + if (preExecutionStack.isPresent()) { + generator.writeFieldName("stack"); + generator.writeStartArray(); + for (final Bytes item : preExecutionStack.get()) { + generator.writeString(toCompactHex(item)); + } + generator.writeEndArray(); + } + + // memory — only capture for memory-touching opcodes + if (options.traceMemory() && frame.memoryWordSize() > 0) { + if (MEMORY_OPCODES.contains(opcodeNumber)) { + generator.writeFieldName("memory"); + generator.writeStartArray(); + for (int i = 0; i < frame.memoryWordSize(); i++) { + generator.writeString(toCompactHex(frame.readMemory(i * 32L, 32))); + } + generator.writeEndArray(); + } + // else: omit memory field entirely for non-memory opcodes + } + + // storage + if (options.traceStorage()) { + writeStorage(frame); + } + + // error + if (haltReason.isPresent()) { + generator.writeFieldName("error"); + generator.writeStartArray(); + generator.writeString(haltReason.get().name()); + generator.writeEndArray(); + } + + // reason (revert) + if (frame.getRevertReason().isPresent()) { + generator.writeStringField("reason", toCompactHex(frame.getRevertReason().get())); + } + + generator.writeEndObject(); + } catch (final IOException e) { + throw new UncheckedIOException(e); + } + + frame.reset(); + } + + @Override + public void tracePrecompileCall( + final MessageFrame frame, final long gasRequirement, final Bytes output) { + // Precompile calls are not part of standard debug_traceBlock structLogs + } + + @Override + public void traceAccountCreationResult( + final MessageFrame frame, final Optional haltReason) { + // Account creation results are not part of standard debug_traceBlock structLogs + } + + @Override + public List getTraceFrames() { + // This tracer streams inline — no frames are accumulated + return List.of(); + } + + public void reset() { + previousOpcode = null; + } + + private boolean traceOpcode(final Operation currentOpcode) { + if (options.traceOpcodes().isEmpty()) { + return true; + } + final boolean traceCurrentOpcode = + options.traceOpcodes().contains(currentOpcode.getName().toLowerCase(Locale.ROOT)); + final boolean tracePreviousOpcode = + previousOpcode != null + && options.traceOpcodes().contains(previousOpcode.getName().toLowerCase(Locale.ROOT)); + if (!traceCurrentOpcode && !tracePreviousOpcode) { + return false; + } + previousOpcode = currentOpcode; + return true; + } + + private Optional captureStack(final MessageFrame frame) { + if (!options.traceStack()) { + return Optional.empty(); + } + final Bytes[] stackContents = new Bytes[frame.stackSize()]; + for (int i = 0; i < stackContents.length; i++) { + stackContents[i] = frame.getStackItem(stackContents.length - i - 1); + } + return Optional.of(stackContents); + } + + private void writeStorage(final MessageFrame frame) throws IOException { + try { + final Map updatedStorage = + frame.getWorldUpdater().getAccount(frame.getRecipientAddress()).getUpdatedStorage(); + if (updatedStorage.isEmpty()) { + return; + } + final Map sorted = new TreeMap<>(updatedStorage); + generator.writeFieldName("storage"); + generator.writeStartObject(); + for (final Map.Entry entry : sorted.entrySet()) { + generator.writeStringField( + toCompactHex(entry.getKey()), toCompactHex(entry.getValue())); + } + generator.writeEndObject(); + } catch (final ModificationNotAllowedException e) { + // Write empty storage + generator.writeFieldName("storage"); + generator.writeStartObject(); + generator.writeEndObject(); + } + } + + /** + * Converts bytes to compact hex string with 0x prefix. Strips leading zeros. Matches the format + * used by StructLog.toCompactHex(). + */ + private static String toCompactHex(final Bytes bytes) { + if (bytes.isEmpty()) { + return "0x0"; + } + final byte[] raw = bytes.toArrayUnsafe(); + final StringBuilder sb = new StringBuilder(raw.length * 2 + 2); + sb.append("0x"); + boolean leadingZero = true; + for (int i = 0; i < raw.length; i++) { + final byte b = raw[i]; + final int hi = (b >> 4) & 0xF; + if (!leadingZero || hi != 0) { + sb.append(HEX_CHARS[hi]); + leadingZero = false; + } + final int lo = b & 0xF; + if (!leadingZero || lo != 0 || i == raw.length - 1) { + sb.append(HEX_CHARS[lo]); + leadingZero = false; + } + } + return sb.toString(); + } +}