trace_block (part 1) (#1714)

cmd/rpcdaemon/commands/trace_adhoc.go

@@ -553,6 +553,10 @@ func (api *TraceAPIImpl) CallMany(ctx context.Context, calls json.RawMessage, bl
 	}
 	defer dbtx.Rollback()
 
+	return api.doCallMany(ctx, dbtx, calls, blockNrOrHash)
+}
+
+func (api *TraceAPIImpl) doCallMany(ctx context.Context, dbtx ethdb.Tx, calls json.RawMessage, blockNrOrHash *rpc.BlockNumberOrHash) ([]*TraceCallResult, error) {
 	chainConfig, err := api.chainConfig(dbtx)
 	if err != nil {
 		return nil, err

cmd/rpcdaemon/commands/trace_filtering.go

@@ -12,6 +12,7 @@ import (
 	"github.com/ledgerwatch/turbo-geth/common/hexutil"
 	"github.com/ledgerwatch/turbo-geth/consensus/ethash"
 	"github.com/ledgerwatch/turbo-geth/core/rawdb"
+	"github.com/ledgerwatch/turbo-geth/core/types"
 	"github.com/ledgerwatch/turbo-geth/eth/tracers"
 	"github.com/ledgerwatch/turbo-geth/ethdb"
 	"github.com/ledgerwatch/turbo-geth/ethdb/bitmapdb"
@@ -71,25 +72,39 @@ func (api *TraceAPIImpl) Get(ctx context.Context, txHash common.Hash, indicies [
 
 // Block implements trace_block
 func (api *TraceAPIImpl) Block(ctx context.Context, blockNr rpc.BlockNumber) (ParityTraces, error) {
-	tx, err := api.kv.BeginRo(ctx)
+	dbtx, err := api.kv.BeginRo(ctx)
 	if err != nil {
 		return nil, err
 	}
-	defer tx.Rollback()
-	blockNum, err := getBlockNumber(blockNr, tx)
+	defer dbtx.Rollback()
+	blockNum, err := getBlockNumber(blockNr, dbtx)
 	if err != nil {
 		return nil, err
 	}
 	bn := hexutil.Uint64(blockNum)
-	var req TraceFilterRequest
-	req.FromBlock = &bn
-	req.ToBlock = &bn
-	req.FromAddress = nil
-	req.ToAddress = nil
-	req.After = nil
-	req.Count = nil
 
-	traces, err := api.Filter(ctx, req)
+	// Extract transactions from block
+	hash, hashErr := rawdb.ReadCanonicalHash(dbtx, blockNum)
+	if hashErr != nil {
+		return nil, hashErr
+	}
+	block, senders, sendersErr := rawdb.ReadBlockWithSenders(ethdb.NewRoTxDb(dbtx), hash, uint64(bn))
+	if sendersErr != nil {
+		return nil, sendersErr
+	}
+	if block == nil {
+		return nil, nil
+	}
+
+	txs := make([]TransactionWithSender, 0, len(senders))
+	for n, tx := range block.Transactions() {
+		txs = append(txs, TransactionWithSender{
+			tx:     *tx,
+			sender: senders[n],
+		})
+	}
+
+	traces, err := api.callManyTransactions(ctx, dbtx, txs, hash, rpc.BlockNumber(bn))
 	if err != nil {
 		return nil, err
 	}
@@ -315,6 +330,52 @@ func (api *TraceAPIImpl) Filter(ctx context.Context, req TraceFilterRequest) (Pa
 	return traces, nil
 }
 
+type TransactionWithSender struct {
+	tx     types.Transaction
+	sender common.Address
+}
+
+func (api *TraceAPIImpl) callManyTransactions(ctx context.Context, dbtx ethdb.Tx, txs []TransactionWithSender, blockHash common.Hash, blockNo rpc.BlockNumber) ([]ParityTrace, error) {
+	toExecute := []interface{}{}
+
+	for _, tx := range txs {
+		gas := hexutil.Uint64(tx.tx.Gas())
+		gasPrice := hexutil.Big(*tx.tx.GasPrice().ToBig())
+		value := hexutil.Big(*tx.tx.Value().ToBig())
+		toExecute = append(toExecute, []interface{}{TraceCallParam{
+			From:     &tx.sender,
+			To:       tx.tx.To(),
+			Gas:      &gas,
+			GasPrice: &gasPrice,
+			Value:    &value,
+			Data:     tx.tx.Data(),
+		}, []string{TraceTypeTrace, TraceTypeStateDiff, TraceTypeVmTrace}})
+	}
+
+	calls, callsErr := json.Marshal(toExecute)
+	if callsErr != nil {
+		return nil, callsErr
+	}
+	traces, cmErr := api.doCallMany(ctx, dbtx, calls, &rpc.BlockNumberOrHash{
+		BlockNumber:      &blockNo,
+		BlockHash:        &blockHash,
+		RequireCanonical: true,
+	})
+
+	if cmErr != nil {
+		return nil, cmErr
+	}
+
+	out := make([]ParityTrace, 0, len(traces))
+	for _, trace := range traces {
+		for _, pt := range trace.Trace {
+			out = append(out, *pt)
+		}
+	}
+
+	return out, nil
+}
+
 func retrieveHistory(tx ethdb.Getter, addr *common.Address, fromBlock uint64, toBlock uint64) ([]uint64, error) {
 	blocks, err := bitmapdb.Get(tx, dbutils.AccountsHistoryBucket, addr.Bytes(), uint32(fromBlock), uint32(toBlock+1))
 	if err != nil {

cmd/rpctest/main.go

@@ -178,6 +178,16 @@ func main() {
 	}
 	with(bench13Cmd, withGethUrl, withTGUrl, withNeedCompare, withBlockNum, withRecord)
 
+	var benchTraceBlockCmd = &cobra.Command{
+		Use:   "benchTraceBlock",
+		Short: "",
+		Long:  ``,
+		Run: func(cmd *cobra.Command, args []string) {
+			rpctest.BenchTraceBlock(tgURL, gethURL, needCompare, blockFrom, blockTo, recordFile)
+		},
+	}
+	with(benchTraceBlockCmd, withGethUrl, withTGUrl, withNeedCompare, withBlockNum, withRecord)
+
 	var proofsCmd = &cobra.Command{
 		Use:   "proofs",
 		Short: "",
@@ -245,6 +255,7 @@ func main() {
 		bench11Cmd,
 		bench12Cmd,
 		bench13Cmd,
+		benchTraceBlockCmd,
 		proofsCmd,
 		fixStateCmd,
 		compareAccountRange,

cmd/rpctest/rpctest/bench_traceblock.go

@@ -0,0 +1,123 @@
+package rpctest
+
+import (
+	"bufio"
+	"fmt"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/ledgerwatch/turbo-geth/common"
+)
+
+// Compares response of TurboGeth with Geth
+// but also can be used for comparing RPCDaemon with Geth
+// parameters:
+// needCompare - if false - doesn't call TurboGeth and doesn't compare responses
+// 		use false value - to generate vegeta files, it's faster but we can generate vegeta files for Geth and Turbogeth
+func BenchTraceBlock(tgURL, oeURL string, needCompare bool, blockFrom uint64, blockTo uint64, recordFile string) {
+	setRoutes(tgURL, oeURL)
+	var client = &http.Client{
+		Timeout: time.Second * 600,
+	}
+	var rec *bufio.Writer
+	if recordFile != "" {
+		f, err := os.Create(recordFile)
+		if err != nil {
+			fmt.Printf("Cannot create file %s for recording: %v\n", recordFile, err)
+			return
+		}
+		defer f.Close()
+		rec = bufio.NewWriter(f)
+		defer rec.Flush()
+	}
+
+	var res CallResult
+	reqGen := &RequestGenerator{
+		client: client,
+	}
+
+	skipTxs := make(map[common.Hash]struct{})
+	for _, txHash := range wrongTxs {
+		skipTxs[common.HexToHash(txHash)] = struct{}{}
+	}
+
+	reqGen.reqID++
+	var blockNumber EthBlockNumber
+	res = reqGen.TurboGeth("eth_blockNumber", reqGen.blockNumber(), &blockNumber)
+	if res.Err != nil {
+		fmt.Printf("Could not get block number: %v\n", res.Err)
+		return
+	}
+	if blockNumber.Error != nil {
+		fmt.Printf("Error getting block number: %d %s\n", blockNumber.Error.Code, blockNumber.Error.Message)
+		return
+	}
+	fmt.Printf("Last block: %d\n", blockNumber.Number)
+	for bn := blockFrom; bn <= blockTo; bn++ {
+		reqGen.reqID++
+		var b EthBlockByNumber
+		res = reqGen.TurboGeth("eth_getBlockByNumber", reqGen.getBlockByNumber(bn), &b)
+		if res.Err != nil {
+			fmt.Printf("Could not retrieve block (turbo-geth) %d: %v\n", bn, res.Err)
+			return
+		}
+
+		if b.Error != nil {
+			fmt.Printf("Error retrieving block (turbo-geth): %d %s\n", b.Error.Code, b.Error.Message)
+			return
+		}
+
+		if needCompare {
+			var bg EthBlockByNumber
+			res = reqGen.Geth("eth_getBlockByNumber", reqGen.getBlockByNumber(bn), &bg)
+			if res.Err != nil {
+				fmt.Printf("Could not retrieve block (OE) %d: %v\n", bn, res.Err)
+				return
+			}
+			if bg.Error != nil {
+				fmt.Printf("Error retrieving block (OE): %d %s\n", bg.Error.Code, bg.Error.Message)
+				return
+			}
+			if !compareBlocks(&b, &bg) {
+				fmt.Printf("Block difference for %d\n", bn)
+				return
+			}
+		}
+
+		recording := rec != nil // This flag will be set to false if recording is not to be performed
+		reqGen.reqID++
+		request := reqGen.traceBlock(bn)
+		res = reqGen.TurboGeth2("trace_block", request)
+		if res.Err != nil {
+			fmt.Printf("Could not trace block (turbo-geth) %d: %v\n", bn, res.Err)
+			return
+		}
+		if errVal := res.Result.Get("error"); errVal != nil {
+			fmt.Printf("Error tracing block (turbo-geth): %d %s\n", errVal.GetInt("code"), errVal.GetStringBytes("message"))
+			return
+		}
+		if needCompare {
+			resg := reqGen.Geth2("trace_block", request)
+			if resg.Err != nil {
+				fmt.Printf("Could not trace block (OE) %d: %v\n", bn, resg.Err)
+				return
+			}
+			if errVal := resg.Result.Get("error"); errVal != nil {
+				fmt.Printf("Error tracing call (OE): %d %s\n", errVal.GetInt("code"), errVal.GetStringBytes("message"))
+				return
+			}
+			if resg.Err == nil && resg.Result.Get("error") == nil {
+				if err := compareResults(res.Result, resg.Result); err != nil {
+					fmt.Printf("Different traces block %d, block %d: %v\n", bn, bn, err)
+					fmt.Printf("\n\nTG response=================================\n%s\n", res.Response)
+					fmt.Printf("\n\nOE response=================================\n%s\n", resg.Response)
+					return
+				}
+			}
+		}
+		if recording {
+			fmt.Fprintf(rec, "%s\n%s\n\n", request, res.Response)
+		}
+	}
+}

cmd/rpctest/rpctest/request_generator.go

@@ -165,6 +165,14 @@ func (g *RequestGenerator) debugTraceCall(from common.Address, to *common.Addres
 	return sb.String()
 }
 
+func (g *RequestGenerator) traceBlock(bn uint64) string {
+	var sb strings.Builder
+	fmt.Fprintf(&sb, `{ "jsonrpc": "2.0", "method": "trace_block", "params": ["0x%x"]`, bn)
+	fmt.Fprintf(&sb, `, "id":%d}`, g.reqID)
+
+	return sb.String()
+}
+
 func (g *RequestGenerator) call(target string, method, body string, response interface{}) CallResult {
 	start := time.Now()
 	err := post(g.client, routes[target], body, response)