mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 19:50:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

rpctest for eth_getTransactionReceipt (#2271)

Browse Source 
* rpctest bench for eth_getTransationReceipt

* Add blockHash to re-generated receipts

* Modify scanReceipts

Co-authored-by: Alexey Sharp <alexeysharp@Alexeys-iMac.local>
This commit is contained in:
ledgerwatch 2021-07-02 14:34:20 +01:00 committed by GitHub
parent 110e14e6d0
commit ba55611f29
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 134 additions and 386 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
cmd/hack/hack.go
View File

@ -21,9 +21,7 @@ import (
"github.com/RoaringBitmap/roaring/roaring64"
"github.com/holiman/uint256"
"github.com/ledgerwatch/erigon/ethdb/cbor"
kv2 "github.com/ledgerwatch/erigon/ethdb/kv"
"github.com/ledgerwatch/erigon/migrations"
"github.com/wcharczuk/go-chart"
"github.com/wcharczuk/go-chart/util"
@ -2070,7 +2068,7 @@ func scanTxs(chaindata string) error {
return nil
}
func scanReceipts(chaindata string) error {
func scanReceipts(chaindata string, block uint64) error {
dbdb := kv2.MustOpen(chaindata).RwKV()
defer dbdb.Close()
txtx, err := dbdb.BeginRw(context.Background())
@ -2085,70 +2083,13 @@ func scanReceipts(chaindata string) error {
} else {
return fmt.Errorf("no transaction")
}
genesisBlock, err := rawdb.ReadBlockByNumber(tx, 0)
if err != nil {
return err
}
chainConfig, cerr := rawdb.ReadChainConfig(tx, genesisBlock.Hash())
if cerr != nil {
return cerr
}
logInterval := 30 * time.Second
logEvery := time.NewTicker(logInterval)
defer logEvery.Stop()
var buf bytes.Buffer
var key [8]byte
var v []byte
var to uint64
if to, err = stages.GetStageProgress(tx, stages.Execution); err != nil {
binary.BigEndian.PutUint64(key[:], block)
if v, err = tx.GetOne(dbutils.BlockReceiptsPrefix, key[:]); err != nil {
return err
}
for blockNum := uint64(1); blockNum <= to; blockNum++ {
binary.BigEndian.PutUint64(key[:], blockNum)
if v, err = tx.GetOne(dbutils.BlockReceiptsPrefix, key[:]); err != nil {
return err
}
if v == nil {
continue
}
//fmt.Printf("blockNum = %d\n", blockNum)
select {
default:
case <-logEvery.C:
log.Info("Scanned receipts up to", "block", blockNum)
}
var receipts types.Receipts
var oldReceipts migrations.OldReceipts
if err = cbor.Unmarshal(&oldReceipts, bytes.NewReader(v)); err != nil {
continue
}
var blockHash common.Hash
if blockHash, err = rawdb.ReadCanonicalHash(tx, blockNum); err != nil {
return err
}
var body *types.Body
if chainConfig.IsBerlin(blockNum) {
body = rawdb.ReadBody(tx, blockHash, blockNum)
}
receipts = make(types.Receipts, len(oldReceipts))
for i, oldReceipt := range oldReceipts {
receipts[i] = new(types.Receipt)
receipts[i].PostState = oldReceipt.PostState
receipts[i].Status = oldReceipt.Status
receipts[i].CumulativeGasUsed = oldReceipt.CumulativeGasUsed
if body != nil {
receipts[i].Type = body.Transactions[i].Type()
}
}
buf.Reset()
if err = cbor.Marshal(&buf, receipts); err != nil {
return err
}
//if err = tx.Put(dbutils.BlockReceiptsPrefix, common.CopyBytes(key[:]), common.CopyBytes(buf.Bytes())); err != nil {
// return err
//}
}
fmt.Printf("blockNum = %d, receipt %x\n", block, v)
return nil
}
@ -2356,7 +2297,7 @@ func main() {
err = scanTxs(*chaindata)
case "scanReceipts":
err = scanReceipts(*chaindata)
err = scanReceipts(*chaindata, uint64(*block))
case "testTxPool":
err = testTxPool()

4
cmd/rpcdaemon/commands/eth_receipts.go
View File

@ -5,9 +5,10 @@ import (
"context"
"encoding/binary"
"fmt"
"github.com/holiman/uint256"
"math/big"
"github.com/holiman/uint256"
"github.com/RoaringBitmap/roaring"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/common/dbutils"
@ -53,6 +54,7 @@ func getReceipts(ctx context.Context, tx ethdb.Tx, chainConfig *params.ChainConf
if err != nil {
return nil, err
}
receipt.BlockHash = block.Hash()
receipts = append(receipts, receipt)
}

24
cmd/rpctest/main.go
View File

@ -22,7 +22,6 @@ func main() {
erigonURL string
blockFrom uint64
blockTo uint64
chaindata string
recordFile string
)
withErigonUrl := func(cmd *cobra.Command) {
@ -198,27 +197,15 @@ func main() {
}
with(benchTraceFilterCmd, withGethUrl, withErigonUrl, withNeedCompare, withBlockNum, withRecord)
var proofsCmd = &cobra.Command{
Use: "proofs",
var benchTxReceiptCmd = &cobra.Command{
Use: "benchTxReceipt",
Short: "",
Long: ``,
Run: func(cmd *cobra.Command, args []string) {
rpctest.Proofs(chaindata, gethURL, blockFrom)
rpctest.BenchTxReceipt(erigonURL, gethURL, needCompare, blockFrom, blockTo, recordFile)
},
}
proofsCmd.Flags().StringVar(&chaindata, "chaindata", "", "")
with(proofsCmd, withGethUrl, withBlockNum)
var fixStateCmd = &cobra.Command{
Use: "fixstate",
Short: "",
Long: ``,
Run: func(cmd *cobra.Command, args []string) {
rpctest.FixState(chaindata, gethURL)
},
}
fixStateCmd.Flags().StringVar(&chaindata, "chaindata", "", "")
with(fixStateCmd, withGethUrl)
with(benchTxReceiptCmd, withGethUrl, withErigonUrl, withNeedCompare, withBlockNum, withRecord)
var replayCmd = &cobra.Command{
Use: "replay",
@ -267,8 +254,7 @@ func main() {
bench13Cmd,
benchTraceBlockCmd,
benchTraceFilterCmd,
proofsCmd,
fixStateCmd,
benchTxReceiptCmd,
compareAccountRange,
replayCmd,
)

121
cmd/rpctest/rpctest/bench_txreceipts.go Normal file
View File

@ -0,0 +1,121 @@
package rpctest
import (
"bufio"
"fmt"
"net/http"
"os"
"time"
)
// benchTxReceipt compares response of Erigon with Geth
// but also can be used for comparing RPCDaemon with Geth
// parameters:
// needCompare - if false - doesn't call Erigon and doesn't compare responses
// use false value - to generate vegeta files, it's faster but we can generate vegeta files for Geth and Erigon
func BenchTxReceipt(erigonURL, gethURL string, needCompare bool, blockFrom uint64, blockTo uint64, recordFile string) {
setRoutes(erigonURL, gethURL)
var client = &http.Client{
Timeout: time.Second * 600,
}
var rec *bufio.Writer
if recordFile != "" {
f, err := os.Create(recordFile)
if err != nil {
fmt.Printf("Cannot create file %s for recording: %v\n", recordFile, err)
return
}
defer f.Close()
rec = bufio.NewWriter(f)
defer rec.Flush()
}
var res CallResult
reqGen := &RequestGenerator{
client: client,
}
reqGen.reqID++
var blockNumber EthBlockNumber
res = reqGen.Erigon("eth_blockNumber", reqGen.blockNumber(), &blockNumber)
if res.Err != nil {
fmt.Printf("Could not get block number: %v\n", res.Err)
return
}
if blockNumber.Error != nil {
fmt.Printf("Error getting block number: %d %s\n", blockNumber.Error.Code, blockNumber.Error.Message)
return
}
fmt.Printf("Last block: %d\n", blockNumber.Number)
for bn := blockFrom; bn <= blockTo; bn++ {
reqGen.reqID++
var b EthBlockByNumber
res = reqGen.Erigon("eth_getBlockByNumber", reqGen.getBlockByNumber(bn), &b)
if res.Err != nil {
fmt.Printf("Could not retrieve block (Erigon) %d: %v\n", bn, res.Err)
return
}
if b.Error != nil {
fmt.Printf("Error retrieving block (Erigon): %d %s\n", b.Error.Code, b.Error.Message)
return
}
if needCompare {
var bg EthBlockByNumber
res = reqGen.Geth("eth_getBlockByNumber", reqGen.getBlockByNumber(bn), &bg)
if res.Err != nil {
fmt.Printf("Could not retrieve block (geth) %d: %v\n", bn, res.Err)
return
}
if bg.Error != nil {
fmt.Printf("Error retrieving block (geth): %d %s\n", bg.Error.Code, bg.Error.Message)
return
}
if !compareBlocks(&b, &bg) {
fmt.Printf("Block difference for %d\n", bn)
return
}
}
for _, tx := range b.Result.Transactions {
reqGen.reqID++
request := reqGen.getTransactionReceipt(tx.Hash)
recording := rec != nil // This flag will be set to false if recording is not to be performed
res = reqGen.Erigon2("eth_getTransactionReceipt", request)
if res.Err != nil {
fmt.Printf("Could not eth getTransactionReceipt (Erigon) %d: %v\n", bn, res.Err)
return
}
if errVal := res.Result.Get("error"); errVal != nil {
fmt.Printf("Error eth getTransactionReceipt (Erigon): %d %s\n", errVal.GetInt("code"), errVal.GetStringBytes("message"))
return
}
if needCompare {
resg := reqGen.Geth2("eth_getTransactionReceipt", request)
if resg.Err != nil {
fmt.Printf("Could not eth getTransactionReceipt (geth) %d: %v\n", bn, resg.Err)
return
}
if errVal := resg.Result.Get("error"); errVal != nil {
fmt.Printf("Error eth getTransactionReceipt (geth): %d %s\n", errVal.GetInt("code"), errVal.GetStringBytes("message"))
return
}
if resg.Err == nil && resg.Result.Get("error") == nil {
if err := compareResults(res.Result, resg.Result); err != nil {
fmt.Printf("Different getTransactionReceipt block %d, tx %s: %v\n", bn, tx.Hash, err)
fmt.Printf("\n\nTG response=================================\n%s\n", res.Response)
fmt.Printf("\n\nG response=================================\n%s\n", resg.Response)
return
}
}
}
if recording {
fmt.Fprintf(rec, "%s\n%s\n\n", request, res.Response)
}
}
}
}

135
cmd/rpctest/rpctest/fixState.go
View File

@ -1,135 +0,0 @@
package rpctest
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"net/http"
"time"
"github.com/ledgerwatch/erigon/common"
"github.com/ledgerwatch/erigon/common/dbutils"
"github.com/ledgerwatch/erigon/core/rawdb"
"github.com/ledgerwatch/erigon/core/state"
"github.com/ledgerwatch/erigon/core/types/accounts"
"github.com/ledgerwatch/erigon/ethdb"
"github.com/ledgerwatch/erigon/ethdb/kv"
"github.com/ledgerwatch/erigon/turbo/trie"
)
func FixState(chaindata string, url string) {
db := kv.MustOpen(chaindata).RwKV()
defer db.Close()
tx, err1 := db.BeginRw(context.Background())
if err1 != nil {
panic(err1)
}
defer tx.Rollback()
currentHeader := rawdb.ReadCurrentHeader(tx)
blockNum := currentHeader.Number.Uint64()
blockHash := currentHeader.Hash()
fmt.Printf("Block number: %d\n", blockNum)
fmt.Printf("Block root hash: %x\n", currentHeader.Root)
reqID := 0
roots := make(map[common.Address]*accounts.Account)
var client = &http.Client{
Timeout: time.Second * 600,
}
c, err := tx.Cursor(dbutils.PlainStateBucket)
if err != nil {
panic(err)
}
defer c.Close()
if err := ethdb.ForEach(c, func(k, v []byte) (bool, error) {
if len(k) == common.AddressLength {
var address common.Address
address.SetBytes(k)
if _, ok := roots[address]; !ok {
if account, err2 := state.NewPlainStateReader(tx).ReadAccountData(address); err2 != nil {
return false, err2
} else {
roots[address] = account
}
}
}
return true, nil
}); err != nil {
panic(err)
}
for address, account := range roots {
if account != nil && account.Root != trie.EmptyRoot {
contractPrefix := make([]byte, common.HashLength+common.IncarnationLength)
addrHash, _ := common.HashData(address.Bytes())
copy(contractPrefix, addrHash[:])
binary.BigEndian.PutUint64(contractPrefix[common.HashLength:], account.Incarnation)
rl := trie.NewRetainList(0)
loader := trie.NewFlatDBTrieLoader("checkRoots")
if err := loader.Reset(rl, nil, nil, false); err != nil {
panic(err)
}
root, err1 := loader.CalcTrieRoot(tx, contractPrefix, nil)
if err1 != nil || root != account.Root {
fmt.Printf("%x: error %v, got hash %x, expected hash %x\n", addrHash, err1, root, account.Root)
template := `{"jsonrpc":"2.0","method":"debug_storageRangeAt","params":["0x%x", %d,"0x%x","0x%x",%d],"id":%d}`
sm := make(map[common.Hash]storageEntry)
nextKey := &common.Hash{}
for nextKey != nil {
reqID++
var sr DebugStorageRange
if err := post(client, url, fmt.Sprintf(template, blockHash, 0, address, *nextKey, 1024, reqID), &sr); err != nil {
fmt.Printf("Could not get storageRange: %v\n", err)
return
}
if sr.Error != nil {
fmt.Printf("Error getting storageRange: %d %s\n", sr.Error.Code, sr.Error.Message)
break
} else {
nextKey = sr.Result.NextKey
for k, v := range sr.Result.Storage {
sm[k] = v
}
}
}
fmt.Printf("Retrieved %d storage items from geth archive node\n", len(sm))
for key, entry := range sm {
var cKey [common.HashLength + common.IncarnationLength + common.HashLength]byte
copy(cKey[:], addrHash[:])
binary.BigEndian.PutUint64(cKey[common.HashLength:], account.Incarnation)
copy(cKey[common.HashLength+common.IncarnationLength:], key[:])
dbValue, _ := tx.GetOne(dbutils.HashedStorageBucket, cKey[:])
value := bytes.TrimLeft(entry.Value[:], "\x00")
if !bytes.Equal(dbValue, value) {
fmt.Printf("Key: %x, value: %x, dbValue: %x\n", key, value, dbValue)
if err := tx.Put(dbutils.HashedStorageBucket, cKey[:], value); err != nil {
fmt.Printf("%v\n", err)
}
}
}
var cKey [common.HashLength + common.IncarnationLength + common.HashLength]byte
copy(cKey[:], addrHash[:])
binary.BigEndian.PutUint64(cKey[common.HashLength:], account.Incarnation)
c2, err := tx.Cursor(dbutils.HashedStorageBucket)
if err != nil {
panic(err)
}
if err := ethdb.Walk(c, cKey[:], 8*(common.HashLength+common.IncarnationLength), func(k, v []byte) (bool, error) {
var kh common.Hash
copy(kh[:], k[common.HashLength+common.IncarnationLength:])
if _, ok := sm[kh]; !ok {
fmt.Printf("Key: %x, dbValue: %x\n", kh, v)
if err := tx.Delete(dbutils.HashedStorageBucket, k, nil); err != nil {
fmt.Printf("%v\n", err)
}
}
return true, nil
}); err != nil {
panic(err)
}
c2.Close()
}
}
}
}

167
cmd/rpctest/rpctest/proofs.go
View File

@ -1,167 +0,0 @@
package rpctest
import (
"context"
"fmt"
"os"
"github.com/ledgerwatch/erigon/ethdb"
"github.com/ledgerwatch/erigon/ethdb/kv"
"github.com/ledgerwatch/erigon/turbo/trie"
)
func Proofs(chaindata string, url string, block uint64) {
fileName := "trie.txt"
db := kv.MustOpen(chaindata)
defer db.Close()
tx, err1 := db.Begin(context.Background(), ethdb.RW)
if err1 != nil {
panic(err1)
}
defer tx.Rollback()
var t *trie.Trie
if _, errf := os.Stat(fileName); errf != nil {
if os.IsNotExist(errf) {
// Resolve 6 top levels of the accounts trie
rl := trie.NewRetainList(6)
loader := trie.NewFlatDBTrieLoader("checkRoots")
if err := loader.Reset(rl, nil, nil, false); err != nil {
panic(err)
}
root, err := loader.CalcTrieRoot(tx.(ethdb.HasTx).Tx().(ethdb.RwTx), []byte{}, nil)
if err != nil {
panic(err)
}
fmt.Printf("Resolved with hash: %x\n", root)
f, err1 := os.Create(fileName)
if err1 == nil {
defer f.Close()
t.Print(f)
} else {
panic(err1)
}
fmt.Printf("Saved trie to file\n")
} else {
panic(errf)
}
} else {
f, err1 := os.Open(fileName)
if err1 == nil {
defer f.Close()
var err2 error
t, err2 = trie.Load(f)
if err2 != nil {
panic(err2)
}
fmt.Printf("Restored from file with hash: %x\n", t.Hash())
} else {
panic(err1)
}
}
/* TODO: migrate to usage cursors api - and to not use preimages
var client = &http.Client{
Timeout: time.Second * 600,
}
reqID := 0
level := 0
diffKeys := [][]byte{{}}
var newDiffKeys [][]byte
for len(diffKeys) > 0 && level < 6 {
fmt.Printf("================================================\n")
fmt.Printf("LEVEL %d, diffKeys: %d\n", level, len(diffKeys))
fmt.Printf("================================================\n")
for _, diffKey := range diffKeys {
// Find account with the suitable hash
var startKey common.Hash
for i, d := range diffKey {
if i%2 == 0 {
startKey[i/2] |= (d << 4)
} else {
startKey[i/2] |= d
}
}
var account common.Address
var found bool
err := tx.Walk(dbutils.PreimagePrefix, startKey[:], 4*len(diffKey), func(k, v []byte) (bool, error) {
if len(v) == common.AddressLength {
copy(account[:], v)
found = true
return false, nil
}
return true, nil
})
if err != nil {
fmt.Printf("Error when looking for suitable account for diffKey %x\n", diffKey)
return
}
if !found {
fmt.Printf("Could not find suitable account for diffKey %x\n", diffKey)
return
}
reqID++
template := `{"jsonrpc":"2.0","method":"eth_getProof","params":["0x%x",[],"0x%x"],"id":%d}`
var proof EthGetProof
if err = post(client, url, fmt.Sprintf(template, account, block, reqID), &proof); err != nil {
fmt.Printf("Could not get block number: %v\n", err)
return
}
if proof.Error != nil {
fmt.Printf("Error retrieving proof: %d %s\n", proof.Error.Code, proof.Error.Message)
return
}
if len(proof.Result.AccountProof) <= len(diffKey) {
fmt.Printf("RPC result needs to be at least %d levels deep\n", len(diffKey)+1)
return
}
p := proof.Result.AccountProof[len(diffKey)]
b := common.FromHex(p)
h := common.BytesToHash(crypto.Keccak256(b))
hE, err := t.HashOfHexKey(diffKey)
if err != nil {
fmt.Printf("Error computing partial hash for %x: %v\n", diffKey, err)
return
}
if h != hE {
fmt.Printf("key %x: %x %x adding nibbles: ", diffKey, h, hE)
var branch [17][]byte
err = rlp.DecodeBytes(b, &branch)
if err != nil {
fmt.Printf("Error decoding: %v\n", err)
fmt.Printf("%s\n", p)
}
// Expand keys further
for nibble := byte(0); nibble < 16; nibble++ {
newDiff := make([]byte, len(diffKey)+1)
copy(newDiff, diffKey)
newDiff[len(diffKey)] = nibble
var proofHash common.Hash
copy(proofHash[:], branch[nibble])
if proofHash != (common.Hash{}) || err != nil {
newHash, err := t.HashOfHexKey(newDiff)
if err != nil {
fmt.Printf("Error computing partial hash for %x: %v\n", newDiff, err)
}
if proofHash != newHash {
newDiffKeys = append(newDiffKeys, newDiff)
fmt.Printf("%x", nibble)
}
}
}
fmt.Printf("\n")
} else {
fmt.Printf("MATCH key %x: %x %x\n", diffKey, h, hE)
}
}
diffKeys = newDiffKeys
newDiffKeys = nil
level++
}
fmt.Printf("\n\nRESULT:\n")
for _, diffKey := range diffKeys {
fmt.Printf("%x\n", diffKey)
}
*/
}