diff --git a/cmd/geth/config.go b/cmd/geth/config.go index ce4c2790f..48c41477f 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -74,9 +74,9 @@ type ethstatsConfig struct { } type gethConfig struct { - Eth eth.Config - Node node.Config - Ethstats ethstatsConfig + Eth eth.Config + Node node.Config + Ethstats ethstatsConfig } func loadConfig(file string, cfg *gethConfig) error { @@ -107,8 +107,8 @@ func defaultNodeConfig() node.Config { func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { // Load defaults. cfg := gethConfig{ - Eth: eth.DefaultConfig, - Node: defaultNodeConfig(), + Eth: eth.DefaultConfig, + Node: defaultNodeConfig(), } // Load config file. diff --git a/cmd/restapi/README.md b/cmd/restapi/README.md new file mode 100644 index 000000000..fae4d8aae --- /dev/null +++ b/cmd/restapi/README.md @@ -0,0 +1,71 @@ +# Turbo-Geth Rest API + +## Build + +``` +make restapi +``` + +## Running + +* Running node with `--remote-db-listen-addr` (e.g `./build/bin/geth --remote-db-listen-addr localhost:9999`). +* Running Restapi: `./build/bin/restapi` (Default Port: 8080) + +## API + +* `/api/v1/remote-db/`: gives remote-db url +* `/api/v1/accounts/:accountID`: gives account data + * accountID is account address + * Reponse: + +```json +{ + + "balance":"BALANCE", + "code_hash":"HASH", + "implementation": + { + "incarnation":NUMBER + }, + "nonce":NUMBER, + "root_hash":"HASH" +} +``` +* `/api/v1/storage/` + * gives the storage + * Response: +```json +[ + {"prefix": "Storage Prefix","value": "Value"}, + ... +] +``` +* `/api/v1/retrace/:chain/:number` + * chain is the name of the chain(mainnet, testnet, goerli and rinkeby) + * number is block number (e.g 98345) + * extract changeSets and readSets for each block + * Response: +```json +[ + { + "storage": { + "reads": [READ, ...], + "writes": [WRITE, ...] + }, + "accounts": { + "reads": [READ, ...], + "writes": [WRITE, ...] + } + } +] +``` +* `/api/v1/intermediate-hash/` + * extract intermediate hashes + * Response: +```json +[ + {"prefix": "Prefix","value": "Value"}, + ... +] +``` + diff --git a/cmd/restapi/apis/remote_reader.go b/cmd/restapi/apis/remote_reader.go new file mode 100644 index 000000000..d44e15b63 --- /dev/null +++ b/cmd/restapi/apis/remote_reader.go @@ -0,0 +1,226 @@ +package apis + +import ( + "bytes" + "math/big" + + "github.com/ledgerwatch/turbo-geth/common" + "github.com/ledgerwatch/turbo-geth/common/dbutils" + "github.com/ledgerwatch/turbo-geth/consensus" + "github.com/ledgerwatch/turbo-geth/core/state" + "github.com/ledgerwatch/turbo-geth/core/types" + "github.com/ledgerwatch/turbo-geth/core/types/accounts" + "github.com/ledgerwatch/turbo-geth/crypto" + "github.com/ledgerwatch/turbo-geth/ethdb" + "github.com/ledgerwatch/turbo-geth/ethdb/remote/remotechain" + "github.com/ledgerwatch/turbo-geth/params" + "github.com/ledgerwatch/turbo-geth/rpc" + "golang.org/x/net/context" +) + +// ChangeSetReader is a mock StateWriter that accumulates changes in-memory into ChangeSets. +type RemoteReader struct { + accountReads map[common.Address]bool + storageReads map[common.Hash]bool + blockNr uint64 + db ethdb.KV +} + +type RemoteContext struct { + db ethdb.KV +} + +type powEngine struct { +} + +func (c *powEngine) VerifyHeader(chain consensus.ChainReader, header *types.Header, seal bool) error { + + panic("must not be called") +} +func (c *powEngine) VerifyHeaders(chain consensus.ChainReader, headers []*types.Header, seals []bool) (chan<- struct{}, <-chan error) { + panic("must not be called") +} +func (c *powEngine) VerifyUncles(chain consensus.ChainReader, block *types.Block) error { + panic("must not be called") +} +func (c *powEngine) VerifySeal(chain consensus.ChainReader, header *types.Header) error { + panic("must not be called") +} +func (c *powEngine) Prepare(chain consensus.ChainReader, header *types.Header) error { + panic("must not be called") +} +func (c *powEngine) Finalize(chainConfig *params.ChainConfig, header *types.Header, state *state.IntraBlockState, txs []*types.Transaction, uncles []*types.Header) { + panic("must not be called") +} +func (c *powEngine) FinalizeAndAssemble(chainConfig *params.ChainConfig, header *types.Header, state *state.IntraBlockState, txs []*types.Transaction, + uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error) { + panic("must not be called") +} +func (c *powEngine) Seal(_ consensus.Cancel, chain consensus.ChainReader, block *types.Block, results chan<- consensus.ResultWithContext, stop <-chan struct{}) error { + panic("must not be called") +} +func (c *powEngine) SealHash(header *types.Header) common.Hash { + panic("must not be called") +} +func (c *powEngine) CalcDifficulty(chain consensus.ChainReader, time uint64, parent *types.Header) *big.Int { + panic("must not be called") +} +func (c *powEngine) APIs(chain consensus.ChainReader) []rpc.API { + panic("must not be called") +} + +func (c *powEngine) Close() error { + panic("must not be called") +} + +func (c *powEngine) Author(header *types.Header) (common.Address, error) { + return header.Coinbase, nil +} + +func NewRemoteReader(db ethdb.KV, blockNr uint64) *RemoteReader { + return &RemoteReader{ + accountReads: make(map[common.Address]bool), + storageReads: make(map[common.Hash]bool), + db: db, + blockNr: blockNr, + } +} + +func (r *RemoteReader) GetAccountReads() [][]byte { + output := make([][]byte, 0) + for key := range r.accountReads { + output = append(output, key.Bytes()) + } + return output +} + +func (r *RemoteReader) GetStorageReads() [][]byte { + output := make([][]byte, 0) + for key := range r.storageReads { + output = append(output, key.Bytes()) + } + return output +} + +func (r *RemoteReader) ReadAccountData(address common.Address) (*accounts.Account, error) { + r.accountReads[address] = true + addrHash, _ := common.HashData(address[:]) + key := addrHash[:] + r.accountReads[address] = true + composite, _ := dbutils.CompositeKeySuffix(key, r.blockNr) + var dat []byte + err := r.db.View(context.Background(), func(tx ethdb.Tx) error { + { + hB := tx.Bucket(dbutils.AccountsHistoryBucket) + hC := hB.Cursor() + hK, hV, err := hC.Seek(composite) + if err != nil { + return err + } + + if hK != nil && bytes.HasPrefix(hK, key) { + dat = make([]byte, len(hV)) + copy(dat, hV) + return nil + } + } + { + b := tx.Bucket(dbutils.AccountsBucket) + c := b.Cursor() + k, v, err := c.Seek(key) + if err != nil { + return err + } + + if k != nil && bytes.Equal(k, key) { + dat = make([]byte, len(v)) + copy(dat, v) + return nil + } + } + + return ethdb.ErrKeyNotFound + }) + if err != nil || dat == nil || len(dat) == 0 { + return nil, nil + } + acc := accounts.NewAccount() + err = acc.DecodeForStorage(dat) + if err != nil { + return nil, err + } + return &acc, nil +} + +func (r *RemoteReader) ReadAccountStorage(address common.Address, incarnation uint64, key *common.Hash) ([]byte, error) { + keyHash, _ := common.HashData(key[:]) + addrHash, _ := common.HashData(address[:]) + + compositeKey := dbutils.GenerateCompositeStorageKey(addrHash, incarnation, keyHash) + var val []byte + err := r.db.View(context.Background(), func(tx ethdb.Tx) error { + b := tx.Bucket(dbutils.StorageBucket) + v, err := b.Get(compositeKey) + val = v + return err + }) + if err != nil { + return nil, err + } + r.storageReads[*key] = true + return val, nil +} + +func (r *RemoteReader) ReadAccountCode(address common.Address, codeHash common.Hash) ([]byte, error) { + if bytes.Equal(codeHash[:], crypto.Keccak256(nil)) { + return nil, nil + } + var val []byte + err := r.db.View(context.Background(), func(tx ethdb.Tx) error { + b := tx.Bucket(dbutils.StorageBucket) + v, err := b.Get(codeHash[:]) + val = v + return err + }) + if err != nil { + return nil, err + } + return val, nil +} + +func (r *RemoteReader) ReadAccountCodeSize(address common.Address, codeHash common.Hash) (int, error) { + if bytes.Equal(codeHash[:], crypto.Keccak256(nil)) { + return 0, nil + } + var val []byte + err := r.db.View(context.Background(), func(tx ethdb.Tx) error { + b := tx.Bucket(dbutils.StorageBucket) + v, err := b.Get(codeHash[:]) + val = v + return err + }) + if err != nil { + return 0, err + } + return len(val), nil +} + +func NewRemoteContext(db ethdb.KV) *RemoteContext { + return &RemoteContext{ + db: db, + } +} + +func (e *RemoteContext) Engine() consensus.Engine { + return &powEngine{} +} + +func (e *RemoteContext) GetHeader(hash common.Hash, number uint64) *types.Header { + var header *types.Header + _ = e.db.View(context.Background(), func(tx ethdb.Tx) error { + h, _ := remotechain.ReadHeader(tx, hash, number) + header = h + return nil + }) + return header +} diff --git a/cmd/restapi/apis/retrace_tx_api.go b/cmd/restapi/apis/retrace_tx_api.go new file mode 100644 index 000000000..491ef501e --- /dev/null +++ b/cmd/restapi/apis/retrace_tx_api.go @@ -0,0 +1,156 @@ +package apis + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/ledgerwatch/turbo-geth/common" + "github.com/ledgerwatch/turbo-geth/common/dbutils" + "github.com/ledgerwatch/turbo-geth/consensus/ethash" + "github.com/ledgerwatch/turbo-geth/consensus/misc" + "github.com/ledgerwatch/turbo-geth/core" + "github.com/ledgerwatch/turbo-geth/core/state" + "github.com/ledgerwatch/turbo-geth/core/types" + "github.com/ledgerwatch/turbo-geth/core/vm" + "github.com/ledgerwatch/turbo-geth/ethdb" + "github.com/ledgerwatch/turbo-geth/ethdb/remote/remotechain" + "github.com/ledgerwatch/turbo-geth/params" +) + +func RegisterRetraceAPI(router *gin.RouterGroup, e *Env) error { + router.GET(":chain/:number", e.GetWritesReads) + return nil +} + +func (e *Env) GetWritesReads(c *gin.Context) { + results, err := Retrace(c.Param("number"), c.Param("chain"), e.DB) + if err != nil { + c.AbortWithError(http.StatusInternalServerError, err) //nolint:errcheck + return + } + c.JSON(http.StatusOK, results) +} + +type WritesReads struct { + Reads []string `json:"reads"` + Writes []string `json:"writes"` +} +type RetraceResponse struct { + Storage WritesReads `json:"storage"` + Account WritesReads `json:"accounts"` +} + +func Retrace(blockNumber, chain string, remoteDB ethdb.KV) (RetraceResponse, error) { + chainConfig := ReadChainConfig(remoteDB, chain) + noOpWriter := state.NewNoopWriter() + bn, err := strconv.Atoi(blockNumber) + if err != nil { + return RetraceResponse{}, err + } + block, err := GetBlockByNumber(remoteDB, uint64(bn)) + chainCtx := NewRemoteContext(remoteDB) + if err != nil { + return RetraceResponse{}, err + } + writer := state.NewChangeSetWriter() + reader := NewRemoteReader(remoteDB, uint64(bn)) + intraBlockState := state.New(reader) + + if err = runBlock(intraBlockState, noOpWriter, writer, chainConfig, chainCtx, block); err != nil { + return RetraceResponse{}, err + } + + var output RetraceResponse + accountChanges, _ := writer.GetAccountChanges() + if err != nil { + return RetraceResponse{}, err + } + for _, ch := range accountChanges.Changes { + output.Account.Writes = append(output.Account.Writes, common.Bytes2Hex(ch.Key)) + } + for _, ch := range reader.GetAccountReads() { + output.Account.Reads = append(output.Account.Reads, common.Bytes2Hex(ch)) + } + + storageChanges, _ := writer.GetStorageChanges() + for _, ch := range storageChanges.Changes { + output.Storage.Writes = append(output.Storage.Writes, common.Bytes2Hex(ch.Key)) + } + for _, ch := range reader.GetStorageReads() { + output.Storage.Reads = append(output.Storage.Reads, common.Bytes2Hex(ch)) + } + return output, nil +} + +func runBlock(ibs *state.IntraBlockState, txnWriter state.StateWriter, blockWriter state.StateWriter, + chainConfig *params.ChainConfig, bcb core.ChainContext, block *types.Block, +) error { + header := block.Header() + vmConfig := vm.Config{} + engine := ethash.NewFullFaker() + gp := new(core.GasPool).AddGas(block.GasLimit()) + usedGas := new(uint64) + var receipts types.Receipts + if chainConfig.DAOForkSupport && chainConfig.DAOForkBlock != nil && chainConfig.DAOForkBlock.Cmp(block.Number()) == 0 { + misc.ApplyDAOHardFork(ibs) + } + for _, tx := range block.Transactions() { + receipt, err := core.ApplyTransaction(chainConfig, bcb, nil, gp, ibs, txnWriter, header, tx, usedGas, vmConfig) + if err != nil { + return fmt.Errorf("tx %x failed: %v", tx.Hash(), err) + } + receipts = append(receipts, receipt) + } + // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) + if _, err := engine.FinalizeAndAssemble(chainConfig, header, ibs, block.Transactions(), block.Uncles(), receipts); err != nil { + return fmt.Errorf("finalize of block %d failed: %v", block.NumberU64(), err) + } + + ctx := chainConfig.WithEIPsFlags(context.Background(), header.Number) + if err := ibs.CommitBlock(ctx, blockWriter); err != nil { + return fmt.Errorf("commiting block %d failed: %v", block.NumberU64(), err) + } + return nil +} + +func GetBlockByNumber(db ethdb.KV, number uint64) (*types.Block, error) { + var block *types.Block + err := db.View(context.Background(), func(tx ethdb.Tx) error { + b, err := remotechain.GetBlockByNumber(tx, number) + block = b + return err + }) + if err != nil { + return nil, err + } + return block, nil +} + +// ReadChainConfig retrieves the consensus settings based on the given genesis hash. +func ReadChainConfig(db ethdb.KV, chain string) *params.ChainConfig { + var k []byte + var data []byte + switch chain { + case "mainnet": + k = params.MainnetGenesisHash[:] + case "testnet": + k = params.TestnetGenesisHash[:] + case "rinkeby": + k = params.RinkebyGenesisHash[:] + case "goerli": + k = params.GoerliGenesisHash[:] + } + _ = db.View(context.Background(), func(tx ethdb.Tx) error { + b := tx.Bucket(dbutils.ConfigPrefix) + d, _ := b.Get(k) + data = d + return nil + }) + var config params.ChainConfig + _ = json.Unmarshal(data, &config) + return &config +} diff --git a/cmd/restapi/rest/serve_rest.go b/cmd/restapi/rest/serve_rest.go index 9c2e70a03..131de7526 100644 --- a/cmd/restapi/rest/serve_rest.go +++ b/cmd/restapi/rest/serve_rest.go @@ -49,6 +49,9 @@ func ServeREST(ctx context.Context, localAddress, remoteDBAddress string) error if err = apis.RegisterStorageAPI(root.Group("storage"), e); err != nil { return err } + if err = apis.RegisterRetraceAPI(root.Group("retrace"), e); err != nil { + return err + } if err = apis.RegisterIntermediateHashAPI(root.Group("intermediate-hash"), e); err != nil { return err } diff --git a/cmd/semantics/main.go b/cmd/semantics/main.go index 64bdd9840..c3ee937d9 100644 --- a/cmd/semantics/main.go +++ b/cmd/semantics/main.go @@ -1,9 +1,9 @@ package main import ( - "fmt" + "fmt" ) func main() { - fmt.Printf("Hello, semantics!\n") -} \ No newline at end of file + fmt.Printf("Hello, semantics!\n") +} diff --git a/go.mod b/go.mod index 7b288525e..ad6f0a706 100644 --- a/go.mod +++ b/go.mod @@ -68,7 +68,7 @@ require ( github.com/wcharczuk/go-chart v2.0.1+incompatible github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208 golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 - golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect + golang.org/x/net v0.0.0-20200301022130-244492dfa37a golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 golang.org/x/text v0.3.2 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0