/* Copyright 2022 Erigon contributors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package commitment import ( "encoding/binary" "encoding/hex" "fmt" "sort" "testing" "github.com/holiman/uint256" "github.com/ledgerwatch/erigon-lib/common" "golang.org/x/crypto/sha3" ) // In memory commitment and state to use with the tests type MockState struct { numBuf [binary.MaxVarintLen64]byte sm map[string][]byte // backbone of the state cm map[string][]byte // backbone of the commitments } func NewMockState() *MockState { return &MockState{ sm: make(map[string][]byte), cm: make(map[string][]byte), } } func (ms MockState) lockFn() { } func (ms MockState) unlockFn() { } func (ms MockState) branchFn(prefix []byte) ([]byte, error) { if exBytes, ok := ms.cm[string(prefix)]; ok { return exBytes, nil } return nil, nil } func (ms MockState) accountFn(plainKey []byte, cell *Cell) ([]byte, error) { exBytes, ok := ms.sm[string(plainKey)] if !ok { return nil, fmt.Errorf("accountFn not found key [%x]", plainKey) } var ex Update pos, err := ex.decode(exBytes, 0) if err != nil { return nil, fmt.Errorf("accountFn decode existing [%x], bytes: [%x]: %w", plainKey, exBytes, err) } if pos != len(exBytes) { return nil, fmt.Errorf("accountFn key [%x] leftover bytes in [%x], comsumed %x", plainKey, exBytes, pos) } if ex.Flags&STORAGE_UPDATE != 0 { return nil, fmt.Errorf("accountFn reading storage item for key [%x]", plainKey) } if ex.Flags&DELETE_UPDATE != 0 { return nil, fmt.Errorf("accountFn reading deleted account for key [%x]", plainKey) } if ex.Flags&BALANCE_UPDATE != 0 { cell.Balance.Set(&ex.Balance) } else { cell.Balance.Clear() } if ex.Flags&NONCE_UPDATE != 0 { cell.Nonce = ex.Nonce } else { cell.Nonce = 0 } if ex.Flags&CODE_UPDATE != 0 { copy(cell.CodeHash[:], ex.CodeHashOrStorage[:]) } else { cell.CodeHash = [32]byte{} } return plainKey, nil } func (ms MockState) storageFn(plainKey []byte, cell *Cell) ([]byte, error) { exBytes, ok := ms.sm[string(plainKey)] if !ok { return nil, fmt.Errorf("storageFn not found key [%x]", plainKey) } var ex Update pos, err := ex.decode(exBytes, 0) if err != nil { return nil, fmt.Errorf("storageFn decode existing [%x], bytes: [%x]: %w", plainKey, exBytes, err) } if pos != len(exBytes) { return nil, fmt.Errorf("storageFn key [%x] leftover bytes in [%x], comsumed %x", plainKey, exBytes, pos) } if ex.Flags&BALANCE_UPDATE != 0 { return nil, fmt.Errorf("storageFn reading balance for key [%x]", plainKey) } if ex.Flags&NONCE_UPDATE != 0 { return nil, fmt.Errorf("storageFn reading nonce for key [%x]", plainKey) } if ex.Flags&CODE_UPDATE != 0 { return nil, fmt.Errorf("storageFn reading codeHash for key [%x]", plainKey) } if ex.Flags&DELETE_UPDATE != 0 { return nil, fmt.Errorf("storageFn reading deleted item for key [%x]", plainKey) } if ex.Flags&STORAGE_UPDATE != 0 { copy(cell.Storage[:], ex.CodeHashOrStorage[:]) } else { cell.Storage = [32]byte{} } return plainKey, nil } func (ms *MockState) applyPlainUpdates(plainKeys [][]byte, updates []Update) error { for i, key := range plainKeys { update := updates[i] if update.Flags&DELETE_UPDATE != 0 { delete(ms.sm, string(key)) } else { if exBytes, ok := ms.sm[string(key)]; ok { var ex Update pos, err := ex.decode(exBytes, 0) if err != nil { return fmt.Errorf("applyPlainUpdates decode existing [%x], bytes: [%x]: %w", key, exBytes, err) } if pos != len(exBytes) { return fmt.Errorf("applyPlainUpdates key [%x] leftover bytes in [%x], comsumed %x", key, exBytes, pos) } if update.Flags&BALANCE_UPDATE != 0 { ex.Flags |= BALANCE_UPDATE ex.Balance.Set(&update.Balance) } if update.Flags&NONCE_UPDATE != 0 { ex.Flags |= NONCE_UPDATE ex.Nonce = update.Nonce } if update.Flags&CODE_UPDATE != 0 { ex.Flags |= CODE_UPDATE copy(ex.CodeHashOrStorage[:], update.CodeHashOrStorage[:]) } if update.Flags&STORAGE_UPDATE != 0 { ex.Flags |= STORAGE_UPDATE copy(ex.CodeHashOrStorage[:], update.CodeHashOrStorage[:]) } ms.sm[string(key)] = ex.encode(nil, ms.numBuf[:]) } else { ms.sm[string(key)] = update.encode(nil, ms.numBuf[:]) } } } return nil } func (ms *MockState) applyBranchNodeUpdates(updates map[string][]byte) { for key, update := range updates { ms.cm[key] = update } } func decodeHex(in string) []byte { payload, err := hex.DecodeString(in) if err != nil { panic(err) } return payload } // UpdateBuilder collects updates to the state // and provides them in properly sorted form type UpdateBuilder struct { balances map[string]*uint256.Int nonces map[string]uint64 codeHashes map[string][32]byte storages map[string]map[string][]byte deletes map[string]struct{} deletes2 map[string]map[string]struct{} keyset map[string]struct{} keyset2 map[string]map[string]struct{} } func NewUpdateBuilder() *UpdateBuilder { return &UpdateBuilder{ balances: make(map[string]*uint256.Int), nonces: make(map[string]uint64), codeHashes: make(map[string][32]byte), storages: make(map[string]map[string][]byte), deletes: make(map[string]struct{}), deletes2: make(map[string]map[string]struct{}), keyset: make(map[string]struct{}), keyset2: make(map[string]map[string]struct{}), } } func (ub *UpdateBuilder) Balance(addr string, balance uint64) *UpdateBuilder { sk := string(decodeHex(addr)) delete(ub.deletes, sk) ub.balances[sk] = uint256.NewInt(balance) ub.keyset[sk] = struct{}{} return ub } func (ub *UpdateBuilder) Nonce(addr string, nonce uint64) *UpdateBuilder { sk := string(decodeHex(addr)) delete(ub.deletes, sk) ub.nonces[sk] = nonce ub.keyset[sk] = struct{}{} return ub } func (ub *UpdateBuilder) CodeHash(addr string, hash [32]byte) *UpdateBuilder { sk := string(decodeHex(addr)) delete(ub.deletes, sk) ub.codeHashes[sk] = hash ub.keyset[sk] = struct{}{} return ub } func (ub *UpdateBuilder) Storage(addr string, loc string, value string) *UpdateBuilder { sk1 := string(decodeHex(addr)) sk2 := string(decodeHex(loc)) v := decodeHex(value) if d, ok := ub.deletes2[sk1]; ok { delete(d, sk2) if len(d) == 0 { delete(ub.deletes2, sk1) } } if k, ok := ub.keyset2[sk1]; ok { k[sk2] = struct{}{} } else { ub.keyset2[sk1] = make(map[string]struct{}) ub.keyset2[sk1][sk2] = struct{}{} } if s, ok := ub.storages[sk1]; ok { s[sk2] = v } else { ub.storages[sk1] = make(map[string][]byte) ub.storages[sk1][sk2] = v } return ub } func (ub *UpdateBuilder) Delete(addr string) *UpdateBuilder { sk := string(decodeHex(addr)) delete(ub.balances, sk) delete(ub.nonces, sk) delete(ub.codeHashes, sk) delete(ub.storages, sk) ub.deletes[sk] = struct{}{} ub.keyset[sk] = struct{}{} return ub } func (ub *UpdateBuilder) DeleteStorage(addr string, loc string) *UpdateBuilder { sk1 := string(decodeHex(addr)) sk2 := string(decodeHex(loc)) if s, ok := ub.storages[sk1]; ok { delete(s, sk2) if len(s) == 0 { delete(ub.storages, sk1) } } if k, ok := ub.keyset2[sk1]; ok { k[sk2] = struct{}{} } else { ub.keyset2[sk1] = make(map[string]struct{}) ub.keyset2[sk1][sk2] = struct{}{} } if d, ok := ub.deletes2[sk1]; ok { d[sk2] = struct{}{} } else { ub.deletes2[sk1] = make(map[string]struct{}) ub.deletes2[sk1][sk2] = struct{}{} } return ub } // Build returns three slices (in the order sorted by the hashed keys) // 1. Plain keys // 2. Corresponding hashed keys // 3. Corresponding updates func (ub *UpdateBuilder) Build() (plainKeys, hashedKeys [][]byte, updates []Update) { var hashed []string preimages := make(map[string][]byte) preimages2 := make(map[string][]byte) keccak := sha3.NewLegacyKeccak256() for key := range ub.keyset { keccak.Reset() keccak.Write([]byte(key)) h := keccak.Sum(nil) hashedKey := make([]byte, len(h)*2) for i, c := range h { hashedKey[i*2] = (c >> 4) & 0xf hashedKey[i*2+1] = c & 0xf } hashed = append(hashed, string(hashedKey)) preimages[string(hashedKey)] = []byte(key) } hashedKey := make([]byte, 128) for sk1, k := range ub.keyset2 { keccak.Reset() keccak.Write([]byte(sk1)) h := keccak.Sum(nil) for i, c := range h { hashedKey[i*2] = (c >> 4) & 0xf hashedKey[i*2+1] = c & 0xf } for sk2 := range k { keccak.Reset() keccak.Write([]byte(sk2)) h2 := keccak.Sum(nil) for i, c := range h2 { hashedKey[64+i*2] = (c >> 4) & 0xf hashedKey[64+i*2+1] = c & 0xf } hs := string(common.Copy(hashedKey)) hashed = append(hashed, hs) preimages[hs] = []byte(sk1) preimages2[hs] = []byte(sk2) } } sort.Strings(hashed) plainKeys = make([][]byte, len(hashed)) hashedKeys = make([][]byte, len(hashed)) updates = make([]Update, len(hashed)) for i, hashedKey := range hashed { hashedKeys[i] = []byte(hashedKey) key := preimages[hashedKey] key2 := preimages2[hashedKey] plainKey := make([]byte, len(key)+len(key2)) copy(plainKey[:], []byte(key)) if key2 != nil { copy(plainKey[len(key):], []byte(key2)) } plainKeys[i] = plainKey u := &updates[i] if key2 == nil { if balance, ok := ub.balances[string(key)]; ok { u.Flags |= BALANCE_UPDATE u.Balance.Set(balance) } if nonce, ok := ub.nonces[string(key)]; ok { u.Flags |= NONCE_UPDATE u.Nonce = nonce } if codeHash, ok := ub.codeHashes[string(key)]; ok { u.Flags |= CODE_UPDATE copy(u.CodeHashOrStorage[:], codeHash[:]) } } else { if sm, ok1 := ub.storages[string(key)]; ok1 { if storage, ok2 := sm[string(key2)]; ok2 { u.Flags |= STORAGE_UPDATE u.CodeHashOrStorage = [32]byte{} copy(u.CodeHashOrStorage[32-len(storage):], storage) } } } } return } func TestEmptyState(t *testing.T) { ms := NewMockState() hph := NewHexPatriciaHashed(1, ms.branchFn, ms.accountFn, ms.storageFn, ms.lockFn, ms.unlockFn) hph.SetTrace(false) plainKeys, hashedKeys, updates := NewUpdateBuilder(). Balance("00", 4). Balance("01", 5). Balance("02", 6). Balance("03", 7). Balance("04", 8). Storage("04", "01", "0401"). Storage("03", "56", "050505"). Storage("03", "57", "060606"). Balance("05", 9). Storage("05", "02", "8989"). Storage("05", "04", "9898"). Build() if err := ms.applyPlainUpdates(plainKeys, updates); err != nil { t.Fatal(err) } branchNodeUpdates, err := hph.ProcessUpdates(plainKeys, hashedKeys, updates) if err != nil { t.Fatal(err) } ms.applyBranchNodeUpdates(branchNodeUpdates) fmt.Printf("1. Generated updates\n") var keys []string for key := range branchNodeUpdates { keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { branchNodeUpdate := branchNodeUpdates[key] fmt.Printf("%x => %s\n", key, branchToString(branchNodeUpdate)) } // More updates hph.Reset() plainKeys, hashedKeys, updates = NewUpdateBuilder(). Storage("03", "58", "050505"). Build() if err := ms.applyPlainUpdates(plainKeys, updates); err != nil { t.Fatal(err) } branchNodeUpdates, err = hph.ProcessUpdates(plainKeys, hashedKeys, updates) if err != nil { t.Fatal(err) } ms.applyBranchNodeUpdates(branchNodeUpdates) fmt.Printf("2. Generated updates\n") keys = keys[:0] for key := range branchNodeUpdates { keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { branchNodeUpdate := branchNodeUpdates[key] fmt.Printf("%x => %s\n", key, branchToString(branchNodeUpdate)) } // More updates hph.Reset() plainKeys, hashedKeys, updates = NewUpdateBuilder(). Storage("03", "58", "070807"). Build() if err := ms.applyPlainUpdates(plainKeys, updates); err != nil { t.Fatal(err) } branchNodeUpdates, err = hph.ProcessUpdates(plainKeys, hashedKeys, updates) if err != nil { t.Fatal(err) } ms.applyBranchNodeUpdates(branchNodeUpdates) fmt.Printf("3. Generated updates\n") keys = keys[:0] for key := range branchNodeUpdates { keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { branchNodeUpdate := branchNodeUpdates[key] fmt.Printf("%x => %s\n", key, branchToString(branchNodeUpdate)) } }