mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-28 14:47:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[erigon2] cleanup obsolete code for bin-commitment-trie (#451)

Browse Source 
* cleanup obsolete code for bin-commitment-trie

* renamed back test functions for hex, renamed test file for bin trie for consistency
This commit is contained in:
Artem Tsebrovskiy 2022-05-19 23:18:03 +03:00 committed by GitHub
parent 7908982ed9
commit 59d810650c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 275 additions and 1869 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

190
aggregator/aggregator.go
View File

@ -1007,7 +1007,6 @@ func NewAggregator(diffDir string, unwindLimit uint64, aggregationStep uint64, c
commitments: commitments,
archHasher: murmur3.New128WithSeed(0), // TODO: Randomise salt
}
a.trace = true
for fType := FirstType; fType < NumberOfTypes; fType++ {
a.files[fType] = btree.New(32)
}
@ -1146,14 +1145,7 @@ func (a *Aggregator) rebuildRecentState(tx kv.RwTx) error {
var commitMerger commitmentMerger
if fType == Commitment {
switch a.hph.Variant() {
case commitment.VariantHexPatriciaTrie, commitment.VariantReducedHexPatriciaTrie:
commitMerger = mergeCommitments
case commitment.VariantBinPatriciaTrie:
commitMerger = mergeBinCommitments
default:
return false
}
commitMerger = mergeCommitments
}
if err = changes.aggregateToBtree(tree, prefixLen, commitMerger); err != nil {
@ -1418,12 +1410,6 @@ func encodeU64(i uint64, to []byte) []byte {
}
}
var spkNotFound = make(map[string]int)
func MarkKeyNotFound(k string) {
spkNotFound[k]++
}
// commitmentValTransform parses the value of the commitment record to extract references
// to accounts and storage items, then looks them up in the new, merged files, and replaces them with
// the updated references
@ -1465,6 +1451,8 @@ func (cvt *CommitmentValTransform) commitmentValTransform(val []byte, transValBu
accountPlainKey = encodeU64(offset, []byte{byte(j - 1)})
//fmt.Printf("replaced account [%x]=>[%x] for file [%d-%d]\n", apkBuf, accountPlainKey, item.startBlock, item.endBlock)
break
} else if j == 0 {
fmt.Printf("could not find replacement key [%x], file=%s.%d-%d]\n\n", apkBuf, Account.String(), item.startBlock, item.endBlock)
}
}
}
@ -1482,11 +1470,7 @@ func (cvt *CommitmentValTransform) commitmentValTransform(val []byte, transValBu
g.Reset(offset)
//fmt.Printf("offsetToKey storage [%x] offset=%d, file=%d-%d\n", storagePlainKey, offset, cvt.pre[Storage][fileI].startBlock, cvt.pre[Storage][fileI].endBlock)
spkBuf, _ = g.Next(spkBuf[:0])
//
}
//if bytes.Equal(storagePlainKey, wantedOfft) || bytes.Equal(spkBuf, wantedOfft) {
// fmt.Printf("WantedOffset replacing storage [%x] => [%x]\n", spkBuf, storagePlainKey)
//}
// Lookup spkBuf in the post storage files
for j := len(cvt.post[Storage]); j > 0; j-- {
item := cvt.post[Storage][j-1]
@ -1500,9 +1484,6 @@ func (cvt *CommitmentValTransform) commitmentValTransform(val []byte, transValBu
if keyMatch, _ := g.Match(spkBuf); keyMatch {
storagePlainKey = encodeU64(offset, []byte{byte(j - 1)})
//fmt.Printf("replacing storage [%x] => [fileI=%d, offset=%d, file=%s.%d-%d]\n", spkBuf, j-1, offset, Storage.String(), item.startBlock, item.endBlock)
//if bytes.Equal(storagePlainKey, wantedOfft) {
// fmt.Printf("OFF replacing storage [%x] => [%x]\n", spkBuf, storagePlainKey)
//}
break
} else if j == 0 {
fmt.Printf("could not find replacement key [%x], file=%s.%d-%d]\n\n", spkBuf, Storage.String(), item.startBlock, item.endBlock)
@ -1517,102 +1498,6 @@ func (cvt *CommitmentValTransform) commitmentValTransform(val []byte, transValBu
return transValBuf, nil
}
//var wanted = []byte{0, 3, 70,113}
// var wanted = []byte{138, 1, 88, 39, 36, 194, 18, 220, 117, 172, 221, 139, 208, 27, 186, 172, 217, 9, 154, 251, 240, 124, 16, 228, 140, 98, 195, 47, 222, 155, 131, 231, 90, 114, 61, 225, 14, 230, 104, 165, 113, 52, 4, 143, 167, 207, 154, 237, 244, 218, 83, 204}
var Wanted = []byte{87, 13, 60, 125, 6, 210, 211, 78, 26, 212, 11, 71, 211, 176, 73, 96, 60, 95, 127, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
//var wantedOfft = encodeU64(6583, []byte{0})
// var wantedOfft = encodeU64(38437, []byte{0})
// commitmentValTransform parses the value of the commitment record to extract references
// to accounts and storage items, then looks them up in the new, merged files, and replaces them with
// the updated references
//
// this is copy of the above function, but with a different plainkeys extractor
func (cvt *CommitmentValTransform) commitmentBinValTransform(val []byte, transValBuf []byte) ([]byte, error) {
if len(val) == 0 {
return transValBuf, nil
}
accountPlainKeys, storagePlainKeys, err := commitment.ExtractBinPlainKeys(val)
if err != nil {
return nil, err
}
var transAccountPks [][]byte
var transStoragePks [][]byte
var apkBuf, spkBuf []byte
for _, accountPlainKey := range accountPlainKeys {
if len(accountPlainKey) == length.Addr {
// Non-optimised key originating from a database record
apkBuf = append(apkBuf[:0], accountPlainKey...)
} else {
// Optimised key referencing a state file record (file number and offset within the file)
fileI := int(accountPlainKey[0])
offset := decodeU64(accountPlainKey[1:])
g := cvt.pre[Account][fileI].getterMerge
g.Reset(offset)
apkBuf, _ = g.Next(apkBuf[:0])
//fmt.Printf("replacing account [%x] from [%x]\n", apkBuf, accountPlainKey)
}
// Look up apkBuf in the post account files
for j := len(cvt.post[Account]); j > 0; j-- {
item := cvt.post[Account][j-1]
if item.index.Empty() {
continue
}
offset := item.readerMerge.Lookup(apkBuf)
g := item.getterMerge
g.Reset(offset)
if g.HasNext() {
if keyMatch, _ := g.Match(apkBuf); keyMatch {
accountPlainKey = encodeU64(offset, []byte{byte(j - 1)})
//fmt.Printf("replaced account [%x]=>[%x] for file [%d-%d]\n", apkBuf, accountPlainKey, item.startBlock, item.endBlock)
break
}
}
}
transAccountPks = append(transAccountPks, accountPlainKey)
}
for _, storagePlainKey := range storagePlainKeys {
if len(storagePlainKey) == length.Addr+length.Hash {
// Non-optimised key originating from a database record
spkBuf = append(spkBuf[:0], storagePlainKey...)
} else {
// Optimised key referencing a state file record (file number and offset within the file)
fileI := int(storagePlainKey[0])
offset := decodeU64(storagePlainKey[1:])
g := cvt.pre[Storage][fileI].getterMerge
g.Reset(offset)
spkBuf, _ = g.Next(spkBuf[:0])
//fmt.Printf("replacing storage [%x] from [%x]\n", spkBuf, storagePlainKey)
}
// Lookup spkBuf in the post storage files
for j := len(cvt.post[Storage]); j > 0; j-- {
item := cvt.post[Storage][j-1]
if item.index.Empty() {
continue
}
offset := item.readerMerge.Lookup(spkBuf)
g := item.getterMerge
g.Reset(offset)
if g.HasNext() {
if keyMatch, _ := g.Match(spkBuf); keyMatch {
storagePlainKey = encodeU64(offset, []byte{byte(j - 1)})
//fmt.Printf("replaced storage [%x]=>[%x]\n", spkBuf, storagePlainKey)
break
}
}
}
transStoragePks = append(transStoragePks, storagePlainKey)
}
if transValBuf, err = commitment.ReplaceBinPlainKeys(val, transAccountPks, transStoragePks, transValBuf); err != nil {
return nil, err
}
return transValBuf, nil
}
func (a *Aggregator) backgroundMerge() {
defer a.mergeWg.Done()
for range a.mergeChannel {
@ -1652,14 +1537,8 @@ func (a *Aggregator) backgroundMerge() {
var valTransform func([]byte, []byte) ([]byte, error)
var mergeFunc func([]byte, []byte, []byte) ([]byte, error)
if fType == Commitment {
switch a.hph.Variant() {
case commitment.VariantBinPatriciaTrie:
valTransform = cvt.commitmentBinValTransform
mergeFunc = mergeBinCommitments
case commitment.VariantHexPatriciaTrie, commitment.VariantReducedHexPatriciaTrie:
valTransform = cvt.commitmentValTransform
mergeFunc = mergeCommitments
}
valTransform = cvt.commitmentValTransform
mergeFunc = mergeCommitments
} else {
mergeFunc = mergeReplace
}
@ -1813,10 +1692,6 @@ func mergeCommitments(preval, val, buf []byte) ([]byte, error) {
return commitment.MergeHexBranches(preval, val, buf)
}
func mergeBinCommitments(preval, val, buf []byte) ([]byte, error) {
return mergeReplace(preval, val, buf)
}
func (a *Aggregator) backgroundHistoryMerge() {
defer a.historyWg.Done()
for range a.historyChannel {
@ -1853,12 +1728,7 @@ func (a *Aggregator) backgroundHistoryMerge() {
case isBitmap:
mergeFunc = mergeBitmaps
case fType == Commitment:
switch a.hph.Variant() {
case commitment.VariantBinPatriciaTrie:
mergeFunc = mergeBinCommitments
case commitment.VariantHexPatriciaTrie, commitment.VariantReducedHexPatriciaTrie:
mergeFunc = mergeCommitments
}
mergeFunc = mergeCommitments
default:
mergeFunc = mergeReplace
}
@ -2104,21 +1974,9 @@ func (a *Aggregator) readFromFiles(fType FileType, lock bool, blockNum uint64, f
})
if fType == Commitment {
var plainKeysExtractor func(branchData []byte) (accountPlainKeys [][]byte, storagePlainKeys [][]byte, err error)
var plainKeysReplacer func(branchData []byte, accountPlainKeys [][]byte, storagePlainKeys [][]byte, newData []byte) ([]byte, error)
switch a.hph.Variant() {
case commitment.VariantHexPatriciaTrie, commitment.VariantReducedHexPatriciaTrie:
plainKeysExtractor = commitment.ExtractPlainKeys
plainKeysReplacer = commitment.ReplacePlainKeys
case commitment.VariantBinPatriciaTrie:
plainKeysExtractor = commitment.ExtractBinPlainKeys
plainKeysReplacer = commitment.ReplaceBinPlainKeys
}
// Transform references
if len(val) > 0 {
accountPlainKeys, storagePlainKeys, err := plainKeysExtractor(val)
accountPlainKeys, storagePlainKeys, err := commitment.ExtractPlainKeys(val)
if err != nil {
panic(fmt.Errorf("value %x: %w", val, err))
}
@ -2151,7 +2009,7 @@ func (a *Aggregator) readFromFiles(fType FileType, lock bool, blockNum uint64, f
}
transStoragePks = append(transStoragePks, spkBuf)
}
if val, err = plainKeysReplacer(val, transAccountPks, transStoragePks, nil); err != nil {
if val, err = commitment.ReplacePlainKeys(val, transAccountPks, transStoragePks, nil); err != nil {
panic(err)
}
}
@ -2608,24 +2466,13 @@ func (w *Writer) computeCommitment(trace bool) ([]byte, error) {
original = prevV[4:]
}
if original != nil {
switch w.a.hph.Variant() {
case commitment.VariantBinPatriciaTrie:
branchNodeUpdate, err = mergeReplace(original, branchNodeUpdate, nil)
if err != nil {
return nil, err
}
// use current version without merge
case commitment.VariantHexPatriciaTrie, commitment.VariantReducedHexPatriciaTrie:
// try to merge previous (original) and current (branchNodeUpdate) into one update
mergedVal, err := commitment.MergeHexBranches(original, branchNodeUpdate, nil)
if err != nil {
return nil, err
}
//fmt.Printf("computeCommitment merge [%x] [%x]+[%x]=>[%x]\n", commitment.CompactToHex(prefix), original, branchNodeUpdate, mergedVal)
branchNodeUpdate = mergedVal
default:
panic(fmt.Errorf("unknown commitment trie variant %s", w.a.hph.Variant()))
// try to merge previous (original) and current (branchNodeUpdate) into one update
mergedVal, err := commitment.MergeHexBranches(original, branchNodeUpdate, nil)
if err != nil {
return nil, err
}
//fmt.Printf("computeCommitment merge [%x] [%x]+[%x]=>[%x]\n", commitment.CompactToHex(prefix), original, branchNodeUpdate, mergedVal)
branchNodeUpdate = mergedVal
//fmt.Printf("bstat: %v\n", w.a.hph.(*commitment.BinPatriciaTrie).StatString())
}
@ -3196,14 +3043,7 @@ func (w *Writer) aggregateUpto(blockFrom, blockTo uint64) error {
var commitMerger commitmentMerger
if fType == Commitment {
switch w.a.hph.Variant() {
case commitment.VariantHexPatriciaTrie, commitment.VariantReducedHexPatriciaTrie:
commitMerger = mergeCommitments
case commitment.VariantBinPatriciaTrie:
commitMerger = mergeBinCommitments
default:
return fmt.Errorf("unknown commitment variant %s: failed to define how to merge commitments", w.a.hph.Variant())
}
commitMerger = mergeCommitments
}
if aggTask.bt[fType], err = aggTask.changes[fType].aggregate(blockFrom, blockTo, prefixLen, w.tx, fType.Table(), commitMerger); err != nil {

25
commitment/bin_patricia_hashed.go
View File

@ -36,6 +36,18 @@ const (
maxChild = 2
)
type bitstring []uint8
func hexToBin(hex []byte) bitstring {
bin := make([]byte, 4*len(hex))
for i := range bin {
if hex[i/4]&(1<<(3-i%4)) != 0 {
bin[i] = 1
}
}
return bin
}
// BinHashed implements commitment based on patricia merkle tree with radix 16,
// with keys pre-hashed by keccak256
type BinHashed struct {
@ -73,18 +85,18 @@ type BinHashed struct {
byteArrayWriter ByteArrayWriter
}
func NewBinHashed(accountKeyLen int,
func NewBinPatriciaHashed(accountKeyLen int,
branchFn func(prefix []byte) ([]byte, error),
accountFn func(plainKey []byte, cell *BinCell) error,
storageFn func(plainKey []byte, cell *BinCell) error,
accountFn func(plainKey []byte, cell *Cell) error,
storageFn func(plainKey []byte, cell *Cell) error,
) *BinHashed {
return &BinHashed{
keccak: sha3.NewLegacyKeccak256().(keccakState),
keccak2: sha3.NewLegacyKeccak256().(keccakState),
accountKeyLen: accountKeyLen,
branchFn: branchFn,
accountFn: accountFn,
storageFn: storageFn,
accountFn: wrapAccountStorageFn(accountFn),
storageFn: wrapAccountStorageFn(storageFn),
}
}
@ -142,7 +154,7 @@ func (hph *BinHashed) ProcessUpdates(plainKeys, hashedKeys [][]byte, updates []U
}
func (hph *BinHashed) Variant() TrieVariant {
return VariantReducedHexPatriciaTrie
return VariantBinPatriciaTrie
}
func (hph *BinHashed) RootHash() ([]byte, error) {
@ -763,7 +775,6 @@ func (hph *BinHashed) fold() ([]byte, []byte, error) {
depth := hph.depths[hph.activeRows-1]
var branchData []byte
var bitmapBuf [4]byte
// updateKey, _ := bitstring(hph.currentKey[:updateKeyLen]).reconstructHex()
updateKey := binToCompact(hph.currentKey[:updateKeyLen])
if hph.trace {
fmt.Printf("touchMap[%d]=%016b, afterMap[%d]=%016b\n", row, hph.touchMap[row], row, hph.afterMap[row])

233
commitment/bin_patricia_hashed_test.go Normal file
View File

@ -0,0 +1,233 @@
package commitment
import (
"encoding/hex"
"fmt"
"testing"
"github.com/ledgerwatch/erigon-lib/common/length"
"github.com/stretchr/testify/require"
"golang.org/x/exp/slices"
)
func Test_BinPatriciaTrie_UniqueRepresentation(t *testing.T) {
t.Skip()
ms := NewMockState(t)
ms2 := NewMockState(t)
trie := NewBinPatriciaHashed(length.Addr, ms.branchFn, ms.accountFn, ms.storageFn)
trieBatch := NewBinPatriciaHashed(length.Addr, ms2.branchFn, ms2.accountFn, ms2.storageFn)
plainKeys, hashedKeys, updates := NewUpdateBuilder().
Balance("e25652aaa6b9417973d325f9a1246b48ff9420bf", 12).
Balance("cdd0a12034e978f7eccda72bd1bd89a8142b704e", 120000).
Nonce("5bb6abae12c87592b940458437526cb6cad60d50", 152512).
Balance("2fcb355beb0ea2b5fcf3b62a24e2faaff1c8d0c0", 100000).
Balance("463510be61a7ccde354509c0ab813e599ee3fc8a", 200000).
Storage("cd3e804beea486038609f88f399140dfbe059ef3", "02", "98").
Balance("82c88c189d5deeba0ad11463b80b44139bd519c1", 300000).
Balance("0647e43e8f9ba3fb8b14ad30796b7553d667c858", 400000).
Delete("cdd0a12034e978f7eccda72bd1bd89a8142b704e").
Balance("06548d648c23b12f2e9bfd1bae274b658be208f4", 500000).
Balance("e5417f49640cf8a0b1d6e38f9dfdc00196e99e8b", 600000).
Nonce("825ac9fa5d015ec7c6b4cbbc50f78d619d255ea7", 184).
Build()
for i := 0; i < len(updates); i++ {
_, err := trie.ProcessUpdates(plainKeys[i:i+1], hashedKeys[i:i+1], updates[i:i+1])
require.NoError(t, err)
}
trieBatch.ProcessUpdates(plainKeys, hashedKeys, updates)
sequentialHash, _ := trie.RootHash()
require.Len(t, sequentialHash, 32)
batchHash, _ := trieBatch.RootHash()
require.EqualValues(t, sequentialHash, batchHash)
}
func Test_BinPatriciaTrie_EmptyState(t *testing.T) {
ms := NewMockState(t)
hph := NewBinPatriciaHashed(1, ms.branchFn, ms.accountFn, ms.storageFn)
hph.SetTrace(false)
plainKeys, hashedKeys, updates := NewUpdateBuilder().
Balance("00", 4).
Balance("01", 5).
Balance("02", 6).
Balance("03", 7).
Balance("04", 8).
Storage("04", "01", "0401").
Storage("03", "56", "050505").
Storage("03", "57", "060606").
Balance("05", 9).
Storage("05", "02", "8989").
Storage("05", "04", "9898").
Build()
if err := ms.applyPlainUpdates(plainKeys, updates); err != nil {
t.Fatal(err)
}
branchNodeUpdates, err := hph.ProcessUpdates(plainKeys, hashedKeys, updates)
if err != nil {
t.Fatal(err)
}
ms.applyBranchNodeUpdates(branchNodeUpdates)
fmt.Printf("1. Generated updates\n")
var keys []string
for key := range branchNodeUpdates {
keys = append(keys, key)
}
slices.Sort(keys)
for _, key := range keys {
branchNodeUpdate := branchNodeUpdates[key]
fmt.Printf("%x => %s\n", CompactToHex([]byte(key)), branchToString(branchNodeUpdate))
}
// More updates
hph.Reset()
hph.SetTrace(false)
plainKeys, hashedKeys, updates = NewUpdateBuilder().
Storage("03", "58", "050505").
Build()
if err := ms.applyPlainUpdates(plainKeys, updates); err != nil {
t.Fatal(err)
}
branchNodeUpdates, err = hph.ProcessUpdates(plainKeys, hashedKeys, updates)
if err != nil {
t.Fatal(err)
}
ms.applyBranchNodeUpdates(branchNodeUpdates)
fmt.Printf("2. Generated updates\n")
keys = keys[:0]
for key := range branchNodeUpdates {
keys = append(keys, key)
}
slices.Sort(keys)
for _, key := range keys {
branchNodeUpdate := branchNodeUpdates[key]
fmt.Printf("%x => %s\n", CompactToHex([]byte(key)), branchToString(branchNodeUpdate))
}
// More updates
hph.Reset()
hph.SetTrace(false)
plainKeys, hashedKeys, updates = NewUpdateBuilder().
Storage("03", "58", "070807").
Build()
if err := ms.applyPlainUpdates(plainKeys, updates); err != nil {
t.Fatal(err)
}
branchNodeUpdates, err = hph.ProcessUpdates(plainKeys, hashedKeys, updates)
if err != nil {
t.Fatal(err)
}
ms.applyBranchNodeUpdates(branchNodeUpdates)
fmt.Printf("3. Generated updates\n")
keys = keys[:0]
for key := range branchNodeUpdates {
keys = append(keys, key)
}
slices.Sort(keys)
for _, key := range keys {
branchNodeUpdate := branchNodeUpdates[key]
fmt.Printf("%x => %s\n", CompactToHex([]byte(key)), branchToString(branchNodeUpdate))
}
}
func Test_BinPatriciaHashed_ProcessUpdates_UniqueRepresentation(t *testing.T) {
t.Skip()
ms := NewMockState(t)
ms2 := NewMockState(t)
plainKeys, hashedKeys, updates := NewUpdateBuilder().
Balance("f4", 4).
Storage("04", "01", "0401").
Balance("ba", 065606).
Balance("00", 4).
Balance("01", 5).
Balance("02", 6).
Balance("03", 7).
Storage("03", "56", "050505").
Balance("05", 9).
Storage("03", "57", "060606").
Balance("b9", 6).
Nonce("ff", 169356).
Storage("05", "02", "8989").
Storage("f5", "04", "9898").
Build()
renderUpdates := func(branchNodeUpdates map[string][]byte) {
var keys []string
for key := range branchNodeUpdates {
keys = append(keys, key)
}
slices.Sort(keys)
for _, key := range keys {
branchNodeUpdate := branchNodeUpdates[key]
fmt.Printf("%x => %s\n", CompactToHex([]byte(key)), branchToString(branchNodeUpdate))
}
}
trieOne := NewBinPatriciaHashed(1, ms.branchFn, ms.accountFn, ms.storageFn)
trieTwo := NewBinPatriciaHashed(1, ms2.branchFn, ms2.accountFn, ms2.storageFn)
trieTwo.Reset()
trieOne.Reset()
trieOne.SetTrace(true)
trieTwo.SetTrace(true)
// single sequential update
roots := make([][]byte, 0)
branchNodeUpdatesOne := make(map[string][]byte)
for i := 0; i < len(updates); i++ {
if err := ms.applyPlainUpdates(plainKeys[i:i+1], updates[i:i+1]); err != nil {
t.Fatal(err)
}
branchNodeUpdates, err := trieOne.ProcessUpdates(plainKeys[i:i+1], hashedKeys[i:i+1], updates[i:i+1])
require.NoError(t, err)
sequentialRoot, err := trieOne.RootHash()
require.NoError(t, err)
roots = append(roots, sequentialRoot)
ms.applyBranchNodeUpdates(branchNodeUpdates)
for br, upd := range branchNodeUpdates {
branchNodeUpdatesOne[br] = upd
}
renderUpdates(branchNodeUpdatesOne)
}
fmt.Printf("1. Trie sequential update generated following branch updates\n")
renderUpdates(branchNodeUpdatesOne)
err := ms2.applyPlainUpdates(plainKeys, updates)
require.NoError(t, err)
fmt.Printf("\n\n")
// batch update
branchNodeUpdatesTwo, err := trieTwo.ProcessUpdates(plainKeys, hashedKeys, updates)
require.NoError(t, err)
ms2.applyBranchNodeUpdates(branchNodeUpdatesTwo)
fmt.Printf("2. Trie batch update generated following branch updates\n")
renderUpdates(branchNodeUpdatesTwo)
sequentialRoot, err := trieOne.RootHash()
require.NoError(t, err)
for i, root := range roots {
fmt.Printf("%d [%s]\n", i, hex.EncodeToString(root))
}
require.NotContainsf(t, roots[:len(roots)-1], sequentialRoot, "sequential root %s found in previous hashes", hex.EncodeToString(sequentialRoot))
batchRoot, err := trieTwo.RootHash()
require.NoError(t, err)
require.EqualValues(t, batchRoot, sequentialRoot,
"expected equal roots, got sequential [%v] != batch [%v]", hex.EncodeToString(sequentialRoot), hex.EncodeToString(batchRoot))
require.Lenf(t, batchRoot, 32, "root hash length should be equal to 32 bytes")
}

914
commitment/bin_patricia_trie.go
View File

@ -1,914 +0,0 @@
package commitment
import (
"bytes"
"encoding/binary"
"encoding/hex"
"fmt"
"strings"
"github.com/ledgerwatch/erigon-lib/common/length"
"golang.org/x/crypto/sha3"
)
type BinPatriciaTrie struct {
root *RootNode
trace bool
stat stat
keccak keccakState
}
type stat struct {
hashesTotal uint64
nodesTotal uint64
}
func NewBinaryPatriciaTrie() *BinPatriciaTrie {
return &BinPatriciaTrie{
keccak: sha3.NewLegacyKeccak256().(keccakState),
}
}
func (t *BinPatriciaTrie) Update(key, value []byte) {
keyPath := newBitstring(key)
if t.root == nil {
t.root = &RootNode{
Node: &Node{
Key: key,
Value: value,
},
CommonPrefix: keyPath,
}
t.stat.nodesTotal++
return
}
edge, keyPathRest, splitAt, latest := t.root.Walk(keyPath, func(_ *Node) {})
if len(edge) == 0 && len(keyPathRest) == 0 {
latest.Value = value
return
}
pathToLatest := keyPath[:len(keyPath)-len(keyPathRest)]
t.stat.nodesTotal++
newLeaf := &Node{P: latest, Key: key, Value: value}
latest.splitEdge(pathToLatest, edge[:splitAt], edge[splitAt:], keyPathRest, newLeaf, t.hash)
if latest.P == nil {
t.root.CommonPrefix = edge[:splitAt]
}
}
// Key describes path in trie to value. When UpdateHashed is used,
// hashed key describes path to the leaf node and plainKey is stored in the leaf node Key field.
func (t *BinPatriciaTrie) UpdateHashed(plainKey, hashedKey, value []byte, isStorage bool) (updates map[string][]byte) {
keyPath := newBitstring(hashedKey)
updates = make(map[string][]byte)
if t.root == nil {
t.root = &RootNode{
Node: &Node{
Key: plainKey,
Value: value,
isStorage: isStorage,
},
CommonPrefix: keyPath,
}
t.stat.nodesTotal++
t.hash(t.root.Node, keyPath, 0)
touchMap := uint16(1 << keyPath[len(keyPath)-1])
updates[keyPath.String()] = encodeNodeUpdate(t.root.Node, touchMap, touchMap, nil)
return updates
}
touchedNodes := make([]*Node, 0)
edge, keyPathRest, splitAt, latest := t.root.Walk(keyPath, func(fn *Node) { fn.hash = fn.hash[:0]; touchedNodes = append(touchedNodes, fn) })
pathToLatest := keyPath[:len(keyPath)-len(keyPathRest)]
var touchMap uint16
if len(edge) == 0 && len(keyPathRest) == 0 { // we found the leaf
latest.Value = value
t.hash(latest, bitstring{}, 0)
touchMap = 1 << edge[len(edge)-1]
updates[pathToLatest.String()] = encodeNodeUpdate(latest, touchMap, touchMap, nil)
return updates
}
// split existing edge and insert new leaf
t.stat.nodesTotal++
newLeaf := &Node{P: latest, Key: plainKey, Value: value, isStorage: isStorage}
updates = latest.splitEdge(pathToLatest, edge[:splitAt], edge[splitAt:], keyPathRest, newLeaf, t.hash)
if latest.P == nil {
t.root.CommonPrefix = edge[:splitAt]
}
return updates
}
// Get returns value stored by provided key.
func (t *BinPatriciaTrie) Get(key []byte) ([]byte, bool) {
keyPath := newBitstring(key)
if t.root == nil {
return nil, false
}
edge, keyPathRest, _, latest := t.root.Walk(keyPath, func(_ *Node) {})
if len(edge) == 0 && len(keyPathRest) == 0 {
if latest.Value == nil {
switch {
case len(latest.LPrefix) == 0 && latest.L != nil:
return latest.L.Value, true
case len(latest.RPrefix) == 0 && latest.R != nil:
return latest.R.Value, true
}
}
return latest.Value, true
}
return nil, false
}
// Get returns value stored by provided key.
func (t *BinPatriciaTrie) getNode(key []byte) *Node {
keyPath := newBitstring(key)
if t.root == nil {
return nil
}
edge, keyPathRest, _, latest := t.root.Walk(keyPath, func(_ *Node) {})
if len(edge) == 0 && len(keyPathRest) == 0 {
return latest
}
return nil
}
func (t *BinPatriciaTrie) RootHash() ([]byte, error) {
if t.root == nil {
return EmptyRootHash, nil
}
return t.hash(t.root.Node, t.root.CommonPrefix, 0), nil
}
func (t *BinPatriciaTrie) ProcessUpdates(plainKeys, hashedKeys [][]byte, updates []Update) (branchNodeUpdates map[string][]byte, err error) {
branchNodeUpdates = make(map[string][]byte)
for i, update := range updates {
account := new(Account)
node := t.getNode(hashedKeys[i]) // check if key exist
if node != nil && !node.isStorage {
account.decode(node.Value)
}
// apply supported updates
if update.Flags == DELETE_UPDATE {
//continue
if node != nil {
if node.P != nil {
meltPrefix := node.P.deleteChild(node)
if node.P.P == nil {
t.root.CommonPrefix = append(t.root.CommonPrefix, meltPrefix...)
}
} else { // remove root
t.root = nil
}
t.stat.nodesTotal--
branchNodeUpdates[hexToBin(hashedKeys[i]).String()] = []byte{}
}
continue
}
if update.Flags&BALANCE_UPDATE != 0 {
account.Balance.Set(&update.Balance)
}
if update.Flags&NONCE_UPDATE != 0 {
account.Nonce = update.Nonce
}
if update.Flags&CODE_UPDATE != 0 {
if account.CodeHash == nil {
account.CodeHash = make([]byte, len(update.CodeHashOrStorage))
}
copy(account.CodeHash, update.CodeHashOrStorage[:])
}
aux := make([]byte, 0)
isStorage := false
if update.Flags&STORAGE_UPDATE != 0 {
isStorage = true
aux = update.CodeHashOrStorage[:update.ValLength]
}
// aux is not empty only when storage update is there
if len(aux) == 0 {
aux = account.encode(aux)
}
ukey := t.UpdateHashed(plainKeys[i], hashedKeys[i], aux, isStorage)
for pref, val := range ukey {
branchNodeUpdates[pref] = val
if val != nil && t.trace {
fmt.Printf("%q => %s\n", pref, branchToString2(val))
}
}
for pref, upd := range t.rootHashWithUpdates() {
v, ex := branchNodeUpdates[pref]
if ex {
upd = append(v[:4], upd[4:]...)
}
branchNodeUpdates[pref] = upd
}
}
return branchNodeUpdates, nil
}
func DecodeNodeFromUpdate(buf []byte) (touch, after uint16, node Node, err error) {
if len(buf) < 5 {
return
}
touch = binary.BigEndian.Uint16(buf[:2])
after = binary.BigEndian.Uint16(buf[2:4])
bits, pos := PartFlags(buf[4]), 5
if bits&ACCOUNT_PLAIN_PART != 0 {
n, aux, err := decodeSizedBuffer(buf[pos:])
if err != nil {
return touch, after, Node{}, fmt.Errorf("decode account plain key: %w", err)
}
pos += n
node.Key = aux
}
if bits&STORAGE_PLAIN_PART != 0 {
n, aux, err := decodeSizedBuffer(buf[pos:])
if err != nil {
return touch, after, Node{}, fmt.Errorf("decode storage plain key: %w", err)
}
pos += n
node.Key = aux
node.isStorage = true
}
if bits&HASH_PART != 0 {
n, aux, err := decodeSizedBuffer(buf[pos:])
if err != nil {
return touch, after, Node{}, fmt.Errorf("decode node hash: %w", err)
}
pos += n
_ = pos
node.hash = aux
}
return
}
func encodeNodeUpdate(node *Node, touched, after uint16, branchData []byte) []byte {
var numBuf [binary.MaxVarintLen64]byte
binary.BigEndian.PutUint16(numBuf[0:], touched)
binary.BigEndian.PutUint16(numBuf[2:], after)
if branchData == nil {
branchData = make([]byte, 4, 32)
}
copy(branchData[:4], numBuf[:])
var fieldBits PartFlags
if node.Value != nil {
fieldBits = ACCOUNT_PLAIN_PART
if node.isStorage {
fieldBits = STORAGE_PLAIN_PART
}
}
if len(node.hash) == length.Hash {
fieldBits |= HASH_PART
}
branchData = append(branchData, byte(fieldBits))
if fieldBits&(ACCOUNT_PLAIN_PART|STORAGE_PLAIN_PART) != 0 {
n := binary.PutUvarint(numBuf[:], uint64(len(node.Key)))
branchData = append(branchData, append(numBuf[:n], node.Key...)...)
}
if fieldBits&HASH_PART > 0 {
n := binary.PutUvarint(numBuf[:], uint64(len(node.hash)))
branchData = append(branchData, append(numBuf[:n], node.hash...)...)
}
return branchData
}
func (t *BinPatriciaTrie) Reset() {
t.root = nil
fmt.Printf("trie %v\n", t.StatString())
}
func (t *BinPatriciaTrie) ResetFns(
branchFn func(prefix []byte) ([]byte, error),
accountFn func(plainKey []byte, cell *Cell) error,
storageFn func(plainKey []byte, cell *Cell) error,
) {
}
func (t *BinPatriciaTrie) Variant() TrieVariant { return VariantBinPatriciaTrie }
func (t *BinPatriciaTrie) SetTrace(b bool) { t.trace = b }
type RootNode struct {
*Node
CommonPrefix bitstring
}
// There are three types of nodes:
// - Leaf (with a value and without branches)
// - Branch (with left and right child)
// - Root - Either leaf or branch. When root is branch, it's Key contains their common prefix as a bitstring.
type Node struct {
L, R, P *Node // left and right child, parent. For root P is nil
LPrefix bitstring // left child prefix, always begins with 0
RPrefix bitstring // right child prefix, always begins with 1
hash []byte // node hash
Key []byte // same as common prefix, useful for debugging, actual key should be reconstructed by path to the node
Value []byte // exists only in LEAF node
isStorage bool // if true, then Value holds storage value for the Key, otherwise it holds encoded account
}
func (n *Node) splitEdge(pathToNode, commonPath, detachedPath, restKeyPath bitstring, newLeaf *Node, hasher func(n *Node, pref bitstring, offt int) []byte) map[string][]byte {
var movedNode *Node
switch {
case n.Value == nil:
movedNode = &Node{ // move existed branch
L: n.L,
R: n.R,
P: n,
LPrefix: n.LPrefix,
RPrefix: n.RPrefix,
hash: n.hash,
}
movedNode.L.P, movedNode.R.P = movedNode, movedNode
default:
movedNode = &Node{ // move existed leaf
P: n,
Key: n.Key,
Value: n.Value,
isStorage: n.isStorage,
hash: n.hash,
}
}
newLeaf.P = n
switch restKeyPath[0] {
case 0:
n.LPrefix, n.L = restKeyPath, newLeaf
n.RPrefix, n.R = detachedPath, movedNode
case 1:
n.LPrefix, n.L = detachedPath, movedNode
n.RPrefix, n.R = restKeyPath, newLeaf
}
// node become extended, reset key and value
n.Key, n.Value, n.hash, n.isStorage = nil, nil, nil, false
hasher(n, commonPath, 0)
nodeTouch := uint16(1 << pathToNode[len(pathToNode)-1])
nodeAfter := uint16(3) // both child has been updated
updates := make(map[string][]byte, 3)
hasher(n, bitstring{}, 0)
updates[pathToNode.String()] = encodeNodeUpdate(n, nodeTouch, nodeAfter, nil)
rtouch := uint16(1 << restKeyPath[0])
updates[append(pathToNode, restKeyPath[0]).String()] = encodeNodeUpdate(newLeaf, rtouch, rtouch, nil)
if n.P == nil {
// commonPath should be set to RootNode.CommonPrefix outside the function
return updates
}
if len(commonPath) > 0 {
switch commonPath[0] {
case 1:
//if n.P != nil {
n.P.RPrefix = commonPath
//}
//n.RPrefix = commonPath
case 0:
//if n.P != nil {
n.P.LPrefix = commonPath
//}
//n.LPrefix = commonPath
}
}
return updates
}
func (n *RootNode) Walk(path bitstring, fn func(cd *Node)) (nodePath, pathRest bitstring, splitAt int, current *Node) {
nodePath = n.CommonPrefix
var bit uint8
var equal bool
for current = n.Node; current != nil; {
fn(current)
splitAt, bit, equal = nodePath.splitPoint(path)
if equal {
return bitstring{}, bitstring{}, 0, current
}
if splitAt < len(nodePath) {
return nodePath, path[splitAt:], splitAt, current
}
if splitAt == 0 || splitAt == len(nodePath) {
path = path[splitAt:]
switch bit {
case 1:
if current.R == nil {
return nodePath, path, splitAt, current
}
nodePath = current.RPrefix
current = current.R
case 0:
if current.L == nil {
return nodePath, path, splitAt, current
}
nodePath = current.LPrefix
current = current.L
}
continue
}
break
}
return nodePath, path, splitAt, current
}
func (n *Node) deleteChild(child *Node) bitstring {
var melt *Node
var meltPrefix bitstring
// remove child data
switch child {
case n.L:
n.L, n.LPrefix = nil, nil
melt = n.R
meltPrefix = n.RPrefix
case n.R:
n.R, n.RPrefix = nil, nil
melt = n.L
meltPrefix = n.LPrefix
default:
panic("could delete only child nodes")
}
melt.P = n.P
// merge parent path to skip this half-branch node
if n.P != nil {
switch {
case n.P.L == n:
n.P.L, n.P.LPrefix = melt, append(n.P.LPrefix, meltPrefix...)
case n.P.R == n:
n.P.R, n.P.RPrefix = melt, append(n.P.RPrefix, meltPrefix...)
default:
panic("failed to merge parent path")
}
} else { // n is root
n.LPrefix, n.RPrefix = melt.LPrefix, melt.RPrefix
n.Key = melt.Key
n.Value = melt.Value
n.L, n.R, n.Value = melt.L, melt.R, melt.Value
n.hash = n.hash[:0]
}
return meltPrefix
}
func (t *BinPatriciaTrie) StatString() string {
s := t.stat
return fmt.Sprintf("hashes_total %d nodes %d", s.hashesTotal, s.nodesTotal)
}
func (t *BinPatriciaTrie) rootHashWithUpdates() map[string][]byte {
//if t.root == nil {
// return EmptyRootHash, nil
//}
//return t.hash(t.root.Node, t.root.CommonPrefix, 0), nil
updates := make(map[string][]byte)
t.hashWithUpdates(t.root.Node, t.root.CommonPrefix, &updates)
return updates
}
func (t *BinPatriciaTrie) hashWithUpdates(n *Node, pref bitstring, updates *map[string][]byte) ([]byte, []byte) {
if len(n.hash) == 32 {
return n.hash, nil
}
t.keccak.Reset()
t.stat.hashesTotal++
var hash []byte
if n.Value == nil {
// This is a branch node, so the rule is
// branch_hash = hash(left_root_hash || right_root_hash)
lkey := bitstring(make([]byte, len(pref)+len(n.LPrefix)))
copy(lkey, pref)
copy(lkey[len(pref):], n.LPrefix)
rkey := bitstring(make([]byte, len(pref)+len(n.RPrefix)))
copy(rkey, pref)
copy(rkey[len(pref):], n.RPrefix)
lh, lupd := t.hashWithUpdates(n.L, lkey, updates)
rh, rupd := t.hashWithUpdates(n.R, rkey, updates)
t.keccak.Write(lh)
t.keccak.Write(rh)
hash = t.keccak.Sum(nil)
if len(lupd) > 0 {
binary.BigEndian.PutUint16(lupd[0:], 1)
(*updates)[lkey.String()] = lupd
}
if len(rupd) > 0 {
binary.BigEndian.PutUint16(rupd[0:], 2)
(*updates)[rkey.String()] = rupd
}
if t.trace {
fmt.Printf("branch %v (%v|%v)\n", hex.EncodeToString(hash), hex.EncodeToString(lh), hex.EncodeToString(rh))
}
t.keccak.Reset()
} else {
// This is a leaf node, so the hashing rule is
// leaf_hash = hash(hash(key) || hash(leaf_value))
t.keccak.Write(n.Key)
kh := t.keccak.Sum(nil)
t.keccak.Reset()
t.keccak.Write(n.Value)
hash = t.keccak.Sum(nil)
t.keccak.Reset()
t.keccak.Write(kh)
t.keccak.Write(hash)
hash = t.keccak.Sum(nil)
t.keccak.Reset()
if t.trace {
fmt.Printf("leaf %v\n", hex.EncodeToString(hash))
}
}
n.hash = hash
upd := encodeNodeUpdate(n, 0, 3, nil)
if n.P == nil {
binary.BigEndian.PutUint16(upd[0:], 3)
(*updates)[pref.String()] = upd
}
return hash, upd
}
func (t *BinPatriciaTrie) hash(n *Node, pref bitstring, off int) []byte {
t.keccak.Reset()
t.stat.hashesTotal++
if len(n.hash) == 32 && n.P != nil {
return n.hash
}
var hash []byte
if n.Value == nil {
// This is a branch node, so the rule is
// branch_hash = hash(left_root_hash || right_root_hash)
lh := t.hash(n.L, n.LPrefix, off+len(pref))
rh := t.hash(n.R, n.RPrefix, off+len(pref))
t.keccak.Write(lh)
t.keccak.Write(rh)
hash = t.keccak.Sum(nil)
if t.trace {
fmt.Printf("branch %v (%v|%v)\n", hex.EncodeToString(hash), hex.EncodeToString(lh), hex.EncodeToString(rh))
}
t.keccak.Reset()
} else {
// This is a leaf node, so the hashing rule is
// leaf_hash = hash(hash(key) || hash(leaf_value))
t.keccak.Write(n.Key)
kh := t.keccak.Sum(nil)
t.keccak.Reset()
t.keccak.Write(n.Value)
hash = t.keccak.Sum(nil)
t.keccak.Reset()
t.keccak.Write(kh)
t.keccak.Write(hash)
hash = t.keccak.Sum(nil)
t.keccak.Reset()
if t.trace {
fmt.Printf("leaf %v\n", hex.EncodeToString(hash))
}
}
//if len(pref) > 1 {
// fpLen := len(pref) + off
// t.keccak.Write([]byte{byte(fpLen), byte(fpLen >> 8)})
// t.keccak.Write(zero30)
// t.keccak.Write(hash)
//
// hash = t.keccak.Sum(nil)
// t.keccak.Reset()
//}
//if t.trace {
// fmt.Printf("hash %v off %d, pref %d\n", hex.EncodeToString(hash), off, len(pref))
//}
n.hash = hash
return hash
}
var Zero30 = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
type bitstring []uint8
func hexToBin(hex []byte) bitstring {
bin := make([]byte, 4*len(hex))
for i := range bin {
if hex[i/4]&(1<<(3-i%4)) != 0 {
bin[i] = 1
}
}
return bin
}
func newBitstring(key []byte) bitstring {
bits := make([]byte, 8*len(key))
for i := range bits {
if key[i/8]&(1<<(7-i%7)) != 0 {
bits[i] = 1
}
}
return bits
}
func bitstringWithPadding(key []byte, _ int) bitstring {
bs := newBitstring(key)
if last := key[len(key)-1]; last&0xf0 != 0 {
padding := int(0xf0 ^ last)
bs = bs[:len(bs)-8-padding]
}
// bs = bs[:len(bs)-padding-1]
return bs
}
func (b bitstring) String() string {
var s string
for _, bit := range b {
switch bit {
case 1:
s += "1"
case 0:
s += "0"
default:
panic(fmt.Errorf("invalid bit %d in bitstring", bit))
}
}
return s
}
func (b bitstring) splitPoint(other bitstring) (at int, bit byte, equal bool) {
for ; at < len(b) && at < len(other); at++ {
if b[at] != other[at] {
return at, other[at], false
}
}
switch {
case len(b) == len(other):
return 0, 0, true
case at == len(b): // b ends before other
return at, other[at], false
case at == len(other): // other ends before b
return at, b[at], false
default:
panic("oro")
}
}
// Converts b into slice of bytes.
// if len of b is not a multiple of 8, we add 1 <= padding <= 7 zeros to the latest byte
// and return amount of added zeros
func (b bitstring) reconstructHex() (re []byte, padding int) {
re = make([]byte, len(b)/8)
var offt, i int
for {
bt, ok := b.readByte(offt)
if !ok {
break
}
re[i] = bt
offt += 8
i++
}
if offt >= len(b) {
return re, 0
}
padding = offt + 8 - len(b)
pd := append(b[offt:], bytes.Repeat([]byte{0}, padding)...)
last, ok := pd.readByte(0)
if !ok {
panic(fmt.Errorf("reconstruct failed: padding %d padded size %d", padding, len(pd)))
}
pad := byte(padding | 0xf0)
return append(re, last, pad), padding
}
func (b bitstring) readByte(offsetBits int) (byte, bool) {
if len(b) <= offsetBits+7 {
return 0, false
}
return b[offsetBits+7] | b[offsetBits+6]<<1 | b[offsetBits+5]<<2 | b[offsetBits+4]<<3 | b[offsetBits+3]<<4 | b[offsetBits+2]<<5 | b[offsetBits+1]<<6 | b[offsetBits]<<7, true
}
// ExtractPlainKeys parses branchData and extract the plain keys for accounts and storage in the same order
// they appear witjin the branchData
func ExtractBinPlainKeys(branchData []byte) (accountPlainKeys [][]byte, storagePlainKeys [][]byte, err error) {
storagePlainKeys = make([][]byte, 0)
accountPlainKeys = make([][]byte, 0)
touchMap := binary.BigEndian.Uint16(branchData[0:])
afterMap := binary.BigEndian.Uint16(branchData[2:])
pos := 4
for bitset, noop := touchMap&afterMap, 0; bitset != 0; noop++ {
if pos >= len(branchData) {
break
}
bit := bitset & -bitset
fieldBits := PartFlags(branchData[pos])
pos++
if fieldBits&ACCOUNT_PLAIN_PART > 0 {
n, aux, err := decodeSizedBuffer(branchData[pos:])
if err != nil {
return nil, nil, fmt.Errorf("extractBinPlainKeys: [%x] account %w", branchData, err)
}
accountPlainKeys = append(accountPlainKeys, aux)
pos += n
}
if fieldBits&STORAGE_PLAIN_PART > 0 {
n, aux, err := decodeSizedBuffer(branchData[pos:])
if err != nil {
return nil, nil, fmt.Errorf("extractBinPlainKeys: storage %w", err)
}
storagePlainKeys = append(storagePlainKeys, aux)
pos += n
}
if fieldBits&HASH_PART > 0 {
n, _, err := decodeSizedBuffer(branchData[pos:])
if err != nil {
return nil, nil, fmt.Errorf("extractBinPlainKeys: hash %w", err)
}
pos += n
}
bitset ^= bit
}
return
}
func decodeSizedBuffer(buf []byte) (int, []byte, error) {
sz, n := binary.Uvarint(buf)
switch {
case n == 0:
return 0, nil, fmt.Errorf("buffer size too small")
case n < 0:
return 0, nil, fmt.Errorf("value overflow")
default:
}
size := int(sz)
if len(buf) < n+size {
return n, []byte{}, fmt.Errorf("encoded size larger than buffer size")
}
return n + size, buf[n : n+size], nil
}
func ReplaceBinPlainKeys(branchData []byte, accountPlainKeys [][]byte, storagePlainKeys [][]byte, newData []byte) ([]byte, error) {
var numBuf [binary.MaxVarintLen64]byte
touchMap := binary.BigEndian.Uint16(branchData[0:])
afterMap := binary.BigEndian.Uint16(branchData[2:])
pos := 4
if cap(newData) < 4 {
newData = make([]byte, 4)
}
copy(newData, branchData[:4])
var accountI, storageI int
for bitset, noop := touchMap&afterMap, 0; bitset != 0; noop++ {
if pos >= len(branchData) {
break
}
bit := bitset & -bitset
fieldBits := PartFlags(branchData[pos])
newData = append(newData, byte(fieldBits))
pos++
if fieldBits == 0 {
continue
}
if fieldBits&ACCOUNT_PLAIN_PART > 0 {
ptr, _, err := decodeSizedBuffer(branchData[pos:])
if err != nil {
return nil, fmt.Errorf("replaceBinPlainKeys: account %w", err)
}
n := binary.PutUvarint(numBuf[:], uint64(len(accountPlainKeys[accountI])))
newData = append(newData, numBuf[:n]...)
newData = append(newData, accountPlainKeys[accountI]...)
accountI++
pos += ptr
}
if fieldBits&STORAGE_PLAIN_PART > 0 {
ptr, _, err := decodeSizedBuffer(branchData[pos:])
if err != nil {
return nil, fmt.Errorf("replaceBinPlainKeys: storage %w", err)
}
n := binary.PutUvarint(numBuf[:], uint64(len(storagePlainKeys[storageI])))
newData = append(newData, numBuf[:n]...)
newData = append(newData, storagePlainKeys[storageI]...)
storageI++
pos += ptr
}
if fieldBits&HASH_PART > 0 {
n, _, err := decodeSizedBuffer(branchData[pos:])
if err != nil {
return nil, fmt.Errorf("extractBinPlainKeys: hash %w", err)
}
newData = append(newData, branchData[pos:pos+n]...)
pos += n
}
bitset ^= bit
}
return newData, nil
}
func branchToString2(branchData []byte) string {
if len(branchData) == 0 {
return "{ DELETED }"
}
touchMap := binary.BigEndian.Uint16(branchData[0:])
afterMap := binary.BigEndian.Uint16(branchData[2:])
pos := 4
var sb strings.Builder
fmt.Fprintf(&sb, "touchMap %016b, afterMap %016b\n", touchMap, afterMap)
for bitset, noop := touchMap&afterMap, 0; bitset != 0; noop++ {
if pos >= len(branchData) {
break
}
bit := bitset & -bitset
if pos >= len(branchData) {
break
}
fieldBits := PartFlags(branchData[pos])
pos++
sb.WriteString("{")
if fieldBits&ACCOUNT_PLAIN_PART > 0 {
n, pk, err := decodeSizedBuffer(branchData[pos:])
if err != nil {
panic(err)
}
fmt.Fprintf(&sb, "accountPlainKey=[%x]", pk)
pos += n
}
if fieldBits&STORAGE_PLAIN_PART > 0 {
n, pk, err := decodeSizedBuffer(branchData[pos:])
if err != nil {
panic(err)
}
fmt.Fprintf(&sb, "storagePlainKey=[%x]", pk)
pos += n
}
if fieldBits&HASH_PART > 0 {
n, hash, err := decodeSizedBuffer(branchData[pos:])
if err != nil {
panic(err)
}
fmt.Fprintf(&sb, " hash=[%x]", hash)
pos += n
}
sb.WriteString(" }\n")
bitset ^= bit
}
return sb.String()
}

612
commitment/bin_patricia_trie_test.go
View File

@ -1,612 +0,0 @@
package commitment
import (
"bytes"
"encoding/hex"
"fmt"
"math/rand"
"testing"
"github.com/holiman/uint256"
"github.com/stretchr/testify/require"
)
func Test_Update(t *testing.T) {
t.Skip()
tests := []struct {
key, value []byte
}{
{key: []byte{12}, value: []byte("notorious")},
{key: []byte{14}, value: []byte("2pac")},
{key: []byte{15}, value: []byte("eminem")},
{key: []byte{11}, value: []byte("big pun")},
{key: []byte{20}, value: []byte("method-man")},
{key: []byte{18}, value: []byte("fat-joe")},
{key: []byte{30}, value: []byte("jay-z")},
{key: []byte{5}, value: []byte("redman")},
}
bt := NewBinaryPatriciaTrie()
for _, test := range tests {
bt.Update(test.key, test.value)
}
require.NotNil(t, bt.root.Node)
stack := make([]*Node, 0)
var stackPtr int
stack = append(stack, bt.root.Node)
stackPtr++
visited := make(map[*Node]struct{})
antipaths := make(map[*Node]string)
antipaths[bt.root.Node] = bitstring(bt.root.CommonPrefix).String()
for len(stack) > 0 {
next := stack[stackPtr-1]
_, seen := visited[next]
if seen {
stack = stack[:stackPtr-1]
stackPtr--
continue
}
visited[next] = struct{}{}
if next.Value == nil {
//require.Truef(t, next.L != nil || next.R != nil, "if node is not a leaf, at least one child should present")
if next.P != nil {
require.True(t, next.R != nil && next.L != nil, "bot child should exist L: %p, R: %p", next.L, next.R)
}
}
if next.L != nil || next.R != nil {
require.Truef(t, next.Value == nil, "if node has childs, node value should be nil, got %v", next.Value)
}
if next.L != nil {
stack = append(stack, next.L)
stackPtr++
curp := antipaths[next]
antipaths[next.L] = curp + bitstring(next.LPrefix).String()
require.Truef(t, bytes.HasPrefix(next.LPrefix, []byte{0}), "left prefix always begins with 0, got %v", next.LPrefix)
}
if next.R != nil {
stack = append(stack, next.R)
stackPtr++
curp := antipaths[next]
antipaths[next.R] = curp + bitstring(next.RPrefix).String()
require.Truef(t, bytes.HasPrefix(next.RPrefix, []byte{1}), "right prefix always begins with 1, got %v", next.RPrefix)
}
if next.Value != nil {
// leaf, go back
stack = stack[:stackPtr-1]
stackPtr--
continue
}
}
for node, path := range antipaths {
if node.Value == nil {
continue
}
if newBitstring(node.Key).String() != path {
t.Fatalf("node key %v- %v, path %v", node.Key, newBitstring(node.Key).String(), path)
}
}
t.Logf("tree total nodes: %d", len(visited))
}
func Test_Get(t *testing.T) {
t.Skip()
bt := NewBinaryPatriciaTrie()
tests := []struct {
key, value []byte
}{
{key: []byte{12}, value: []byte("notorious")},
{key: []byte{14}, value: []byte("2pac")},
{key: []byte{15}, value: []byte("eminem")},
{key: []byte{11}, value: []byte("big pun")},
{key: []byte{20}, value: []byte("method-man")},
{key: []byte{18}, value: []byte("fat-joe")},
{key: []byte{30}, value: []byte("jay-z")},
{key: []byte{5}, value: []byte("redman")},
}
for _, test := range tests {
bt.Update(test.key, test.value)
}
require.NotNil(t, bt.root.Node)
for _, test := range tests {
buf, ok := bt.Get(test.key)
require.Truef(t, ok, "key %v not found", test.key)
require.EqualValues(t, test.value, buf)
}
}
func Test_BinaryPatriciaTrie_ProcessUpdates(t *testing.T) {
bt := NewBinaryPatriciaTrie()
builder := NewUpdateBuilder().
Balance("9a", 100000).
Balance("e8", 200000).
Balance("a2", 300000).
Balance("f0", 400000).
Balance("af", 500000).
Balance("33", 600000).
Nonce("aa", 184)
plainKeys, hashedKeys, updates := builder.Build()
bt.SetTrace(true)
_, err := bt.ProcessUpdates(plainKeys, hashedKeys, updates)
require.NoError(t, err)
require.NotNil(t, bt.root.Node)
stack := make([]*Node, 0)
var stackPtr int
stack = append(stack, bt.root.Node)
stackPtr++
visited := make(map[*Node]struct{})
// validity check
for len(stack) > 0 {
next := stack[stackPtr-1]
_, seen := visited[next]
if seen {
stack = stack[:stackPtr-1]
stackPtr--
continue
}
visited[next] = struct{}{}
if next.Value == nil {
require.Truef(t, next.L != nil || next.R != nil, "if node is not a leaf, at least one child should present")
if next.P != nil {
require.True(t, next.R != nil && next.L != nil, "bot child should exist L: %p, R: %p", next.L, next.R)
}
}
if next.L != nil || next.R != nil {
require.Truef(t, next.Value == nil, "if node has childs, node value should be nil, got %v", next.Value)
}
if next.L != nil {
stack = append(stack, next.L)
stackPtr++
require.Truef(t, bytes.HasPrefix(next.LPrefix, []byte{0}), "left prefix always begins with 0, got %v", next.LPrefix)
}
if next.R != nil {
stack = append(stack, next.R)
stackPtr++
require.Truef(t, bytes.HasPrefix(next.RPrefix, []byte{1}), "right prefix always begins with 1, got %v", next.RPrefix)
}
if next.Value != nil {
// leaf, go back
stack = stack[:stackPtr-1]
stackPtr--
continue
}
}
rootHash, _ := bt.RootHash()
require.Len(t, rootHash, 32)
fmt.Printf("%+v\n", hex.EncodeToString(rootHash))
t.Logf("tree total nodes: %d", len(visited))
}
func Test_BinaryPatriciaTrie_UniqueRepresentation(t *testing.T) {
trieSequential := NewBinaryPatriciaTrie()
builder := NewUpdateBuilder().
Balance("9a", 100000).
Balance("e8", 200000).
Balance("a2", 300000).
Balance("f0", 400000).
Balance("af", 500000).
Balance("33", 600000).
Nonce("aa", 184)
plainKeys, hashedKeys, updates := builder.Build()
emptyHash, _ := trieSequential.RootHash()
require.EqualValues(t, EmptyRootHash, emptyHash)
for i := 0; i < len(plainKeys); i++ {
trieSequential.ProcessUpdates(plainKeys[i:i+1], hashedKeys[i:i+1], updates[i:i+1])
sequentialHash, _ := trieSequential.RootHash()
require.Len(t, sequentialHash, 32)
}
trieBatch := NewBinaryPatriciaTrie()
trieBatch.SetTrace(true)
trieBatch.ProcessUpdates(plainKeys, hashedKeys, updates)
sequentialHash, _ := trieSequential.RootHash()
batchHash, _ := trieBatch.RootHash()
require.EqualValues(t, batchHash, sequentialHash)
}
func Test_BinaryPatriciaTrie_BranchEncoding(t *testing.T) {
builder := NewUpdateBuilder().
Balance("9a", 100000).
Balance("e8", 200000).
Balance("a2", 300000).
Balance("f0", 400000).
Balance("af", 500000).
Balance("33", 600000).
Nonce("aa", 184)
plainKeys, hashedKeys, updates := builder.Build()
trie := NewBinaryPatriciaTrie()
emptyHash, _ := trie.RootHash()
require.EqualValues(t, EmptyRootHash, emptyHash)
trie.SetTrace(true)
branchUpdates, err := trie.ProcessUpdates(plainKeys, hashedKeys, updates)
require.NoError(t, err)
require.NotEmpty(t, branchUpdates)
//sequentialHash, _ := trie.RootHash()
//expectedRoot, _ := hex.DecodeString("87809bbb5282c01ac13cac744db5fee083882e93f781d6af2ad028455d5bdaac")
//
//require.EqualValues(t, expectedRoot, sequentialHash)
for pref, update := range branchUpdates {
account, _, _ := ExtractBinPlainKeys(update)
t.Logf("pref %v: accounts:", pref)
for _, acc := range account {
t.Logf("\t%s\n", hex.EncodeToString(acc))
}
}
}
func Test_ReplaceBinPlainKeys(t *testing.T) {
v, err := hex.DecodeString("0000000310ea0300000000000001010000000000000001000100000000010001010000000000010101000000000101000100000000000101000000000000010001000000000100010000000000010000000000000000010000000000000100010000000000000100010000000001000000000000000100000100000000000000010000000000000101000000000100010000000000010001000000000001000000000000000101010000000000010101000000000000000101000000000000000000000000000000000000000001000000000000000001010100000000000100000000000000010001000000000000000000000000010101010000000001010100000000000000000100000000010000010000000001010101000000000100010000000000000101010000000001010000000000000000010100000000010100010000000000000100000000000000010000000000000000000000000001000101000000000101010100000000000001000000000001000001000000000101010000000000010001010000000001010101000000000101010000000000010001000000000000000001000000000001010000000000000001010000000001000100000000000100010000000000010101000000000000010100000000000101010100000000000000000000000001010000ea03010000000000010001000000000000010101000000000000000100000000010001010000000000010101000000000000000000000000010000010000000001010101000000000101010000000000010000000000000001000101000000000000010100000000000101010000000000010100000000000000010100000000010100000000000001010100000000000100010000000000010101010000000001010101000000000001000000000000010000000000000000000001000000000000000100000000000100000000000000000101000000000101000100000000000100000000000000010100000000000001000000000000000001000000000000010101000000000001000100000000000001010000000001010101000000000100000000000000010100010000000001000101000000000101010100000000010001000000000000010100000000000101010100000000000101000000000000000100000000000000000100000000010100010000000000010100000000000000010000000000000001000000000001010100000000000000000000000000010000010000000001010000000000000100000000000000010001010000000000010000000000000100000000000000000100000000000001000100000000000001010000000000010001010214b910f4453d5fa062f828caf3b0e2adff3824407c24e68080a000000000000000000000000000000000000000000000000000000000000000000214d2798468b343c4b104ecc2585e1c1b57b7c1807424e68080a00000000000000000000000000000000000000000000000000000000000000000")
require.NoError(t, err)
accountPlainKeys := make([][]byte, 0)
accountPlainKeys = append(accountPlainKeys, []byte("1a"))
accountPlainKeys = append(accountPlainKeys, []byte("b1"))
storageKeys := make([][]byte, 0)
buf := make([]byte, 0)
fin, err := ReplaceBinPlainKeys(v, accountPlainKeys, storageKeys, buf)
require.NoError(t, err)
require.NotEmpty(t, fin)
}
func Test_ReplaceBinPlainKeys2(t *testing.T) {
//rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
key := make([]byte, 52)
_, err := rand.Read(key)
require.NoError(t, err)
L := &Node{Key: key, Value: []byte("aa"), isStorage: true}
buf := encodeNodeUpdate(L, 1, 1, nil)
accountPlainKeys, storageKeys, err := ExtractBinPlainKeys(buf)
require.NoError(t, err)
require.Empty(t, accountPlainKeys)
require.Contains(t, storageKeys, key)
newStorageKeys := make([][]byte, 0)
newStorageKeys = append(newStorageKeys, []byte("1aa0"))
//fin, err := ReplacePlainKeys(v, accountPlainKeys, storageKeys, buf)
fin := make([]byte, 0)
fin, err = ReplaceBinPlainKeys(buf, accountPlainKeys, newStorageKeys, fin)
require.NoError(t, err)
require.NotEmpty(t, fin)
require.NotEqualValues(t, fin, buf)
accountPlainKeys, storageKeys, err = ExtractBinPlainKeys(fin)
require.NoError(t, err)
require.Empty(t, accountPlainKeys)
require.Contains(t, storageKeys, newStorageKeys[0])
}
func Test_EncodeUpdate_Storage(t *testing.T) {
L := &Node{Key: []byte("1aa0"), Value: []byte("aa"), isStorage: true}
buf := encodeNodeUpdate(L, 1, 1, nil)
accountPlainKeys, storagePlainKeys, err := ExtractBinPlainKeys(buf)
require.NoError(t, err)
require.Len(t, storagePlainKeys, 1)
require.Len(t, accountPlainKeys, 0)
require.Contains(t, storagePlainKeys, L.Key)
newAccountPlainKeys := make([][]byte, 0)
newAccountPlainKeys = append(newAccountPlainKeys, []byte("11a"))
newStorageKeys := make([][]byte, 0)
newStorageKeys = append(newStorageKeys, []byte("7770"))
fin, err := ReplaceBinPlainKeys(buf, newAccountPlainKeys, newStorageKeys, []byte{})
require.NoError(t, err)
require.NotEmpty(t, fin)
accountPlainKeys, storagePlainKeys, err = ExtractBinPlainKeys(fin)
require.NoError(t, err)
require.Len(t, storagePlainKeys, 1)
require.Len(t, accountPlainKeys, 0)
require.Contains(t, storagePlainKeys, newStorageKeys[0])
// ====================== replace account plain key
R := &Node{Key: []byte("1a"), Value: []byte("bb")}
buf = encodeNodeUpdate(R, 2, 2, nil)
accountPlainKeys, storagePlainKeys, err = ExtractBinPlainKeys(buf)
require.NoError(t, err)
require.Len(t, storagePlainKeys, 0)
require.Len(t, accountPlainKeys, 1)
require.Contains(t, accountPlainKeys, R.Key)
fin, err = ReplaceBinPlainKeys(buf, newAccountPlainKeys, newStorageKeys, []byte{})
require.NoError(t, err)
require.NotEmpty(t, fin)
accountPlainKeys, storagePlainKeys, err = ExtractBinPlainKeys(fin)
require.NoError(t, err)
require.Len(t, storagePlainKeys, 0)
require.Len(t, accountPlainKeys, 1)
require.Contains(t, accountPlainKeys, newAccountPlainKeys[0])
}
func Test_bitstring_encode_decode_padding(t *testing.T) {
t.Skip()
key, err := hex.DecodeString("db3164534fec08b5a86ae5dda0a997a63f2ee408")
require.NoError(t, err)
bs := newBitstring(key)
re, padding := bs.reconstructHex()
require.Zerof(t, padding, "padding should be zero")
require.EqualValues(t, key, re)
}
func Test_bitstring_encode_decode_empty(t *testing.T) {
re, pad := bitstring{}.reconstructHex()
require.EqualValues(t, bitstring{}, re)
require.EqualValues(t, 0, pad)
}
func Test_bitstring_encode_decode_one_padding(t *testing.T) {
bs := bitstring{1}
re, pad := bs.reconstructHex()
require.EqualValues(t, 7, pad)
require.EqualValues(t, []byte{1 << 7, byte(0xf0 | pad)}, re)
bs2 := bitstringWithPadding(re, pad)
require.EqualValues(t, bs, bs2)
}
func Test_bitstring_encode_decode_padding_notzero(t *testing.T) {
t.Skip("Failing")
key, err := hex.DecodeString("db3164534fec08b5a86ae5dda0a997a63f2ee408")
require.NoError(t, err)
bs := newBitstring(key)
offt := 3 // last byte is 08 => 1000, chop last three zeros
chop := bs[len(bs)-offt:]
bs = bs[:len(bs)-offt]
_ = chop
re, padding := bs.reconstructHex() // during reconstruction padding will be applied - add 3 chopped zero
require.EqualValues(t, offt, padding)
require.EqualValues(t, key, re)
}
func Test_BinaryPatriciaTrie_ProcessUpdatesDelete(t *testing.T) {
bt := NewBinaryPatriciaTrie()
builder := NewUpdateBuilder().
Balance("ffff", 200000).
Balance("feff", 300000).
Balance("ffdf", 400000).
Balance("fedf", 400000)
plainKeys, hashedKeys, updates := builder.Build()
bt.SetTrace(true)
_, err := bt.ProcessUpdates(plainKeys, hashedKeys, updates)
require.NotNil(t, bt.root.Node)
require.NoError(t, err)
t.Logf("trie stat: %s", bt.StatString())
builder = NewUpdateBuilder().
Delete("fedf").
Delete("ffff")
plainKeys, hashedKeys, updates = builder.Build()
require.NoError(t, err)
_, err = bt.ProcessUpdates(plainKeys, hashedKeys, updates)
require.NotNil(t, bt.root.Node)
require.NoError(t, err)
for i, key := range hashedKeys {
v, ok := bt.Get(key) // keys "af" and "e8" should be deleted
if len(v) > 0 {
t.Logf("key %x: %v", hashedKeys[i], new(Account).decode(v).String())
}
require.Emptyf(t, v, "value for key %x should be not empty", plainKeys[i])
require.False(t, ok)
}
stack := make([]*Node, 0)
var stackPtr int
stack = append(stack, bt.root.Node)
stackPtr++
visited := make(map[*Node]struct{})
// validity check
for len(stack) > 0 {
next := stack[stackPtr-1]
_, seen := visited[next]
if seen {
stack = stack[:stackPtr-1]
stackPtr--
continue
}
visited[next] = struct{}{}
if next.Value == nil {
require.Truef(t, next.L != nil || next.R != nil, "if node is not a leaf, at least one child should present")
if next.P != nil {
require.True(t, next.R != nil && next.L != nil, "bot child should exist L: %p, R: %p", next.L, next.R)
}
}
if next.L != nil || next.R != nil {
require.Truef(t, next.Value == nil, "if node has childs, node value should be nil, got %v", next.Value)
}
if next.L != nil {
stack = append(stack, next.L)
stackPtr++
require.Truef(t, bytes.HasPrefix(next.LPrefix, []byte{0}), "left prefix always begins with 0, got %v", next.LPrefix)
}
if next.R != nil {
stack = append(stack, next.R)
stackPtr++
require.Truef(t, bytes.HasPrefix(next.RPrefix, []byte{1}), "right prefix always begins with 1, got %v", next.RPrefix)
}
if next.Value != nil {
// leaf, go back
stack = stack[:stackPtr-1]
stackPtr--
continue
}
}
rootHash, _ := bt.RootHash()
require.Len(t, rootHash, 32)
fmt.Printf("%+v\n", hex.EncodeToString(rootHash))
t.Logf("tree total nodes: %d", len(visited))
}
func Test_BinaryPatriciaTrie_ProcessStorageUpdates(t *testing.T) {
bt := NewBinaryPatriciaTrie()
builder := NewUpdateBuilder().
Storage("e8", "02", "98").
Balance("e8", 1337)
plainKeys, hashedKeys, updates := builder.Build()
bt.SetTrace(true)
_, err := bt.ProcessUpdates(plainKeys, hashedKeys, updates)
require.NoError(t, err)
require.NotNil(t, bt.root.Node)
checkPlainKeys, checkHashedKeys, _ := NewUpdateBuilder().Delete("e8").Build()
av, exist := bt.Get(checkHashedKeys[0])
require.Truef(t, exist, "key %x should exist", checkPlainKeys[0])
acc := new(Account).decode(av)
require.Truef(t, acc.Balance.Eq(uint256.NewInt(1337)), "balance should be 1337, got %v", acc.Balance)
accountPlainKey, accountHashedKey, upd := NewUpdateBuilder().DeleteStorage("e8", "02").Build()
_, err = bt.ProcessUpdates(accountPlainKey, accountHashedKey, upd)
require.NoError(t, err)
for i, key := range accountHashedKey {
v, ok := bt.Get(key)
require.Emptyf(t, v, "value for key %x should be empty", accountPlainKey[i])
require.False(t, ok)
}
stack := make([]*Node, 0)
var stackPtr int
stack = append(stack, bt.root.Node)
stackPtr++
visited := make(map[*Node]struct{})
// validity check
for len(stack) > 0 {
next := stack[stackPtr-1]
_, seen := visited[next]
if seen {
stack = stack[:stackPtr-1]
stackPtr--
continue
}
visited[next] = struct{}{}
if next.Value == nil {
require.Truef(t, next.L != nil || next.R != nil, "if node is not a leaf, at least one child should present")
if next.P != nil {
require.True(t, next.R != nil && next.L != nil, "bot child should exist L: %p, R: %p", next.L, next.R)
}
}
if next.L != nil || next.R != nil {
require.Truef(t, next.Value == nil, "if node has childs, node value should be nil, got %v", next.Value)
}
if next.L != nil {
stack = append(stack, next.L)
stackPtr++
if len(next.LPrefix) == 0 {
require.NotNilf(t, next.L.Value, "if left prefix is empty, left child MUST be leaf and there MUST be another child down on path, got branch")
} else {
require.Truef(t, bytes.HasPrefix(next.LPrefix, []byte{0}), "left prefix always begins with 0, got %v", next.LPrefix)
}
}
if next.R != nil {
stack = append(stack, next.R)
stackPtr++
if len(next.RPrefix) == 0 {
require.NotNilf(t, next.R.Value, "if right prefix is nil, right child MUST be leaf, got branch")
} else {
require.Truef(t, bytes.HasPrefix(next.RPrefix, []byte{1}), "right prefix always begins with 1, got %v", next.RPrefix)
}
}
if next.Value != nil {
// leaf, go back
stack = stack[:stackPtr-1]
stackPtr--
continue
}
}
rootHash, _ := bt.RootHash()
require.Len(t, rootHash, 32)
fmt.Printf("%+v\n", hex.EncodeToString(rootHash))
t.Logf("tree total nodes: %d", len(visited))
}
func Test_encodeNode(t *testing.T) {
t.Skip()
builder := NewUpdateBuilder().
Balance("ff", 255).
Balance("fd", 253).
Balance("fe", 254)
apk, _, upd := builder.Build()
trie := NewBinaryPatriciaTrie()
trie.trace = true
for i := 0; i < len(upd); i++ {
updates, err := trie.ProcessUpdates(apk[i:i+1], apk[i:i+1], upd[i:i+1])
require.NoError(t, err)
require.NotEmpty(t, updates)
fmt.Printf("-----\n")
}
}

82
commitment/commitment.go
View File

@ -1,12 +1,6 @@
package commitment
import (
"bytes"
"encoding/binary"
"fmt"
"github.com/holiman/uint256"
"github.com/ledgerwatch/erigon-lib/common/length"
)
@ -37,8 +31,7 @@ type TrieVariant string
const (
// HexPatriciaHashed used as default commitment approach
VariantHexPatriciaTrie TrieVariant = "hex-patricia-hashed"
VariantReducedHexPatriciaTrie TrieVariant = "modified-hex-patricia-hashed"
VariantHexPatriciaTrie TrieVariant = "hex-patricia-hashed"
// Experimental mode with binary key representation
VariantBinPatriciaTrie TrieVariant = "bin-patricia-hashed"
)
@ -46,81 +39,10 @@ const (
func InitializeTrie(tv TrieVariant) Trie {
switch tv {
case VariantBinPatriciaTrie:
return NewBinaryPatriciaTrie()
case VariantReducedHexPatriciaTrie:
return NewBinHashed(length.Addr, nil, nil, nil)
return NewBinPatriciaHashed(length.Addr, nil, nil, nil)
case VariantHexPatriciaTrie:
fallthrough
default:
return NewHexPatriciaHashed(length.Addr, nil, nil, nil)
}
}
type Account struct {
CodeHash []byte // hash of the bytecode
Nonce uint64
Balance uint256.Int
}
func (a *Account) String() string {
return fmt.Sprintf("Account{Nonce: %d, Balance: %s, CodeHash: %x}", a.Nonce, a.Balance.String(), a.CodeHash)
}
func (a *Account) decode(buffer []byte) *Account {
var pos int
if buffer[pos] < 128 {
a.Nonce = uint64(buffer[pos])
pos++
} else {
var nonce uint64
sizeBytes := int(buffer[pos] - 128)
pos++
nonce, n := binary.Uvarint(buffer[pos : pos+sizeBytes])
a.Nonce = nonce
pos += n
}
if buffer[pos] < 128 {
b := uint256.NewInt(uint64(buffer[pos]))
a.Balance = *b
pos++
} else {
bc := int(buffer[pos] - 128)
pos++
a.Balance.SetBytes(buffer[pos : pos+bc])
pos += bc
}
codeSize := int(buffer[pos] - 128)
if codeSize > 0 {
pos++
a.CodeHash = make([]byte, codeSize)
copy(a.CodeHash, buffer[pos:pos+codeSize])
}
return a
}
func (a *Account) encode(bb []byte) []byte {
buffer := bytes.NewBuffer(bb)
// Encoding nonce
if a.Nonce < 128 && a.Nonce != 0 {
buffer.WriteByte(byte(a.Nonce))
} else {
aux := [binary.MaxVarintLen64]byte{}
n := binary.PutUvarint(aux[:], a.Nonce)
buffer.WriteByte(byte(128 + n))
buffer.Write(aux[:n])
}
// Encoding balance
if a.Balance.LtUint64(128) && !a.Balance.IsZero() {
buffer.WriteByte(byte(a.Balance.Uint64()))
} else {
buffer.WriteByte(byte(128 + a.Balance.ByteLen()))
buffer.Write(a.Balance.Bytes())
}
buffer.WriteByte(byte(128 + len(a.CodeHash)))
buffer.Write(a.CodeHash)
return buffer.Bytes()
}

74
commitment/commitment_test.go
View File

@ -1,74 +0,0 @@
package commitment
import (
"crypto/rand"
"testing"
"github.com/holiman/uint256"
"github.com/stretchr/testify/require"
)
func Test_AccountEncodeDecode(t *testing.T) {
balance := uint256.NewInt(1002020020)
acc := &Account{
Nonce: 1913453,
CodeHash: []byte{10, 20, 30, 10},
Balance: *balance,
}
rand.Read(acc.CodeHash[:])
aux := make([]byte, 0)
aux = acc.encode(aux)
require.NotEmpty(t, aux)
bcc := new(Account)
bcc.decode(aux)
c := new(Account) //.decode([]byte{128, 0, 1, 128})
ff := c.encode(nil)
require.NotEmpty(t, ff)
_ = c
require.EqualValues(t, acc.Nonce, bcc.Nonce)
require.True(t, acc.Balance.Eq(&bcc.Balance))
require.EqualValues(t, acc.CodeHash, bcc.CodeHash)
}
func Test_BinPatriciaTrie_UniqueRepresentation(t *testing.T) {
trie := NewBinaryPatriciaTrie()
trieBatch := NewBinaryPatriciaTrie()
plainKeys, hashedKeys, updates := NewUpdateBuilder().
Balance("01", 12).
Balance("f1", 120000).
Nonce("aa", 152512).
Balance("9a", 100000).
Balance("e8", 200000).
Balance("a2", 300000).
Balance("f0", 400000).
Balance("af", 500000).
Balance("33", 600000).
Nonce("aa", 184).
Build()
for i := 0; i < len(updates); i++ {
_, err := trie.ProcessUpdates(plainKeys[i:i+1], hashedKeys[i:i+1], updates[i:i+1])
require.NoError(t, err)
}
trieBatch.ProcessUpdates(plainKeys, hashedKeys, updates)
hash, _ := trie.RootHash()
require.Len(t, hash, 32)
batchHash, _ := trieBatch.RootHash()
require.EqualValues(t, hash, batchHash)
for i, hkey := range hashedKeys {
buf, ok := trie.Get(hkey)
require.Truef(t, ok, "key %x should be present, but not found", plainKeys[i])
buf2, ok := trieBatch.Get(hkey)
require.True(t, ok)
require.EqualValues(t, buf2, buf)
}
}

14
commitment/hex_patricia_hashed_test.go
View File

@ -542,25 +542,25 @@ func Test_HexPatriciaHashed_EmptyUpdateState(t *testing.T) {
require.EqualValues(t, hashBeforeEmptyUpdate, hashAfterEmptyUpdate)
}
func TestHexPatriciaHashed_ProcessUpdates_UniqueRepresentation(t *testing.T) {
func Test_HexPatriciaHashed_ProcessUpdates_UniqueRepresentation(t *testing.T) {
ms := NewMockState(t)
ms2 := NewMockState(t)
plainKeys, hashedKeys, updates := NewUpdateBuilder().
Balance("f4", 4).
//Storage("04", "01", "0401").
Storage("04", "01", "0401").
Balance("ba", 065606).
Balance("00", 4).
Balance("01", 5).
Balance("02", 6).
Balance("03", 7).
//Storage("03", "56", "050505").
// Storage("03", "56", "050505").
Balance("05", 9).
//Storage("03", "57", "060606").
// Storage("03", "57", "060606").
Balance("b9", 6).
//Nonce("ff", 169356).
//Storage("05", "02", "8989").
//Storage("f5", "04", "9898").
Nonce("ff", 169356).
Storage("05", "02", "8989").
Storage("f5", "04", "9898").
Build()
renderUpdates := func(branchNodeUpdates map[string][]byte) {