diff --git a/common/debug/experiments.go b/common/debug/experiments.go index cf091ae65..524917d34 100644 --- a/common/debug/experiments.go +++ b/common/debug/experiments.go @@ -7,6 +7,21 @@ import ( "time" ) +var ( + writeMap bool + writeMapOnce sync.Once +) + +func WriteMap() bool { + writeMapOnce.Do(func() { + v, _ := os.LookupEnv("WRITE_MAP") + if v == "true" { + writeMap = true + } + }) + return writeMap +} + var ( bigRoTx uint getBigRoTx sync.Once diff --git a/core/state/rw22.go b/core/state/rw22.go index f2dad7ad1..f28c79073 100644 --- a/core/state/rw22.go +++ b/core/state/rw22.go @@ -85,17 +85,17 @@ type State22 struct { triggerLock sync.RWMutex queue TxTaskQueue queueLock sync.Mutex - changes map[string]*btree.BTreeG[pair] + changes map[string]*btree.BTreeG[statePair] sizeEstimate uint64 txsDone uint64 finished bool } -type pair struct { +type statePair struct { key, val []byte } -func stateItemLess(i, j pair) bool { +func stateItemLess(i, j statePair) bool { return bytes.Compare(i.key, j.key) < 0 } @@ -103,7 +103,7 @@ func NewState22() *State22 { rs := &State22{ triggers: map[uint64]*TxTask{}, senderTxNums: map[common.Address]uint64{}, - changes: map[string]*btree.BTreeG[pair]{}, + changes: map[string]*btree.BTreeG[statePair]{}, } rs.receiveWork = sync.NewCond(&rs.queueLock) return rs @@ -112,16 +112,14 @@ func NewState22() *State22 { func (rs *State22) put(table string, key, val []byte) { t, ok := rs.changes[table] if !ok { - t = btree.NewG[pair](32, stateItemLess) + t = btree.NewG[statePair](32, stateItemLess) rs.changes[table] = t } - item := pair{key: key, val: val} + item := statePair{key: key, val: val} t.ReplaceOrInsert(item) - rs.sizeEstimate += SizeOfItem + uint64(len(key)) + uint64(len(val)) + rs.sizeEstimate += PairSize + uint64(len(key)) + uint64(len(val)) } -const SizeOfItem = uint64(48) //unsafe.Sizeof(pair{}) - func (rs *State22) Get(table string, key []byte) []byte { rs.lock.RLock() defer rs.lock.RUnlock() @@ -133,7 +131,7 @@ func (rs *State22) get(table string, key []byte) []byte { if !ok { return nil } - if i, ok := t.Get(pair{key: key}); ok { + if i, ok := t.Get(statePair{key: key}); ok { return i.val } return nil @@ -144,7 +142,7 @@ func (rs *State22) Flush(rwTx kv.RwTx) error { defer rs.lock.Unlock() for table, t := range rs.changes { var err error - t.Ascend(func(item pair) bool { + t.Ascend(func(item statePair) bool { if len(item.val) == 0 { if err = rwTx.Delete(table, item.key); err != nil { return false @@ -310,7 +308,7 @@ func (rs *State22) Apply(roTx kv.Tx, txTask *TxTask, agg *libstate.Aggregator22) } psChanges := rs.changes[kv.PlainState] if psChanges != nil { - psChanges.AscendGreaterOrEqual(pair{key: addr1}, func(item pair) bool { + psChanges.AscendGreaterOrEqual(statePair{key: addr1}, func(item statePair) bool { if !bytes.HasPrefix(item.key, addr1) { return false } @@ -528,7 +526,7 @@ func (rs *State22) ReadsValid(readLists map[string]*KvList) bool { //fmt.Printf("ValidReads\n") for table, list := range readLists { //fmt.Printf("Table %s\n", table) - var t *btree.BTreeG[pair] + var t *btree.BTreeG[statePair] var ok bool if table == CodeSizeTable { t, ok = rs.changes[kv.Code] @@ -540,7 +538,7 @@ func (rs *State22) ReadsValid(readLists map[string]*KvList) bool { } for i, key := range list.Keys { val := list.Vals[i] - if item, ok := t.Get(pair{key: key}); ok { + if item, ok := t.Get(statePair{key: key}); ok { //fmt.Printf("key [%x] => [%x] vs [%x]\n", key, val, rereadVal) if table == CodeSizeTable { if binary.BigEndian.Uint64(val) != uint64(len(item.val)) { @@ -728,15 +726,16 @@ func (r *StateReader22) SetTrace(trace bool) { } func (r *StateReader22) ReadAccountData(address common.Address) (*accounts.Account, error) { - enc := r.rs.Get(kv.PlainState, address.Bytes()) + addr := address.Bytes() + enc := r.rs.Get(kv.PlainState, addr) if enc == nil { var err error - enc, err = r.tx.GetOne(kv.PlainState, address.Bytes()) + enc, err = r.tx.GetOne(kv.PlainState, addr) if err != nil { return nil, err } } - r.readLists[kv.PlainState].Keys = append(r.readLists[kv.PlainState].Keys, address.Bytes()) + r.readLists[kv.PlainState].Keys = append(r.readLists[kv.PlainState].Keys, addr) r.readLists[kv.PlainState].Vals = append(r.readLists[kv.PlainState].Vals, common2.Copy(enc)) if len(enc) == 0 { return nil, nil diff --git a/core/state/state_recon_writer.go b/core/state/state_recon_writer.go index b3c0a731d..1f59baa9b 100644 --- a/core/state/state_recon_writer.go +++ b/core/state/state_recon_writer.go @@ -80,7 +80,7 @@ func (rs *ReconState) Put(table string, key1, key2, val []byte, txNum uint64) { rs.sizeEstimate += PairSize + uint64(len(key1)) + uint64(len(key2)) + uint64(len(val)) } -const PairSize = uint64(48) // uint64(unsafe.Sizeof(item)) +const PairSize = uint64(48) // uint64(unsafe.Sizeof(reconPair{})) func (rs *ReconState) Get(table string, key1, key2 []byte, txNum uint64) []byte { rs.lock.RLock() diff --git a/core/types/receipt.go b/core/types/receipt.go index ae64be28b..4c13dae0e 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -22,7 +22,6 @@ import ( "fmt" "io" "math/big" - "unsafe" "github.com/ledgerwatch/erigon/common" "github.com/ledgerwatch/erigon/common/hexutil" @@ -286,17 +285,6 @@ func (r *Receipt) statusEncoding() []byte { return r.PostState } -// Size returns the approximate memory used by all internal contents. It is used -// to approximate and limit the memory consumption of various caches. -func (r *Receipt) Size() common.StorageSize { - size := common.StorageSize(unsafe.Sizeof(*r)) + common.StorageSize(len(r.PostState)) - size += common.StorageSize(len(r.Logs)) * common.StorageSize(unsafe.Sizeof(Log{})) - for _, log := range r.Logs { - size += common.StorageSize(len(log.Topics)*common.HashLength + len(log.Data)) - } - return size -} - // Copy creates a deep copy of the Receipt. func (r *Receipt) Copy() *Receipt { postState := make([]byte, len(r.PostState)) diff --git a/node/node.go b/node/node.go index 076b41110..71bcbedd4 100644 --- a/node/node.go +++ b/node/node.go @@ -28,6 +28,7 @@ import ( "sync" "github.com/c2h5oh/datasize" + "github.com/ledgerwatch/erigon/common/debug" "github.com/ledgerwatch/erigon/node/nodecfg" "github.com/ledgerwatch/erigon/params" "golang.org/x/sync/semaphore" @@ -327,7 +328,8 @@ func OpenDatabase(config *nodecfg.Config, logger log.Logger, label kv.Label) (kv roTxsLimiter := semaphore.NewWeighted(roTxLimit) // 1 less than max to allow unlocking to happen opts := mdbx.NewMDBX(logger). WriteMergeThreshold(4 * 8192). - Path(dbPath).Label(label).DBVerbosity(config.DatabaseVerbosity).RoTxsLimiter(roTxsLimiter) + Path(dbPath).Label(label). + DBVerbosity(config.DatabaseVerbosity).RoTxsLimiter(roTxsLimiter) if exclusive { opts = opts.Exclusive() } @@ -336,6 +338,9 @@ func OpenDatabase(config *nodecfg.Config, logger log.Logger, label kv.Label) (kv } else { opts = opts.GrowthStep(16 * datasize.MB) } + if debug.WriteMap() { + opts = opts.WriteMap() + } return opts.Open() } var err error