mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 03:30:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

option to disable runtime.ReadMemStats (#4200)

Browse Source
This commit is contained in:
Alex Sharov 2022-05-19 11:48:07 +07:00 committed by GitHub
parent 64508a7d4e
commit 726ef081ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 59 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
cmd/downloader/downloader/downloader.go
View File

@ -5,7 +5,6 @@ import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
"time"
@ -283,7 +282,6 @@ func MainLoop(ctx context.Context, d *Downloader, silent bool) {
}
}()
var m runtime.MemStats
logEvery := time.NewTicker(20 * time.Second)
defer logEvery.Stop()
@ -309,14 +307,12 @@ func MainLoop(ctx context.Context, d *Downloader, silent bool) {
continue
}
runtime.ReadMemStats(&m)
if stats.Completed {
log.Info("[Snapshots] Seeding",
"up", common2.ByteCount(stats.UploadRate)+"/s",
"peers", stats.PeersUnique,
"connections", stats.ConnectionsTotal,
"files", stats.FilesTotal,
"alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys))
"files", stats.FilesTotal)
continue
}
@ -326,8 +322,7 @@ func MainLoop(ctx context.Context, d *Downloader, silent bool) {
"upload", common2.ByteCount(stats.UploadRate)+"/s",
"peers", stats.PeersUnique,
"connections", stats.ConnectionsTotal,
"files", stats.FilesTotal,
"alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys))
"files", stats.FilesTotal)
if stats.PeersUnique == 0 {
ips := d.Torrent().BadPeerIPs()
if len(ips) > 0 {

5
cmd/evm/runner.go
View File

@ -30,6 +30,7 @@ import (
"time"
"github.com/holiman/uint256"
common2 "github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/kv/memdb"
"github.com/urfave/cli"
@ -94,11 +95,11 @@ func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) (output []by
stats.bytesAllocated = result.AllocedBytesPerOp()
} else {
var memStatsBefore, memStatsAfter goruntime.MemStats
goruntime.ReadMemStats(&memStatsBefore)
common2.ReadMemStats(&memStatsBefore)
startTime := time.Now()
output, gasLeft, err = execFunc()
stats.time = time.Since(startTime)
goruntime.ReadMemStats(&memStatsAfter)
common2.ReadMemStats(&memStatsAfter)
stats.allocs = int64(memStatsAfter.Mallocs - memStatsBefore.Mallocs)
stats.bytesAllocated = int64(memStatsAfter.TotalAlloc - memStatsBefore.TotalAlloc)
}

3
cmd/hack/flow/flow.go
View File

@ -14,6 +14,7 @@ import (
"strings"
"time"
"github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon/cmd/hack/tool"
"github.com/ledgerwatch/erigon/common/debug"
"github.com/ledgerwatch/erigon/core/vm"
@ -93,7 +94,7 @@ func worker(code []byte) {
defer debug.LogPanic()
for {
var m runtime.MemStats
runtime.ReadMemStats(&m)
common.ReadMemStats(&m)
// For info on each, see: https://golang.org/pkg/runtime/#MemStats
if !*quiet {
fmt.Printf("Alloc = %v MiB", bToMb(m.Alloc))

12
cmd/hack/hack.go
View File

@ -1009,7 +1009,7 @@ func testGetProof(chaindata string, address common.Address, rewind int, regen bo
}
storageKeys := []string{}
var m runtime.MemStats
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
db := mdbx.MustOpen(chaindata)
defer db.Close()
tx, err1 := db.BeginRo(context.Background())
@ -1048,7 +1048,7 @@ func testGetProof(chaindata string, address common.Address, rewind int, regen bo
}); err != nil {
return err
}
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
log.Info("Constructed account map", "size", len(accountMap),
"alloc", libcommon.ByteCount(m.Alloc), "sys", libcommon.ByteCount(m.Sys))
storageMap := make(map[string][]byte)
@ -1065,7 +1065,7 @@ func testGetProof(chaindata string, address common.Address, rewind int, regen bo
}); err != nil {
return err
}
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
log.Info("Constructed storage map", "size", len(storageMap),
"alloc", libcommon.ByteCount(m.Alloc), "sys", libcommon.ByteCount(m.Sys))
var unfurlList = make([]string, len(accountMap)+len(storageMap))
@ -1110,7 +1110,7 @@ func testGetProof(chaindata string, address common.Address, rewind int, regen bo
}
}
slices.Sort(unfurlList)
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
log.Info("Constructed account unfurl lists",
"alloc", libcommon.ByteCount(m.Alloc), "sys", libcommon.ByteCount(m.Sys))
@ -1129,13 +1129,13 @@ func testGetProof(chaindata string, address common.Address, rewind int, regen bo
if err != nil {
return err
}
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
log.Info("Loaded subtries",
"alloc", libcommon.ByteCount(m.Alloc), "sys", libcommon.ByteCount(m.Sys))
hash, err := rawdb.ReadCanonicalHash(tx, block)
tool.Check(err)
header := rawdb.ReadHeader(tx, hash, block)
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
log.Info("Constructed trie",
"alloc", libcommon.ByteCount(m.Alloc), "sys", libcommon.ByteCount(m.Sys))
fmt.Printf("Resulting root: %x, expected root: %x\n", root, header.Root)

2
cmd/state/commands/erigon2.go
View File

@ -377,7 +377,7 @@ func (s *stat) print(aStats aggregator.FilesStats, logger log.Logger) {
func (s *stat) delta(aStats aggregator.FilesStats, blockNum uint64) *stat {
currentTime := time.Now()
runtime.ReadMemStats(&s.mem)
libcommon.ReadMemStats(&s.mem)
interval := currentTime.Sub(s.prevTime).Seconds()
s.blockNum = blockNum

4
common/debug/memstats.go
View File

@ -3,11 +3,13 @@ package debug
import (
"fmt"
"runtime"
"github.com/ledgerwatch/erigon-lib/common"
)
func PrintMemStats(short bool) {
var m runtime.MemStats
runtime.ReadMemStats(&m)
common.ReadMemStats(&m)
// For info on each, see: https://golang.org/pkg/runtime/#MemStats
if short {
fmt.Printf("HeapInuse: %vMb\n", ByteToMb(m.HeapInuse))

2
eth/stagedsync/stage_bodies.go
View File

@ -254,7 +254,7 @@ func logProgressBodies(logPrefix string, committed uint64, prevDeliveredCount, d
speed := (deliveredCount - prevDeliveredCount) / float64(logInterval/time.Second)
wastedSpeed := (wastedCount - prevWastedCount) / float64(logInterval/time.Second)
var m runtime.MemStats
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
log.Info(fmt.Sprintf("[%s] Wrote block bodies", logPrefix),
"block_num", committed,
"delivery/sec", libcommon.ByteCount(uint64(speed)),

8
eth/stagedsync/stage_call_traces.go
View File

@ -136,7 +136,7 @@ func promoteCallTraces(logPrefix string, tx kv.RwTx, startBlock, endBlock uint64
default:
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
speed := float64(blockNum-prev) / float64(logInterval/time.Second)
prev = blockNum
@ -184,7 +184,7 @@ func promoteCallTraces(logPrefix string, tx kv.RwTx, startBlock, endBlock uint64
default:
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
log.Info(fmt.Sprintf("[%s] Pruning call trace intermediate table", logPrefix), "number", blockNum,
"alloc", libcommon.ByteCount(m.Alloc), "sys", libcommon.ByteCount(m.Sys))
}
@ -333,7 +333,7 @@ func DoUnwindCallTraces(logPrefix string, db kv.RwTx, from, to uint64, ctx conte
select {
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
speed := float64(blockNum-prev) / float64(logInterval/time.Second)
prev = blockNum
@ -432,7 +432,7 @@ func pruneCallTraces(tx kv.RwTx, logPrefix string, pruneTo uint64, ctx context.C
select {
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
log.Info(fmt.Sprintf("[%s] Progress", logPrefix), "number", blockNum, "alloc", libcommon.ByteCount(m.Alloc), "sys", libcommon.ByteCount(m.Sys))
case <-ctx.Done():
return libcommon.ErrStopped

4
eth/stagedsync/stage_execute.go
View File

@ -366,7 +366,7 @@ func logProgress(logPrefix string, prevBlock uint64, prevTime time.Time, current
speedMgas := float64(gas) / 1_000_000 / (float64(interval) / float64(time.Second))
var m runtime.MemStats
runtime.ReadMemStats(&m)
common.ReadMemStats(&m)
var logpairs = []interface{}{
"number", currentBlock,
"blk/s", fmt.Sprintf("%.1f", speed),
@ -439,12 +439,10 @@ func unwindExecutionStage(u *UnwindState, s *StageState, tx kv.RwTx, quit <-chan
changes := etl.NewCollector(logPrefix, cfg.tmpdir, etl.NewOldestEntryBuffer(etl.BufferOptimalSize))
defer changes.Close()
t := time.Now()
errRewind := changeset.RewindData(tx, s.BlockNumber, u.UnwindPoint, changes, quit)
if errRewind != nil {
return fmt.Errorf("getting rewind data: %w", errRewind)
}
fmt.Printf("rewind: %s\n", time.Since(t))
if err := changes.Load(tx, stateBucket, func(k, v []byte, table etl.CurrentTableReader, next etl.LoadNextFunc) error {
if len(k) == 20 {

2
eth/stagedsync/stage_hashstate.go
View File

@ -226,7 +226,7 @@ func readPlainStateOnce(
select {
default:
case <-logEvery.C:
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
log.Info(fmt.Sprintf("[%s] ETL [1/2] Extracting", logPrefix), "current key", fmt.Sprintf("%x...", k[:6]), "alloc", libcommon.ByteCount(m.Alloc), "sys", libcommon.ByteCount(m.Sys))
}
}

6
eth/stagedsync/stage_headers.go
View File

@ -1011,7 +1011,7 @@ func HeadersUnwind(u *UnwindState, s *StageState, tx kv.RwTx, cfg HeadersCfg, te
func logProgressHeaders(logPrefix string, prev, now uint64) uint64 {
speed := float64(now-prev) / float64(logInterval/time.Second)
var m runtime.MemStats
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
log.Info(fmt.Sprintf("[%s] Wrote block headers", logPrefix),
"number", now,
"blk/second", speed,
@ -1131,7 +1131,7 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R
}
var m runtime.MemStats
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
log.Info("[Snapshots] Stat", "blocks", cfg.snapshots.BlocksAvailable(), "alloc", libcommon.ByteCount(m.Alloc), "sys", libcommon.ByteCount(m.Sys))
// Create .idx files
@ -1273,7 +1273,7 @@ Loop:
log.Info(fmt.Sprintf("[Snapshots] Waiting for torrents metadata: %d/%d", stats.MetadataReady, stats.FilesTotal))
continue
}
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
log.Info("[Snapshots] download",
"progress", fmt.Sprintf("%.2f%% %s/%s", stats.Progress, libcommon.ByteCount(stats.BytesCompleted), libcommon.ByteCount(stats.BytesTotal)),
"download", libcommon.ByteCount(stats.DownloadRate)+"/s",

4
eth/stagedsync/stage_indexes.go
View File

@ -154,7 +154,7 @@ func promoteHistory(logPrefix string, tx kv.RwTx, changesetBucket string, start,
default:
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
log.Info(fmt.Sprintf("[%s] Progress", logPrefix), "number", blockN, "alloc", libcommon.ByteCount(m.Alloc), "sys", libcommon.ByteCount(m.Sys))
case <-checkFlushEvery.C:
if needFlush64(updates, cfg.bufLimit) {
@ -291,7 +291,7 @@ func unwindHistory(logPrefix string, db kv.RwTx, csBucket string, to uint64, cfg
select {
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
log.Info(fmt.Sprintf("[%s] Progress", logPrefix), "number", blockN, "alloc", libcommon.ByteCount(m.Alloc), "sys", libcommon.ByteCount(m.Sys))
case <-quitCh:
return libcommon.ErrStopped

2
eth/stagedsync/stage_log_index.go
View File

@ -126,7 +126,7 @@ func promoteLogIndex(logPrefix string, tx kv.RwTx, start uint64, cfg LogIndexCfg
default:
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
log.Info(fmt.Sprintf("[%s] Progress", logPrefix), "number", blockNum, "alloc", libcommon.ByteCount(m.Alloc), "sys", libcommon.ByteCount(m.Sys))
case <-checkFlushEvery.C:
if needFlush(topics, cfg.bufLimit) {

2
eth/stagedsync/stage_tevm.go
View File

@ -256,7 +256,7 @@ func logTEVMProgress(logPrefix string, prevContract uint64, prevTime time.Time,
interval := currentTime.Sub(prevTime)
speed := float64(currentContract-prevContract) / float64(interval/time.Second)
var m runtime.MemStats
runtime.ReadMemStats(&m)
libcommon.ReadMemStats(&m)
var logpairs = []interface{}{
"number", currentContract,
"contracts/s", speed,

4
go.mod
View File

@ -36,11 +36,12 @@ require (
github.com/json-iterator/go v1.1.12
github.com/julienschmidt/httprouter v1.3.0
github.com/kevinburke/go-bindata v3.21.0+incompatible
github.com/ledgerwatch/erigon-lib v0.0.0-20220517074816-b1dc1bfbbf82
github.com/ledgerwatch/erigon-lib v0.0.0-20220519043706-488fb254d8d9
github.com/ledgerwatch/log/v3 v3.4.1
github.com/ledgerwatch/secp256k1 v1.0.0
github.com/pelletier/go-toml v1.9.5
github.com/pelletier/go-toml/v2 v2.0.0
github.com/pion/stun v0.3.5
github.com/quasilyte/go-ruleguard/dsl v0.3.19
github.com/rs/cors v1.8.2
github.com/spf13/cobra v1.4.0
@ -131,7 +132,6 @@ require (
github.com/pion/sctp v1.8.2 // indirect
github.com/pion/sdp/v3 v3.0.4 // indirect
github.com/pion/srtp/v2 v2.0.5 // indirect
github.com/pion/stun v0.3.5 // indirect
github.com/pion/transport v0.13.0 // indirect
github.com/pion/turn/v2 v2.0.6 // indirect
github.com/pion/udp v0.1.1 // indirect

4
go.sum
View File

@ -456,8 +456,8 @@ github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3P
github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/ledgerwatch/erigon-lib v0.0.0-20220517074816-b1dc1bfbbf82 h1:Zb9QP/RrwH3vfDMaudeaOy14jZQTi1qGb/L5vTdaf1M=
github.com/ledgerwatch/erigon-lib v0.0.0-20220517074816-b1dc1bfbbf82/go.mod h1:F5LaFSAgKHxkKjf5bHbtAhU424n+WNRREBssXYNC6Xo=
github.com/ledgerwatch/erigon-lib v0.0.0-20220519043706-488fb254d8d9 h1:WcfTgkeZFxHCFLqE7TCEdiNfSQYOaeyoRs6YTBXxnqw=
github.com/ledgerwatch/erigon-lib v0.0.0-20220519043706-488fb254d8d9/go.mod h1:F5LaFSAgKHxkKjf5bHbtAhU424n+WNRREBssXYNC6Xo=
github.com/ledgerwatch/log/v3 v3.4.1 h1:/xGwlVulXnsO9Uq+tzaExc8OWmXXHU0dnLalpbnY5Bc=
github.com/ledgerwatch/log/v3 v3.4.1/go.mod h1:VXcz6Ssn6XEeU92dCMc39/g1F0OYAjw1Mt+dGP5DjXY=
github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=

3
internal/debug/api.go
View File

@ -34,6 +34,7 @@ import (
"sync"
"time"
"github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/log/v3"
)
@ -74,7 +75,7 @@ func (*HandlerT) BacktraceAt(location string) error {
// MemStats returns detailed runtime memory statistics.
func (*HandlerT) MemStats() *runtime.MemStats {
s := new(runtime.MemStats)
runtime.ReadMemStats(s)
common.ReadMemStats(s)
return s
}

4
metrics/runtime_test.go
View File

@ -4,6 +4,8 @@ import (
"runtime"
"testing"
"time"
"github.com/ledgerwatch/erigon-lib/common"
)
func TestRuntimeMemStatsBlocking(t *testing.T) {
@ -14,7 +16,7 @@ func TestRuntimeMemStatsBlocking(t *testing.T) {
go testRuntimeMemStatsBlocking(ch)
var memStats runtime.MemStats
t0 := time.Now()
runtime.ReadMemStats(&memStats)
common.ReadMemStats(&memStats)
t1 := time.Now()
t.Log("i++ during runtime.ReadMemStats:", <-ch)
go testRuntimeMemStatsBlocking(ch)

2
migrations/txs_begin_end.go
View File

@ -60,7 +60,7 @@ var txsBeginEnd = Migration{
select {
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
common2.ReadMemStats(&m)
log.Info("[database version migration] Adding system-txs",
"progress", fmt.Sprintf("%.2f%%", 100-100*float64(blockNum)/float64(latestBlock)), "block_num", blockNum,
"alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys))

24
turbo/snapshotsync/block_snapshots.go
View File

@ -612,7 +612,9 @@ func BuildIndices(ctx context.Context, s *RoSnapshots, chainID uint256.Int, tmpD
return
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
if lvl >= log.LvlInfo {
common2.ReadMemStats(&m)
}
log.Log(lvl, "[snapshots] HeadersIdx", "blockNum", blockTo,
"alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys))
default:
@ -659,7 +661,9 @@ func BuildIndices(ctx context.Context, s *RoSnapshots, chainID uint256.Int, tmpD
return
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
if lvl >= log.LvlInfo {
common2.ReadMemStats(&m)
}
log.Log(lvl, "[snapshots] BodiesIdx", "blockNum", blockTo,
"alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys))
default:
@ -713,7 +717,9 @@ func BuildIndices(ctx context.Context, s *RoSnapshots, chainID uint256.Int, tmpD
return
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
if lvl >= log.LvlInfo {
common2.ReadMemStats(&m)
}
log.Log(lvl, "[Snapshots Indexing] TransactionsIdx", "blockNum", blockTo,
"alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys))
default:
@ -1129,7 +1135,9 @@ func DumpTxs(ctx context.Context, db kv.RoDB, segmentFile, tmpDir string, blockF
return false, ctx.Err()
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
if lvl >= log.LvlInfo {
common2.ReadMemStats(&m)
}
log.Log(lvl, "[snapshots] Dumping txs", "block num", blockNum,
"alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys),
)
@ -1197,7 +1205,9 @@ func DumpHeaders(ctx context.Context, db kv.RoDB, segmentFilePath, tmpDir string
return false, ctx.Err()
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
if lvl >= log.LvlInfo {
common2.ReadMemStats(&m)
}
log.Log(lvl, "[snapshots] Dumping headers", "block num", blockNum,
"alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys),
)
@ -1252,7 +1262,9 @@ func DumpBodies(ctx context.Context, db kv.RoDB, segmentFilePath, tmpDir string,
return false, ctx.Err()
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
if lvl >= log.LvlInfo {
common2.ReadMemStats(&m)
}
log.Log(lvl, "[snapshots] Wrote into file", "block num", blockNum,
"alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys),
)