add flag --db.size.limit (#7325)

cmd/utils/flags.go

@@ -714,6 +714,10 @@ var (
 		Usage: "set mdbx pagesize on db creation: must be power of 2 and '256b <= pagesize <= 64kb'. default: equal to OperationSystem's pageSize",
 		Value: datasize.ByteSize(kv.DefaultPageSize()).String(),
 	}
+	DbSizeLimitFlag = cli.StringFlag{
+		Name:  "db.size.limit",
+		Value: (8 * datasize.TB).String(),
+	}
 
 	HealthCheckFlag = cli.BoolFlag{
 		Name:  "healthcheck",
@@ -1148,10 +1152,17 @@ func setDataDir(ctx *cli.Context, cfg *nodecfg.Config) {
 	if err := cfg.MdbxPageSize.UnmarshalText([]byte(ctx.String(DbPageSizeFlag.Name))); err != nil {
 		panic(err)
 	}
+	if err := cfg.MdbxDBSizeLimit.UnmarshalText([]byte(ctx.String(DbSizeLimitFlag.Name))); err != nil {
+		panic(err)
+	}
 	sz := cfg.MdbxPageSize.Bytes()
 	if !isPowerOfTwo(sz) || sz < 256 || sz > 64*1024 {
 		panic(fmt.Errorf("invalid --db.pagesize: %s=%d, see: %s", ctx.String(DbPageSizeFlag.Name), sz, DbPageSizeFlag.Usage))
 	}
+	szLimit := cfg.MdbxDBSizeLimit.Bytes()
+	if !isPowerOfTwo(szLimit) || szLimit < 256 {
+		panic(fmt.Errorf("invalid --db.size.limit: %s=%d, see: %s", ctx.String(DbSizeLimitFlag.Name), sz, DbSizeLimitFlag.Usage))
+	}
 }
 
 func isPowerOfTwo(n uint64) bool {

core/genesis_write.go

@@ -504,7 +504,7 @@ func GenesisToBlock(g *types.Genesis, tmpDir string) (*types.Block, *state.Intra
 		genesisDBLock.Lock()
 		defer genesisDBLock.Unlock()
 		if genesisTmpDB == nil {
-			genesisTmpDB = mdbx.NewMDBX(log.New()).InMem(tmpDir).MapSize(2 * datasize.GB).PageSize(2 * 4096).MustOpen()
+			genesisTmpDB = mdbx.NewMDBX(log.New()).InMem(tmpDir).MapSize(2 * datasize.GB).MustOpen()
 		}
 		var tx kv.RwTx
 		if tx, err = genesisTmpDB.BeginRw(context.Background()); err != nil {

go.mod

@@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon
 go 1.19
 
 require (
-	github.com/ledgerwatch/erigon-lib v0.0.0-20230417053659-99b6a25a1698
+	github.com/ledgerwatch/erigon-lib v0.0.0-20230417120239-f419601a7490
 	github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336
 	github.com/ledgerwatch/log/v3 v3.7.0
 	github.com/ledgerwatch/secp256k1 v1.0.0

go.sum

@@ -436,8 +436,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
 github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
-github.com/ledgerwatch/erigon-lib v0.0.0-20230417053659-99b6a25a1698 h1:HDW/k+A6p7mjmmD9npVO43HEDrFSXlhhIdoj4b5p/bo=
+github.com/ledgerwatch/erigon-lib v0.0.0-20230417120239-f419601a7490 h1:9MK5bucfEbWIrYFmnY3rV7QrC3Hyd4iVD6afkovKQsQ=
-github.com/ledgerwatch/erigon-lib v0.0.0-20230417053659-99b6a25a1698/go.mod h1:D05f9OXc/2cnYxCyBexlu5HeIeQW9GKXynyWYzJ1F5I=
+github.com/ledgerwatch/erigon-lib v0.0.0-20230417120239-f419601a7490/go.mod h1:D05f9OXc/2cnYxCyBexlu5HeIeQW9GKXynyWYzJ1F5I=
 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336 h1:Yxmt4Wyd0RCLr7UJJAl0ApCP/f5qkWfvHfgPbnI8ghM=
 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230404044759-5dec854ce336/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
 github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og=

node/node.go

@@ -316,7 +316,7 @@ func OpenDatabase(config *nodecfg.Config, label kv.Label) (kv.RwDB, error) {
 			opts = opts.Exclusive()
 		}
 		if label == kv.ChainDB {
-			opts = opts.PageSize(config.MdbxPageSize.Bytes()).MapSize(8 * datasize.TB)
+			opts = opts.PageSize(config.MdbxPageSize.Bytes()).MapSize(config.MdbxDBSizeLimit)
 		} else {
 			opts = opts.GrowthStep(16 * datasize.MB)
 		}

node/nodecfg/config.go

@@ -163,7 +163,8 @@ type Config struct {
 	TLSKeyFile          string
 	TLSCACert           string
 
-	MdbxPageSize datasize.ByteSize
+	MdbxPageSize    datasize.ByteSize
+	MdbxDBSizeLimit datasize.ByteSize
 
 	// HealthCheck enables standard grpc health check
 	HealthCheck bool

p2p/enode/nodedb.go

@@ -102,7 +102,7 @@ func bucketsConfig(_ kv.TableCfg) kv.TableCfg {
 func newMemoryDB(logger log.Logger, tmpDir string) (*DB, error) {
 	db := &DB{quit: make(chan struct{})}
 	var err error
-	db.kv, err = mdbx.NewMDBX(logger).InMem(tmpDir).Label(kv.SentryDB).WithTableCfg(bucketsConfig).Open()
+	db.kv, err = mdbx.NewMDBX(logger).InMem(tmpDir).Label(kv.SentryDB).WithTableCfg(bucketsConfig).MapSize(1 * datasize.GB).Open()
 	if err != nil {
 		return nil, err
 	}

turbo/cli/default_flags.go

@@ -86,6 +86,7 @@ var DefaultFlags = []cli.Flag{
 	&utils.SnapKeepBlocksFlag,
 	&utils.SnapStopFlag,
 	&utils.DbPageSizeFlag,
+	&utils.DbSizeLimitFlag,
 	&utils.TorrentPortFlag,
 	&utils.TorrentMaxPeersFlag,
 	&utils.TorrentConnsPerFileFlag,