From 7f4876880cf1623275eaa9af7e902f803d5177f7 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Wed, 14 Dec 2022 09:50:31 +0700 Subject: [PATCH] e3: more aggressive prune (#780) --- state/aggregator22.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/state/aggregator22.go b/state/aggregator22.go index 7da0f0573..0692162f7 100644 --- a/state/aggregator22.go +++ b/state/aggregator22.go @@ -429,6 +429,25 @@ func (sf Agg22StaticFiles) Close() { sf.tracesTo.Close() } +func (a *Aggregator22) BuildFiles(ctx context.Context, db kv.RoDB) (err error) { + if (a.txNum.Load() + 1) <= a.maxTxNum.Load()+a.aggregationStep+a.keepInDB { // Leave one step worth in the DB + return nil + } + + // trying to create as much small-step-files as possible: + // - to reduce amount of small merges + // - to remove old data from db as early as possible + // - during files build, may happen commit of new data. on each loop step getting latest id in db + step := a.EndTxNumMinimax() / a.aggregationStep + for ; step < lastIdInDB(db, a.accounts.indexKeysTable)/a.aggregationStep; step++ { + if err := a.buildFilesInBackground(ctx, step, db); err != nil { + log.Warn("buildFilesInBackground", "err", err) + break + } + } + return nil +} + func (a *Aggregator22) buildFilesInBackground(ctx context.Context, step uint64, db kv.RoDB) (err error) { closeAll := true log.Info("[snapshots] history build", "step", fmt.Sprintf("%d-%d", step, step+1)) @@ -655,7 +674,7 @@ func (a *Aggregator22) PruneWithTiemout(ctx context.Context, timeout time.Durati } func (a *Aggregator22) Prune(ctx context.Context, limit uint64) error { - //a.Warmup(0, cmp.Max(a.aggregationStep, limit)) // warmup is asyn and moving faster than data deletion + a.Warmup(0, cmp.Max(a.aggregationStep, limit)) // warmup is asyn and moving faster than data deletion return a.prune(ctx, 0, a.maxTxNum.Load(), limit) }