From 11637a5c7d3b4404bff50c1cc91efd2da4aa3024 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 2 Jan 2023 10:10:48 +0700 Subject: [PATCH] e3: force merge snapshots before reconst (#6484) --- eth/stagedsync/exec3.go | 15 +++++++++++++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index f05eba6a9..43c245cc1 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -36,6 +36,7 @@ import ( "github.com/ledgerwatch/erigon/core/state" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/eth/ethconfig" + "github.com/ledgerwatch/erigon/eth/ethconfig/estimate" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/turbo/services" @@ -1210,6 +1211,13 @@ func ReconstituteState(ctx context.Context, s *StageState, dirs datadir.Dirs, wo startTime := time.Now() defer agg.EnableMadvNormal().DisableReadAhead() blockSnapshots := blockReader.(WithSnapshots).Snapshots() + defer blockSnapshots.EnableReadAhead().DisableReadAhead() + + // force merge snapshots before reconstitution, to allign domains progress + // un-finished merge can happen at "kill -9" during merge + if err := agg.MergeLoop(ctx, estimate.CompressSnapshot.Workers()); err != nil { + return err + } var ok bool var blockNum uint64 // First block which is not covered by the history snapshot files @@ -1260,8 +1268,11 @@ func ReconstituteState(ctx context.Context, s *StageState, dirs datadir.Dirs, wo defer os.RemoveAll(reconDbPath) // Incremental reconstitution, step by step (snapshot range by snapshot range) - defer blockSnapshots.EnableReadAhead().DisableReadAhead() - aggSteps := agg.MakeSteps() + + aggSteps, err := agg.MakeSteps() + if err != nil { + return err + } for step, as := range aggSteps { log.Info("Step of incremental reconstitution", "step", step+1, "out of", len(aggSteps), "workers", workerCount) if err := reconstituteStep(step+1 == len(aggSteps), workerCount, ctx, db, diff --git a/go.mod b/go.mod index cc9545840..712f247dc 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230101044116-61e3747169bf + github.com/ledgerwatch/erigon-lib v0.0.0-20230102030350-e98139900454 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20221223003841-487873d31492 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index d5afbd479..2f76b2c05 100644 --- a/go.sum +++ b/go.sum @@ -563,8 +563,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230101044116-61e3747169bf h1:PgINp2BaCQYjOM7Raj9iepD2oSJRWueMs8S8mO4jlRI= -github.com/ledgerwatch/erigon-lib v0.0.0-20230101044116-61e3747169bf/go.mod h1:HaO/Rz8JnrwEhJWEZxRyy3H3DAImgoK05l9ZhULXU2k= +github.com/ledgerwatch/erigon-lib v0.0.0-20230102030350-e98139900454 h1:Q7HDFrWmQZafwQI4C7duP2eS5wGQSzUsr804LhzzMmQ= +github.com/ledgerwatch/erigon-lib v0.0.0-20230102030350-e98139900454/go.mod h1:HaO/Rz8JnrwEhJWEZxRyy3H3DAImgoK05l9ZhULXU2k= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20221223003841-487873d31492 h1:SSYvbAzdreVrXdy8z8A92ug36c7zsGQLzXFrSiw92Zc= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20221223003841-487873d31492/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og=