mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 19:50:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

integration mdbx_to_mdbx (#4897)

Browse Source
This commit is contained in:
Alex Sharov 2022-08-02 13:37:57 +07:00 committed by GitHub
parent 7199dcf7a2
commit 9dc15ac5bb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -122,7 +122,7 @@ all: erigon $(COMMANDS)
db-tools: git-submodules db-tools: git-submodules
@echo "Building db-tools" @echo "Building db-tools"
# hub.docker.com setup incorrect gitpath for git modules. Just remove it and re-init submodule. @# hub.docker.com setup incorrect gitpath for git modules. Just remove it and re-init submodule.
rm -rf libmdbx rm -rf libmdbx
git submodule update --init --recursive --force libmdbx git submodule update --init --recursive --force libmdbx

12
cmd/integration/commands/refetence_db.go
View File

@ -349,7 +349,10 @@ MainLoop:
func mdbxToMdbx(ctx context.Context, logger log.Logger, from, to string) error { func mdbxToMdbx(ctx context.Context, logger log.Logger, from, to string) error {
_ = os.RemoveAll(to) _ = os.RemoveAll(to)
src := mdbx2.NewMDBX(logger).Path(from).Flags(func(flags uint) uint { return mdbx.Readonly | mdbx.Accede }).MustOpen() src := mdbx2.NewMDBX(logger).Path(from).Flags(func(flags uint) uint { return mdbx.Readonly | mdbx.Accede }).MustOpen()
dst := mdbx2.NewMDBX(logger).Path(to).MustOpen() dst := mdbx2.NewMDBX(logger).Path(to).
WriteMap().
Flags(func(flags uint) uint { return flags | mdbx.NoMemInit }).
MustOpen()
return kv2kv(ctx, src, dst) return kv2kv(ctx, src, dst)
} }
@ -368,6 +371,7 @@ func kv2kv(ctx context.Context, src, dst kv.RwDB) error {
commitEvery := time.NewTicker(30 * time.Second) commitEvery := time.NewTicker(30 * time.Second)
defer commitEvery.Stop() defer commitEvery.Stop()
var total uint64
for name, b := range src.AllBuckets() { for name, b := range src.AllBuckets() {
if b.IsDeprecated { if b.IsDeprecated {
continue continue
@ -382,7 +386,10 @@ func kv2kv(ctx context.Context, src, dst kv.RwDB) error {
if err != nil { if err != nil {
return err return err
} }
total, _ = srcC.Count()
casted, isDupsort := c.(kv.RwCursorDupSort) casted, isDupsort := c.(kv.RwCursorDupSort)
i := uint64(0)
for k, v, err := srcC.First(); k != nil; k, v, err = srcC.Next() { for k, v, err := srcC.First(); k != nil; k, v, err = srcC.Next() {
if err != nil { if err != nil {
@ -399,11 +406,12 @@ func kv2kv(ctx context.Context, src, dst kv.RwDB) error {
} }
} }
i++
select { select {
case <-ctx.Done(): case <-ctx.Done():
return ctx.Err() return ctx.Err()
case <-commitEvery.C: case <-commitEvery.C:
log.Info("Progress", "bucket", name, "key", fmt.Sprintf("%x", k)) log.Info("Progress", "bucket", name, "progress", fmt.Sprintf("%.1fm/%.1fm", float64(i)/1_000_000, float64(total)/1_000_000), "key", fmt.Sprintf("%x", k))
if err2 := dstTx.Commit(); err2 != nil { if err2 := dstTx.Commit(); err2 != nil {
return err2 return err2
} }