From d2bde8c096e95c18014cf5c366d1c7d77b136ced Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Fri, 11 Aug 2023 11:54:59 +0600 Subject: [PATCH] Recsplit: cancelable build (#7994) --- .github/workflows/ci.yml | 2 +- Makefile | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- .../freezeblocks/block_snapshots.go | 21 ++++++++++--------- .../freezeblocks/block_snapshots_test.go | 4 ++-- 6 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2470c3099..f71c8f329 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: if: runner.os == 'Linux' uses: golangci/golangci-lint-action@v3 with: - version: v1.52 + version: v1.54 - name: Test run: make test diff --git a/Makefile b/Makefile index 55bf87a6e..52bfceafd 100644 --- a/Makefile +++ b/Makefile @@ -165,7 +165,7 @@ lintci: ## lintci-deps: (re)installs golangci-lint to build/bin/golangci-lint lintci-deps: rm -f ./build/bin/golangci-lint - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./build/bin v1.53.3 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./build/bin v1.54.0 ## clean: cleans the go cache, build dir, libmdbx db dir clean: @@ -213,7 +213,7 @@ git-submodules: @git submodule update --quiet --init --recursive --force || true PACKAGE_NAME := github.com/ledgerwatch/erigon -GOLANG_CROSS_VERSION ?= v1.20.5 +GOLANG_CROSS_VERSION ?= v1.20.7 .PHONY: release-dry-run release-dry-run: git-submodules diff --git a/go.mod b/go.mod index 357b02f74..03f4b07da 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.19 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230806094003-563a68124b44 + github.com/ledgerwatch/erigon-lib v0.0.0-20230811055104-e22b2e3ec100 github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230810173239-feb52fae58d9 github.com/ledgerwatch/log/v3 v3.8.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index b1d53b9ff..3bc6abd1e 100644 --- a/go.sum +++ b/go.sum @@ -497,8 +497,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230806094003-563a68124b44 h1:5tmiUuLlj94snkO1Ljq12dmqBDG+ncO2IQNPe9fU0HA= -github.com/ledgerwatch/erigon-lib v0.0.0-20230806094003-563a68124b44/go.mod h1:vMETmlckriMRtrg81+YGcmA4/V3XFmjScMqjCojPr3g= +github.com/ledgerwatch/erigon-lib v0.0.0-20230811055104-e22b2e3ec100 h1:DRQDvUPhWfTHCinPxJHKHvoRoUPIEstVHkdf2TfPmyQ= +github.com/ledgerwatch/erigon-lib v0.0.0-20230811055104-e22b2e3ec100/go.mod h1:vMETmlckriMRtrg81+YGcmA4/V3XFmjScMqjCojPr3g= github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230810173239-feb52fae58d9 h1:fG8PozTh9rKBRtWwZsoCA8kJ0M/B6SiG4Vo1sF29Inw= github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230810173239-feb52fae58d9/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.8.0 h1:gCpp7uGtIerEz1jKVPeDnbIopFPud9ZnCpBLlLBGqPU= diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go index 33002f6d7..4b2757baf 100644 --- a/turbo/snapshotsync/freezeblocks/block_snapshots.go +++ b/turbo/snapshotsync/freezeblocks/block_snapshots.go @@ -1795,7 +1795,7 @@ RETRY: return fmt.Errorf("TransactionsIdx: at=%d-%d, post index building, expect: %d, got %d", blockFrom, blockTo, expectedCount, i) } - if err := txnHashIdx.Build(); err != nil { + if err := txnHashIdx.Build(ctx); err != nil { if errors.Is(err, recsplit.ErrCollision) { logger.Warn("Building recsplit. Collision happened. It's ok. Restarting with another salt...", "err", err) txnHashIdx.ResetNextSalt() @@ -1804,7 +1804,7 @@ RETRY: } return fmt.Errorf("txnHashIdx: %w", err) } - if err := txnHash2BlockNumIdx.Build(); err != nil { + if err := txnHash2BlockNumIdx.Build(ctx); err != nil { if errors.Is(err, recsplit.ErrCollision) { logger.Warn("Building recsplit. Collision happened. It's ok. Restarting with another salt...", "err", err) txnHashIdx.ResetNextSalt() @@ -1895,13 +1895,14 @@ func Idx(ctx context.Context, d *compress.Decompressor, firstDataID uint64, tmpD var idxFilePath = segmentFileName[0:len(segmentFileName)-len(extension)] + ".idx" rs, err := recsplit.NewRecSplit(recsplit.RecSplitArgs{ - KeyCount: d.Count(), - Enums: true, - BucketSize: 2000, - LeafSize: 8, - TmpDir: tmpDir, - IndexFile: idxFilePath, - BaseDataID: firstDataID, + KeyCount: d.Count(), + Enums: true, + BucketSize: 2000, + LeafSize: 8, + TmpDir: tmpDir, + IndexFile: idxFilePath, + BaseDataID: firstDataID, + EtlBufLimit: etl.BufferOptimalSize / 2, }, logger) if err != nil { return err @@ -1928,7 +1929,7 @@ RETRY: default: } } - if err = rs.Build(); err != nil { + if err = rs.Build(ctx); err != nil { if errors.Is(err, recsplit.ErrCollision) { logger.Info("Building recsplit. Collision happened. It's ok. Restarting with another salt...", "err", err) rs.ResetNextSalt() diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots_test.go b/turbo/snapshotsync/freezeblocks/block_snapshots_test.go index ff70e2d7c..7e8294cfd 100644 --- a/turbo/snapshotsync/freezeblocks/block_snapshots_test.go +++ b/turbo/snapshotsync/freezeblocks/block_snapshots_test.go @@ -38,7 +38,7 @@ func createTestSegmentFile(t *testing.T, from, to uint64, name snaptype.Type, di defer idx.Close() err = idx.AddKey([]byte{1}, 0) require.NoError(t, err) - err = idx.Build() + err = idx.Build(context.Background()) require.NoError(t, err) if name == snaptype.Transactions { idx, err := recsplit.NewRecSplit(recsplit.RecSplitArgs{ @@ -51,7 +51,7 @@ func createTestSegmentFile(t *testing.T, from, to uint64, name snaptype.Type, di require.NoError(t, err) err = idx.AddKey([]byte{1}, 0) require.NoError(t, err) - err = idx.Build() + err = idx.Build(context.Background()) require.NoError(t, err) defer idx.Close() }