From 4f95342036fa031405a7eb0bb0c38d5675814093 Mon Sep 17 00:00:00 2001 From: milen <94537774+taratorio@users.noreply.github.com> Date: Mon, 18 Dec 2023 19:13:21 +0200 Subject: [PATCH] freezeblocks: fix blockreader last frozen bor span and event ids (#9018) During testing we run into a "span 7813 not found (db)" due to a very large unwind (1 million blocks). This is because the block reader's `LastFrozenSpanID` and `LastFrozenEventID` returned results that are not consistent with `FrozenBorBlocks`. The latter is taking into account the existence of `.idx` files while the former 2 functions were not. Note such a large unwind is not likely to happen normally unless there is a bug in our unwind logic or an operator is manually unwinding very far back due to reasons like chain halts (ie mumbai bug problem from few months ago), devel testing or anything else along these lines. Regardless, it exposed the above discrepancy which is best to be fixed. --- .../snapshotsync/freezeblocks/block_reader.go | 24 +- .../freezeblocks/block_reader_test.go | 224 ++++++++++++++++++ 2 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 turbo/snapshotsync/freezeblocks/block_reader_test.go diff --git a/turbo/snapshotsync/freezeblocks/block_reader.go b/turbo/snapshotsync/freezeblocks/block_reader.go index 6d14bb4a8..bd6662266 100644 --- a/turbo/snapshotsync/freezeblocks/block_reader.go +++ b/turbo/snapshotsync/freezeblocks/block_reader.go @@ -1076,7 +1076,17 @@ func (r *BlockReader) LastFrozenEventID() uint64 { if len(segments) == 0 { return 0 } - lastSegment := segments[len(segments)-1] + // find the last segment which has a built index + var lastSegment *BorEventSegment + for i := len(segments) - 1; i >= 0; i-- { + if segments[i].IdxBorTxnHash != nil { + lastSegment = segments[i] + break + } + } + if lastSegment == nil { + return 0 + } var lastEventID uint64 gg := lastSegment.seg.MakeGetter() var buf []byte @@ -1094,7 +1104,17 @@ func (r *BlockReader) LastFrozenSpanID() uint64 { if len(segments) == 0 { return 0 } - lastSegment := segments[len(segments)-1] + // find the last segment which has a built index + var lastSegment *BorSpanSegment + for i := len(segments) - 1; i >= 0; i-- { + if segments[i].idx != nil { + lastSegment = segments[i] + break + } + } + if lastSegment == nil { + return 0 + } var lastSpanID uint64 if lastSegment.ranges.to > zerothSpanEnd { lastSpanID = (lastSegment.ranges.to - zerothSpanEnd - 1) / spanLength diff --git a/turbo/snapshotsync/freezeblocks/block_reader_test.go b/turbo/snapshotsync/freezeblocks/block_reader_test.go new file mode 100644 index 000000000..ccd53c29c --- /dev/null +++ b/turbo/snapshotsync/freezeblocks/block_reader_test.go @@ -0,0 +1,224 @@ +package freezeblocks + +import ( + "context" + "encoding/binary" + "os" + "path/filepath" + "testing" + + "github.com/ledgerwatch/log/v3" + "github.com/stretchr/testify/require" + + "github.com/ledgerwatch/erigon-lib/common/length" + "github.com/ledgerwatch/erigon-lib/compress" + "github.com/ledgerwatch/erigon-lib/downloader/snaptype" + "github.com/ledgerwatch/erigon-lib/recsplit" + "github.com/ledgerwatch/erigon/eth/ethconfig" + "github.com/ledgerwatch/erigon/turbo/testlog" +) + +func TestBlockReaderLastFrozenSpanIDWhenSegmentFilesArePresent(t *testing.T) { + t.Parallel() + + logger := testlog.Logger(t, log.LvlInfo) + dir := t.TempDir() + createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger) + createTestSegmentFile(t, 0, 500_000, snaptype.BorSpans, dir, logger) + borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger) + defer borRoSnapshots.Close() + err := borRoSnapshots.ReopenFolder() + require.NoError(t, err) + + blockReader := &BlockReader{borSn: borRoSnapshots} + require.Equal(t, uint64(78), blockReader.LastFrozenSpanID()) +} + +func TestBlockReaderLastFrozenSpanIDWhenSegmentFilesAreNotPresent(t *testing.T) { + t.Parallel() + + logger := testlog.Logger(t, log.LvlInfo) + dir := t.TempDir() + borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger) + defer borRoSnapshots.Close() + err := borRoSnapshots.ReopenFolder() + require.NoError(t, err) + + blockReader := &BlockReader{borSn: borRoSnapshots} + require.Equal(t, uint64(0), blockReader.LastFrozenSpanID()) +} + +func TestBlockReaderLastFrozenSpanIDReturnsLastSegWithIdx(t *testing.T) { + t.Parallel() + + logger := testlog.Logger(t, log.LvlInfo) + dir := t.TempDir() + createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger) + createTestBorEventSegmentFile(t, 500_000, 1_000_000, 264, dir, logger) + createTestBorEventSegmentFile(t, 1_000_000, 1_500_000, 528, dir, logger) + createTestSegmentFile(t, 0, 500_000, snaptype.BorSpans, dir, logger) + createTestSegmentFile(t, 500_000, 1_000_000, snaptype.BorSpans, dir, logger) + createTestSegmentFile(t, 1_000_000, 1_500_000, snaptype.BorSpans, dir, logger) + // delete idx file for last bor span segment to simulate segment with missing idx file + idxFileToDelete := filepath.Join(dir, snaptype.IdxFileName(1_000_000, 1_500_000, snaptype.BorSpans.String())) + err := os.Remove(idxFileToDelete) + require.NoError(t, err) + borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger) + defer borRoSnapshots.Close() + err = borRoSnapshots.ReopenFolder() + require.NoError(t, err) + + blockReader := &BlockReader{borSn: borRoSnapshots} + require.Equal(t, uint64(156), blockReader.LastFrozenSpanID()) +} + +func TestBlockReaderLastFrozenSpanIDReturnsZeroWhenAllSegmentsDoNotHaveIdx(t *testing.T) { + t.Parallel() + + logger := testlog.Logger(t, log.LvlInfo) + dir := t.TempDir() + createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger) + createTestBorEventSegmentFile(t, 500_000, 1_000_000, 264, dir, logger) + createTestBorEventSegmentFile(t, 1_000_000, 1_500_000, 528, dir, logger) + createTestSegmentFile(t, 0, 500_000, snaptype.BorSpans, dir, logger) + createTestSegmentFile(t, 500_000, 1_000_000, snaptype.BorSpans, dir, logger) + createTestSegmentFile(t, 1_000_000, 1_500_000, snaptype.BorSpans, dir, logger) + // delete idx file for all bor span segments to simulate segments with missing idx files + idxFileToDelete := filepath.Join(dir, snaptype.IdxFileName(0, 500_000, snaptype.BorSpans.String())) + err := os.Remove(idxFileToDelete) + require.NoError(t, err) + idxFileToDelete = filepath.Join(dir, snaptype.IdxFileName(500_000, 1_000_000, snaptype.BorSpans.String())) + err = os.Remove(idxFileToDelete) + require.NoError(t, err) + idxFileToDelete = filepath.Join(dir, snaptype.IdxFileName(1_000_000, 1_500_000, snaptype.BorSpans.String())) + err = os.Remove(idxFileToDelete) + require.NoError(t, err) + borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger) + defer borRoSnapshots.Close() + err = borRoSnapshots.ReopenFolder() + require.NoError(t, err) + + blockReader := &BlockReader{borSn: borRoSnapshots} + require.Equal(t, uint64(0), blockReader.LastFrozenSpanID()) +} + +func TestBlockReaderLastFrozenEventIDWhenSegmentFilesArePresent(t *testing.T) { + t.Parallel() + + logger := testlog.Logger(t, log.LvlInfo) + dir := t.TempDir() + createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger) + createTestSegmentFile(t, 0, 500_000, snaptype.BorSpans, dir, logger) + borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger) + defer borRoSnapshots.Close() + err := borRoSnapshots.ReopenFolder() + require.NoError(t, err) + + blockReader := &BlockReader{borSn: borRoSnapshots} + require.Equal(t, uint64(132), blockReader.LastFrozenEventID()) +} + +func TestBlockReaderLastFrozenEventIDWhenSegmentFilesAreNotPresent(t *testing.T) { + t.Parallel() + + logger := testlog.Logger(t, log.LvlInfo) + dir := t.TempDir() + borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger) + defer borRoSnapshots.Close() + err := borRoSnapshots.ReopenFolder() + require.NoError(t, err) + + blockReader := &BlockReader{borSn: borRoSnapshots} + require.Equal(t, uint64(0), blockReader.LastFrozenEventID()) +} + +func TestBlockReaderLastFrozenEventIDReturnsLastSegWithIdx(t *testing.T) { + t.Parallel() + + logger := testlog.Logger(t, log.LvlInfo) + dir := t.TempDir() + createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger) + createTestBorEventSegmentFile(t, 500_000, 1_000_000, 264, dir, logger) + createTestBorEventSegmentFile(t, 1_000_000, 1_500_000, 528, dir, logger) + createTestSegmentFile(t, 0, 500_000, snaptype.BorSpans, dir, logger) + createTestSegmentFile(t, 500_000, 1_000_000, snaptype.BorSpans, dir, logger) + createTestSegmentFile(t, 1_000_000, 1_500_000, snaptype.BorSpans, dir, logger) + // delete idx file for last bor events segment to simulate segment with missing idx file + idxFileToDelete := filepath.Join(dir, snaptype.IdxFileName(1_000_000, 1_500_000, snaptype.BorEvents.String())) + err := os.Remove(idxFileToDelete) + require.NoError(t, err) + borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger) + defer borRoSnapshots.Close() + err = borRoSnapshots.ReopenFolder() + require.NoError(t, err) + + blockReader := &BlockReader{borSn: borRoSnapshots} + require.Equal(t, uint64(264), blockReader.LastFrozenEventID()) +} + +func TestBlockReaderLastFrozenEventIDReturnsZeroWhenAllSegmentsDoNotHaveIdx(t *testing.T) { + t.Parallel() + + logger := testlog.Logger(t, log.LvlInfo) + dir := t.TempDir() + createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger) + createTestBorEventSegmentFile(t, 500_000, 1_000_000, 264, dir, logger) + createTestBorEventSegmentFile(t, 1_000_000, 1_500_000, 528, dir, logger) + createTestSegmentFile(t, 0, 500_000, snaptype.BorSpans, dir, logger) + createTestSegmentFile(t, 500_000, 1_000_000, snaptype.BorSpans, dir, logger) + createTestSegmentFile(t, 1_000_000, 1_500_000, snaptype.BorSpans, dir, logger) + // delete idx files for all bor events segment to simulate segment files with missing idx files + idxFileToDelete := filepath.Join(dir, snaptype.IdxFileName(0, 500_000, snaptype.BorEvents.String())) + err := os.Remove(idxFileToDelete) + require.NoError(t, err) + idxFileToDelete = filepath.Join(dir, snaptype.IdxFileName(500_000, 1_000_000, snaptype.BorEvents.String())) + err = os.Remove(idxFileToDelete) + require.NoError(t, err) + idxFileToDelete = filepath.Join(dir, snaptype.IdxFileName(1_000_000, 1_500_000, snaptype.BorEvents.String())) + err = os.Remove(idxFileToDelete) + require.NoError(t, err) + borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger) + defer borRoSnapshots.Close() + err = borRoSnapshots.ReopenFolder() + require.NoError(t, err) + + blockReader := &BlockReader{borSn: borRoSnapshots} + require.Equal(t, uint64(0), blockReader.LastFrozenEventID()) +} + +func createTestBorEventSegmentFile(t *testing.T, from, to, eventId uint64, dir string, logger log.Logger) { + compressor, err := compress.NewCompressor( + context.Background(), + "test", + filepath.Join(dir, snaptype.SegmentFileName(from, to, snaptype.BorEvents)), + dir, + 100, + 1, + log.LvlDebug, + logger, + ) + require.NoError(t, err) + defer compressor.Close() + data := make([]byte, length.Hash+length.BlockNum+8) + binary.BigEndian.PutUint64(data[length.Hash+length.BlockNum:length.Hash+length.BlockNum+8], eventId) + err = compressor.AddWord(data) + require.NoError(t, err) + err = compressor.Compress() + require.NoError(t, err) + idx, err := recsplit.NewRecSplit( + recsplit.RecSplitArgs{ + KeyCount: 1, + BucketSize: 10, + TmpDir: dir, + IndexFile: filepath.Join(dir, snaptype.IdxFileName(from, to, snaptype.BorEvents.String())), + LeafSize: 8, + }, + logger, + ) + require.NoError(t, err) + defer idx.Close() + err = idx.AddKey([]byte{1}, 0) + require.NoError(t, err) + err = idx.Build(context.Background()) + require.NoError(t, err) +}