mirror of
https://gitlab.com/pulsechaincom/erigon-pulse.git
synced 2025-01-07 03:22:18 +00:00
4f95342036
During testing we run into a "span 7813 not found (db)" due to a very large unwind (1 million blocks). This is because the block reader's `LastFrozenSpanID` and `LastFrozenEventID` returned results that are not consistent with `FrozenBorBlocks`. The latter is taking into account the existence of `.idx` files while the former 2 functions were not. Note such a large unwind is not likely to happen normally unless there is a bug in our unwind logic or an operator is manually unwinding very far back due to reasons like chain halts (ie mumbai bug problem from few months ago), devel testing or anything else along these lines. Regardless, it exposed the above discrepancy which is best to be fixed.
225 lines
8.7 KiB
Go
225 lines
8.7 KiB
Go
package freezeblocks
|
|
|
|
import (
|
|
"context"
|
|
"encoding/binary"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/ledgerwatch/log/v3"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/ledgerwatch/erigon-lib/common/length"
|
|
"github.com/ledgerwatch/erigon-lib/compress"
|
|
"github.com/ledgerwatch/erigon-lib/downloader/snaptype"
|
|
"github.com/ledgerwatch/erigon-lib/recsplit"
|
|
"github.com/ledgerwatch/erigon/eth/ethconfig"
|
|
"github.com/ledgerwatch/erigon/turbo/testlog"
|
|
)
|
|
|
|
func TestBlockReaderLastFrozenSpanIDWhenSegmentFilesArePresent(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
logger := testlog.Logger(t, log.LvlInfo)
|
|
dir := t.TempDir()
|
|
createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger)
|
|
createTestSegmentFile(t, 0, 500_000, snaptype.BorSpans, dir, logger)
|
|
borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger)
|
|
defer borRoSnapshots.Close()
|
|
err := borRoSnapshots.ReopenFolder()
|
|
require.NoError(t, err)
|
|
|
|
blockReader := &BlockReader{borSn: borRoSnapshots}
|
|
require.Equal(t, uint64(78), blockReader.LastFrozenSpanID())
|
|
}
|
|
|
|
func TestBlockReaderLastFrozenSpanIDWhenSegmentFilesAreNotPresent(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
logger := testlog.Logger(t, log.LvlInfo)
|
|
dir := t.TempDir()
|
|
borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger)
|
|
defer borRoSnapshots.Close()
|
|
err := borRoSnapshots.ReopenFolder()
|
|
require.NoError(t, err)
|
|
|
|
blockReader := &BlockReader{borSn: borRoSnapshots}
|
|
require.Equal(t, uint64(0), blockReader.LastFrozenSpanID())
|
|
}
|
|
|
|
func TestBlockReaderLastFrozenSpanIDReturnsLastSegWithIdx(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
logger := testlog.Logger(t, log.LvlInfo)
|
|
dir := t.TempDir()
|
|
createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger)
|
|
createTestBorEventSegmentFile(t, 500_000, 1_000_000, 264, dir, logger)
|
|
createTestBorEventSegmentFile(t, 1_000_000, 1_500_000, 528, dir, logger)
|
|
createTestSegmentFile(t, 0, 500_000, snaptype.BorSpans, dir, logger)
|
|
createTestSegmentFile(t, 500_000, 1_000_000, snaptype.BorSpans, dir, logger)
|
|
createTestSegmentFile(t, 1_000_000, 1_500_000, snaptype.BorSpans, dir, logger)
|
|
// delete idx file for last bor span segment to simulate segment with missing idx file
|
|
idxFileToDelete := filepath.Join(dir, snaptype.IdxFileName(1_000_000, 1_500_000, snaptype.BorSpans.String()))
|
|
err := os.Remove(idxFileToDelete)
|
|
require.NoError(t, err)
|
|
borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger)
|
|
defer borRoSnapshots.Close()
|
|
err = borRoSnapshots.ReopenFolder()
|
|
require.NoError(t, err)
|
|
|
|
blockReader := &BlockReader{borSn: borRoSnapshots}
|
|
require.Equal(t, uint64(156), blockReader.LastFrozenSpanID())
|
|
}
|
|
|
|
func TestBlockReaderLastFrozenSpanIDReturnsZeroWhenAllSegmentsDoNotHaveIdx(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
logger := testlog.Logger(t, log.LvlInfo)
|
|
dir := t.TempDir()
|
|
createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger)
|
|
createTestBorEventSegmentFile(t, 500_000, 1_000_000, 264, dir, logger)
|
|
createTestBorEventSegmentFile(t, 1_000_000, 1_500_000, 528, dir, logger)
|
|
createTestSegmentFile(t, 0, 500_000, snaptype.BorSpans, dir, logger)
|
|
createTestSegmentFile(t, 500_000, 1_000_000, snaptype.BorSpans, dir, logger)
|
|
createTestSegmentFile(t, 1_000_000, 1_500_000, snaptype.BorSpans, dir, logger)
|
|
// delete idx file for all bor span segments to simulate segments with missing idx files
|
|
idxFileToDelete := filepath.Join(dir, snaptype.IdxFileName(0, 500_000, snaptype.BorSpans.String()))
|
|
err := os.Remove(idxFileToDelete)
|
|
require.NoError(t, err)
|
|
idxFileToDelete = filepath.Join(dir, snaptype.IdxFileName(500_000, 1_000_000, snaptype.BorSpans.String()))
|
|
err = os.Remove(idxFileToDelete)
|
|
require.NoError(t, err)
|
|
idxFileToDelete = filepath.Join(dir, snaptype.IdxFileName(1_000_000, 1_500_000, snaptype.BorSpans.String()))
|
|
err = os.Remove(idxFileToDelete)
|
|
require.NoError(t, err)
|
|
borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger)
|
|
defer borRoSnapshots.Close()
|
|
err = borRoSnapshots.ReopenFolder()
|
|
require.NoError(t, err)
|
|
|
|
blockReader := &BlockReader{borSn: borRoSnapshots}
|
|
require.Equal(t, uint64(0), blockReader.LastFrozenSpanID())
|
|
}
|
|
|
|
func TestBlockReaderLastFrozenEventIDWhenSegmentFilesArePresent(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
logger := testlog.Logger(t, log.LvlInfo)
|
|
dir := t.TempDir()
|
|
createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger)
|
|
createTestSegmentFile(t, 0, 500_000, snaptype.BorSpans, dir, logger)
|
|
borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger)
|
|
defer borRoSnapshots.Close()
|
|
err := borRoSnapshots.ReopenFolder()
|
|
require.NoError(t, err)
|
|
|
|
blockReader := &BlockReader{borSn: borRoSnapshots}
|
|
require.Equal(t, uint64(132), blockReader.LastFrozenEventID())
|
|
}
|
|
|
|
func TestBlockReaderLastFrozenEventIDWhenSegmentFilesAreNotPresent(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
logger := testlog.Logger(t, log.LvlInfo)
|
|
dir := t.TempDir()
|
|
borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger)
|
|
defer borRoSnapshots.Close()
|
|
err := borRoSnapshots.ReopenFolder()
|
|
require.NoError(t, err)
|
|
|
|
blockReader := &BlockReader{borSn: borRoSnapshots}
|
|
require.Equal(t, uint64(0), blockReader.LastFrozenEventID())
|
|
}
|
|
|
|
func TestBlockReaderLastFrozenEventIDReturnsLastSegWithIdx(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
logger := testlog.Logger(t, log.LvlInfo)
|
|
dir := t.TempDir()
|
|
createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger)
|
|
createTestBorEventSegmentFile(t, 500_000, 1_000_000, 264, dir, logger)
|
|
createTestBorEventSegmentFile(t, 1_000_000, 1_500_000, 528, dir, logger)
|
|
createTestSegmentFile(t, 0, 500_000, snaptype.BorSpans, dir, logger)
|
|
createTestSegmentFile(t, 500_000, 1_000_000, snaptype.BorSpans, dir, logger)
|
|
createTestSegmentFile(t, 1_000_000, 1_500_000, snaptype.BorSpans, dir, logger)
|
|
// delete idx file for last bor events segment to simulate segment with missing idx file
|
|
idxFileToDelete := filepath.Join(dir, snaptype.IdxFileName(1_000_000, 1_500_000, snaptype.BorEvents.String()))
|
|
err := os.Remove(idxFileToDelete)
|
|
require.NoError(t, err)
|
|
borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger)
|
|
defer borRoSnapshots.Close()
|
|
err = borRoSnapshots.ReopenFolder()
|
|
require.NoError(t, err)
|
|
|
|
blockReader := &BlockReader{borSn: borRoSnapshots}
|
|
require.Equal(t, uint64(264), blockReader.LastFrozenEventID())
|
|
}
|
|
|
|
func TestBlockReaderLastFrozenEventIDReturnsZeroWhenAllSegmentsDoNotHaveIdx(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
logger := testlog.Logger(t, log.LvlInfo)
|
|
dir := t.TempDir()
|
|
createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger)
|
|
createTestBorEventSegmentFile(t, 500_000, 1_000_000, 264, dir, logger)
|
|
createTestBorEventSegmentFile(t, 1_000_000, 1_500_000, 528, dir, logger)
|
|
createTestSegmentFile(t, 0, 500_000, snaptype.BorSpans, dir, logger)
|
|
createTestSegmentFile(t, 500_000, 1_000_000, snaptype.BorSpans, dir, logger)
|
|
createTestSegmentFile(t, 1_000_000, 1_500_000, snaptype.BorSpans, dir, logger)
|
|
// delete idx files for all bor events segment to simulate segment files with missing idx files
|
|
idxFileToDelete := filepath.Join(dir, snaptype.IdxFileName(0, 500_000, snaptype.BorEvents.String()))
|
|
err := os.Remove(idxFileToDelete)
|
|
require.NoError(t, err)
|
|
idxFileToDelete = filepath.Join(dir, snaptype.IdxFileName(500_000, 1_000_000, snaptype.BorEvents.String()))
|
|
err = os.Remove(idxFileToDelete)
|
|
require.NoError(t, err)
|
|
idxFileToDelete = filepath.Join(dir, snaptype.IdxFileName(1_000_000, 1_500_000, snaptype.BorEvents.String()))
|
|
err = os.Remove(idxFileToDelete)
|
|
require.NoError(t, err)
|
|
borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, logger)
|
|
defer borRoSnapshots.Close()
|
|
err = borRoSnapshots.ReopenFolder()
|
|
require.NoError(t, err)
|
|
|
|
blockReader := &BlockReader{borSn: borRoSnapshots}
|
|
require.Equal(t, uint64(0), blockReader.LastFrozenEventID())
|
|
}
|
|
|
|
func createTestBorEventSegmentFile(t *testing.T, from, to, eventId uint64, dir string, logger log.Logger) {
|
|
compressor, err := compress.NewCompressor(
|
|
context.Background(),
|
|
"test",
|
|
filepath.Join(dir, snaptype.SegmentFileName(from, to, snaptype.BorEvents)),
|
|
dir,
|
|
100,
|
|
1,
|
|
log.LvlDebug,
|
|
logger,
|
|
)
|
|
require.NoError(t, err)
|
|
defer compressor.Close()
|
|
data := make([]byte, length.Hash+length.BlockNum+8)
|
|
binary.BigEndian.PutUint64(data[length.Hash+length.BlockNum:length.Hash+length.BlockNum+8], eventId)
|
|
err = compressor.AddWord(data)
|
|
require.NoError(t, err)
|
|
err = compressor.Compress()
|
|
require.NoError(t, err)
|
|
idx, err := recsplit.NewRecSplit(
|
|
recsplit.RecSplitArgs{
|
|
KeyCount: 1,
|
|
BucketSize: 10,
|
|
TmpDir: dir,
|
|
IndexFile: filepath.Join(dir, snaptype.IdxFileName(from, to, snaptype.BorEvents.String())),
|
|
LeafSize: 8,
|
|
},
|
|
logger,
|
|
)
|
|
require.NoError(t, err)
|
|
defer idx.Close()
|
|
err = idx.AddKey([]byte{1}, 0)
|
|
require.NoError(t, err)
|
|
err = idx.Build(context.Background())
|
|
require.NoError(t, err)
|
|
}
|