mirror/go-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/go-pulse.git synced 2024-12-22 11:31:02 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

internal/era: update block index format to be based on record offset (#28959)

Browse Source 
As mentioned in #26621, the block index format for era1 is not in line with the regular era block index. This change modifies the index so all relative offsets are based against the beginning of the block index record.
This commit is contained in:
lightclient 2024-02-08 23:42:50 -07:00 committed by GitHub
parent ac5aa672d3
commit 85938dda09
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 19 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/utils/history_test.go
View File

@ -134,7 +134,7 @@ func TestHistoryImportAndExport(t *testing.T) {
for j := 0; it.Next(); j++ { for j := 0; it.Next(); j++ {
n := i*int(step) + j n := i*int(step) + j
if it.Error() != nil { if it.Error() != nil {
t.Fatalf("error reading block entry %d: %v", n, err) t.Fatalf("error reading block entry %d: %v", n, it.Error())
} }
block, receipts, err := it.BlockAndReceipts() block, receipts, err := it.BlockAndReceipts()
if err != nil { if err != nil {

24
internal/era/builder.go
View File

@ -49,7 +49,7 @@ import (
// CompressedBody = { type: [0x04, 0x00], data: snappyFramed(rlp(body)) } // CompressedBody = { type: [0x04, 0x00], data: snappyFramed(rlp(body)) }
// CompressedReceipts = { type: [0x05, 0x00], data: snappyFramed(rlp(receipts)) } // CompressedReceipts = { type: [0x05, 0x00], data: snappyFramed(rlp(receipts)) }
// TotalDifficulty = { type: [0x06, 0x00], data: uint256(header.total_difficulty) } // TotalDifficulty = { type: [0x06, 0x00], data: uint256(header.total_difficulty) }
// Accumulator = { type: [0x07, 0x00], data: accumulator-root } // AccumulatorRoot = { type: [0x07, 0x00], data: accumulator-root }
// BlockIndex = { type: [0x32, 0x66], data: block-index } // BlockIndex = { type: [0x32, 0x66], data: block-index }
// //
// Accumulator is computed by constructing an SSZ list of header-records of length at most // Accumulator is computed by constructing an SSZ list of header-records of length at most
@ -64,8 +64,8 @@ import (
// block-index := starting-number | index | index | index ... | count // block-index := starting-number | index | index | index ... | count
// //
// starting-number is the first block number in the archive. Every index is a // starting-number is the first block number in the archive. Every index is a
// defined relative to index's location in the file. The total number of block // defined relative to beginning of the record. The total number of block
// entries in the file is recorded in count. // entries in the file is recorded with count.
// //
// Due to the accumulator size limit of 8192, the maximum number of blocks in // Due to the accumulator size limit of 8192, the maximum number of blocks in
// an Era1 batch is also 8192. // an Era1 batch is also 8192.
@ -115,12 +115,14 @@ func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int)
func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error { func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error {
// Write Era1 version entry before first block. // Write Era1 version entry before first block.
if b.startNum == nil { if b.startNum == nil {
if err := writeVersion(b.w); err != nil { n, err := b.w.Write(TypeVersion, nil)
if err != nil {
return err return err
} }
n := number startNum := number
b.startNum = &n b.startNum = &startNum
b.startTd = new(big.Int).Sub(td, difficulty) b.startTd = new(big.Int).Sub(td, difficulty)
b.written += n
} }
if len(b.indexes) >= MaxEra1Size { if len(b.indexes) >= MaxEra1Size {
return fmt.Errorf("exceeds maximum batch size of %d", MaxEra1Size) return fmt.Errorf("exceeds maximum batch size of %d", MaxEra1Size)
@ -169,7 +171,7 @@ func (b *Builder) Finalize() (common.Hash, error) {
return common.Hash{}, fmt.Errorf("error writing accumulator: %w", err) return common.Hash{}, fmt.Errorf("error writing accumulator: %w", err)
} }
// Get beginning of index entry to calculate block relative offset. // Get beginning of index entry to calculate block relative offset.
base := int64(b.written + (3 * 8)) // skip e2store header (type, length) and start block base := int64(b.written)
// Construct block index. Detailed format described in Builder // Construct block index. Detailed format described in Builder
// documentation, but it is essentially encoded as: // documentation, but it is essentially encoded as:
@ -186,7 +188,7 @@ func (b *Builder) Finalize() (common.Hash, error) {
// relative offset, the corresponding block can be quickly read by // relative offset, the corresponding block can be quickly read by
// performing a seek relative to the current position. // performing a seek relative to the current position.
for i, offset := range b.indexes { for i, offset := range b.indexes {
relative := int64(offset) - (base + int64(i)*8) relative := int64(offset) - base
binary.LittleEndian.PutUint64(index[8+i*8:], uint64(relative)) binary.LittleEndian.PutUint64(index[8+i*8:], uint64(relative))
} }
binary.LittleEndian.PutUint64(index[8+count*8:], uint64(count)) binary.LittleEndian.PutUint64(index[8+count*8:], uint64(count))
@ -220,9 +222,3 @@ func (b *Builder) snappyWrite(typ uint16, in []byte) error {
} }
return nil return nil
} }
// writeVersion writes a version entry to e2store.
func writeVersion(w *e2store.Writer) error {
_, err := w.Write(TypeVersion, nil)
return err
}

13
internal/era/era.go
View File

@ -221,9 +221,10 @@ func (e *Era) Count() uint64 {
// is the absolute block number desired. // is the absolute block number desired.
func (e *Era) readOffset(n uint64) (int64, error) { func (e *Era) readOffset(n uint64) (int64, error) {
var ( var (
firstIndex = -8 - int64(e.m.count)*8 // size of count - index entries blockIndexRecordOffset = e.m.length - 24 - int64(e.m.count)*8 // skips start, count, and header
firstIndex = blockIndexRecordOffset + 16 // first index after header / start-num
indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes
offOffset = e.m.length + firstIndex + indexOffset // offset of block offset offOffset = firstIndex + indexOffset // offset of block offset
) )
e.mu.Lock() e.mu.Lock()
defer e.mu.Unlock() defer e.mu.Unlock()
@ -231,10 +232,10 @@ func (e *Era) readOffset(n uint64) (int64, error) {
if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil { if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil {
return 0, err return 0, err
} }
// Since the block offset is relative from its location + size of index // Since the block offset is relative from the start of the block index record
// value (8), we need to add it to it's offset to get the block's // we need to add the record offset to it's offset to get the block's absolute
// absolute offset. // offset.
return offOffset + 8 + int64(binary.LittleEndian.Uint64(e.buf[:])), nil return blockIndexRecordOffset + int64(binary.LittleEndian.Uint64(e.buf[:])), nil
} }
// newReader returns a snappy.Reader for the e2store entry value at off. // newReader returns a snappy.Reader for the e2store entry value at off.