erigon-pulse/migrations/receipts.go

264 lines
8.3 KiB
Go
Raw Normal View History

package migrations
import (
"bytes"
"encoding/binary"
"fmt"
"os"
"runtime"
"time"
"github.com/ledgerwatch/turbo-geth/common"
"github.com/ledgerwatch/turbo-geth/common/dbutils"
"github.com/ledgerwatch/turbo-geth/common/etl"
2020-10-28 03:24:17 +00:00
"github.com/ledgerwatch/turbo-geth/core/rawdb"
"github.com/ledgerwatch/turbo-geth/core/types"
"github.com/ledgerwatch/turbo-geth/ethdb"
"github.com/ledgerwatch/turbo-geth/ethdb/cbor"
"github.com/ledgerwatch/turbo-geth/log"
"github.com/ledgerwatch/turbo-geth/rlp"
)
var receiptsCborEncode = Migration{
Name: "receipts_cbor_encode",
Up: func(db ethdb.Database, tmpdir string, progress []byte, CommitProgress etl.LoadCommitHandler) error {
logEvery := time.NewTicker(30 * time.Second)
defer logEvery.Stop()
buf := bytes.NewBuffer(make([]byte, 0, 100_000))
const loadStep = "load"
collector, err1 := etl.NewCollectorFromFiles(tmpdir)
if err1 != nil {
return err1
}
switch string(progress) {
case "":
if collector != nil { // can't use files if progress field not set
_ = os.RemoveAll(tmpdir)
collector = nil
}
case loadStep:
if collector == nil {
return ErrMigrationETLFilesDeleted
}
goto LoadStep
}
collector = etl.NewCriticalCollector(tmpdir, etl.NewSortableBuffer(etl.BufferOptimalSize))
if err1 = db.Walk(dbutils.BlockReceiptsPrefix, nil, 0, func(k, v []byte) (bool, error) {
blockNum := binary.BigEndian.Uint64(k[:8])
select {
default:
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
log.Info("Migration progress", "blockNum", blockNum, "alloc", common.StorageSize(m.Alloc), "sys", common.StorageSize(m.Sys))
}
// Convert the receipts from their storage form to their internal representation
storageReceipts := []*types.ReceiptForStorage{}
if err := rlp.DecodeBytes(v, &storageReceipts); err != nil {
return false, fmt.Errorf("invalid receipt array RLP: %w, k=%x", err, k)
}
buf.Reset()
if err := cbor.Marshal(buf, storageReceipts); err != nil {
return false, err
}
if err := collector.Collect(k, buf.Bytes()); err != nil {
return false, fmt.Errorf("collecting key %x: %w", k, err)
}
return true, nil
}); err1 != nil {
return err1
}
if err := db.(ethdb.BucketsMigrator).ClearBuckets(dbutils.BlockReceiptsPrefix); err != nil {
return fmt.Errorf("clearing the receipt bucket: %w", err)
}
// Commit clearing of the bucket - freelist should now be written to the database
if err := CommitProgress(db, []byte(loadStep), false); err != nil {
2020-10-28 03:18:10 +00:00
return fmt.Errorf("committing the removal of receipt table: %w", err)
}
LoadStep:
// Commit again
if err := CommitProgress(db, []byte(loadStep), false); err != nil {
2020-10-28 03:18:10 +00:00
return fmt.Errorf("committing again to create a stable view the removal of receipt table: %w", err)
}
// Now transaction would have been re-opened, and we should be re-using the space
if err := collector.Load("receipts_cbor_encode", db, dbutils.BlockReceiptsPrefix, etl.IdentityLoadFunc, etl.TransformArgs{OnLoadCommit: CommitProgress}); err != nil {
return fmt.Errorf("loading the transformed data back into the receipts table: %w", err)
}
return nil
},
}
var receiptsOnePerTx = Migration{
Name: "receipts_store_logs_separately",
Up: func(db ethdb.Database, tmpdir string, progress []byte, CommitProgress etl.LoadCommitHandler) (err error) {
logEvery := time.NewTicker(30 * time.Second)
defer logEvery.Stop()
ChangeSets dupsort (#1342) * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * change_set_dup * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * working version * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * aa * squash * squash * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * history_early_stop * history_early_stop * vmConfig with ReadOnly false * auto_increment * auto_increment * rebase master Co-authored-by: Alexey Akhunov <akhounov@gmail.com>
2020-11-16 12:08:28 +00:00
logPrefix := "data migration: receipts_store_logs_separately"
// Recently was introduced receipts serialization problem
// Code was not generated well for types.Log type
// So, to fix this problem - need deserialize by reflection (LegacyReceipt doesn't have generated code)
// then serialize by generated code - types.Receipts and types.Log have generated code now
type LegacyReceipt struct {
PostState []byte `codec:"1"`
Status uint64 `codec:"2"`
CumulativeGasUsed uint64 `codec:"3"`
Logs []*types.Log `codec:"4"`
}
buf := bytes.NewBuffer(make([]byte, 0, 100_000))
reader := bytes.NewReader(nil)
const loadStep = "load"
collectorR, err1 := etl.NewCollectorFromFiles(tmpdir + "1")
if err1 != nil {
return err1
}
collectorL, err1 := etl.NewCollectorFromFiles(tmpdir + "2")
if err1 != nil {
return err1
}
switch string(progress) {
case "":
// can't use files if progress field not set, clear them
if collectorR != nil {
collectorR.Close(logPrefix)
collectorR = nil
}
if collectorL != nil {
collectorL.Close(logPrefix)
collectorL = nil
}
case loadStep:
if collectorR == nil || collectorL == nil {
return ErrMigrationETLFilesDeleted
}
defer func() {
// don't clean if error or panic happened
if err != nil {
return
}
if rec := recover(); rec != nil {
panic(rec)
}
collectorR.Close(logPrefix)
collectorL.Close(logPrefix)
}()
goto LoadStep
}
collectorR = etl.NewCriticalCollector(tmpdir+"1", etl.NewSortableBuffer(etl.BufferOptimalSize*2))
collectorL = etl.NewCriticalCollector(tmpdir+"2", etl.NewSortableBuffer(etl.BufferOptimalSize*2))
defer func() {
// don't clean if error or panic happened
if err != nil {
return
}
if rec := recover(); rec != nil {
panic(rec)
}
collectorR.Close(logPrefix)
collectorL.Close(logPrefix)
}()
if err := db.Walk(dbutils.BlockReceiptsPrefix, nil, 0, func(k, v []byte) (bool, error) {
blockNum := binary.BigEndian.Uint64(k[:8])
2020-10-28 03:24:17 +00:00
canonicalHash, err := rawdb.ReadCanonicalHash(db, blockNum)
if err != nil {
return false, err
}
if !bytes.Equal(k[8:], canonicalHash[:]) {
return true, nil
}
select {
default:
case <-logEvery.C:
var m runtime.MemStats
runtime.ReadMemStats(&m)
log.Info(fmt.Sprintf("[%s] Progress", logPrefix), "blockNum", blockNum, "alloc", common.StorageSize(m.Alloc), "sys", common.StorageSize(m.Sys))
}
// Convert the receipts from their storage form to their internal representation
var legacyReceipts []*LegacyReceipt
reader.Reset(v)
if err := cbor.Unmarshal(&legacyReceipts, reader); err != nil {
return false, err
}
// Convert the receipts from their storage form to their internal representation
receipts := make(types.Receipts, len(legacyReceipts))
for i := range legacyReceipts {
receipts[i] = &types.Receipt{}
receipts[i].PostState = legacyReceipts[i].PostState
receipts[i].Status = legacyReceipts[i].Status
receipts[i].CumulativeGasUsed = legacyReceipts[i].CumulativeGasUsed
receipts[i].Logs = legacyReceipts[i].Logs
}
for txId, r := range receipts {
if len(r.Logs) == 0 {
continue
}
newK := make([]byte, 8+4)
copy(newK, k[:8])
binary.BigEndian.PutUint32(newK[8:], uint32(txId))
buf.Reset()
if err := cbor.Marshal(buf, r.Logs); err != nil {
return false, err
}
if err := collectorL.Collect(newK, buf.Bytes()); err != nil {
return false, fmt.Errorf("collecting key %x: %w", k, err)
}
}
buf.Reset()
if err := cbor.Marshal(buf, receipts); err != nil {
return false, err
}
if err := collectorR.Collect(common.CopyBytes(k[:8]), buf.Bytes()); err != nil {
return false, fmt.Errorf("collecting key %x: %w", k, err)
}
return true, nil
}); err != nil {
return err
}
if err := db.(ethdb.BucketsMigrator).ClearBuckets(dbutils.BlockReceiptsPrefix); err != nil {
return fmt.Errorf("clearing the receipt bucket: %w", err)
}
// Commit clearing of the bucket - freelist should now be written to the database
if err := CommitProgress(db, []byte(loadStep), false); err != nil {
2020-10-28 03:18:10 +00:00
return fmt.Errorf("committing the removal of receipt table: %w", err)
}
LoadStep:
// Commit again
if err := CommitProgress(db, []byte(loadStep), false); err != nil {
2020-10-28 03:18:10 +00:00
return fmt.Errorf("committing the removal of receipt table: %w", err)
}
// Now transaction would have been re-opened, and we should be re-using the space
if err := collectorR.Load(logPrefix, db, dbutils.BlockReceiptsPrefix, etl.IdentityLoadFunc, etl.TransformArgs{}); err != nil {
return fmt.Errorf("loading the transformed data back into the receipts table: %w", err)
}
if err := collectorL.Load(logPrefix, db, dbutils.Log, etl.IdentityLoadFunc, etl.TransformArgs{OnLoadCommit: CommitProgress}); err != nil {
return fmt.Errorf("loading the transformed data back into the receipts table: %w", err)
}
return nil
},
}