package migrations import ( "fmt" "github.com/davecgh/go-spew/spew" "github.com/ledgerwatch/turbo-geth/common" "github.com/ledgerwatch/turbo-geth/common/changeset" "github.com/ledgerwatch/turbo-geth/common/dbutils" "github.com/ledgerwatch/turbo-geth/ethdb" "testing" ) func TestChangeSetMigrationSuccess(t *testing.T) { f := func(t *testing.T, n, batchSize int) { db := ethdb.NewMemDatabase() accCS, storageCS := generateChangeSets(t, n) for i, v := range accCS { enc, err := changeset.EncodeChangeSet(v) if err != nil { t.Error(err) } err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), enc) if err != nil { t.Error(err) } } for i, v := range storageCS { enc, err := changeset.EncodeChangeSet(v) if err != nil { t.Error(err) } err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.StorageHistoryBucket), enc) if err != nil { t.Error(err) } } migrator := NewMigrator() migrator.Migrations = []Migration{splitChangeSetMigration(batchSize)} err := migrator.Apply(db, false, false, false, false, false) if err != nil { t.Error(err) } err = db.Walk(ChangeSetBucket, []byte{}, 0, func(k, v []byte) (b bool, e error) { ts, bucket := dbutils.DecodeTimestamp(k) return false, fmt.Errorf("changeset bucket is not empty block %v bucket %v", ts, string(bucket)) }) if err != nil { t.Error(err) } err = db.Walk(dbutils.AccountChangeSetBucket, []byte{}, 0, func(k, v []byte) (b bool, e error) { blockNum, _ := dbutils.DecodeTimestamp(k) cs, innerErr := changeset.DecodeChangeSet(v) if innerErr != nil { return false, fmt.Errorf("decode fails block - %v err: %v", blockNum, innerErr) } if !cs.Equals(accCS[blockNum]) { spew.Dump(cs) spew.Dump(accCS[blockNum]) return false, fmt.Errorf("not equal (%v)", blockNum) } return true, nil }) if err != nil { t.Error(err) } err = db.Walk(dbutils.StorageHistoryBucket, []byte{}, 0, func(k, v []byte) (b bool, e error) { blockNum, _ := dbutils.DecodeTimestamp(k) cs, innerErr := changeset.DecodeChangeSet(v) if innerErr != nil { return false, fmt.Errorf("decode fails block - %v err: %v", blockNum, innerErr) } if !cs.Equals(storageCS[blockNum]) { spew.Dump(cs) spew.Dump(storageCS[blockNum]) return false, fmt.Errorf("not equal (%v)", blockNum) } return true, nil }) if err != nil { t.Error(err) } } t.Run("less batch", func(t *testing.T) { f(t, 50, 60) }) t.Run("more batch", func(t *testing.T) { f(t, 100, 60) }) t.Run("two batches", func(t *testing.T) { f(t, 100, 50) }) } func TestChangeSetMigrationThinHistorySuccess(t *testing.T) { f := func(t *testing.T, n, batchSize int) { db := ethdb.NewMemDatabase() accCS, storageCS := generateChangeSets(t, n) for i, v := range accCS { enc, err := changeset.EncodeChangeSet(v) if err != nil { t.Error(err) } err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), enc) if err != nil { t.Error(err) } } for i, v := range storageCS { enc, err := changeset.EncodeChangeSet(v) if err != nil { t.Error(err) } err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.StorageHistoryBucket), enc) if err != nil { t.Error(err) } } migrator := NewMigrator() migrator.Migrations = []Migration{splitChangeSetMigration(batchSize)} err := migrator.Apply(db, false, false, false, false, true) if err != nil { t.Error(err) } err = db.Walk(ChangeSetBucket, []byte{}, 0, func(k, v []byte) (b bool, e error) { ts, bucket := dbutils.DecodeTimestamp(k) return false, fmt.Errorf("changeset bucket is not empty block %v bucket %v", ts, string(bucket)) }) if err != nil { t.Error(err) } err = db.Walk(dbutils.AccountChangeSetBucket, []byte{}, 0, func(k, v []byte) (b bool, e error) { blockNum, _ := dbutils.DecodeTimestamp(k) cs, innerErr := changeset.DecodeAccounts(v) if innerErr != nil { return false, innerErr } if !cs.Equals(accCS[blockNum]) { spew.Dump(cs) spew.Dump(accCS[blockNum]) return false, fmt.Errorf("not equal %v", blockNum) } return true, nil }) if err != nil { t.Error(err) } err = db.Walk(dbutils.StorageHistoryBucket, []byte{}, 0, func(k, v []byte) (b bool, e error) { blockNum, _ := dbutils.DecodeTimestamp(k) cs, innerErr := changeset.DecodeStorage(v) if innerErr != nil { t.Error(innerErr, blockNum) } if !cs.Equals(storageCS[blockNum]) { spew.Dump(cs) spew.Dump(storageCS[blockNum]) return false, fmt.Errorf("not equal (%v)", blockNum) } return true, nil }) if err != nil { t.Error(err) } } t.Run("less batch", func(t *testing.T) { f(t, 50, 60) }) t.Run("more batch", func(t *testing.T) { f(t, 100, 60) }) t.Run("two batches", func(t *testing.T) { f(t, 100, 50) }) } func TestChangeSetMigrationFail(t *testing.T) { db := ethdb.NewMemDatabase() accCS, storageCS := generateChangeSets(t, 50) for i, v := range accCS { enc, err := changeset.EncodeChangeSet(v) if err != nil { t.Error(err) } if i == 25 { //incorrect value err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), enc[:5]) if err != nil { t.Error(err) } } else { err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.AccountsHistoryBucket), enc) if err != nil { t.Error(err) } } } for i, v := range storageCS { enc, err := changeset.EncodeChangeSet(v) if err != nil { t.Error(err) } err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(i)), dbutils.StorageHistoryBucket), enc) if err != nil { t.Error(err) } } migrator := NewMigrator() migrator.Migrations = []Migration{splitChangeSetMigration(20)} err := migrator.Apply(db, false, false, false, false, true) if err == nil { t.Error("should fail") } _, err = db.Get(dbutils.DatabaseInfoBucket, dbutils.LastAppliedMigration) if err == nil { t.Error("should fail") } //fix incorrect changeset enc, err := changeset.EncodeChangeSet(accCS[25]) if err != nil { t.Error(err) } err = db.Put(ChangeSetBucket, dbutils.CompositeChangeSetKey(dbutils.EncodeTimestamp(uint64(25)), dbutils.AccountsHistoryBucket), enc) if err != nil { t.Error(err) } err = migrator.Apply(db, false, false, false, false, true) if err != nil { t.Error(err) } err = db.Walk(ChangeSetBucket, []byte{}, 0, func(k, v []byte) (b bool, e error) { ts, bucket := dbutils.DecodeTimestamp(k) return false, fmt.Errorf("changeset bucket is not empty block %v bucket %v", ts, string(bucket)) }) if err != nil { t.Error(err) } err = db.Walk(dbutils.AccountChangeSetBucket, []byte{}, 0, func(k, v []byte) (b bool, e error) { blockNum, _ := dbutils.DecodeTimestamp(k) cs, innerErr := changeset.DecodeAccounts(v) if innerErr != nil { return false, innerErr } if !cs.Equals(accCS[blockNum]) { spew.Dump(cs) spew.Dump(accCS[blockNum]) return false, fmt.Errorf("not equal %v", blockNum) } return true, nil }) if err != nil { t.Error(err) } err = db.Walk(dbutils.StorageHistoryBucket, []byte{}, 0, func(k, v []byte) (b bool, e error) { blockNum, _ := dbutils.DecodeTimestamp(k) cs, innerErr := changeset.DecodeStorage(v) if innerErr != nil { t.Error(innerErr, blockNum) } if !cs.Equals(storageCS[blockNum]) { spew.Dump(cs) spew.Dump(storageCS[blockNum]) return false, fmt.Errorf("not equal (%v)", blockNum) } return true, nil }) if err != nil { t.Error(err) } } func generateChangeSets(t *testing.T, n int) ([]*changeset.ChangeSet, []*changeset.ChangeSet) { t.Helper() accChangeset := make([]*changeset.ChangeSet, n) storageChangeset := make([]*changeset.ChangeSet, n) for i := 0; i < n; i++ { csAcc := changeset.NewChangeSet() hash := common.Hash{uint8(i)} err := csAcc.Add(hash.Bytes(), hash.Bytes()) if err != nil { t.Fatal(err) } accChangeset[i] = csAcc csStorage := changeset.NewChangeSet() err = csStorage.Add( dbutils.GenerateCompositeStorageKey( hash, ^uint64(1), hash, ), hash.Bytes(), ) if err != nil { t.Fatal(err) } storageChangeset[i] = csStorage } return accChangeset, storageChangeset }