prysm-pulse/beacon-chain/db/kv/migration_state_validators_test.go
Mohamed Zahoor c6bd7a2f83
Efficient Historical State Representation (#9155)
* added migration in to new schema

* gazel fix

* goimports fix

* construct state with validator entries from a seperate bucket

* save state with validator entry indirection

* fixed save and retreieve issues

* fixed more test cases related to DeepEqual

* added save benchmark

* gazel fix

* organize benchmarks

* add cache and improve state construction  performance

* gazel fix

* check type

* remove cache checking from Has

* fix decoding when cache is off

* fix slice bugs

* add migration testcases

* linter fix

* fix few review comments

* fix review feedback

* gazel fix

* satisfy deepsource

* added the feature flag

* lint fix

* fix usage of featureconfig Inti in testcases

* call resetConfig method

* add feature flag in  migration testcases

* fix formatting

* change bucket name from blockRootValidatorKeysIndexBucket to blockRootValidatorHashesBucket

* remove from cache when state deleted

* fixed few more comments

* added to devModeFlags

* added cache delete under the feature flag

* fix lint

* change cache sizes and improve documentation

* fiexed few more review coments

* not using hash anymore and using a new SaveStates function

* satisfu deepsource

* run gazel

* fix feature flag related stuff

* fixing merge conflict fix

* few goodies

* improve UX and dont swallow error

* merge fix

* import format fix

* fix migrationion when flag not given issue

Co-authored-by: Radosław Kapka <rkapka@wp.pl>
Co-authored-by: Nishant Das <nishdas93@gmail.com>
2021-07-27 20:02:53 +00:00

231 lines
8.2 KiB
Go

package kv
import (
"bytes"
"context"
"testing"
"github.com/golang/snappy"
v1 "github.com/prysmaticlabs/prysm/beacon-chain/state/v1"
v1alpha1 "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
"github.com/prysmaticlabs/prysm/shared/featureconfig"
"github.com/prysmaticlabs/prysm/shared/testutil"
"github.com/prysmaticlabs/prysm/shared/testutil/assert"
"github.com/prysmaticlabs/prysm/shared/testutil/require"
"go.etcd.io/bbolt"
)
func Test_migrateStateValidators(t *testing.T) {
tests := []struct {
name string
setup func(t *testing.T, dbStore *Store, state *v1.BeaconState, vals []*v1alpha1.Validator)
eval func(t *testing.T, dbStore *Store, state *v1.BeaconState, vals []*v1alpha1.Validator)
}{
{
name: "only runs once",
setup: func(t *testing.T, dbStore *Store, state *v1.BeaconState, vals []*v1alpha1.Validator) {
// create some new buckets that should be present for this migration
err := dbStore.db.Update(func(tx *bbolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(stateValidatorsBucket)
assert.NoError(t, err)
_, err = tx.CreateBucketIfNotExists(blockRootValidatorHashesBucket)
assert.NoError(t, err)
return nil
})
assert.NoError(t, err)
// save the state
blockRoot := [32]byte{'A'}
require.NoError(t, dbStore.SaveState(context.Background(), state, blockRoot))
// set the migration as over
err = dbStore.db.Update(func(tx *bbolt.Tx) error {
return tx.Bucket(migrationsBucket).Put(migrationStateValidatorsKey, migrationCompleted)
})
assert.NoError(t, err)
},
eval: func(t *testing.T, dbStore *Store, state *v1.BeaconState, vals []*v1alpha1.Validator) {
// check if the migration is completed, per migration table.
err := dbStore.db.View(func(tx *bbolt.Tx) error {
migrationCompleteOrNot := tx.Bucket(migrationsBucket).Get(migrationStateValidatorsKey)
assert.DeepEqual(t, migrationCompleted, migrationCompleteOrNot, "migration is not complete")
return nil
})
assert.NoError(t, err)
},
},
{
name: "once migrated, always enable flag",
setup: func(t *testing.T, dbStore *Store, state *v1.BeaconState, vals []*v1alpha1.Validator) {
// create some new buckets that should be present for this migration
err := dbStore.db.Update(func(tx *bbolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(stateValidatorsBucket)
assert.NoError(t, err)
_, err = tx.CreateBucketIfNotExists(blockRootValidatorHashesBucket)
assert.NoError(t, err)
return nil
})
assert.NoError(t, err)
// set the migration as over
err = dbStore.db.Update(func(tx *bbolt.Tx) error {
return tx.Bucket(migrationsBucket).Put(migrationStateValidatorsKey, migrationCompleted)
})
assert.NoError(t, err)
},
eval: func(t *testing.T, dbStore *Store, state *v1.BeaconState, vals []*v1alpha1.Validator) {
// disable the flag and see if the code mandates that flag.
resetCfg := featureconfig.InitWithReset(&featureconfig.Flags{
EnableHistoricalSpaceRepresentation: false,
})
defer resetCfg()
// check if the migration is completed, per migration table.
err := dbStore.db.View(func(tx *bbolt.Tx) error {
migrationCompleteOrNot := tx.Bucket(migrationsBucket).Get(migrationStateValidatorsKey)
assert.DeepEqual(t, migrationCompleted, migrationCompleteOrNot, "migration is not complete")
return nil
})
assert.NoError(t, err)
// create a new state and save it
blockRoot := [32]byte{'B'}
st, err := testutil.NewBeaconState()
newValidators := validators(10)
assert.NoError(t, err)
assert.NoError(t, st.SetSlot(101))
assert.NoError(t, st.SetValidators(newValidators))
assert.NoError(t, dbStore.SaveState(context.Background(), st, blockRoot))
assert.NoError(t, err)
// now check if this newly saved state followed the migrated code path
// by checking if the new validators are saved in the validator bucket.
var individualHashes [][]byte
for _, val := range newValidators {
hash, hashErr := val.HashTreeRoot()
assert.NoError(t, hashErr)
individualHashes = append(individualHashes, hash[:])
}
pbState, err := v1.ProtobufBeaconState(st.InnerStateUnsafe())
assert.NoError(t, err)
validatorsFoundCount := 0
for _, val := range pbState.Validators {
hash, hashErr := val.HashTreeRoot()
assert.NoError(t, hashErr)
found := false
for _, h := range individualHashes {
if bytes.Equal(hash[:], h) {
found = true
}
}
require.Equal(t, true, found)
validatorsFoundCount++
}
require.Equal(t, len(vals), validatorsFoundCount)
},
},
{
name: "migrates validators and adds them to new buckets",
setup: func(t *testing.T, dbStore *Store, state *v1.BeaconState, vals []*v1alpha1.Validator) {
// create some new buckets that should be present for this migration
err := dbStore.db.Update(func(tx *bbolt.Tx) error {
_, err := tx.CreateBucketIfNotExists(stateValidatorsBucket)
assert.NoError(t, err)
_, err = tx.CreateBucketIfNotExists(blockRootValidatorHashesBucket)
assert.NoError(t, err)
return nil
})
assert.NoError(t, err)
// add a state with the given validators
blockRoot := [32]byte{'A'}
st, err := testutil.NewBeaconState()
assert.NoError(t, err)
assert.NoError(t, st.SetSlot(100))
assert.NoError(t, st.SetValidators(vals))
assert.NoError(t, dbStore.SaveState(context.Background(), st, blockRoot))
assert.NoError(t, err)
},
eval: func(t *testing.T, dbStore *Store, state *v1.BeaconState, vals []*v1alpha1.Validator) {
// check whether the new buckets are present
err := dbStore.db.View(func(tx *bbolt.Tx) error {
valBkt := tx.Bucket(stateValidatorsBucket)
assert.NotNil(t, valBkt)
idxBkt := tx.Bucket(blockRootValidatorHashesBucket)
assert.NotNil(t, idxBkt)
return nil
})
assert.NoError(t, err)
// check if the migration worked
blockRoot := [32]byte{'A'}
rcvdState, err := dbStore.State(context.Background(), blockRoot)
assert.NoError(t, err)
require.DeepSSZEqual(t, rcvdState.InnerStateUnsafe(), state.InnerStateUnsafe(), "saved state with validators and retrieved state are not matching")
// find hashes of the validators that are set as part of the state
var hashes []byte
var individualHashes [][]byte
for _, val := range vals {
hash, hashErr := val.HashTreeRoot()
assert.NoError(t, hashErr)
hashes = append(hashes, hash[:]...)
individualHashes = append(individualHashes, hash[:])
}
// check if all the validators that were in the state, are stored properly in the validator bucket
pbState, err := v1.ProtobufBeaconState(rcvdState.InnerStateUnsafe())
assert.NoError(t, err)
validatorsFoundCount := 0
for _, val := range pbState.Validators {
hash, hashErr := val.HashTreeRoot()
assert.NoError(t, hashErr)
found := false
for _, h := range individualHashes {
if bytes.Equal(hash[:], h) {
found = true
}
}
require.Equal(t, true, found)
validatorsFoundCount++
}
require.Equal(t, len(vals), validatorsFoundCount)
// check if the state validator indexes are stored properly
err = dbStore.db.View(func(tx *bbolt.Tx) error {
rcvdValhashBytes := tx.Bucket(blockRootValidatorHashesBucket).Get(blockRoot[:])
rcvdValHashes, sErr := snappy.Decode(nil, rcvdValhashBytes)
assert.NoError(t, sErr)
require.DeepEqual(t, hashes, rcvdValHashes)
return nil
})
assert.NoError(t, err)
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
dbStore := setupDB(t)
// enable historical state representation flag to test this
resetCfg := featureconfig.InitWithReset(&featureconfig.Flags{
EnableHistoricalSpaceRepresentation: true,
})
defer resetCfg()
// add a state with the given validators
vals := validators(10)
blockRoot := [32]byte{'A'}
st, err := testutil.NewBeaconState()
assert.NoError(t, err)
assert.NoError(t, st.SetSlot(100))
assert.NoError(t, st.SetValidators(vals))
assert.NoError(t, dbStore.SaveState(context.Background(), st, blockRoot))
assert.NoError(t, err)
tt.setup(t, dbStore, st, vals)
tt.eval(t, dbStore, st, vals)
})
}
}