From b30a0895488106560643f9f8b079ab15403c0e3b Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Thu, 19 Mar 2020 16:30:40 -0400 Subject: [PATCH] Add fetching validators by indices and public keys (#5141) * update ethereumapis with patch * Add indices and pubkeys to ListValidators request * Add sorting * Merge branch 'master' into validators-by-keys-indices * Rename to index * Merge branch 'validators-by-keys-indices' of https://github.com/prysmaticlabs/prysm into validators-by-keys-indices * Add comment --- WORKSPACE | 2 +- beacon-chain/rpc/beacon/validators.go | 68 +++++++++++++++---- beacon-chain/rpc/beacon/validators_test.go | 54 +++++++++++++++ ...thub_prysmaticlabs_ethereumapis-tags.patch | 12 ++-- 4 files changed, 116 insertions(+), 20 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 0998eaf54..526b44e2f 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1300,7 +1300,7 @@ go_repository( go_repository( name = "com_github_prysmaticlabs_ethereumapis", - commit = "fca4d6f69bedb8615c2fc916d1a68f2692285caa", + commit = "25f267e475788bf8e5e01cb9d73cfd0c87020822", importpath = "github.com/prysmaticlabs/ethereumapis", patch_args = ["-p1"], patches = [ diff --git a/beacon-chain/rpc/beacon/validators.go b/beacon-chain/rpc/beacon/validators.go index 8da995e34..1616db8c6 100644 --- a/beacon-chain/rpc/beacon/validators.go +++ b/beacon-chain/rpc/beacon/validators.go @@ -81,13 +81,10 @@ func (bs *Server) ListValidatorBalances( if len(pubKey) == 0 { continue } - - index, ok, err := bs.BeaconDB.ValidatorIndex(ctx, pubKey) - if err != nil { - return nil, status.Errorf(codes.Internal, "Could not retrieve validator index: %v", err) - } + pubkeyBytes := bytesutil.ToBytes48(pubKey) + index, ok := headState.ValidatorIndexByPubkey(pubkeyBytes) if !ok { - return nil, status.Errorf(codes.NotFound, "Could not find validator index for public key %#x", pubKey) + return nil, status.Errorf(codes.NotFound, "Could not find validator index for public key %#x", pubkeyBytes) } filtered[index] = true @@ -124,6 +121,10 @@ func (bs *Server) ListValidatorBalances( } balancesCount = len(res) } + // Depending on the indices and public keys given, results might not be sorted. + sort.Slice(res, func(i, j int) bool { + return res[i].Index < res[j].Index + }) // If there are no balances, we simply return a response specifying this. // Otherwise, attempting to paginate 0 balances below would result in an error. @@ -199,16 +200,55 @@ func (bs *Server) ListValidators( } validatorList := make([]*ethpb.Validators_ValidatorContainer, 0) - for i := 0; i < headState.NumValidators(); i++ { - val, err := headState.ValidatorAtIndex(uint64(i)) + + for _, index := range req.Indices { + val, err := headState.ValidatorAtIndex(index) if err != nil { return nil, status.Error(codes.Internal, "Could not get validator") } validatorList = append(validatorList, ðpb.Validators_ValidatorContainer{ - Index: uint64(i), + Index: index, Validator: val, }) } + + for _, pubKey := range req.PublicKeys { + // Skip empty public key. + if len(pubKey) == 0 { + continue + } + pubkeyBytes := bytesutil.ToBytes48(pubKey) + index, ok := headState.ValidatorIndexByPubkey(pubkeyBytes) + if !ok { + continue + } + val, err := headState.ValidatorAtIndex(index) + if err != nil { + return nil, status.Error(codes.Internal, "Could not get validator") + } + validatorList = append(validatorList, ðpb.Validators_ValidatorContainer{ + Index: index, + Validator: val, + }) + } + // Depending on the indices and public keys given, results might not be sorted. + sort.Slice(validatorList, func(i, j int) bool { + return validatorList[i].Index < validatorList[j].Index + }) + + if len(req.PublicKeys) == 0 && len(req.Indices) == 0 { + for i := 0; i < headState.NumValidators(); i++ { + val, err := headState.ValidatorAtIndex(uint64(i)) + if err != nil { + return nil, status.Error(codes.Internal, "Could not get validator") + } + validatorList = append(validatorList, ðpb.Validators_ValidatorContainer{ + Index: uint64(i), + Validator: val, + }) + } + } + if requestedEpoch < currentEpoch { stopIdx := len(validatorList) for idx, item := range validatorList { @@ -611,13 +651,15 @@ func (bs *Server) GetValidatorPerformance( correctlyVotedHead := make([]bool, 0, reqPubKeysCount) missingValidators := make([][]byte, 0, reqPubKeysCount) + headState, err := bs.HeadFetcher.HeadState(ctx) + if err != nil { + return nil, status.Error(codes.Internal, "Could not get head state") + } // Convert the list of validator public keys to list of validator indices. // Also track missing validators using public keys. for _, key := range req.PublicKeys { - idx, ok, err := bs.BeaconDB.ValidatorIndex(ctx, key) - if err != nil { - return nil, status.Errorf(codes.Internal, "Could not fetch validator idx for public key %#x: %v", key, err) - } + pubkeyBytes := bytesutil.ToBytes48(key) + idx, ok := headState.ValidatorIndexByPubkey(pubkeyBytes) if !ok { missingValidators = append(missingValidators, key) continue diff --git a/beacon-chain/rpc/beacon/validators_test.go b/beacon-chain/rpc/beacon/validators_test.go index e916e2fe7..df253772c 100644 --- a/beacon-chain/rpc/beacon/validators_test.go +++ b/beacon-chain/rpc/beacon/validators_test.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "fmt" "reflect" + "sort" "strconv" "strings" "testing" @@ -694,6 +695,59 @@ func TestServer_ListValidators_NoPagination(t *testing.T) { } } +func TestServer_ListValidators_IndicesPubKeys(t *testing.T) { + db := dbTest.SetupDB(t) + defer dbTest.TeardownDB(t, db) + + validators, _ := setupValidators(t, db, 100) + indicesWanted := []uint64{2, 7, 11, 17} + pubkeyIndicesWanted := []uint64{3, 5, 9, 15} + allIndicesWanted := append(indicesWanted, pubkeyIndicesWanted...) + want := make([]*ethpb.Validators_ValidatorContainer, len(allIndicesWanted)) + for i, idx := range allIndicesWanted { + want[i] = ðpb.Validators_ValidatorContainer{ + Index: idx, + Validator: validators[idx], + } + } + sort.Slice(want, func(i int, j int) bool { + return want[i].Index < want[j].Index + }) + + headState, err := db.HeadState(context.Background()) + if err != nil { + t.Fatal(err) + } + + bs := &Server{ + HeadFetcher: &mock.ChainService{ + State: headState, + }, + FinalizationFetcher: &mock.ChainService{ + FinalizedCheckPoint: ðpb.Checkpoint{ + Epoch: 0, + }, + }, + } + + pubKeysWanted := make([][]byte, len(pubkeyIndicesWanted)) + for i, indice := range pubkeyIndicesWanted { + pubKeysWanted[i] = pubKey(indice) + } + req := ðpb.ListValidatorsRequest{ + Indices: indicesWanted, + PublicKeys: pubKeysWanted, + } + received, err := bs.ListValidators(context.Background(), req) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(want, received.ValidatorList) { + t.Fatal("Incorrect respond of validators") + } +} + func TestServer_ListValidators_Pagination(t *testing.T) { db := dbTest.SetupDB(t) defer dbTest.TeardownDB(t, db) diff --git a/third_party/com_github_prysmaticlabs_ethereumapis-tags.patch b/third_party/com_github_prysmaticlabs_ethereumapis-tags.patch index 627605f48..5af807f06 100644 --- a/third_party/com_github_prysmaticlabs_ethereumapis-tags.patch +++ b/third_party/com_github_prysmaticlabs_ethereumapis-tags.patch @@ -298,7 +298,7 @@ index 2ce5c34..4cbb276 100644 + bytes signature = 3 [(gogoproto.moretags) = "ssz-size:\"96\""]; } diff --git a/eth/v1alpha1/beacon_chain.proto b/eth/v1alpha1/beacon_chain.proto -index 0099328..8b8c6eb 100644 +index 32d0b16..1679371 100644 --- a/eth/v1alpha1/beacon_chain.proto +++ b/eth/v1alpha1/beacon_chain.proto @@ -15,6 +15,7 @@ syntax = "proto3"; @@ -363,7 +363,7 @@ index 0099328..8b8c6eb 100644 // Validator's index in the validator set. uint64 index = 2; -@@ -552,7 +553,7 @@ message GetValidatorRequest { +@@ -560,7 +561,7 @@ message GetValidatorRequest { uint64 index = 1; // 48 byte validator public key. @@ -372,7 +372,7 @@ index 0099328..8b8c6eb 100644 } } -@@ -594,26 +595,25 @@ message ActiveSetChanges { +@@ -602,26 +603,25 @@ message ActiveSetChanges { uint64 epoch = 1; // 48 byte validator public keys that have been activated in the given epoch. @@ -405,7 +405,7 @@ index 0099328..8b8c6eb 100644 // Indices of validators ejected in the given epoch. repeated uint64 ejected_indices = 9; -@@ -663,11 +663,11 @@ message ValidatorQueue { +@@ -671,11 +671,11 @@ message ValidatorQueue { // Ordered list of 48 byte public keys awaiting activation. 0th index is the // next key to be processed. @@ -419,7 +419,7 @@ index 0099328..8b8c6eb 100644 } message ListValidatorAssignmentsRequest { -@@ -679,7 +679,7 @@ message ListValidatorAssignmentsRequest { +@@ -687,7 +687,7 @@ message ListValidatorAssignmentsRequest { bool genesis = 2; } // 48 byte validator public keys to filter assignments for the given epoch. @@ -428,7 +428,7 @@ index 0099328..8b8c6eb 100644 // Validator indicies to filter assignments for the given epoch. repeated uint64 indices = 4; -@@ -714,7 +714,7 @@ message ValidatorAssignments { +@@ -722,7 +722,7 @@ message ValidatorAssignments { uint64 proposer_slot = 4; // 48 byte BLS public key.