diff --git a/beacon-chain/p2p/discovery.go b/beacon-chain/p2p/discovery.go index 558363e8e..dda954a63 100644 --- a/beacon-chain/p2p/discovery.go +++ b/beacon-chain/p2p/discovery.go @@ -85,6 +85,8 @@ func (s *Service) listenForNewNodes() { log.WithError(err).Error("Could not convert to peer info") continue } + // Make sure that peer is not dialed too often, for each connection attempt there's a backoff period. + s.Peers().RandomizeBackOff(peerInfo.ID) go func(info *peer.AddrInfo) { if err := s.connectWithPeer(s.ctx, *info); err != nil { log.WithError(err).Tracef("Could not connect with peer %s", info.String()) diff --git a/beacon-chain/p2p/peers/BUILD.bazel b/beacon-chain/p2p/peers/BUILD.bazel index 447c9924f..cacf31568 100644 --- a/beacon-chain/p2p/peers/BUILD.bazel +++ b/beacon-chain/p2p/peers/BUILD.bazel @@ -12,6 +12,7 @@ go_library( "//beacon-chain/p2p/peers/scorers:go_default_library", "//proto/beacon/p2p/v1:go_default_library", "//shared/params:go_default_library", + "//shared/rand:go_default_library", "//shared/timeutils:go_default_library", "@com_github_ethereum_go_ethereum//p2p/enr:go_default_library", "@com_github_gogo_protobuf//proto:go_default_library", diff --git a/beacon-chain/p2p/peers/status.go b/beacon-chain/p2p/peers/status.go index 48d68f80c..1af875f7d 100644 --- a/beacon-chain/p2p/peers/status.go +++ b/beacon-chain/p2p/peers/status.go @@ -24,6 +24,7 @@ package peers import ( "context" + "math" "sort" "time" @@ -40,6 +41,7 @@ import ( "github.com/prysmaticlabs/prysm/beacon-chain/p2p/peers/scorers" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" "github.com/prysmaticlabs/prysm/shared/params" + "github.com/prysmaticlabs/prysm/shared/rand" "github.com/prysmaticlabs/prysm/shared/timeutils" ) @@ -63,6 +65,13 @@ const ( // InboundRatio is the proportion of our connected peer limit at which we will allow inbound peers. InboundRatio = float64(0.8) + + // MinBackOffDuration minimum amount (in milliseconds) to wait before peer is re-dialed. + // When node and peer are dialing each other simultaneously connection may fail. In order, to break + // of constant dialing, peer is assigned some backoff period, and only dialed again once that backoff is up. + MinBackOffDuration = 100 + // MaxBackOffDuration maximum amount (in milliseconds) to wait before peer is re-dialed. + MaxBackOffDuration = 5000 ) // Status is the structure holding the peer status information. @@ -71,6 +80,7 @@ type Status struct { scorers *scorers.Service store *peerdata.Store ipTracker map[string]uint64 + rand *rand.Rand } // StatusConfig represents peer status service params. @@ -91,6 +101,9 @@ func NewStatus(ctx context.Context, config *StatusConfig) *Status { store: store, scorers: scorers.NewService(ctx, store, config.ScorerParams), ipTracker: map[string]uint64{}, + // Random generator used to calculate dial backoff period. + // It is ok to use deterministic generator, no need for true entropy. + rand: rand.NewDeterministicGenerator(), } } @@ -336,6 +349,22 @@ func (p *Status) SetNextValidTime(pid peer.ID, nextTime time.Time) { peerData.NextValidTime = nextTime } +// RandomizeBackOff adds extra backoff period during which peer will not be dialed. +func (p *Status) RandomizeBackOff(pid peer.ID) { + p.store.Lock() + defer p.store.Unlock() + + peerData := p.store.PeerDataGetOrCreate(pid) + + // No need to add backoff period, if the previous one hasn't expired yet. + if !time.Now().After(peerData.NextValidTime) { + return + } + + duration := time.Duration(math.Max(MinBackOffDuration, float64(p.rand.Intn(MaxBackOffDuration)))) * time.Millisecond + peerData.NextValidTime = time.Now().Add(duration) +} + // IsReadyToDial checks where the given peer is ready to be // dialed again. func (p *Status) IsReadyToDial(pid peer.ID) bool { diff --git a/endtoend/BUILD.bazel b/endtoend/BUILD.bazel index 249f8b86e..de7c7b722 100644 --- a/endtoend/BUILD.bazel +++ b/endtoend/BUILD.bazel @@ -7,6 +7,7 @@ go_test( size = "large", testonly = True, srcs = [ + "connectivity_e2e_test.go", "endtoend_test.go", "minimal_e2e_test.go", "minimal_slashing_e2e_test.go", diff --git a/endtoend/connectivity_e2e_test.go b/endtoend/connectivity_e2e_test.go new file mode 100644 index 000000000..a39fb226a --- /dev/null +++ b/endtoend/connectivity_e2e_test.go @@ -0,0 +1,31 @@ +package endtoend + +import ( + "testing" + + ev "github.com/prysmaticlabs/prysm/endtoend/evaluators" + e2eParams "github.com/prysmaticlabs/prysm/endtoend/params" + "github.com/prysmaticlabs/prysm/endtoend/types" + "github.com/prysmaticlabs/prysm/shared/params" + "github.com/prysmaticlabs/prysm/shared/testutil" + "github.com/prysmaticlabs/prysm/shared/testutil/require" +) + +func TestEndToEnd_Connectivity(t *testing.T) { + // This test isolates all the preliminary networking setup necessary for other e2e tests. + // This allows easier checks for connectivity, discovery and peering issues. + testutil.ResetCache() + params.UseE2EConfig() + require.NoError(t, e2eParams.Init(e2eParams.StandardBeaconCount)) + + testConfig := &types.E2EConfig{ + BeaconFlags: []string{}, + ValidatorFlags: []string{}, + EpochsToRun: 2, + Evaluators: []types.Evaluator{ + ev.PeersConnect, + }, + } + + newTestRunner(t, testConfig).run() +}