Update Shuffle Function (#1055)

beacon-chain/chaintest/backend/simulated_backend_test.go

@@ -27,7 +27,7 @@ func TestRunShuffleTest(t *testing.T) {
 	}
 	testCase := &ShuffleTestCase{
 		Input:  []uint32{1, 2, 3, 4, 5},
-		Output: []uint32{3, 2, 5, 1, 4},
+		Output: []uint32{2, 5, 3, 1, 4},
 		Seed:   "abcde",
 	}
 	if err := sb.RunShuffleTest(testCase); err != nil {

beacon-chain/core/state/state_transition_test.go

@@ -246,10 +246,10 @@ func TestNewValidatorSetRecalculationsInvalid(t *testing.T) {
 		t.Fatalf("Failed to initialize state: %v", err)
 	}
 	// Negative test case, shuffle validators with more than MaxValidatorRegistry.
-	size := params.BeaconConfig().ModuloBias + 1
+	size := 1<<(params.BeaconConfig().RandBytes*8) - 1
 	validators := make([]*pb.ValidatorRecord, size)
 	validator := &pb.ValidatorRecord{Status: uint64(params.Active)}
-	for i := uint64(0); i < size; i++ {
+	for i := 0; i < size; i++ {
 		validators[i] = validator
 	}
 	beaconState.SetValidatorRegistry(validators)

beacon-chain/core/validators/committees_test.go

@@ -39,10 +39,11 @@ func TestGetShardAndCommitteesForSlots(t *testing.T) {
 
 func TestExceedingMaxValidatorRegistryFails(t *testing.T) {
 	// Create more validators than ModuloBias defined in config, this should fail.
-	size := params.BeaconConfig().ModuloBias + 1
+	size := 1<<(params.BeaconConfig().RandBytes*8) - 1
+
 	validators := make([]*pb.ValidatorRecord, size)
 	validator := &pb.ValidatorRecord{Status: uint64(params.Active)}
-	for i := uint64(0); i < size; i++ {
+	for i := 0; i < size; i++ {
 		validators[i] = validator
 	}
 
@@ -55,7 +56,9 @@ func TestExceedingMaxValidatorRegistryFails(t *testing.T) {
 func BenchmarkMaxValidatorRegistry(b *testing.B) {
 	var validators []*pb.ValidatorRecord
 	validator := &pb.ValidatorRecord{}
-	for i := uint64(0); i < params.BeaconConfig().ModuloBias; i++ {
+	size := 1<<(params.BeaconConfig().RandBytes*8) - 1
+
+	for i := 0; i < size; i++ {
 		validators = append(validators, validator)
 	}
 

beacon-chain/simulator/service_test.go

@@ -44,7 +44,6 @@ func (mpow *mockPOWChainService) LatestBlockHash() common.Hash {
 
 func setupSimulator(t *testing.T, beaconDB *db.BeaconDB) (*Simulator, *mockP2P) {
 	ctx := context.Background()
-
 	p2pService := &mockP2P{}
 
 	err := beaconDB.InitializeState(nil)

beacon-chain/utils/shuffle.go

@@ -10,30 +10,53 @@ import (
 )
 
 // ShuffleIndices returns a list of pseudorandomly sampled
-// indices. This is used to use to select attesters and proposers.
-func ShuffleIndices(seed common.Hash, validatorList []uint32) ([]uint32, error) {
-	// Since we are consuming 3 bytes of entropy at a time in the loop,
+// indices. This is used to shuffle validators on ETH2.0 beacon chain.
+func ShuffleIndices(seed common.Hash, indicesList []uint32) ([]uint32, error) {
+	// Each entropy is consumed from the seed in randBytes chunks.
+	randBytes := params.BeaconConfig().RandBytes
+
+	maxValidatorsPerRandBytes := params.BeaconConfig().MaxNumLog2Validators / randBytes
+	upperBound := 1<<(randBytes*maxValidatorsPerRandBytes) - 1
+	// Since we are consuming randBytes of entropy at a time in the loop,
 	// we have a bias at 2**24, this check defines our max list size and is used to remove the bias.
 	// more info on modulo bias: https://stackoverflow.com/questions/10984974/why-do-people-say-there-is-modulo-bias-when-using-a-random-number-generator.
-	if uint64(len(validatorList)) >= params.BeaconConfig().ModuloBias {
-		return nil, errors.New("exceeded upper bound for validator shuffle")
+	if len(indicesList) >= upperBound {
+		return nil, errors.New("input list exceeded upper bound and reached modulo bias")
 	}
 
+	// Rehash the seed to obtain a new pattern of bytes.
 	hashSeed := hashutil.Hash(seed[:])
-	validatorCount := len(validatorList)
+	totalCount := len(indicesList)
+	index := 0
+	for index < totalCount-1 {
+		// Iterate through the hashSeed bytes in chunks of size randBytes.
+		for i := 0; i < 32-(32%int(randBytes)); i += int(randBytes) {
+			// Determine the number of indices remaining and exit if last index reached.
+			remaining := totalCount - index
+			if remaining == 1 {
+				break
+			}
+			// Read randBytes of hashSeed as a maxValidatorsPerRandBytes x randBytes big-endian integer.
+			randChunk := hashSeed[i : i+int(randBytes)]
+			var randValue int
+			for j := 0; j < int(randBytes); j++ {
+				randValue |= int(randChunk[j])
+			}
 
-	// Shuffle stops at the second to last index.
-	for i := 0; i < validatorCount-1; i++ {
-		// Convert every 3 bytes to random number, replace validator index with that number.
-		for j := 0; j+3 < len(hashSeed); j += 3 {
-			swapNum := int(hashSeed[j] + hashSeed[j+1] + hashSeed[j+2])
-			remaining := validatorCount - i
-			swapPos := swapNum%remaining + i
-			validatorList[i], validatorList[swapPos] = validatorList[swapPos], validatorList[i]
+			// Sample values greater than or equal to sampleMax will cause
+			// modulo bias when mapped into the remaining range.
+			randMax := upperBound - upperBound%remaining
+
+			// Perform swap if the consumed entropy will not cause modulo bias.
+			if randValue < randMax {
+				// Select replacement index from the current index.
+				replacementIndex := (randValue % remaining) + index
+				indicesList[index], indicesList[replacementIndex] = indicesList[replacementIndex], indicesList[index]
+				index++
+			}
 		}
-		hashSeed = hashutil.Hash(seed[:])
 	}
-	return validatorList, nil
+	return indicesList, nil
 }
 
 // SplitIndices splits a list into n pieces.

beacon-chain/utils/shuffle_test.go

@@ -11,7 +11,9 @@ import (
 func TestFaultyShuffleIndices(t *testing.T) {
 	var list []uint32
 
-	for i := uint64(0); i < params.BeaconConfig().ModuloBias+1; i++ {
+	upperBound := 1<<(params.BeaconConfig().RandBytes*8) - 1
+
+	for i := 0; i < upperBound+1; i++ {
 		list = append(list, uint32(i))
 	}
 
@@ -25,7 +27,7 @@ func TestShuffleIndices(t *testing.T) {
 	hash2 := common.BytesToHash([]byte{'1', '2', '3', '4', '5', '6', '7', '1', '2', '3', '4', '5', '6', '7', '1', '2', '3', '4', '5', '6', '7', '1', '2', '3', '4', '5', '6', '7', '1', '2', '3', '4', '5', '6', '7'})
 	var list1 []uint32
 
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		list1 = append(list1, uint32(i))
 	}
 
@@ -45,6 +47,12 @@ func TestShuffleIndices(t *testing.T) {
 	if reflect.DeepEqual(list1, list2) {
 		t.Errorf("2 shuffled lists shouldn't be equal")
 	}
+	if !reflect.DeepEqual(list1, []uint32{5, 4, 9, 6, 7, 3, 0, 1, 8, 2}) {
+		t.Errorf("list 1 was incorrectly shuffled")
+	}
+	if !reflect.DeepEqual(list2, []uint32{9, 0, 1, 5, 3, 2, 4, 7, 8, 6}) {
+		t.Errorf("list 2 was incorrectly shuffled")
+	}
 }
 
 func TestSplitIndices(t *testing.T) {

shared/params/config.go

@@ -46,10 +46,11 @@ type BeaconChainConfig struct {
 	InitialForkVersion                      uint64         // InitialForkVersion is used to track fork version between state transitions.
 	InitialForkSlot                         uint64         // InitialForkSlot is used to initialize the fork slot in the initial Beacon state.
 	SimulatedBlockRandao                    [32]byte       // SimulatedBlockRandao is a RANDAO seed stubbed in side simulated block to advance local beacon chain.
-	ModuloBias                              uint64         // ModuloBias is the upper bound of validator shuffle function. Can shuffle validator lists up to that size.
+	RandBytes                               uint64         // RandBytes is the number of bytes used as entropy to shuffle validators.
 	BootstrappedValidatorsCount             uint64         // BootstrappedValidatorsCount is the number of validators we seed to start beacon chain.
 	SyncPollingInterval                     int64          // SyncPollingInterval queries network nodes for sync status.
 	GenesisTime                             time.Time      // GenesisTime used by the protocol.
+	MaxNumLog2Validators                    uint64         // Max number of validators in Log2 can exist given total ETH supply.
 }
 
 // ShardChainConfig contains configs for node to participate in shard chains.
@@ -77,10 +78,11 @@ var defaultBeaconConfig = &BeaconChainConfig{
 	MaxValidatorChurnQuotient:     uint64(32),
 	InitialForkVersion:            0,
 	InitialForkSlot:               0,
-	ModuloBias:                    16777216 - 1,
+	RandBytes:                     3,
 	BootstrappedValidatorsCount:   16384,
 	SyncPollingInterval:           16 * 4, // Query nodes over the network every 4 slots for sync status.
 	GenesisTime:                   time.Date(2018, 9, 0, 0, 0, 0, 0, time.UTC),
+	MaxNumLog2Validators:          24,
 }
 
 var demoBeaconConfig = &BeaconChainConfig{
@@ -101,11 +103,12 @@ var demoBeaconConfig = &BeaconChainConfig{
 	BaseRewardQuotient:            uint64(32768),
 	MaxValidatorChurnQuotient:     uint64(32),
 	InitialForkVersion:            0,
+	RandBytes:                     3,
 	InitialForkSlot:               defaultBeaconConfig.InitialForkSlot,
-	ModuloBias:                    16777216 - 1,
 	SimulatedBlockRandao:          [32]byte{'S', 'I', 'M', 'U', 'L', 'A', 'T', 'E', 'R'},
 	SyncPollingInterval:           2 * 4, // Query nodes over the network every 4 slots for sync status.
 	GenesisTime:                   time.Now(),
+	MaxNumLog2Validators:          24,
 }
 
 var defaultShardConfig = &ShardChainConfig{