mirror of
https://gitlab.com/pulsechaincom/prysm-pulse.git
synced 2025-01-08 18:51:19 +00:00
Refactor generated benchmark files to allow for more general usage (#4436)
* Begin to refactor benchmark files to testutil * Complete most of refactoring * Fix file path * gofmt * Fix path * Move generatego to tools/ * Move gen util to tools/benchmark-files-gen * Add comments to pregen funcs * Make function names consistent * Update README * Redo benchmarks with 16384 validators
This commit is contained in:
Ivan Martinez
GitHub
parent
0605118686
commit
d04399ea96
@ -16,6 +16,7 @@ go_library(
|
||||
visibility = [
|
||||
"//beacon-chain:__subpackages__",
|
||||
"//shared/testutil:__pkg__",
|
||||
"//shared/benchutil/benchmark_files:__subpackages__",
|
||||
"//slasher:__subpackages__",
|
||||
"//tools:__subpackages__",
|
||||
"//validator:__subpackages__",
|
||||
|
||||
@ -14,6 +14,7 @@ go_library(
|
||||
"//beacon-chain:__subpackages__",
|
||||
"//shared/interop:__pkg__",
|
||||
"//shared/testutil:__pkg__",
|
||||
"//tools/benchmark-files-gen:__pkg__",
|
||||
"//tools/genesis-state-gen:__pkg__",
|
||||
],
|
||||
deps = [
|
||||
@ -46,19 +47,23 @@ go_test(
|
||||
name = "go_default_test",
|
||||
size = "small",
|
||||
srcs = [
|
||||
"benchmarks_test.go",
|
||||
"skip_slot_cache_test.go",
|
||||
"state_test.go",
|
||||
"transition_test.go",
|
||||
],
|
||||
data = ["//shared/benchutil/benchmark_files:benchmark_data"],
|
||||
embed = [":go_default_library"],
|
||||
deps = [
|
||||
"//beacon-chain/core/blocks:go_default_library",
|
||||
"//beacon-chain/core/helpers:go_default_library",
|
||||
"//proto/beacon/p2p/v1:go_default_library",
|
||||
"//shared/benchutil:go_default_library",
|
||||
"//shared/bls:go_default_library",
|
||||
"//shared/featureconfig:go_default_library",
|
||||
"//shared/hashutil:go_default_library",
|
||||
"//shared/params:go_default_library",
|
||||
"//shared/stateutil:go_default_library",
|
||||
"//shared/testutil:go_default_library",
|
||||
"//shared/trieutil:go_default_library",
|
||||
"@com_github_gogo_protobuf//proto:go_default_library",
|
||||
|
||||
@ -1,28 +0,0 @@
|
||||
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = ["benchmarks_test.go"],
|
||||
data = ["//beacon-chain/core/state/benchmarks/benchmark_files:benchmark_data"],
|
||||
embed = [":go_default_library"],
|
||||
deps = [
|
||||
"//beacon-chain/core/helpers:go_default_library",
|
||||
"//beacon-chain/core/state:go_default_library",
|
||||
"//proto/beacon/p2p/v1:go_default_library",
|
||||
"//shared/params:go_default_library",
|
||||
"//shared/stateutil:go_default_library",
|
||||
"@com_github_gogo_protobuf//proto:go_default_library",
|
||||
"@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
|
||||
"@com_github_prysmaticlabs_go_ssz//:go_default_library",
|
||||
"@io_bazel_rules_go//go/tools/bazel:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
testonly = True,
|
||||
srcs = ["config.go"],
|
||||
importpath = "github.com/prysmaticlabs/prysm/beacon-chain/core/state/benchmarks",
|
||||
visibility = ["//beacon-chain/core/state/benchmarks:__subpackages__"],
|
||||
deps = ["//shared/params:go_default_library"],
|
||||
)
|
||||
@ -1,51 +0,0 @@
|
||||
# Benchmarks for Prysm State Transition
|
||||
This package contains the functionality needed for benchmarking Prysms state transitions, this includes its block processing (with and without caching) and epoch processing functions. There is also a benchmark for HashTreeRoot on a large beacon state.
|
||||
|
||||
## Benchmark Configuration
|
||||
The following configs are in `config.go`:
|
||||
* `ValidatorCount`: Sets the amount of active validators to perform the benchmarks with. Default is 65536.
|
||||
* `AttestationsPerEpoch`: Sets the amount of attestations per epoch for the benchmarks to perform with, this affects the amount of attestations in a full block and the amount of attestations per epoch in the state for the `ProcessEpoch` and `HashTreeRoot` benchmark. Default is 128.
|
||||
|
||||
## Generating new SSZ files
|
||||
Due to the sheer size of the benchmarking configurations (65536 validators), the files used for benchmarking are pregenerated so there's no wasted computations on generating a genesis state with 65536 validators. This should only be needed if there is a breaking spec change and the tests fail from SSZ issues.
|
||||
|
||||
To generate new files to use for benchmarking, run the below command in the root of Prysm.
|
||||
```
|
||||
go run beacon-chain/core/state/benchmarks/benchmark_files/generate_bench_files.go
|
||||
```
|
||||
|
||||
Bazel does not allow writing to the project directory, so running with `go run` is needed.
|
||||
|
||||
## Running the benchmarks
|
||||
To run the ExecuteStateTransition benchmark:
|
||||
|
||||
```bazel test //beacon-chain/core/state/benchmarks:go_default_test --test_filter=BenchmarkExecuteStateTransition_FullBlock --test_arg=-test.bench=BenchmarkExecuteStateTransition_FullBlock```
|
||||
|
||||
To run the ExecuteStateTransition (with cache) benchmark:
|
||||
|
||||
```bazel test //beacon-chain/core/state/benchmarks:go_default_test --test_filter=BenchmarkExecuteStateTransition_WithCache --test_arg=-test.bench=BenchmarkExecuteStateTransition_WithCache```
|
||||
|
||||
To run the ProcessEpoch benchmark:
|
||||
|
||||
```bazel test //beacon-chain/core/state/benchmarks:go_default_test --test_filter=BenchmarkProcessEpoch_2FullEpochs --test_arg=-test.bench=BenchmarkProcessEpoch_2FullEpochs```
|
||||
|
||||
To run the HashTreeRoot benchmark:
|
||||
|
||||
```bazel test //beacon-chain/core/state/benchmarks:go_default_test --test_filter=BenchmarkHashTreeRoot_FullState --test_arg=-test.bench=BenchmarkHashTreeRoot_FullState```
|
||||
|
||||
To run the HashTreeRootState benchmark:
|
||||
|
||||
```bazel test //beacon-chain/core/state/benchmarks:go_default_test --test_filter=BenchmarkHashTreeRootState_FullState --test_arg=-test.bench=BenchmarkHashTreeRootState_FullState```
|
||||
|
||||
Extra flags needed to benchmark properly:
|
||||
|
||||
```--nocache_test_results --test_arg=-test.v --test_timeout=2000 --test_arg=-test.cpuprofile=/tmp/cpu.profile --test_arg=-test.memprofile=/tmp/mem.profile --test_output=streamed```
|
||||
|
||||
## Current Results as of November 2019
|
||||
```
|
||||
BenchmarkExecuteStateTransition-4 20 33020593584 ns/op
|
||||
BenchmarkExecuteStateTransition_WithCache-4 20 21272276477 ns/op
|
||||
BenchmarkProcessEpoch_2FullEpochs-4 5 158161708836 ns/op
|
||||
BenchmarkHashTreeRoot_FullState-4 50 1509721280 ns/op
|
||||
BenchmarkHashTreeRootState_FullState-4 50 67622586 ns/op
|
||||
```
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -1,48 +0,0 @@
|
||||
package benchmarks
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/prysmaticlabs/prysm/shared/params"
|
||||
)
|
||||
|
||||
// ValidatorCount is for declaring how many validators the benchmarks will be
|
||||
// performed with. Default is 65536 or 2M ETH staked.
|
||||
var ValidatorCount = uint64(65536)
|
||||
|
||||
// AttestationsPerEpoch represents the requested amount attestations in an epoch.
|
||||
// This affects the amount of attestations in a fully attested for block and the amount
|
||||
// of attestations in the state per epoch, so a full 2 epochs should result in twice
|
||||
// this amount of attestations in the state. Default is 128.
|
||||
var AttestationsPerEpoch = uint64(128)
|
||||
|
||||
// GenesisFileName is the generated genesis beacon state file name.
|
||||
var GenesisFileName = fmt.Sprintf("benchmark_files/bStateGenesis-%dAtts-%dVals.ssz", AttestationsPerEpoch, ValidatorCount)
|
||||
|
||||
// BState1EpochFileName is the generated beacon state after 1 skipped epoch file name.
|
||||
var BState1EpochFileName = fmt.Sprintf("benchmark_files/bState1Epoch-%dAtts-%dVals.ssz", AttestationsPerEpoch, ValidatorCount)
|
||||
|
||||
// BState2EpochFileName is the generated beacon state after 2 full epochs file name.
|
||||
var BState2EpochFileName = fmt.Sprintf("benchmark_files/bState2Epochs-%dAtts-%dVals.ssz", AttestationsPerEpoch, ValidatorCount)
|
||||
|
||||
// FullBlockFileName is the generated full block file name.
|
||||
var FullBlockFileName = fmt.Sprintf("benchmark_files/fullBlock-%dAtts-%dVals.ssz", AttestationsPerEpoch, ValidatorCount)
|
||||
|
||||
// FilePath prefixes the file path to the file names.
|
||||
func FilePath(fileName string) string {
|
||||
return fmt.Sprintf("beacon-chain/core/state/benchmarks/%s", fileName)
|
||||
}
|
||||
|
||||
// SetConfig changes the beacon config to match the requested amount of
|
||||
// attestations set to AttestationsPerEpoch.
|
||||
func SetConfig() {
|
||||
maxAtts := AttestationsPerEpoch
|
||||
slotsPerEpoch := params.BeaconConfig().SlotsPerEpoch
|
||||
committeeSize := (ValidatorCount / slotsPerEpoch) / (maxAtts / slotsPerEpoch)
|
||||
c := params.BeaconConfig()
|
||||
c.PersistentCommitteePeriod = 0
|
||||
c.MinValidatorWithdrawabilityDelay = 0
|
||||
c.TargetCommitteeSize = committeeSize
|
||||
c.MaxAttestations = maxAtts
|
||||
params.OverrideBeaconConfig(c)
|
||||
}
|
||||
@ -1,17 +1,15 @@
|
||||
package benchmarks
|
||||
package state
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
|
||||
"github.com/bazelbuild/rules_go/go/tools/bazel"
|
||||
"github.com/gogo/protobuf/proto"
|
||||
ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
|
||||
"github.com/prysmaticlabs/go-ssz"
|
||||
"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
|
||||
"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
|
||||
pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
|
||||
"github.com/prysmaticlabs/prysm/shared/benchutil"
|
||||
"github.com/prysmaticlabs/prysm/shared/featureconfig"
|
||||
"github.com/prysmaticlabs/prysm/shared/params"
|
||||
"github.com/prysmaticlabs/prysm/shared/stateutil"
|
||||
)
|
||||
@ -19,30 +17,29 @@ import (
|
||||
var runAmount = 25
|
||||
|
||||
func TestBenchmarkExecuteStateTransition(t *testing.T) {
|
||||
t.Skip("TODO(4098): Regenerate test data with v0.9.2 spec")
|
||||
SetConfig()
|
||||
beaconState, err := beaconState1Epoch()
|
||||
benchutil.SetBenchmarkConfig()
|
||||
beaconState, err := benchutil.PreGenState1Epoch()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
block, err := fullBlock()
|
||||
block, err := benchutil.PreGenFullBlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := state.ExecuteStateTransition(context.Background(), beaconState, block); err != nil {
|
||||
if _, err := ExecuteStateTransition(context.Background(), beaconState, block); err != nil {
|
||||
t.Fatalf("failed to process block, benchmarks will fail: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkExecuteStateTransition_FullBlock(b *testing.B) {
|
||||
SetConfig()
|
||||
beaconState, err := beaconState1Epoch()
|
||||
benchutil.SetBenchmarkConfig()
|
||||
beaconState, err := benchutil.PreGenState1Epoch()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
cleanStates := clonedStates(beaconState)
|
||||
block, err := fullBlock()
|
||||
block, err := benchutil.PreGenFullBlock()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@ -50,21 +47,26 @@ func BenchmarkExecuteStateTransition_FullBlock(b *testing.B) {
|
||||
b.N = runAmount
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
if _, err := state.ExecuteStateTransition(context.Background(), cleanStates[i], block); err != nil {
|
||||
if _, err := ExecuteStateTransition(context.Background(), cleanStates[i], block); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkExecuteStateTransition_WithCache(b *testing.B) {
|
||||
SetConfig()
|
||||
config := &featureconfig.Flags{
|
||||
EnableProposerIndexCache: true,
|
||||
EnableAttestationCache: true,
|
||||
}
|
||||
featureconfig.Init(config)
|
||||
benchutil.SetBenchmarkConfig()
|
||||
|
||||
beaconState, err := beaconState1Epoch()
|
||||
beaconState, err := benchutil.PreGenState1Epoch()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
cleanStates := clonedStates(beaconState)
|
||||
block, err := fullBlock()
|
||||
block, err := benchutil.PreGenFullBlock()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@ -78,22 +80,27 @@ func BenchmarkExecuteStateTransition_WithCache(b *testing.B) {
|
||||
}
|
||||
beaconState.Slot = currentSlot
|
||||
// Run the state transition once to populate the cache.
|
||||
if _, err := state.ExecuteStateTransition(context.Background(), beaconState, block); err != nil {
|
||||
if _, err := ExecuteStateTransition(context.Background(), beaconState, block); err != nil {
|
||||
b.Fatalf("failed to process block, benchmarks will fail: %v", err)
|
||||
}
|
||||
|
||||
b.N = runAmount
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
if _, err := state.ExecuteStateTransition(context.Background(), cleanStates[i], block); err != nil {
|
||||
if _, err := ExecuteStateTransition(context.Background(), cleanStates[i], block); err != nil {
|
||||
b.Fatalf("failed to process block, benchmarks will fail: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkProcessEpoch_2FullEpochs(b *testing.B) {
|
||||
SetConfig()
|
||||
beaconState, err := beaconState2FullEpochs()
|
||||
config := &featureconfig.Flags{
|
||||
EnableProposerIndexCache: true,
|
||||
EnableAttestationCache: true,
|
||||
}
|
||||
featureconfig.Init(config)
|
||||
benchutil.SetBenchmarkConfig()
|
||||
beaconState, err := benchutil.PreGenState2FullEpochs()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@ -113,14 +120,15 @@ func BenchmarkProcessEpoch_2FullEpochs(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
// ProcessEpochPrecompute is the optimized version of process epoch. It's enabled by default
|
||||
// at run time.
|
||||
if _, err := state.ProcessEpochPrecompute(context.Background(), cleanStates[i]); err != nil {
|
||||
b.Log(i)
|
||||
if _, err := ProcessEpochPrecompute(context.Background(), cleanStates[i]); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkHashTreeRoot_FullState(b *testing.B) {
|
||||
beaconState, err := beaconState2FullEpochs()
|
||||
beaconState, err := benchutil.PreGenState2FullEpochs()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@ -135,7 +143,7 @@ func BenchmarkHashTreeRoot_FullState(b *testing.B) {
|
||||
}
|
||||
|
||||
func BenchmarkHashTreeRootState_FullState(b *testing.B) {
|
||||
beaconState, err := beaconState2FullEpochs()
|
||||
beaconState, err := benchutil.PreGenState2FullEpochs()
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@ -161,51 +169,3 @@ func clonedStates(beaconState *pb.BeaconState) []*pb.BeaconState {
|
||||
}
|
||||
return clonedStates
|
||||
}
|
||||
|
||||
func beaconState1Epoch() (*pb.BeaconState, error) {
|
||||
path, err := bazel.Runfile(BState1EpochFileName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
beaconBytes, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
beaconState := &pb.BeaconState{}
|
||||
if err := ssz.Unmarshal(beaconBytes, beaconState); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return beaconState, nil
|
||||
}
|
||||
|
||||
func beaconState2FullEpochs() (*pb.BeaconState, error) {
|
||||
path, err := bazel.Runfile(BState2EpochFileName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
beaconBytes, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
beaconState := &pb.BeaconState{}
|
||||
if err := ssz.Unmarshal(beaconBytes, beaconState); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return beaconState, nil
|
||||
}
|
||||
|
||||
func fullBlock() (*ethpb.SignedBeaconBlock, error) {
|
||||
path, err := bazel.Runfile(FullBlockFileName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
blockBytes, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
beaconBlock := ðpb.SignedBeaconBlock{}
|
||||
if err := ssz.Unmarshal(blockBytes, beaconBlock); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return beaconBlock, nil
|
||||
}
|
||||
22
shared/benchutil/BUILD.bazel
Normal file
22
shared/benchutil/BUILD.bazel
Normal file
@ -0,0 +1,22 @@
|
||||
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = ["pregen.go"],
|
||||
data = ["//shared/benchutil/benchmark_files:benchmark_data"],
|
||||
importpath = "github.com/prysmaticlabs/prysm/shared/benchutil",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//proto/beacon/p2p/v1:go_default_library",
|
||||
"//shared/params:go_default_library",
|
||||
"@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
|
||||
"@com_github_prysmaticlabs_go_ssz//:go_default_library",
|
||||
"@io_bazel_rules_go//go/tools/bazel:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = ["pregen_test.go"],
|
||||
embed = [":go_default_library"],
|
||||
)
|
||||
48
shared/benchutil/README.md
Normal file
48
shared/benchutil/README.md
Normal file
@ -0,0 +1,48 @@
|
||||
# Benchmarks for Prysm State Transition
|
||||
This package contains the functionality needed for benchmarking Prysms state transitions, this includes its block processing (with and without caching) and epoch processing functions. There is also a benchmark for HashTreeRoot on a large beacon state.
|
||||
|
||||
## Benchmark Configuration
|
||||
The following configs are in `config.go`:
|
||||
* `ValidatorCount`: Sets the amount of active validators to perform the benchmarks with. Default is 16384.
|
||||
* `AttestationsPerEpoch`: Sets the amount of attestations per epoch for the benchmarks to perform with, this affects the amount of attestations in a full block and the amount of attestations per epoch in the state for the `ProcessEpoch` and `HashTreeRoot` benchmark. Default is 128.
|
||||
|
||||
## Generating new SSZ files
|
||||
Due to the sheer size of the benchmarking configurations (16384 validators), the files used for benchmarking are pregenerated so there's no wasted computations on generating a genesis state with 16384 validators. This should only be needed if there is a breaking spec change and the tests fail from SSZ issues.
|
||||
|
||||
To generate new files to use for benchmarking, run the below command in the root of Prysm.
|
||||
```
|
||||
bazel run //tools/benchmark-files-gen -- --output-dir $PRYSMPATH/shared/benchutil/benchmark_files/ --overwrite
|
||||
```
|
||||
|
||||
## Running the benchmarks
|
||||
To run the ExecuteStateTransition benchmark:
|
||||
|
||||
```bazel test //beacon-chain/core/state:go_default_test --test_filter=BenchmarkExecuteStateTransition_FullBlock --test_arg=-test.bench=BenchmarkExecuteStateTransition_FullBlock```
|
||||
|
||||
To run the ExecuteStateTransition (with cache) benchmark:
|
||||
|
||||
```bazel test //beacon-chain/core/state:go_default_test --test_filter=BenchmarkExecuteStateTransition_WithCache --test_arg=-test.bench=BenchmarkExecuteStateTransition_WithCache```
|
||||
|
||||
To run the ProcessEpoch benchmark:
|
||||
|
||||
```bazel test //beacon-chain/core/state:go_default_test --test_filter=BenchmarkProcessEpoch_2FullEpochs --test_arg=-test.bench=BenchmarkProcessEpoch_2FullEpochs```
|
||||
|
||||
To run the HashTreeRoot benchmark:
|
||||
|
||||
```bazel test //beacon-chain/core/state:go_default_test --test_filter=BenchmarkHashTreeRoot_FullState --test_arg=-test.bench=BenchmarkHashTreeRoot_FullState```
|
||||
|
||||
To run the HashTreeRootState benchmark:
|
||||
|
||||
```bazel test //beacon-chain/core/state:go_default_test --test_filter=BenchmarkHashTreeRootState_FullState --test_arg=-test.bench=BenchmarkHashTreeRootState_FullState```
|
||||
|
||||
Extra flags needed to benchmark properly:
|
||||
|
||||
```--nocache_test_results --test_arg=-test.v --test_timeout=2000 --test_arg=-test.cpuprofile=/tmp/cpu.profile --test_arg=-test.memprofile=/tmp/mem.profile --test_output=streamed```
|
||||
|
||||
## Current Results as of January 2020
|
||||
```
|
||||
BenchmarkExecuteStateTransition_FullBlock-4 20 2031438030 ns/op
|
||||
BenchmarkExecuteStateTransition_WithCache-4 20 1857290454 ns/op
|
||||
BenchmarkHashTreeRoot_FullState-4 50 297655834 ns/op
|
||||
BenchmarkHashTreeRootState_FullState-4 50 155535883 ns/op
|
||||
```
|
||||
10
shared/benchutil/benchmark_files/BUILD.bazel
Normal file
10
shared/benchutil/benchmark_files/BUILD.bazel
Normal file
@ -0,0 +1,10 @@
|
||||
filegroup(
|
||||
name = "benchmark_data",
|
||||
srcs = glob([
|
||||
"*.ssz",
|
||||
]),
|
||||
visibility = [
|
||||
"//beacon-chain/core/state:__pkg__",
|
||||
"//shared/benchutil:__subpackages__",
|
||||
],
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
BIN
shared/benchutil/benchmark_files/fullBlock-128Atts-16384Vals.ssz
Normal file
BIN
shared/benchutil/benchmark_files/fullBlock-128Atts-16384Vals.ssz
Normal file
Binary file not shown.
103
shared/benchutil/pregen.go
Normal file
103
shared/benchutil/pregen.go
Normal file
@ -0,0 +1,103 @@
|
||||
package benchutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/bazelbuild/rules_go/go/tools/bazel"
|
||||
ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
|
||||
"github.com/prysmaticlabs/go-ssz"
|
||||
pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
|
||||
"github.com/prysmaticlabs/prysm/shared/params"
|
||||
)
|
||||
|
||||
// ValidatorCount is for declaring how many validators the benchmarks will be
|
||||
// performed with. Default is 16384 or 524K ETH staked.
|
||||
var ValidatorCount = uint64(16384)
|
||||
|
||||
// AttestationsPerEpoch represents the requested amount attestations in an epoch.
|
||||
// This affects the amount of attestations in a fully attested for block and the amount
|
||||
// of attestations in the state per epoch, so a full 2 epochs should result in twice
|
||||
// this amount of attestations in the state. Default is 128.
|
||||
var AttestationsPerEpoch = uint64(128)
|
||||
|
||||
// GenesisFileName is the generated genesis beacon state file name.
|
||||
var GenesisFileName = fmt.Sprintf("bStateGenesis-%dAtts-%dVals.ssz", AttestationsPerEpoch, ValidatorCount)
|
||||
|
||||
// BState1EpochFileName is the generated beacon state after 1 skipped epoch file name.
|
||||
var BState1EpochFileName = fmt.Sprintf("bState1Epoch-%dAtts-%dVals.ssz", AttestationsPerEpoch, ValidatorCount)
|
||||
|
||||
// BState2EpochFileName is the generated beacon state after 2 full epochs file name.
|
||||
var BState2EpochFileName = fmt.Sprintf("bState2Epochs-%dAtts-%dVals.ssz", AttestationsPerEpoch, ValidatorCount)
|
||||
|
||||
// FullBlockFileName is the generated full block file name.
|
||||
var FullBlockFileName = fmt.Sprintf("fullBlock-%dAtts-%dVals.ssz", AttestationsPerEpoch, ValidatorCount)
|
||||
|
||||
func filePath(path string) string {
|
||||
return fmt.Sprintf("shared/benchutil/benchmark_files/%s", path)
|
||||
}
|
||||
|
||||
// PreGenState1Epoch unmarshals the pre-generated beacon state after 1 epoch of block processing and returns it.
|
||||
func PreGenState1Epoch() (*pb.BeaconState, error) {
|
||||
path, err := bazel.Runfile(filePath(BState1EpochFileName))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
beaconBytes, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
beaconState := &pb.BeaconState{}
|
||||
if err := ssz.Unmarshal(beaconBytes, beaconState); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return beaconState, nil
|
||||
}
|
||||
|
||||
// PreGenState2FullEpochs unmarshals the pre-generated beacon state after 2 epoch of full block processing and returns it.
|
||||
func PreGenState2FullEpochs() (*pb.BeaconState, error) {
|
||||
path, err := bazel.Runfile(filePath(BState2EpochFileName))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
beaconBytes, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
beaconState := &pb.BeaconState{}
|
||||
if err := ssz.Unmarshal(beaconBytes, beaconState); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return beaconState, nil
|
||||
}
|
||||
|
||||
// PreGenFullBlock unmarshals the pre-generated signed beacon block containing an epochs worth of attestations and returns it.
|
||||
func PreGenFullBlock() (*ethpb.SignedBeaconBlock, error) {
|
||||
path, err := bazel.Runfile(filePath(FullBlockFileName))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
blockBytes, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
beaconBlock := ðpb.SignedBeaconBlock{}
|
||||
if err := ssz.Unmarshal(blockBytes, beaconBlock); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return beaconBlock, nil
|
||||
}
|
||||
|
||||
// SetBenchmarkConfig changes the beacon config to match the requested amount of
|
||||
// attestations set to AttestationsPerEpoch.
|
||||
func SetBenchmarkConfig() {
|
||||
maxAtts := AttestationsPerEpoch
|
||||
slotsPerEpoch := params.BeaconConfig().SlotsPerEpoch
|
||||
committeeSize := (ValidatorCount / slotsPerEpoch) / (maxAtts / slotsPerEpoch)
|
||||
c := params.BeaconConfig()
|
||||
c.PersistentCommitteePeriod = 0
|
||||
c.MinValidatorWithdrawabilityDelay = 0
|
||||
c.TargetCommitteeSize = committeeSize
|
||||
c.MaxAttestations = maxAtts
|
||||
params.OverrideBeaconConfig(c)
|
||||
}
|
||||
26
shared/benchutil/pregen_test.go
Normal file
26
shared/benchutil/pregen_test.go
Normal file
@ -0,0 +1,26 @@
|
||||
package benchutil
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPreGenFullBlock(t *testing.T) {
|
||||
_, err := PreGenFullBlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreGenState1Epoch(t *testing.T) {
|
||||
_, err := PreGenFullBlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreGenState2FullEpochs(t *testing.T) {
|
||||
_, err := PreGenFullBlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@ -3,34 +3,26 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
testonly = True,
|
||||
srcs = ["generate_bench_files.go"],
|
||||
data = glob(["*.ssz"]),
|
||||
importpath = "github.com/prysmaticlabs/prysm/beacon-chain/core/state/benchmarks/benchmark_files",
|
||||
visibility = ["//beacon-chain/core/state/benchmarks:__subpackages__"],
|
||||
srcs = ["main.go"],
|
||||
importpath = "github.com/prysmaticlabs/prysm/tools/benchmark-files-gen",
|
||||
visibility = ["//visibility:private"],
|
||||
deps = [
|
||||
"//beacon-chain/core/helpers:go_default_library",
|
||||
"//beacon-chain/core/state:go_default_library",
|
||||
"//beacon-chain/core/state/benchmarks:go_default_library",
|
||||
"//proto/beacon/p2p/v1:go_default_library",
|
||||
"//shared/benchutil:go_default_library",
|
||||
"//shared/interop:go_default_library",
|
||||
"//shared/params:go_default_library",
|
||||
"//shared/testutil:go_default_library",
|
||||
"@com_github_pkg_errors//:go_default_library",
|
||||
"@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
|
||||
"@com_github_prysmaticlabs_go_ssz//:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "benchmark_data",
|
||||
srcs = glob([
|
||||
"*.ssz",
|
||||
]),
|
||||
visibility = ["//beacon-chain/core/state/benchmarks:__subpackages__"],
|
||||
)
|
||||
|
||||
go_binary(
|
||||
name = "benchmark_files",
|
||||
name = "benchmark-files-gen",
|
||||
testonly = True,
|
||||
embed = [":go_default_library"],
|
||||
visibility = ["//visibility:private"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
@ -2,43 +2,73 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
|
||||
"github.com/prysmaticlabs/go-ssz"
|
||||
"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
|
||||
"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
|
||||
bench "github.com/prysmaticlabs/prysm/beacon-chain/core/state/benchmarks"
|
||||
pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
|
||||
"github.com/prysmaticlabs/prysm/shared/benchutil"
|
||||
"github.com/prysmaticlabs/prysm/shared/interop"
|
||||
"github.com/prysmaticlabs/prysm/shared/params"
|
||||
"github.com/prysmaticlabs/prysm/shared/testutil"
|
||||
)
|
||||
|
||||
var (
|
||||
outputDir = flag.String("output-dir", "", "Directory to write SSZ files to")
|
||||
overwrite = flag.Bool("overwrite", false, "If SSZ files exist in the output directory, they will be overwritten")
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
if *outputDir == "" {
|
||||
log.Fatal("Please specify --output-dir to write SSZ files to")
|
||||
}
|
||||
|
||||
if !*overwrite {
|
||||
if _, err := os.Stat(path.Join(*outputDir, benchutil.BState1EpochFileName)); err == nil {
|
||||
log.Fatal("The file exists. Use a different file name or the --overwrite flag")
|
||||
}
|
||||
if _, err := os.Stat(path.Join(*outputDir, benchutil.BState2EpochFileName)); err == nil {
|
||||
log.Fatal("The file exists. Use a different file name or the --overwrite flag")
|
||||
}
|
||||
if _, err := os.Stat(path.Join(*outputDir, benchutil.FullBlockFileName)); err == nil {
|
||||
log.Fatal("The file exists. Use a different file name or the --overwrite flag")
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(*outputDir, os.ModePerm); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
log.Printf("Output dir is: %s", *outputDir)
|
||||
log.Println("Generating genesis state")
|
||||
// Generating this for the 2 following states.
|
||||
if err := generateGenesisBeaconState(); err != nil {
|
||||
log.Fatal(err)
|
||||
log.Fatalf("Could not generate genesis state: %v", err)
|
||||
}
|
||||
log.Println("Generating full block and state after 1 skipped epoch")
|
||||
if err := generateMarshalledFullStateAndBlock(); err != nil {
|
||||
log.Fatal(err)
|
||||
log.Fatalf("Could not generate full state and block: %v", err)
|
||||
}
|
||||
log.Println("Generating state after 2 fully attested epochs")
|
||||
if err := generate2FullEpochState(); err != nil {
|
||||
log.Fatal(err)
|
||||
log.Fatalf("Could not generate 2 full epoch state: %v", err)
|
||||
}
|
||||
// Removing this since its 10MB large and no longer needed.
|
||||
if err := os.Remove(bench.FilePath(bench.GenesisFileName)); err != nil {
|
||||
// Removing the genesis state SSZ since its 10MB large and no longer needed.
|
||||
if err := os.Remove(path.Join(*outputDir, benchutil.GenesisFileName)); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func generateGenesisBeaconState() error {
|
||||
genesisState, _, err := interop.GenerateGenesisState(0, bench.ValidatorCount)
|
||||
genesisState, _, err := interop.GenerateGenesisState(0, benchutil.ValidatorCount)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -46,20 +76,20 @@ func generateGenesisBeaconState() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ioutil.WriteFile(bench.FilePath(bench.GenesisFileName), beaconBytes, 0644); err != nil {
|
||||
if err := ioutil.WriteFile(path.Join(*outputDir, benchutil.GenesisFileName), beaconBytes, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func generateMarshalledFullStateAndBlock() error {
|
||||
bench.SetConfig()
|
||||
benchutil.SetBenchmarkConfig()
|
||||
beaconState, err := genesisBeaconState()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
privs, _, err := interop.DeterministicallyGenerateKeys(0, bench.ValidatorCount)
|
||||
privs, _, err := interop.DeterministicallyGenerateKeys(0, benchutil.ValidatorCount)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -78,7 +108,7 @@ func generateMarshalledFullStateAndBlock() error {
|
||||
}
|
||||
|
||||
attConfig := &testutil.BlockGenConfig{
|
||||
NumAttestations: bench.AttestationsPerEpoch / slotsPerEpoch,
|
||||
NumAttestations: benchutil.AttestationsPerEpoch / slotsPerEpoch,
|
||||
}
|
||||
|
||||
atts := []*ethpb.Attestation{}
|
||||
@ -92,18 +122,18 @@ func generateMarshalledFullStateAndBlock() error {
|
||||
|
||||
block, err = testutil.GenerateFullBlock(beaconState, privs, attConfig, beaconState.Slot)
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Wrap(err, "could not generate full block")
|
||||
}
|
||||
block.Block.Body.Attestations = append(atts, block.Block.Body.Attestations...)
|
||||
|
||||
s, err := state.CalculateStateRoot(context.Background(), beaconState, block)
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Wrap(err, "could not calculate state root")
|
||||
}
|
||||
block.Block.StateRoot = s[:]
|
||||
blockRoot, err := ssz.HashTreeRoot(block.Block)
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Wrap(err, "could not get signing root of block")
|
||||
}
|
||||
// Temporarily incrementing the beacon state slot here since BeaconProposerIndex is a
|
||||
// function deterministic on beacon state slot.
|
||||
@ -120,7 +150,7 @@ func generateMarshalledFullStateAndBlock() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ioutil.WriteFile(bench.FilePath(bench.BState1EpochFileName), beaconBytes, 0644); err != nil {
|
||||
if err := ioutil.WriteFile(path.Join(*outputDir, benchutil.BState1EpochFileName), beaconBytes, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -134,26 +164,26 @@ func generateMarshalledFullStateAndBlock() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ioutil.WriteFile(bench.FilePath(bench.FullBlockFileName), blockBytes, 0644); err != nil {
|
||||
if err := ioutil.WriteFile(path.Join(*outputDir, benchutil.FullBlockFileName), blockBytes, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func generate2FullEpochState() error {
|
||||
bench.SetConfig()
|
||||
benchutil.SetBenchmarkConfig()
|
||||
beaconState, err := genesisBeaconState()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
privs, _, err := interop.DeterministicallyGenerateKeys(0, bench.ValidatorCount)
|
||||
privs, _, err := interop.DeterministicallyGenerateKeys(0, benchutil.ValidatorCount)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
attConfig := &testutil.BlockGenConfig{
|
||||
NumAttestations: bench.AttestationsPerEpoch / params.BeaconConfig().SlotsPerEpoch,
|
||||
NumAttestations: benchutil.AttestationsPerEpoch / params.BeaconConfig().SlotsPerEpoch,
|
||||
}
|
||||
|
||||
for i := uint64(0); i < params.BeaconConfig().SlotsPerEpoch*2-1; i++ {
|
||||
@ -171,20 +201,20 @@ func generate2FullEpochState() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ioutil.WriteFile(bench.FilePath(bench.BState2EpochFileName), beaconBytes, 0644); err != nil {
|
||||
if err := ioutil.WriteFile(path.Join(*outputDir, benchutil.BState2EpochFileName), beaconBytes, 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func genesisBeaconState() (*pb.BeaconState, error) {
|
||||
beaconBytes, err := ioutil.ReadFile(bench.FilePath(bench.GenesisFileName))
|
||||
beaconBytes, err := ioutil.ReadFile(path.Join(*outputDir, benchutil.GenesisFileName))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, errors.Wrap(err, "cannot read genesis state file")
|
||||
}
|
||||
genesisState := &pb.BeaconState{}
|
||||
if err := ssz.Unmarshal(beaconBytes, genesisState); err != nil {
|
||||
return nil, err
|
||||
return nil, errors.Wrap(err, "cannot unmarshal genesis state file")
|
||||
}
|
||||
return genesisState, nil
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user