From acc307b959b36e7464416b8d6e608fe0b77fbbfe Mon Sep 17 00:00:00 2001 From: Joel Rousseau Date: Thu, 28 Mar 2024 01:15:39 +0900 Subject: [PATCH] Command-line interface for visualizing min/max span bucket (#13748) * add max/min span visualisation tool cli * go mod tidy * lint imports * remove typo * fix epoch table value * fix deepsource * add dep to bazel * fix dep import order * change command name from span to slasher-span-display * change command args style using - instead of _ * sed s/CONFIGURATION/SLASHER PARAMS// * change double neg to double pos condition * remove unused anonymous func * better function naming * add range condition * [deepsource] Fix Empty slice literal used to declare a variable GO-W1027 * correct typo * do not show incorrect epochs due to round robin * fix import --------- Co-authored-by: Manu NALEPA --- beacon-chain/db/iface/interface.go | 2 +- beacon-chain/db/slasherkv/slasher.go | 12 +- beacon-chain/db/slasherkv/slasher_test.go | 2 +- beacon-chain/slasher/BUILD.bazel | 3 + beacon-chain/slasher/detect_attestations.go | 88 +++--- beacon-chain/slasher/helpers.go | 93 ++++++ beacon-chain/slasher/params.go | 29 +- beacon-chain/slasher/params_test.go | 2 +- beacon-chain/slasher/service.go | 2 +- beacon-chain/slasher/types/BUILD.bazel | 5 +- beacon-chain/slasher/types/types.go | 12 + cmd/prysmctl/db/BUILD.bazel | 5 + cmd/prysmctl/db/cmd.go | 1 + cmd/prysmctl/db/span.go | 304 ++++++++++++++++++++ deps.bzl | 6 + go.mod | 5 +- go.sum | 10 +- 17 files changed, 525 insertions(+), 56 deletions(-) create mode 100644 cmd/prysmctl/db/span.go diff --git a/beacon-chain/db/iface/interface.go b/beacon-chain/db/iface/interface.go index 75da4d87d..9f56e2444 100644 --- a/beacon-chain/db/iface/interface.go +++ b/beacon-chain/db/iface/interface.go @@ -118,7 +118,7 @@ type HeadAccessDatabase interface { // SlasherDatabase interface for persisting data related to detecting slashable offenses on Ethereum. type SlasherDatabase interface { io.Closer - SaveLastEpochsWrittenForValidators( + SaveLastEpochWrittenForValidators( ctx context.Context, epochByValidator map[primitives.ValidatorIndex]primitives.Epoch, ) error SaveAttestationRecordsForValidators( diff --git a/beacon-chain/db/slasherkv/slasher.go b/beacon-chain/db/slasherkv/slasher.go index 05af590d9..e840d0a55 100644 --- a/beacon-chain/db/slasherkv/slasher.go +++ b/beacon-chain/db/slasherkv/slasher.go @@ -70,12 +70,12 @@ func (s *Store) LastEpochWrittenForValidators( return attestedEpochs, err } -// SaveLastEpochsWrittenForValidators updates the latest epoch a slice -// of validator indices has attested to. -func (s *Store) SaveLastEpochsWrittenForValidators( +// SaveLastEpochWrittenForValidators saves the latest epoch +// that each validator has attested to in the provided map. +func (s *Store) SaveLastEpochWrittenForValidators( ctx context.Context, epochByValIndex map[primitives.ValidatorIndex]primitives.Epoch, ) error { - ctx, span := trace.StartSpan(ctx, "BeaconDB.SaveLastEpochsWrittenForValidators") + ctx, span := trace.StartSpan(ctx, "BeaconDB.SaveLastEpochWrittenForValidators") defer span.End() const batchSize = 10000 @@ -157,7 +157,7 @@ func (s *Store) CheckAttesterDoubleVotes( attRecordsBkt := tx.Bucket(attestationRecordsBucket) encEpoch := encodeTargetEpoch(attToProcess.IndexedAttestation.Data.Target.Epoch) - localDoubleVotes := []*slashertypes.AttesterDoubleVote{} + localDoubleVotes := make([]*slashertypes.AttesterDoubleVote, 0) for _, valIdx := range attToProcess.IndexedAttestation.AttestingIndices { // Check if there is signing root in the database for this combination @@ -166,7 +166,7 @@ func (s *Store) CheckAttesterDoubleVotes( validatorEpochKey := append(encEpoch, encIdx...) attRecordsKey := signingRootsBkt.Get(validatorEpochKey) - // An attestation record key is comprised of a signing root (32 bytes). + // An attestation record key consists of a signing root (32 bytes). if len(attRecordsKey) < attestationRecordKeySize { // If there is no signing root for this combination, // then there is no double vote. We can continue to the next validator. diff --git a/beacon-chain/db/slasherkv/slasher_test.go b/beacon-chain/db/slasherkv/slasher_test.go index 2567b18c4..2bbac54cf 100644 --- a/beacon-chain/db/slasherkv/slasher_test.go +++ b/beacon-chain/db/slasherkv/slasher_test.go @@ -89,7 +89,7 @@ func TestStore_LastEpochWrittenForValidators(t *testing.T) { require.NoError(t, err) require.Equal(t, 0, len(attestedEpochs)) - err = beaconDB.SaveLastEpochsWrittenForValidators(ctx, epochsByValidator) + err = beaconDB.SaveLastEpochWrittenForValidators(ctx, epochsByValidator) require.NoError(t, err) retrievedEpochs, err := beaconDB.LastEpochWrittenForValidators(ctx, indices) diff --git a/beacon-chain/slasher/BUILD.bazel b/beacon-chain/slasher/BUILD.bazel index b2d89da53..00bf4aaae 100644 --- a/beacon-chain/slasher/BUILD.bazel +++ b/beacon-chain/slasher/BUILD.bazel @@ -19,6 +19,7 @@ go_library( importpath = "github.com/prysmaticlabs/prysm/v5/beacon-chain/slasher", visibility = [ "//beacon-chain:__subpackages__", + "//cmd/prysmctl:__subpackages__", "//testing/slasher/simulator:__subpackages__", ], deps = [ @@ -27,6 +28,7 @@ go_library( "//beacon-chain/core/blocks:go_default_library", "//beacon-chain/core/feed/state:go_default_library", "//beacon-chain/db:go_default_library", + "//beacon-chain/db/slasherkv:go_default_library", "//beacon-chain/operations/slashings:go_default_library", "//beacon-chain/slasher/types:go_default_library", "//beacon-chain/startup:go_default_library", @@ -45,6 +47,7 @@ go_library( "@com_github_prysmaticlabs_fastssz//:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", "@io_opencensus_go//trace:go_default_library", + "@org_golang_x_exp//maps:go_default_library", ], ) diff --git a/beacon-chain/slasher/detect_attestations.go b/beacon-chain/slasher/detect_attestations.go index 3b695d0f4..8bc349c9b 100644 --- a/beacon-chain/slasher/detect_attestations.go +++ b/beacon-chain/slasher/detect_attestations.go @@ -11,6 +11,7 @@ import ( "github.com/prysmaticlabs/prysm/v5/consensus-types/primitives" ethpb "github.com/prysmaticlabs/prysm/v5/proto/prysm/v1alpha1" "go.opencensus.io/trace" + "golang.org/x/exp/maps" ) // Takes in a list of indexed attestation wrappers and returns any @@ -131,7 +132,7 @@ func (s *Service) checkSurroundVotes( } // Update the latest updated epoch for all validators involved to the current chunk. - indexes := s.params.validatorIndexesInChunk(validatorChunkIndex) + indexes := s.params.ValidatorIndexesInChunk(validatorChunkIndex) for _, index := range indexes { s.latestEpochUpdatedForValidator[index] = currentEpoch } @@ -272,44 +273,20 @@ func (s *Service) updatedChunkByChunkIndex( // minFirstEpochToUpdate is set to the smallest first epoch to update for all validators in the chunk // corresponding to the `validatorChunkIndex`. - var minFirstEpochToUpdate *primitives.Epoch + var ( + minFirstEpochToUpdate *primitives.Epoch + neededChunkIndexesMap map[uint64]bool - neededChunkIndexesMap := map[uint64]bool{} + err error + ) + validatorIndexes := s.params.ValidatorIndexesInChunk(validatorChunkIndex) - validatorIndexes := s.params.validatorIndexesInChunk(validatorChunkIndex) - for _, validatorIndex := range validatorIndexes { - // Retrieve the first epoch to write for the validator index. - isAnEpochToUpdate, firstEpochToUpdate, err := s.firstEpochToUpdate(validatorIndex, currentEpoch) - if err != nil { - return nil, errors.Wrapf(err, "could not get first epoch to write for validator index %d with current epoch %d", validatorIndex, currentEpoch) - } - - if !isAnEpochToUpdate { - // If there is no epoch to write, skip. - continue - } - - // If, for this validator index, the chunk corresponding to the first epoch to write - // (and all following epochs until the current epoch) are already flagged as needed, - // skip. - if minFirstEpochToUpdate != nil && *minFirstEpochToUpdate <= firstEpochToUpdate { - continue - } - - minFirstEpochToUpdate = &firstEpochToUpdate - - // Add new needed chunk indexes to the map. - for i := firstEpochToUpdate; i <= currentEpoch; i++ { - chunkIndex := s.params.chunkIndex(i) - neededChunkIndexesMap[chunkIndex] = true - } + if neededChunkIndexesMap, err = s.findNeededChunkIndexes(validatorIndexes, currentEpoch, minFirstEpochToUpdate); err != nil { + return nil, errors.Wrap(err, "could not find the needed chunk indexed") } - // Get the list of needed chunk indexes. - neededChunkIndexes := make([]uint64, 0, len(neededChunkIndexesMap)) - for chunkIndex := range neededChunkIndexesMap { - neededChunkIndexes = append(neededChunkIndexes, chunkIndex) - } + // Transform the map of needed chunk indexes to a slice. + neededChunkIndexes := maps.Keys(neededChunkIndexesMap) // Retrieve needed chunks from the database. chunkByChunkIndex, err := s.loadChunksFromDisk(ctx, validatorChunkIndex, chunkKind, neededChunkIndexes) @@ -332,7 +309,7 @@ func (s *Service) updatedChunkByChunkIndex( epochToUpdate := firstEpochToUpdate for epochToUpdate <= currentEpoch { - // Get the chunk index for the ecpoh to write. + // Get the chunk index for the epoch to write. chunkIndex := s.params.chunkIndex(epochToUpdate) // Get the chunk corresponding to the chunk index from the `chunkByChunkIndex` map. @@ -363,6 +340,45 @@ func (s *Service) updatedChunkByChunkIndex( return chunkByChunkIndex, nil } +// findNeededChunkIndexes returns a map of chunk indexes +// it loops over the validator indexes and finds the first epoch to update for each validator index. +func (s *Service) findNeededChunkIndexes( + validatorIndexes []primitives.ValidatorIndex, + currentEpoch primitives.Epoch, + minFirstEpochToUpdate *primitives.Epoch, +) (map[uint64]bool, error) { + neededChunkIndexesMap := map[uint64]bool{} + + for _, validatorIndex := range validatorIndexes { + // Retrieve the first epoch to write for the validator index. + isAnEpochToUpdate, firstEpochToUpdate, err := s.firstEpochToUpdate(validatorIndex, currentEpoch) + if err != nil { + return nil, errors.Wrapf(err, "could not get first epoch to write for validator index %d with current epoch %d", validatorIndex, currentEpoch) + } + + if !isAnEpochToUpdate { + // If there is no epoch to write, skip. + continue + } + + // If, for this validator index, the chunk corresponding to the first epoch to write + // (and all following epochs until the current epoch) are already flagged as needed, + // skip. + if minFirstEpochToUpdate != nil && *minFirstEpochToUpdate <= firstEpochToUpdate { + continue + } + + minFirstEpochToUpdate = &firstEpochToUpdate + + // Add new needed chunk indexes to the map. + for i := firstEpochToUpdate; i <= currentEpoch; i++ { + chunkIndex := s.params.chunkIndex(i) + neededChunkIndexesMap[chunkIndex] = true + } + } + return neededChunkIndexesMap, nil +} + // firstEpochToUpdate, given a validator index and the current epoch, returns a boolean indicating // if there is an epoch to write. If it is the case, it returns the first epoch to write. func (s *Service) firstEpochToUpdate(validatorIndex primitives.ValidatorIndex, currentEpoch primitives.Epoch) (bool, primitives.Epoch, error) { diff --git a/beacon-chain/slasher/helpers.go b/beacon-chain/slasher/helpers.go index 534d020d3..26768f3d9 100644 --- a/beacon-chain/slasher/helpers.go +++ b/beacon-chain/slasher/helpers.go @@ -2,8 +2,11 @@ package slasher import ( "bytes" + "context" + "fmt" "strconv" + "github.com/prysmaticlabs/prysm/v5/beacon-chain/db/slasherkv" slashertypes "github.com/prysmaticlabs/prysm/v5/beacon-chain/slasher/types" fieldparams "github.com/prysmaticlabs/prysm/v5/config/fieldparams" "github.com/prysmaticlabs/prysm/v5/config/params" @@ -159,3 +162,93 @@ func isDoubleProposal(incomingSigningRoot, existingSigningRoot [32]byte) bool { } return incomingSigningRoot != existingSigningRoot } + +type GetChunkFromDatabaseFilters struct { + ChunkKind slashertypes.ChunkKind + ValidatorIndex primitives.ValidatorIndex + SourceEpoch primitives.Epoch + IsDisplayAllValidatorsInChunk bool + IsDisplayAllEpochsInChunk bool +} + +// GetChunkFromDatabase Utility function aiming at retrieving a chunk from the +// database. +func GetChunkFromDatabase( + ctx context.Context, + dbPath string, + filters GetChunkFromDatabaseFilters, + params *Parameters, +) (lastEpochForValidatorIndex primitives.Epoch, chunkIndex, validatorChunkIndex uint64, chunk Chunker, err error) { + // init store + d, err := slasherkv.NewKVStore(ctx, dbPath) + if err != nil { + return lastEpochForValidatorIndex, chunkIndex, validatorChunkIndex, chunk, fmt.Errorf("could not open database at path %s: %w", dbPath, err) + } + defer closeDB(d) + + // init service + s := Service{ + params: params, + serviceCfg: &ServiceConfig{ + Database: d, + }, + } + + // variables + validatorIndex := filters.ValidatorIndex + sourceEpoch := filters.SourceEpoch + chunkKind := filters.ChunkKind + validatorChunkIndex = s.params.validatorChunkIndex(validatorIndex) + chunkIndex = s.params.chunkIndex(sourceEpoch) + + // before getting the chunk, we need to verify if the requested epoch is in database + lastEpochForValidator, err := s.serviceCfg.Database.LastEpochWrittenForValidators(ctx, []primitives.ValidatorIndex{validatorIndex}) + if err != nil { + return lastEpochForValidatorIndex, + chunkIndex, + validatorChunkIndex, + chunk, + fmt.Errorf("could not get last epoch written for validator %d: %w", validatorIndex, err) + } + + if len(lastEpochForValidator) == 0 { + return lastEpochForValidatorIndex, + chunkIndex, + validatorChunkIndex, + chunk, + fmt.Errorf("could not get information at epoch %d for validator %d: there's no record found in slasher database", + sourceEpoch, validatorIndex, + ) + } + lastEpochForValidatorIndex = lastEpochForValidator[0].Epoch + + // if the epoch requested is within the range, we can proceed to get the chunk, otherwise return error + atBestSmallestEpoch := lastEpochForValidatorIndex.Sub(uint64(params.historyLength)) + if sourceEpoch < atBestSmallestEpoch || sourceEpoch > lastEpochForValidatorIndex { + return lastEpochForValidatorIndex, + chunkIndex, + validatorChunkIndex, + chunk, + fmt.Errorf("requested epoch %d is outside the slasher history length %d, data can be provided within the epoch range [%d:%d] for validator %d", + sourceEpoch, params.historyLength, atBestSmallestEpoch, lastEpochForValidatorIndex, validatorIndex, + ) + } + + // fetch chunk from DB + chunk, err = s.getChunkFromDatabase(ctx, chunkKind, validatorChunkIndex, chunkIndex) + if err != nil { + return lastEpochForValidatorIndex, + chunkIndex, + validatorChunkIndex, + chunk, + fmt.Errorf("could not get chunk at index %d: %w", chunkIndex, err) + } + + return lastEpochForValidatorIndex, chunkIndex, validatorChunkIndex, chunk, nil +} + +func closeDB(d *slasherkv.Store) { + if err := d.Close(); err != nil { + log.WithError(err).Error("could not close database") + } +} diff --git a/beacon-chain/slasher/params.go b/beacon-chain/slasher/params.go index 5512c5411..59cf044e9 100644 --- a/beacon-chain/slasher/params.go +++ b/beacon-chain/slasher/params.go @@ -16,6 +16,21 @@ type Parameters struct { historyLength primitives.Epoch // H - defines how many epochs we keep of min or max spans. } +// ChunkSize returns the chunk size. +func (p *Parameters) ChunkSize() uint64 { + return p.chunkSize +} + +// ValidatorChunkSize returns the validator chunk size. +func (p *Parameters) ValidatorChunkSize() uint64 { + return p.validatorChunkSize +} + +// HistoryLength returns the history length. +func (p *Parameters) HistoryLength() primitives.Epoch { + return p.historyLength +} + // DefaultParams defines default values for slasher's important parameters, defined // based on optimization analysis for best and worst case scenarios for // slasher's performance. @@ -32,7 +47,15 @@ func DefaultParams() *Parameters { } } -// Validator min and max spans are split into chunks of length C = chunkSize. +func NewParams(chunkSize, validatorChunkSize uint64, historyLength primitives.Epoch) *Parameters { + return &Parameters{ + chunkSize: chunkSize, + validatorChunkSize: validatorChunkSize, + historyLength: historyLength, + } +} + +// ChunkIndex Validator min and max spans are split into chunks of length C = chunkSize. // That is, if we are keeping N epochs worth of attesting history, finding what // chunk a certain epoch, e, falls into can be computed as (e % N) / C. For example, // if we are keeping 6 epochs worth of data, and we have chunks of size 2, then epoch @@ -139,9 +162,9 @@ func (p *Parameters) flatSliceID(validatorChunkIndex, chunkIndex uint64) []byte return ssz.MarshalUint64(make([]byte, 0), uint64(width.Mul(validatorChunkIndex).Add(chunkIndex))) } -// Given a validator chunk index, we determine all of the validator +// ValidatorIndexesInChunk Given a validator chunk index, we determine all the validators // indices that will belong in that chunk. -func (p *Parameters) validatorIndexesInChunk(validatorChunkIndex uint64) []primitives.ValidatorIndex { +func (p *Parameters) ValidatorIndexesInChunk(validatorChunkIndex uint64) []primitives.ValidatorIndex { validatorIndices := make([]primitives.ValidatorIndex, 0) low := validatorChunkIndex * p.validatorChunkSize high := (validatorChunkIndex + 1) * p.validatorChunkSize diff --git a/beacon-chain/slasher/params_test.go b/beacon-chain/slasher/params_test.go index 67140c4e1..bfff8fc55 100644 --- a/beacon-chain/slasher/params_test.go +++ b/beacon-chain/slasher/params_test.go @@ -468,7 +468,7 @@ func TestParams_validatorIndicesInChunk(t *testing.T) { c := &Parameters{ validatorChunkSize: tt.fields.validatorChunkSize, } - if got := c.validatorIndexesInChunk(tt.validatorChunkIdx); !reflect.DeepEqual(got, tt.want) { + if got := c.ValidatorIndexesInChunk(tt.validatorChunkIdx); !reflect.DeepEqual(got, tt.want) { t.Errorf("validatorIndicesInChunk() = %v, want %v", got, tt.want) } }) diff --git a/beacon-chain/slasher/service.go b/beacon-chain/slasher/service.go index be3ddfe72..5de7c859d 100644 --- a/beacon-chain/slasher/service.go +++ b/beacon-chain/slasher/service.go @@ -161,7 +161,7 @@ func (s *Service) Stop() error { ctx, innerCancel := context.WithTimeout(context.Background(), shutdownTimeout) defer innerCancel() log.Info("Flushing last epoch written for each validator to disk, please wait") - if err := s.serviceCfg.Database.SaveLastEpochsWrittenForValidators( + if err := s.serviceCfg.Database.SaveLastEpochWrittenForValidators( ctx, s.latestEpochUpdatedForValidator, ); err != nil { log.Error(err) diff --git a/beacon-chain/slasher/types/BUILD.bazel b/beacon-chain/slasher/types/BUILD.bazel index 9578171f3..e8b935388 100644 --- a/beacon-chain/slasher/types/BUILD.bazel +++ b/beacon-chain/slasher/types/BUILD.bazel @@ -4,7 +4,10 @@ go_library( name = "go_default_library", srcs = ["types.go"], importpath = "github.com/prysmaticlabs/prysm/v5/beacon-chain/slasher/types", - visibility = ["//beacon-chain:__subpackages__"], + visibility = [ + "//beacon-chain:__subpackages__", + "//cmd/prysmctl:__subpackages__", + ], deps = [ "//consensus-types/primitives:go_default_library", "//proto/prysm/v1alpha1:go_default_library", diff --git a/beacon-chain/slasher/types/types.go b/beacon-chain/slasher/types/types.go index 882cc6d39..c1699227e 100644 --- a/beacon-chain/slasher/types/types.go +++ b/beacon-chain/slasher/types/types.go @@ -14,6 +14,18 @@ const ( MaxSpan ) +// String returns the string representation of the chunk kind. +func (c ChunkKind) String() string { + switch c { + case MinSpan: + return "minspan" + case MaxSpan: + return "maxspan" + default: + return "unknown" + } +} + // IndexedAttestationWrapper contains an indexed attestation with its // data root to reduce duplicated computation. type IndexedAttestationWrapper struct { diff --git a/cmd/prysmctl/db/BUILD.bazel b/cmd/prysmctl/db/BUILD.bazel index bbecb3eb6..fddfd2336 100644 --- a/cmd/prysmctl/db/BUILD.bazel +++ b/cmd/prysmctl/db/BUILD.bazel @@ -6,13 +6,18 @@ go_library( "buckets.go", "cmd.go", "query.go", + "span.go", ], importpath = "github.com/prysmaticlabs/prysm/v5/cmd/prysmctl/db", visibility = ["//visibility:public"], deps = [ "//beacon-chain/db/kv:go_default_library", + "//beacon-chain/slasher:go_default_library", + "//beacon-chain/slasher/types:go_default_library", "//config/params:go_default_library", + "//consensus-types/primitives:go_default_library", "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library", + "@com_github_jedib0t_go_pretty_v6//table:go_default_library", "@com_github_pkg_errors//:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", "@com_github_urfave_cli_v2//:go_default_library", diff --git a/cmd/prysmctl/db/cmd.go b/cmd/prysmctl/db/cmd.go index d32112356..bb0206e20 100644 --- a/cmd/prysmctl/db/cmd.go +++ b/cmd/prysmctl/db/cmd.go @@ -9,6 +9,7 @@ var Commands = []*cli.Command{ Subcommands: []*cli.Command{ queryCmd, bucketsCmd, + spanCmd, }, }, } diff --git a/cmd/prysmctl/db/span.go b/cmd/prysmctl/db/span.go new file mode 100644 index 000000000..0af75000f --- /dev/null +++ b/cmd/prysmctl/db/span.go @@ -0,0 +1,304 @@ +package db + +import ( + "fmt" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/pkg/errors" + "github.com/prysmaticlabs/prysm/v5/beacon-chain/slasher" + "github.com/prysmaticlabs/prysm/v5/beacon-chain/slasher/types" + "github.com/prysmaticlabs/prysm/v5/consensus-types/primitives" + "github.com/urfave/cli/v2" +) + +const DefaultChunkKind = types.MinSpan + +var ( + f = struct { + Path string + ValidatorIndex uint64 + Epoch uint64 + ChunkKind string + ChunkSize uint64 + ValidatorChunkSize uint64 + HistoryLength uint64 + IsDisplayAllValidatorsInChunk bool + IsDisplayAllEpochsInChunk bool + }{} + + slasherDefaultParams = slasher.DefaultParams() +) + +var spanCmd = &cli.Command{ + Name: "slasher-span-display", + Usage: "visualise values in db span bucket", + Action: func(c *cli.Context) error { + if err := spanAction(c); err != nil { + return errors.Wrapf(err, "visualise values in db span bucket failed") + } + return nil + }, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "db-path-directory", + Usage: "path to directory containing slasher.db", + Destination: &f.Path, + Required: true, + }, + &cli.Uint64Flag{ + Name: "validator-index", + Usage: "filter by validator index", + Destination: &f.ValidatorIndex, + Required: true, + }, + &cli.Uint64Flag{ + Name: "epoch", + Usage: "filter by epoch", + Destination: &f.Epoch, + Required: true, + }, + &cli.StringFlag{ + Name: "chunk-kind", + Usage: "chunk kind to query (maxspan|minspan)", + Destination: &f.ChunkKind, + Value: DefaultChunkKind.String(), + DefaultText: DefaultChunkKind.String(), + }, + &cli.Uint64Flag{ + Name: "chunk-size", + Usage: "chunk size to query", + Destination: &f.ChunkSize, + DefaultText: fmt.Sprintf("%d", slasherDefaultParams.ChunkSize()), + }, + &cli.Uint64Flag{ + Name: "validator-chunk-size", + Usage: "validator chunk size to query", + Destination: &f.ValidatorChunkSize, + DefaultText: fmt.Sprintf("%d", slasherDefaultParams.ValidatorChunkSize()), + }, + &cli.Uint64Flag{ + Name: "history-length", + Usage: "history length to query", + Destination: &f.HistoryLength, + DefaultText: fmt.Sprintf("%d", slasherDefaultParams.HistoryLength()), + }, + &cli.BoolFlag{ + Name: "display-all-validators-in-chunk", + Usage: "display all validators in chunk", + Destination: &f.IsDisplayAllValidatorsInChunk, + }, + &cli.BoolFlag{ + Name: "display-all-epochs-in-chunk", + Usage: "display all epochs in chunk", + Destination: &f.IsDisplayAllEpochsInChunk, + }, + }, +} + +func spanAction(cliCtx *cli.Context) error { + var ( + chunk slasher.Chunker + validatorChunkIdx uint64 + lastEpochForValidatorIndex primitives.Epoch + + err error + ) + + // context + ctx := cliCtx.Context + + // variables + chunkKind := getChunkKind() + params := getSlasherParams() + i := primitives.ValidatorIndex(f.ValidatorIndex) + epoch := primitives.Epoch(f.Epoch) + + // display configuration + fmt.Printf("############################# SLASHER PARAMS ###############################\n") + fmt.Printf("# Chunk Size: %d\n", params.ChunkSize()) + fmt.Printf("# Validator Chunk Size: %d\n", params.ValidatorChunkSize()) + fmt.Printf("# History Length: %d\n", params.HistoryLength()) + fmt.Printf("# DB: %s\n", f.Path) + fmt.Printf("# Chunk Kind: %s\n", chunkKind) + fmt.Printf("# Validator: %d\n", i) + fmt.Printf("# Epoch: %d\n", epoch) + fmt.Printf("############################################################################\n") + + // fetch chunk in database + if lastEpochForValidatorIndex, _, validatorChunkIdx, chunk, err = slasher.GetChunkFromDatabase( + ctx, + f.Path, + slasher.GetChunkFromDatabaseFilters{ + ChunkKind: chunkKind, + ValidatorIndex: i, + SourceEpoch: epoch, + IsDisplayAllValidatorsInChunk: f.IsDisplayAllValidatorsInChunk, + IsDisplayAllEpochsInChunk: f.IsDisplayAllEpochsInChunk, + }, + params, + ); err != nil { + return errors.Wrapf(err, "could not get chunk from database") + } + + // fetch information related to chunk + fmt.Printf("\n################################ CHUNK #####################################\n") + firstValidator := params.ValidatorIndexesInChunk(validatorChunkIdx)[0] + firstEpoch := epoch - (epoch.Mod(params.ChunkSize())) + fmt.Printf("# First validator in chunk: %d\n", firstValidator) + fmt.Printf("# First epoch in chunk: %d\n\n", firstEpoch) + fmt.Printf("# Last epoch found in database for validator(%d): %d\n", i, lastEpochForValidatorIndex) + fmt.Printf("############################################################################\n\n") + + // init table + tw := table.NewWriter() + + minLowerBound := lastEpochForValidatorIndex.Sub(uint64(params.HistoryLength())) + if f.IsDisplayAllValidatorsInChunk { + if f.IsDisplayAllEpochsInChunk { + // display all validators and epochs in chunk + + // headers + addEpochsHeader(tw, params.ChunkSize(), firstEpoch) + + // rows + b := chunk.Chunk() + c := uint64(0) + for z := uint64(0); z < uint64(len(b)); z += params.ChunkSize() { + end := z + params.ChunkSize() + if end > uint64(len(b)) { + end = uint64(len(b)) + } + subChunk := b[z:end] + + row := make(table.Row, params.ChunkSize()+1) + title := firstValidator + primitives.ValidatorIndex(c) + row[0] = title + for y, span := range subChunk { + row[y+1] = getSpanOrNonApplicable(firstEpoch, y, minLowerBound, lastEpochForValidatorIndex, span) + } + tw.AppendRow(row) + + c++ + } + } else { + // display all validators but only the requested epoch in chunk + indexEpochInChunk := epoch - firstEpoch + + // headers + addEpochsHeader(tw, 1, firstEpoch) + + // rows + b := chunk.Chunk() + c := uint64(0) + for z := uint64(0); z < uint64(len(b)); z += params.ChunkSize() { + end := z + params.ChunkSize() + if end > uint64(len(b)) { + end = uint64(len(b)) + } + subChunk := b[z:end] + + row := make(table.Row, 2) + title := firstValidator + primitives.ValidatorIndex(c) + row[0] = title + row[1] = subChunk[indexEpochInChunk] + tw.AppendRow(row) + + c++ + } + } + } else { + if f.IsDisplayAllEpochsInChunk { + // display only the requested validator with all epochs in chunk + + // headers + addEpochsHeader(tw, params.ChunkSize(), firstEpoch) + + // rows + b := chunk.Chunk() + validatorFirstEpochIdx := uint64(i.Mod(params.ValidatorChunkSize())) * params.ChunkSize() + subChunk := b[validatorFirstEpochIdx : validatorFirstEpochIdx+params.ChunkSize()] + row := make(table.Row, params.ChunkSize()+1) + title := i + row[0] = title + for y, span := range subChunk { + row[y+1] = getSpanOrNonApplicable(firstEpoch, y, minLowerBound, lastEpochForValidatorIndex, span) + } + tw.AppendRow(row) + } else { + // display only the requested validator and epoch in chunk + + // headers + addEpochsHeader(tw, 1, epoch) + + // rows + b := chunk.Chunk() + validatorFirstEpochIdx := uint64(i.Mod(params.ValidatorChunkSize())) * params.ChunkSize() + subChunk := b[validatorFirstEpochIdx : validatorFirstEpochIdx+params.ChunkSize()] + row := make(table.Row, 2) + title := i + row[0] = title + indexEpochInChunk := epoch - firstEpoch + row[1] = subChunk[indexEpochInChunk] + tw.AppendRow(row) + } + } + + // display table + displayTable(tw) + + return nil +} + +// getSpanOrNonApplicable checks if there's some epoch that are not correct in chunk due to the round robin +// nature of 2D chunking when an epoch gets overwritten by an epoch eg. (params.historyLength + next_epoch) > params.historyLength +// if we are out of the range, we display a n/a value otherwise the span value +func getSpanOrNonApplicable(firstEpoch primitives.Epoch, y int, minLowerBound primitives.Epoch, lastEpochForValidatorIndex primitives.Epoch, span uint16) string { + if firstEpoch.Add(uint64(y)) < minLowerBound || firstEpoch.Add(uint64(y)) > lastEpochForValidatorIndex { + return "-" + } + return fmt.Sprintf("%d", span) +} + +func displayTable(tw table.Writer) { + tw.AppendSeparator() + fmt.Println(tw.Render()) +} + +func addEpochsHeader(tw table.Writer, nbEpoch uint64, firstEpoch primitives.Epoch) { + header := table.Row{"Validator / Epoch"} + for y := 0; uint64(y) < nbEpoch; y++ { + header = append(header, firstEpoch+primitives.Epoch(y)) + } + tw.AppendHeader(header) +} + +func getChunkKind() types.ChunkKind { + chunkKind := types.MinSpan + if f.ChunkKind == "maxspan" { + chunkKind = types.MaxSpan + } + return chunkKind +} + +func getSlasherParams() *slasher.Parameters { + var ( + chunkSize, validatorChunkSize uint64 + historyLength primitives.Epoch + ) + if f.ChunkSize != 0 && f.ChunkSize != slasherDefaultParams.ChunkSize() { + chunkSize = f.ChunkSize + } else { + chunkSize = slasherDefaultParams.ChunkSize() + } + if f.ValidatorChunkSize != 0 && f.ValidatorChunkSize != slasherDefaultParams.ValidatorChunkSize() { + validatorChunkSize = f.ValidatorChunkSize + } else { + validatorChunkSize = slasherDefaultParams.ValidatorChunkSize() + } + if f.HistoryLength != 0 && f.HistoryLength != uint64(slasherDefaultParams.HistoryLength()) { + historyLength = primitives.Epoch(f.HistoryLength) + } else { + historyLength = slasherDefaultParams.HistoryLength() + } + return slasher.NewParams(chunkSize, validatorChunkSize, historyLength) +} diff --git a/deps.bzl b/deps.bzl index 3a0e8ad7b..5138a698d 100644 --- a/deps.bzl +++ b/deps.bzl @@ -5265,6 +5265,12 @@ def prysm_deps(): sum = "h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=", version = "v1.26.0", ) + go_repository( + name = "com_github_jedib0t_go_pretty_v6", + importpath = "github.com/jedib0t/go-pretty/v6", + sum = "h1:gOGo0613MoqUcf0xCj+h/V3sHDaZasfv152G6/5l91s=", + version = "v6.5.4", + ) http_archive( name = "com_github_supranational_blst", diff --git a/go.mod b/go.mod index 7125cb59c..b37225eb4 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( github.com/holiman/uint256 v1.2.4 github.com/ianlancetaylor/cgosymbolizer v0.0.0-20200424224625-be1b05b0b279 github.com/ipfs/go-log/v2 v2.5.1 + github.com/jedib0t/go-pretty/v6 v6.5.4 github.com/joonix/log v0.0.0-20200409080653-9c1d2ceb5f1d github.com/json-iterator/go v1.1.12 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 @@ -237,7 +238,7 @@ require ( golang.org/x/exp/typeparams v0.0.0-20231108232855-2478ac86f678 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect @@ -258,7 +259,7 @@ require ( github.com/go-playground/validator/v10 v10.13.0 github.com/peterh/liner v1.2.0 // indirect github.com/prysmaticlabs/gohashtree v0.0.4-beta - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.16.0 // indirect google.golang.org/api v0.44.0 // indirect google.golang.org/appengine v1.6.7 // indirect k8s.io/klog/v2 v2.80.0 // indirect diff --git a/go.sum b/go.sum index bca04ee0f..60ac9b9c6 100644 --- a/go.sum +++ b/go.sum @@ -569,6 +569,8 @@ github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+ github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jedib0t/go-pretty/v6 v6.5.4 h1:gOGo0613MoqUcf0xCj+h/V3sHDaZasfv152G6/5l91s= +github.com/jedib0t/go-pretty/v6 v6.5.4/go.mod h1:5LQIxa52oJ/DlDSLv0HEkWOFMDGoWkJb9ss5KqPpJBg= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jhump/protoreflect v1.8.1/go.mod h1:7GcYQDdMU/O/BBrl/cX6PNHpXh6cenjd8pneu5yW7Tg= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -1412,13 +1414,13 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=