mirror of
https://gitlab.com/pulsechaincom/prysm-pulse.git
synced 2024-12-25 04:47:18 +00:00
92932ae58e
* Change LowestSignedProposal to Also Return a Boolean for Slashing Protection (#8020) * amend to use bools * ineff assign * comment * Update `LowestSignedTargetEpoch` to include exists (#8004) * Replace highest with lowerest * Update validator/db/kv/attestation_history_v2.go Co-authored-by: Raul Jordan <raul@prysmaticlabs.com> * Update validator/db/kv/attestation_history_v2.go Co-authored-by: Raul Jordan <raul@prysmaticlabs.com> * Invert equality for saveLowestSourceTargetToDB * Add eip checks to ensure epochs cant be lower than db ones * Should be less than equal to * Check if epoch exists in DB getters * Revert run time checks Co-authored-by: Raul Jordan <raul@prysmaticlabs.com> Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com> * Export Attesting History for Slashing Interchange Standard (#8027) * added in att history checks * logic for export * export return nil * test for export atts * round trip passes first try! * rem println * fix up tests * pass test * Validate Proposers Are Not Slashable With Regard to Data Within Slasher Interchange JSON (#8031) * filter slashable blocks and atts in same json stub * add filter blocks func * add test for filtering out the bad public keys * Export Slashing Protection History Via CLI (#8040) * include cli entrypoint for history exports * builds properly * test to confirm we export the data as expected * abstract helpers properly * full test suite * gaz * better errors * marshal ident * Add the additional eip-3076 attestation checks (#7966) * Replace highest with lowerest * Update validator/db/kv/attestation_history_v2.go Co-authored-by: Raul Jordan <raul@prysmaticlabs.com> * Update validator/db/kv/attestation_history_v2.go Co-authored-by: Raul Jordan <raul@prysmaticlabs.com> * Invert equality for saveLowestSourceTargetToDB * Add eip checks to ensure epochs cant be lower than db ones * Should be less than equal to * Check if epoch exists in DB getters Co-authored-by: Raul Jordan <raul@prysmaticlabs.com> * Add EIP-3076 Invariants for Proposer Slashing Protection (#8067) * add invariant for proposer protection * write different test cases * pass tests * Add EIP-3076 Interchange JSON CLI command to validator (#7880) * Import JSON CLI * CLI impotr * f * Begin adding new commands in slashing protection * Move testing helpers to separate packae * Add command for importing slashing protection JSONs * fix import cycle * fix test * Undo cleaning changes * Improvements * Add better prompts * Fix prompt * Fix * Fix * Fix * Fix conflict * Fix * Fixes * Fixes * Fix exported func * test func * Fixes * fix test * simplify import and standardize with export * add round trip test * true integration test works * fix up comments * logrus * better error * fix build * build fix * Update validator/slashing-protection/cli_export.go Co-authored-by: Preston Van Loon <preston@prysmaticlabs.com> * Update validator/slashing-protection/cli_import.go Co-authored-by: Preston Van Loon <preston@prysmaticlabs.com> * fmt Co-authored-by: Raul Jordan <raul@prysmaticlabs.com> Co-authored-by: Preston Van Loon <preston@prysmaticlabs.com> * Filter Slashable Attester Public Keys in Slashing Interchange Import (#8051) * filter slashable attesters from the same JSON * builds * fix up initially broken test * circular dep * import fix * giz * added in attesting history package * add test for filter slashable attester keys * pass tests * Save Slashable Keys to Disk in the Validator Client (#8082) * begin db funcs * add in test and bucket * gaz * rem changes to import * ineff assign * add godoc * Properly Handle Duplicate Public Key Entries in Slashing Interchange Imports (#8089) * Prevent Blacklisted Public Keys from Slashing Protection Imports from Having Duties at Runtime (#8084) * tests on update duties * ensure the slashable public keys are filtered out from update duties via test * begin test * attempt test * rename for better context * pass tests * deep source * ensure tests pass * Check for Signing Root Mismatch When Submitting Proposals and Importing Proposals in Slashing Interchange (#8085) * flexible signing root * add test * add tests * fix test * Preston's comments * res tests * ensure we consider the case for minimum proposals * pass test * tests passing * rem unused code * Set Empty Epochs in Between Attestations as FAR_FUTURE_EPOCH in Attesting History (#8113) * set target data * all tests passing * ineff assign * signing root * Add Slashing Interchange, EIP-3076, Spec Tests to Prysm (#7858) * Add interchange test framework * add checks for attestations * Import genesis root if necessary * flexible signing root * add test * Sync * fix up test build * only 3 failing tests now * two failing * attempting to debug problems in conformity tests * include latest changes * protect test in validator/client passing * pass tests * imports * spec tests passing with bazel * gh archive link to spectests using tar.gz suffix * rev * rev more comment changes * fix sha * godoc * add back save Co-authored-by: Raul Jordan <raul@prysmaticlabs.com> * Implement Migration for Unattested Epochs in Attesting History Database (#8121) * migrate attesting history backbone done * begin migration logic * implement migration logic * migration test * add test * migration logic * bazel * migration to its own file * Handle empty blocks and attestations in interchange json and sort interchange json by public key (#8132) * Handle empty blocks and attestations in interchange json * add test * sort json * easier empty arrays * pass test Co-authored-by: Raul Jordan <raul@prysmaticlabs.com> * builds * more tests finally build * Align Slashing Interchange With Optimized Slashing Protection (#8268) * attestation history should account for multiple targets per source * attempt at some fixes * attempt some test fixes * experimenting with sorting * only one more failing test * tests now pass * slash protect tests passing * only few tests now failing * only spec tests failing now * spec tests passing * all tests passing * helper function for verifying double votes * use helper * gaz * deep source * tests fixed * expect specific number of times for domain data calls * final comments * Batch Save Imported EIP-3076 Attestations (#8304) * optimize save * test added * add test for sad path Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com> * revert bad find replace * add comment to db func Co-authored-by: terence tsao <terence@prysmaticlabs.com> Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com> Co-authored-by: Ivan Martinez <ivanthegreatdev@gmail.com> Co-authored-by: Preston Van Loon <preston@prysmaticlabs.com> Co-authored-by: Shay Zluf <thezluf@gmail.com>
176 lines
5.3 KiB
Go
176 lines
5.3 KiB
Go
// Package kv defines a persistent backend for the validator service.
|
|
package kv
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
prombolt "github.com/prysmaticlabs/prombbolt"
|
|
"github.com/prysmaticlabs/prysm/shared/event"
|
|
"github.com/prysmaticlabs/prysm/shared/fileutil"
|
|
"github.com/prysmaticlabs/prysm/shared/params"
|
|
bolt "go.etcd.io/bbolt"
|
|
)
|
|
|
|
const (
|
|
// Number of attestation records we can hold in memory
|
|
// before we flush them to the database. Roughly corresponds
|
|
// to the max number of keys per validator client, but there is no
|
|
// detriment if there are more keys than this capacity, as attestations
|
|
// for those keys will simply be flushed at the next flush interval.
|
|
attestationBatchCapacity = 2048
|
|
// Time interval after which we flush attestation records to the database
|
|
// from a batch kept in memory for slashing protection.
|
|
attestationBatchWriteInterval = time.Millisecond * 100
|
|
)
|
|
|
|
// ProtectionDbFileName Validator slashing protection db file name.
|
|
var (
|
|
ProtectionDbFileName = "validator.db"
|
|
)
|
|
|
|
// Store defines an implementation of the Prysm Database interface
|
|
// using BoltDB as the underlying persistent kv-store for eth2.
|
|
type Store struct {
|
|
db *bolt.DB
|
|
databasePath string
|
|
batchedAttestations []*AttestationRecord
|
|
batchedAttestationsChan chan *AttestationRecord
|
|
batchAttestationsFlushedFeed *event.Feed
|
|
}
|
|
|
|
// Close closes the underlying boltdb database.
|
|
func (s *Store) Close() error {
|
|
prometheus.Unregister(createBoltCollector(s.db))
|
|
return s.db.Close()
|
|
}
|
|
|
|
func (s *Store) update(fn func(*bolt.Tx) error) error {
|
|
return s.db.Update(fn)
|
|
}
|
|
func (s *Store) view(fn func(*bolt.Tx) error) error {
|
|
return s.db.View(fn)
|
|
}
|
|
|
|
// ClearDB removes any previously stored data at the configured data directory.
|
|
func (s *Store) ClearDB() error {
|
|
if _, err := os.Stat(s.databasePath); os.IsNotExist(err) {
|
|
return nil
|
|
}
|
|
prometheus.Unregister(createBoltCollector(s.db))
|
|
return os.Remove(filepath.Join(s.databasePath, ProtectionDbFileName))
|
|
}
|
|
|
|
// DatabasePath at which this database writes files.
|
|
func (s *Store) DatabasePath() string {
|
|
return s.databasePath
|
|
}
|
|
|
|
func createBuckets(tx *bolt.Tx, buckets ...[]byte) error {
|
|
for _, bucket := range buckets {
|
|
if _, err := tx.CreateBucketIfNotExists(bucket); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// NewKVStore initializes a new boltDB key-value store at the directory
|
|
// path specified, creates the kv-buckets based on the schema, and stores
|
|
// an open connection db object as a property of the Store struct.
|
|
func NewKVStore(ctx context.Context, dirPath string, pubKeys [][48]byte) (*Store, error) {
|
|
hasDir, err := fileutil.HasDir(dirPath)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !hasDir {
|
|
if err := fileutil.MkdirAll(dirPath); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
datafile := filepath.Join(dirPath, ProtectionDbFileName)
|
|
boltDB, err := bolt.Open(datafile, params.BeaconIoConfig().ReadWritePermissions, &bolt.Options{Timeout: params.BeaconIoConfig().BoltTimeout})
|
|
if err != nil {
|
|
if errors.Is(err, bolt.ErrTimeout) {
|
|
return nil, errors.New("cannot obtain database lock, database may be in use by another process")
|
|
}
|
|
return nil, err
|
|
}
|
|
|
|
kv := &Store{
|
|
db: boltDB,
|
|
databasePath: dirPath,
|
|
batchedAttestations: make([]*AttestationRecord, 0, attestationBatchCapacity),
|
|
batchedAttestationsChan: make(chan *AttestationRecord, attestationBatchCapacity),
|
|
batchAttestationsFlushedFeed: new(event.Feed),
|
|
}
|
|
|
|
if err := kv.db.Update(func(tx *bolt.Tx) error {
|
|
return createBuckets(
|
|
tx,
|
|
genesisInfoBucket,
|
|
deprecatedAttestationHistoryBucket,
|
|
historicProposalsBucket,
|
|
lowestSignedSourceBucket,
|
|
lowestSignedTargetBucket,
|
|
lowestSignedProposalsBucket,
|
|
highestSignedProposalsBucket,
|
|
slashablePublicKeysBucket,
|
|
pubKeysBucket,
|
|
migrationsBucket,
|
|
)
|
|
}); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Initialize the required public keys into the DB to ensure they're not empty.
|
|
if pubKeys != nil {
|
|
if err := kv.UpdatePublicKeysBuckets(pubKeys); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
// Prune attesting records older than the current weak subjectivity period.
|
|
if err := kv.PruneAttestationsOlderThanCurrentWeakSubjectivity(ctx); err != nil {
|
|
return nil, errors.Wrap(err, "could not prune old attestations from DB")
|
|
}
|
|
|
|
// Batch save attestation records for slashing protection at timed
|
|
// intervals to our database.
|
|
go kv.batchAttestationWrites(ctx)
|
|
|
|
return kv, prometheus.Register(createBoltCollector(kv.db))
|
|
}
|
|
|
|
// UpdatePublicKeysBuckets for a specified list of keys.
|
|
func (s *Store) UpdatePublicKeysBuckets(pubKeys [][48]byte) error {
|
|
return s.update(func(tx *bolt.Tx) error {
|
|
bucket := tx.Bucket(historicProposalsBucket)
|
|
for _, pubKey := range pubKeys {
|
|
if _, err := bucket.CreateBucketIfNotExists(pubKey[:]); err != nil {
|
|
return errors.Wrap(err, "failed to create proposal history bucket")
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// Size returns the db size in bytes.
|
|
func (s *Store) Size() (int64, error) {
|
|
var size int64
|
|
err := s.db.View(func(tx *bolt.Tx) error {
|
|
size = tx.Size()
|
|
return nil
|
|
})
|
|
return size, err
|
|
}
|
|
|
|
// createBoltCollector returns a prometheus collector specifically configured for boltdb.
|
|
func createBoltCollector(db *bolt.DB) prometheus.Collector {
|
|
return prombolt.New("boltDB", db)
|
|
}
|