diff --git a/beacon-chain/db/filesystem/BUILD.bazel b/beacon-chain/db/filesystem/BUILD.bazel index b12ed9a7d..2d186a74f 100644 --- a/beacon-chain/db/filesystem/BUILD.bazel +++ b/beacon-chain/db/filesystem/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "blob.go", "ephemeral.go", + "metrics.go", ], importpath = "github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem", visibility = ["//visibility:public"], @@ -20,6 +21,8 @@ go_library( "//time/slots:go_default_library", "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library", "@com_github_pkg_errors//:go_default_library", + "@com_github_prometheus_client_golang//prometheus:go_default_library", + "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", "@com_github_spf13_afero//:go_default_library", ], diff --git a/beacon-chain/db/filesystem/blob.go b/beacon-chain/db/filesystem/blob.go index 5210c87a4..eba0295e8 100644 --- a/beacon-chain/db/filesystem/blob.go +++ b/beacon-chain/db/filesystem/blob.go @@ -78,6 +78,7 @@ type BlobStorage struct { // Save saves blobs given a list of sidecars. func (bs *BlobStorage) Save(sidecar blocks.VerifiedROBlob) error { + startTime := time.Now() fname := namerForSidecar(sidecar) sszPath := fname.path() exists, err := afero.Exists(bs.fs, sszPath) @@ -142,6 +143,8 @@ func (bs *BlobStorage) Save(sidecar blocks.VerifiedROBlob) error { if err != nil { return errors.Wrap(err, "failed to rename partial file to final name") } + blobsTotalGauge.Inc() + blobSaveLatency.Observe(time.Since(startTime).Seconds()) return nil } @@ -149,6 +152,7 @@ func (bs *BlobStorage) Save(sidecar blocks.VerifiedROBlob) error { // Since BlobStorage only writes blobs that have undergone full verification, the return // value is always a VerifiedROBlob. func (bs *BlobStorage) Get(root [32]byte, idx uint64) (blocks.VerifiedROBlob, error) { + startTime := time.Now() expected := blobNamer{root: root, index: idx} encoded, err := afero.ReadFile(bs.fs, expected.path()) var v blocks.VerifiedROBlob @@ -163,6 +167,9 @@ func (bs *BlobStorage) Get(root [32]byte, idx uint64) (blocks.VerifiedROBlob, er if err != nil { return blocks.VerifiedROBlob{}, err } + defer func() { + blobFetchLatency.Observe(time.Since(startTime).Seconds()) + }() return verification.BlobSidecarNoop(ro) } @@ -249,9 +256,15 @@ func (bs *BlobStorage) Prune(currentSlot primitives.Slot) error { } for _, folder := range folders { if folder.IsDir() { + num, err := bs.countFiles(folder.Name()) + if err != nil { + return err + } if err := bs.processFolder(folder, currentSlot, retentionSlots); err != nil { return err } + blobsPrunedCounter.Add(float64(num)) + blobsTotalGauge.Add(-float64(num)) } } pruneTime := time.Since(t) diff --git a/beacon-chain/db/filesystem/metrics.go b/beacon-chain/db/filesystem/metrics.go new file mode 100644 index 000000000..8d5fd21df --- /dev/null +++ b/beacon-chain/db/filesystem/metrics.go @@ -0,0 +1,68 @@ +package filesystem + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prysmaticlabs/prysm/v4/consensus-types/primitives" + "github.com/spf13/afero" +) + +var ( + blobBuckets = []float64{0.00003, 0.00005, 0.00007, 0.00009, 0.00011, 0.00013, 0.00015} + blobSaveLatency = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "blob_storage_save_latency", + Help: "Latency of blob storage save operations in seconds", + Buckets: blobBuckets, + }) + blobFetchLatency = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "blob_storage_get_latency", + Help: "Latency of blob storage get operations in seconds", + Buckets: blobBuckets, + }) + blobsPrunedCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "blob_pruned_blobs_total", + Help: "Total number of pruned blobs.", + }) + blobsTotalGauge = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "blobs_on_disk_total", + Help: "Total number of blobs in filesystem.", + }) +) + +func (bs *BlobStorage) Initialize(lastFinalizedSlot primitives.Slot) error { + if err := bs.Prune(lastFinalizedSlot); err != nil { + return err + } + if err := bs.collectTotalBlobMetric(); err != nil { + return err + } + return nil +} + +// CollectTotalBlobMetric set the number of blobs currently present in the filesystem +// to the blobsTotalGauge metric. +func (bs *BlobStorage) collectTotalBlobMetric() error { + totalBlobs := 0 + folders, err := afero.ReadDir(bs.fs, ".") + if err != nil { + return err + } + for _, folder := range folders { + num, err := bs.countFiles(folder.Name()) + if err != nil { + return err + } + totalBlobs = totalBlobs + num + } + blobsTotalGauge.Set(float64(totalBlobs)) + return nil +} + +// countFiles returns the length of blob files for a given directory. +func (bs *BlobStorage) countFiles(folderName string) (int, error) { + files, err := afero.ReadDir(bs.fs, folderName) + if err != nil { + return 0, err + } + return len(files), nil +} diff --git a/beacon-chain/node/node.go b/beacon-chain/node/node.go index 28bd249f9..a31ebb427 100644 --- a/beacon-chain/node/node.go +++ b/beacon-chain/node/node.go @@ -227,7 +227,7 @@ func New(cliCtx *cli.Context, cancel context.CancelFunc, opts ...Option) (*Beaco } if beacon.finalizedStateAtStartUp != nil { - if err := beacon.BlobStorage.Prune(beacon.finalizedStateAtStartUp.Slot()); err != nil { + if err := beacon.BlobStorage.Initialize(beacon.finalizedStateAtStartUp.Slot()); err != nil { return nil, err } } else {