Blob filesystem metrics (#13316)

* Add metrics

* Replace counter with gauge

* Preston's comments

* Remove hardcoded number

* Count blob files

* Fix count order

* Fixes

* Cleanup

* Add blob bucket

* Update beacon-chain/node/node.go

Co-authored-by: Preston Van Loon <pvanloon@offchainlabs.com>

* Rename

---------

Co-authored-by: Preston Van Loon <pvanloon@offchainlabs.com>

beacon-chain/db/filesystem/BUILD.bazel

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "blob.go",
         "ephemeral.go",
+        "metrics.go",
     ],
     importpath = "github.com/prysmaticlabs/prysm/v4/beacon-chain/db/filesystem",
     visibility = ["//visibility:public"],
@@ -20,6 +21,8 @@ go_library(
         "//time/slots:go_default_library",
         "@com_github_ethereum_go_ethereum//common/hexutil:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_spf13_afero//:go_default_library",
     ],

beacon-chain/db/filesystem/blob.go

@@ -78,6 +78,7 @@ type BlobStorage struct {
 
 // Save saves blobs given a list of sidecars.
 func (bs *BlobStorage) Save(sidecar blocks.VerifiedROBlob) error {
+	startTime := time.Now()
 	fname := namerForSidecar(sidecar)
 	sszPath := fname.path()
 	exists, err := afero.Exists(bs.fs, sszPath)
@@ -142,6 +143,8 @@ func (bs *BlobStorage) Save(sidecar blocks.VerifiedROBlob) error {
 	if err != nil {
 		return errors.Wrap(err, "failed to rename partial file to final name")
 	}
+	blobsTotalGauge.Inc()
+	blobSaveLatency.Observe(time.Since(startTime).Seconds())
 	return nil
 }
 
@@ -149,6 +152,7 @@ func (bs *BlobStorage) Save(sidecar blocks.VerifiedROBlob) error {
 // Since BlobStorage only writes blobs that have undergone full verification, the return
 // value is always a VerifiedROBlob.
 func (bs *BlobStorage) Get(root [32]byte, idx uint64) (blocks.VerifiedROBlob, error) {
+	startTime := time.Now()
 	expected := blobNamer{root: root, index: idx}
 	encoded, err := afero.ReadFile(bs.fs, expected.path())
 	var v blocks.VerifiedROBlob
@@ -163,6 +167,9 @@ func (bs *BlobStorage) Get(root [32]byte, idx uint64) (blocks.VerifiedROBlob, er
 	if err != nil {
 		return blocks.VerifiedROBlob{}, err
 	}
+	defer func() {
+		blobFetchLatency.Observe(time.Since(startTime).Seconds())
+	}()
 	return verification.BlobSidecarNoop(ro)
 }
 
@@ -249,9 +256,15 @@ func (bs *BlobStorage) Prune(currentSlot primitives.Slot) error {
 	}
 	for _, folder := range folders {
 		if folder.IsDir() {
+			num, err := bs.countFiles(folder.Name())
+			if err != nil {
+				return err
+			}
 			if err := bs.processFolder(folder, currentSlot, retentionSlots); err != nil {
 				return err
 			}
+			blobsPrunedCounter.Add(float64(num))
+			blobsTotalGauge.Add(-float64(num))
 		}
 	}
 	pruneTime := time.Since(t)

beacon-chain/db/filesystem/metrics.go

@@ -0,0 +1,68 @@
+package filesystem
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prysmaticlabs/prysm/v4/consensus-types/primitives"
+	"github.com/spf13/afero"
+)
+
+var (
+	blobBuckets     = []float64{0.00003, 0.00005, 0.00007, 0.00009, 0.00011, 0.00013, 0.00015}
+	blobSaveLatency = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name:    "blob_storage_save_latency",
+		Help:    "Latency of blob storage save operations in seconds",
+		Buckets: blobBuckets,
+	})
+	blobFetchLatency = promauto.NewHistogram(prometheus.HistogramOpts{
+		Name:    "blob_storage_get_latency",
+		Help:    "Latency of blob storage get operations in seconds",
+		Buckets: blobBuckets,
+	})
+	blobsPrunedCounter = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "blob_pruned_blobs_total",
+		Help: "Total number of pruned blobs.",
+	})
+	blobsTotalGauge = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "blobs_on_disk_total",
+		Help: "Total number of blobs in filesystem.",
+	})
+)
+
+func (bs *BlobStorage) Initialize(lastFinalizedSlot primitives.Slot) error {
+	if err := bs.Prune(lastFinalizedSlot); err != nil {
+		return err
+	}
+	if err := bs.collectTotalBlobMetric(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// CollectTotalBlobMetric set the number of blobs currently present in the filesystem
+// to the blobsTotalGauge metric.
+func (bs *BlobStorage) collectTotalBlobMetric() error {
+	totalBlobs := 0
+	folders, err := afero.ReadDir(bs.fs, ".")
+	if err != nil {
+		return err
+	}
+	for _, folder := range folders {
+		num, err := bs.countFiles(folder.Name())
+		if err != nil {
+			return err
+		}
+		totalBlobs = totalBlobs + num
+	}
+	blobsTotalGauge.Set(float64(totalBlobs))
+	return nil
+}
+
+// countFiles returns the length of blob files for a given directory.
+func (bs *BlobStorage) countFiles(folderName string) (int, error) {
+	files, err := afero.ReadDir(bs.fs, folderName)
+	if err != nil {
+		return 0, err
+	}
+	return len(files), nil
+}

beacon-chain/node/node.go

@@ -227,7 +227,7 @@ func New(cliCtx *cli.Context, cancel context.CancelFunc, opts ...Option) (*Beaco
 	}
 
 	if beacon.finalizedStateAtStartUp != nil {
-		if err := beacon.BlobStorage.Prune(beacon.finalizedStateAtStartUp.Slot()); err != nil {
+		if err := beacon.BlobStorage.Initialize(beacon.finalizedStateAtStartUp.Slot()); err != nil {
 			return nil, err
 		}
 	} else {