From dbc9686d159ba62c51961457e81bc4e9ab1d13a3 Mon Sep 17 00:00:00 2001
From: Preston Van Loon <preston@prysmaticlabs.com>
Date: Fri, 10 Jul 2020 10:07:49 -0700
Subject: [PATCH] Add roughtime offset histogram metric, log on large offsets
 (#6546)

* Add roughtime_offset_nsec metric. Log a warning if offset is greater than 2 seconds
* gofmt
* use math.abs, add help text
* gofmt
---
 shared/roughtime/BUILD.bazel  |  2 ++
 shared/roughtime/roughtime.go | 20 ++++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/shared/roughtime/BUILD.bazel b/shared/roughtime/BUILD.bazel
index e83c71635..58f91e0a3 100644
--- a/shared/roughtime/BUILD.bazel
+++ b/shared/roughtime/BUILD.bazel
@@ -8,6 +8,8 @@ go_library(
     deps = [
         "//shared/runutil:go_default_library",
         "@com_github_cloudflare_roughtime//:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus:go_default_library",
+        "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
     ],
 )
diff --git a/shared/roughtime/roughtime.go b/shared/roughtime/roughtime.go
index 8f459efec..43d5c03ea 100644
--- a/shared/roughtime/roughtime.go
+++ b/shared/roughtime/roughtime.go
@@ -3,9 +3,12 @@ package roughtime
 
 import (
 	"context"
+	"math"
 	"time"
 
 	rt "github.com/cloudflare/roughtime"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/prysmaticlabs/prysm/shared/runutil"
 	"github.com/sirupsen/logrus"
 )
@@ -20,6 +23,19 @@ var offset time.Duration
 
 var log = logrus.WithField("prefix", "roughtime")
 
+var offsetHistogram = promauto.NewHistogram(prometheus.HistogramOpts{
+	Name: "roughtime_offset_nsec",
+	Help: "The absolute value delta between roughtime computed clock time and the system clock time.",
+	Buckets: []float64{
+		float64(50 * time.Millisecond),
+		float64(100 * time.Millisecond),
+		float64(500 * time.Millisecond),
+		float64(1 * time.Second),
+		float64(2 * time.Second),
+		float64(10 * time.Second),
+	},
+})
+
 func init() {
 	recalibrateRoughtime()
 	runutil.RunEvery(context.Background(), RecalibrationInterval, recalibrateRoughtime)
@@ -36,6 +52,10 @@ func recalibrateRoughtime() {
 	if err != nil {
 		log.WithError(err).Error("Failed to calculate roughtime offset")
 	}
+	offsetHistogram.Observe(math.Abs(float64(offset)))
+	if offset > 2*time.Second {
+		log.WithField("offset", offset).Warn("Roughtime reports your clock is off by more than 2 seconds")
+	}
 }
 
 // Since returns the duration since t, based on the roughtime response