Add roughtime offset histogram metric, log on large offsets (#6546)

* Add roughtime_offset_nsec metric. Log a warning if offset is greater than 2 seconds
* gofmt
* use math.abs, add help text
* gofmt
This commit is contained in:
Preston Van Loon 2020-07-10 10:07:49 -07:00 committed by GitHub
parent cc8b3e349d
commit dbc9686d15
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 22 additions and 0 deletions

View File

@ -8,6 +8,8 @@ go_library(
deps = [
"//shared/runutil:go_default_library",
"@com_github_cloudflare_roughtime//:go_default_library",
"@com_github_prometheus_client_golang//prometheus:go_default_library",
"@com_github_prometheus_client_golang//prometheus/promauto:go_default_library",
"@com_github_sirupsen_logrus//:go_default_library",
],
)

View File

@ -3,9 +3,12 @@ package roughtime
import (
"context"
"math"
"time"
rt "github.com/cloudflare/roughtime"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prysmaticlabs/prysm/shared/runutil"
"github.com/sirupsen/logrus"
)
@ -20,6 +23,19 @@ var offset time.Duration
var log = logrus.WithField("prefix", "roughtime")
var offsetHistogram = promauto.NewHistogram(prometheus.HistogramOpts{
Name: "roughtime_offset_nsec",
Help: "The absolute value delta between roughtime computed clock time and the system clock time.",
Buckets: []float64{
float64(50 * time.Millisecond),
float64(100 * time.Millisecond),
float64(500 * time.Millisecond),
float64(1 * time.Second),
float64(2 * time.Second),
float64(10 * time.Second),
},
})
func init() {
recalibrateRoughtime()
runutil.RunEvery(context.Background(), RecalibrationInterval, recalibrateRoughtime)
@ -36,6 +52,10 @@ func recalibrateRoughtime() {
if err != nil {
log.WithError(err).Error("Failed to calculate roughtime offset")
}
offsetHistogram.Observe(math.Abs(float64(offset)))
if offset > 2*time.Second {
log.WithField("offset", offset).Warn("Roughtime reports your clock is off by more than 2 seconds")
}
}
// Since returns the duration since t, based on the roughtime response