From dbc37b5660003d4611d3d827529b4f1c7dc3d451 Mon Sep 17 00:00:00 2001
From: ledgerwatch <akhounov@gmail.com>
Date: Tue, 24 Dec 2019 13:26:29 +0000
Subject: [PATCH] bolt vs badger log parser and chart (#284)

---
 cmd/hack/hack.go                   | 47 +++++++++++++++---------
 cmd/hack/scripts/geth_log_parse.py | 59 ++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+), 18 deletions(-)
 create mode 100644 cmd/hack/scripts/geth_log_parse.py

diff --git a/cmd/hack/hack.go b/cmd/hack/hack.go
index dd35b8337..1331ccadb 100644
--- a/cmd/hack/hack.go
+++ b/cmd/hack/hack.go
@@ -136,10 +136,10 @@ func days() []chart.GridLine {
 }
 
 func mychart() {
-	blocks, hours, dbsize, trienodes, heap := readData("geth.csv")
-	//blocks0, hours0, _, _, _ := readData("geth.csv")
+	blocks, hours, dbsize, trienodes, heap := readData("bolt.csv")
+	blocks0, hours0, dbsize0, _, _ := readData("badger.csv")
 	mainSeries := &chart.ContinuousSeries{
-		Name: "Cumulative sync time (SSD)",
+		Name: "Cumulative sync time (bolt)",
 		Style: chart.Style{
 			Show:        true,
 			StrokeColor: chart.ColorBlue,
@@ -148,20 +148,18 @@ func mychart() {
 		XValues: blocks,
 		YValues: hours,
 	}
-	/*
-		hddSeries := &chart.ContinuousSeries{
-			Name: "Cumulative sync time (HDD)",
-			Style: chart.Style{
-				Show:        true,
-				StrokeColor: chart.ColorRed,
-				FillColor:   chart.ColorRed.WithAlpha(100),
-			},
-			XValues: blocks0,
-			YValues: hours0,
-		}
-	*/
+	badgerSeries := &chart.ContinuousSeries{
+		Name: "Cumulative sync time (badger)",
+		Style: chart.Style{
+			Show:        true,
+			StrokeColor: chart.ColorRed,
+			FillColor:   chart.ColorRed.WithAlpha(100),
+		},
+		XValues: blocks0,
+		YValues: hours0,
+	}
 	dbsizeSeries := &chart.ContinuousSeries{
-		Name: "Database size",
+		Name: "Database size (bolt)",
 		Style: chart.Style{
 			Show:        true,
 			StrokeColor: chart.ColorBlack,
@@ -170,6 +168,16 @@ func mychart() {
 		XValues: blocks,
 		YValues: dbsize,
 	}
+	dbsizeSeries0 := &chart.ContinuousSeries{
+		Name: "Database size (badger)",
+		Style: chart.Style{
+			Show:        true,
+			StrokeColor: chart.ColorOrange,
+		},
+		YAxis:   chart.YAxisSecondary,
+		XValues: blocks,
+		YValues: dbsize0,
+	}
 
 	graph1 := chart.Chart{
 		Width:  1280,
@@ -223,8 +231,9 @@ func mychart() {
 		},
 		Series: []chart.Series{
 			mainSeries,
-			//hddSeries,
+			badgerSeries,
 			dbsizeSeries,
+			dbsizeSeries0,
 		},
 	}
 
@@ -1180,7 +1189,9 @@ func main() {
 	if *action == "bucketStats" {
 		bucketStats(*chaindata)
 	}
-	//mychart()
+	if *action == "syncChart" {
+		mychart()
+	}
 	//testRebuild()
 	if *action == "testRewind" {
 		testRewind(*chaindata, *block, *rewind)
diff --git a/cmd/hack/scripts/geth_log_parse.py b/cmd/hack/scripts/geth_log_parse.py
new file mode 100644
index 000000000..af879b2b4
--- /dev/null
+++ b/cmd/hack/scripts/geth_log_parse.py
@@ -0,0 +1,59 @@
+import datetime
+
+with open('/Users/alexeyakhunov/workspace/src/github.com/ledgerwatch/turbo-geth/badger.log') as f:
+    lines = f.readlines()
+
+print(len(lines))
+
+seconds_before_reset = 0
+seconds_since_reset = 0
+last_reset = None
+last_block = 0
+trie_nodes = 0
+db_size = 0
+heap = 0
+
+with open('/Users/alexeyakhunov/workspace/src/github.com/ledgerwatch/turbo-geth/badger.csv', 'w') as w:
+
+    for l in lines:
+        if l.startswith('Oct'):
+            l = l[49:]
+        if l.startswith('INFO ['):
+            # Line with date time, parse it INFO [01-29|01:31:18]
+            # datetime.datetime(year, month, day[, hour[, minute[, second[, microsecond[, tzinfo]]]]])
+            dt = datetime.datetime(2018, int(l[6:8]), int(l[9:11]), int(l[12:14]), int(l[15:17]), int(l[18:20]))
+        if 'Block synchronisation started' in l:
+            print(l)
+            last_reset = dt
+            print('Before Reset. Hours before reset:', (seconds_before_reset/3600.0), '. Hours since last reset:', (seconds_since_reset/3600.0))
+            seconds_before_reset += seconds_since_reset
+            seconds_since_reset = 0
+            print('After Reset. Hours before reset:', (seconds_before_reset / 3600.0), '. Hours since last reset:', (seconds_since_reset / 3600.0))
+        if ('Memory' in l) and ('nodes=' in l):
+            ns = l.index('nodes=') + len('nodes=')
+            ne = len(l)
+            if ' ' in l[ns:]:
+                ne = l.index(' ', ns)
+            trie_nodes = int(l[ns:ne])
+        if ('Database' in l) and ('size=' in l):
+            ns = l.index('size=') + len('size=')
+            ws = l.index(' written=', ns)
+            s = int(l[ns:ws])
+            if s > db_size:
+                db_size = s
+        if ('Memory' in l) and ('alloc=' in l):
+            ns = l.index('alloc=') + len('alloc=')
+            ne = l.index(' ', ns)
+            heap = int(l[ns:ne])
+        if 'Imported new chain segment' in l:
+            ns = l.index('number=') + len('number=')
+            ne = l.index(' ', ns)
+            block = int(l[ns:ne])
+            if block > last_block:
+                if not last_reset is None:
+                    seconds_since_reset = (dt - last_reset).total_seconds()
+                print(block/1000000.0, ',', \
+                    (seconds_before_reset + seconds_since_reset)/3600.0, ',', \
+                    db_size/1024.0/1024.0/1024.0, ',', \
+                    trie_nodes/1000000.0, ',', \
+                    heap/1024.0/1024.0, file=w)