mirror of
https://gitlab.com/pulsechaincom/erigon-pulse.git
synced 2024-12-24 20:47:16 +00:00
447 lines
11 KiB
Go
447 lines
11 KiB
Go
package stateless
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"math"
|
|
"math/big"
|
|
"time"
|
|
|
|
"github.com/ledgerwatch/turbo-geth/common/dbutils"
|
|
|
|
"github.com/ledgerwatch/bolt"
|
|
|
|
"github.com/ledgerwatch/turbo-geth/common"
|
|
"github.com/ledgerwatch/turbo-geth/core/types/accounts"
|
|
"github.com/ledgerwatch/turbo-geth/crypto"
|
|
|
|
"image"
|
|
"image/color"
|
|
|
|
"github.com/llgcode/draw2d"
|
|
"github.com/llgcode/draw2d/draw2dimg"
|
|
"github.com/petar/GoLLRB/llrb"
|
|
//"sort"
|
|
)
|
|
|
|
type KeyItem struct {
|
|
key common.Hash
|
|
}
|
|
|
|
func (a *KeyItem) Less(b llrb.Item) bool {
|
|
bi := b.(*KeyItem)
|
|
return bytes.Compare(a.key[:], bi.key[:]) < 0
|
|
}
|
|
|
|
func storageRoot(db *bolt.DB, contract common.Address) (common.Hash, error) {
|
|
var storageRoot common.Hash
|
|
err := db.View(func(tx *bolt.Tx) error {
|
|
b := tx.Bucket(dbutils.AccountsBucket)
|
|
if b == nil {
|
|
return fmt.Errorf("Could not find accounts bucket")
|
|
}
|
|
enc, _ := b.Get(crypto.Keccak256(contract[:]))
|
|
if enc == nil {
|
|
return fmt.Errorf("Could find account %x\n", contract)
|
|
}
|
|
var account accounts.Account
|
|
if err := account.DecodeForStorage(enc); err != nil {
|
|
return err
|
|
}
|
|
storageRoot = account.Root
|
|
return nil
|
|
})
|
|
return storageRoot, err
|
|
}
|
|
|
|
func actualContractSize(db *bolt.DB, contract common.Address) (int, error) {
|
|
var fk [52]byte
|
|
copy(fk[:], contract[:])
|
|
actual := 0
|
|
if err := db.View(func(tx *bolt.Tx) error {
|
|
b := tx.Bucket(dbutils.StorageBucket)
|
|
c := b.Cursor()
|
|
for k, _ := c.Seek(fk[:]); k != nil && bytes.HasPrefix(k, contract[:]); k, _ = c.Next() {
|
|
actual++
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
return 0, err
|
|
}
|
|
return actual, nil
|
|
}
|
|
|
|
func estimateContractSize(seed common.Hash, current *llrb.LLRB, probes int, probeWidth int, trace bool) (int, error) {
|
|
if trace {
|
|
fmt.Printf("-----------------------------\n")
|
|
}
|
|
var seekkey KeyItem
|
|
var large [33]byte
|
|
large[0] = 1
|
|
largeInt := big.NewInt(0).SetBytes(large[:])
|
|
sectorSize := big.NewInt(0).Div(largeInt, big.NewInt(int64(probes)))
|
|
probeKeyHash := seed[:]
|
|
probe := big.NewInt(0).SetBytes(seed[:])
|
|
samples := make(map[[32]byte]*big.Int)
|
|
curr := big.NewInt(0)
|
|
prev := big.NewInt(0)
|
|
allSteps := big.NewInt(0)
|
|
|
|
for i := 0; i < probes; i++ {
|
|
if trace {
|
|
fmt.Printf("i==%d\n", i)
|
|
}
|
|
prev.SetBytes(probeKeyHash)
|
|
allSteps.SetUint64(0)
|
|
for ci := 0; ci < 32-len(probeKeyHash); ci++ {
|
|
seekkey.key[ci] = 0
|
|
}
|
|
copy(seekkey.key[32-len(probeKeyHash):], probeKeyHash)
|
|
if trace {
|
|
fmt.Printf("seekkey: %x\n", seekkey.key)
|
|
}
|
|
var firstK *KeyItem
|
|
for j := 0; j <= probeWidth; {
|
|
if trace {
|
|
fmt.Printf("Start with j == %d\n", j)
|
|
}
|
|
current.AscendGreaterOrEqual(&seekkey, func(item llrb.Item) bool {
|
|
if j > probeWidth {
|
|
return false
|
|
}
|
|
k := item.(*KeyItem)
|
|
if trace {
|
|
fmt.Printf("j == %d, %x\n", j, k.key)
|
|
}
|
|
curr.SetBytes(k.key[:])
|
|
diff := big.NewInt(0)
|
|
if prev.Cmp(curr) < 0 {
|
|
diff.Sub(curr, prev)
|
|
} else {
|
|
diff.Sub(prev, curr)
|
|
diff.Sub(largeInt, diff)
|
|
}
|
|
allSteps.Add(allSteps, diff)
|
|
if _, ok := samples[k.key]; !ok || j > 0 {
|
|
samples[k.key] = diff
|
|
}
|
|
prev.SetBytes(k.key[:])
|
|
j++
|
|
if firstK == nil {
|
|
firstK = k
|
|
} else if k == firstK {
|
|
j = probeWidth + 1
|
|
}
|
|
return true
|
|
})
|
|
if j <= probeWidth {
|
|
for ci := 0; ci < 32; ci++ {
|
|
seekkey.key[ci] = 0
|
|
}
|
|
if trace {
|
|
fmt.Printf("Looping at j == %d\n", j)
|
|
}
|
|
}
|
|
}
|
|
if allSteps.Cmp(sectorSize) < 0 {
|
|
probe.Add(probe, sectorSize)
|
|
} else {
|
|
if trace {
|
|
fmt.Printf("Move by allSteps\n")
|
|
}
|
|
probe.Add(probe, allSteps)
|
|
}
|
|
for probe.Cmp(largeInt) >= 0 {
|
|
probe.Sub(probe, largeInt)
|
|
}
|
|
probeKeyHash = probe.Bytes()
|
|
}
|
|
total := big.NewInt(0)
|
|
for _, sample := range samples {
|
|
total.Add(total, sample)
|
|
}
|
|
sampleCount := len(samples)
|
|
estimatedInt := big.NewInt(0)
|
|
if sampleCount > 0 {
|
|
estimatedInt.Mul(largeInt, big.NewInt(int64(sampleCount)))
|
|
estimatedInt.Div(estimatedInt, total)
|
|
}
|
|
if trace {
|
|
fmt.Printf("probes: %d, probeWidth: %d, sampleCount: %d, estimate: %d\n", probes, probeWidth, sampleCount, estimatedInt)
|
|
}
|
|
return int(estimatedInt.Int64()), nil
|
|
}
|
|
|
|
func getHeatMapColor(value float64) (red, green, blue float64) {
|
|
const NUM_COLORS int = 4
|
|
color := [NUM_COLORS][3]float64{
|
|
{0, 0, 1},
|
|
{0, 1, 0},
|
|
{1, 1, 0},
|
|
{1, 0, 0},
|
|
}
|
|
// A static array of 4 colors: (blue, green, yellow, red) using {r,g,b} for each.
|
|
|
|
var idx1 int // |-- Our desired color will be between these two indexes in "color".
|
|
var idx2 int // |
|
|
var fractBetween float64 // Fraction between "idx1" and "idx2" where our value is.
|
|
|
|
if value <= 0 {
|
|
idx1 = 0
|
|
idx2 = 0
|
|
} else if value >= 1 {
|
|
idx1 = NUM_COLORS - 1
|
|
idx2 = NUM_COLORS - 1
|
|
} else {
|
|
value = value * float64(NUM_COLORS-1) // Will multiply value by 3.
|
|
idx1 = int(value) // Our desired color will be after this index.
|
|
idx2 = idx1 + 1 // ... and before this index (inclusive).
|
|
fractBetween = value - float64(idx1) // Distance between the two indexes (0-1).
|
|
}
|
|
|
|
if idx1 >= len(color) || idx1 < 0 {
|
|
fmt.Printf("value: %f, idx1: %d\n", value, idx1)
|
|
}
|
|
if idx2 >= len(color) || idx2 < 0 {
|
|
fmt.Printf("value: %f, idx2: %d\n", value, idx2)
|
|
}
|
|
red = (color[idx2][0]-color[idx1][0])*fractBetween + color[idx1][0]
|
|
green = (color[idx2][1]-color[idx1][1])*fractBetween + color[idx1][1]
|
|
blue = (color[idx2][2]-color[idx1][2])*fractBetween + color[idx1][2]
|
|
return
|
|
}
|
|
|
|
func estimateContract(
|
|
idx int,
|
|
current *llrb.LLRB,
|
|
seed common.Hash,
|
|
valMap map[int][][]float64,
|
|
maxValMap map[int][][]float64,
|
|
valCount map[int]int,
|
|
maxi, maxj int,
|
|
trace bool,
|
|
) bool {
|
|
maxAllVals := maxValMap[0]
|
|
allVals := valMap[0]
|
|
actual := current.Len()
|
|
if trace {
|
|
fmt.Printf("Actual size: %d\n", actual)
|
|
}
|
|
if actual < 2 {
|
|
return false
|
|
}
|
|
category := int(math.Log2(float64(actual)))
|
|
if category != 1 {
|
|
//return false
|
|
}
|
|
//fmt.Printf("%d\n", idx)
|
|
maxVals, ok := maxValMap[category]
|
|
if !ok {
|
|
maxVals = make([][]float64, maxi)
|
|
for i := 1; i < maxi; i++ {
|
|
maxVals[i] = make([]float64, maxj)
|
|
}
|
|
maxValMap[category] = maxVals
|
|
}
|
|
vals, ok := valMap[category]
|
|
if !ok {
|
|
vals = make([][]float64, maxi)
|
|
for i := 1; i < maxi; i++ {
|
|
vals[i] = make([]float64, maxj)
|
|
}
|
|
valMap[category] = vals
|
|
}
|
|
for i := 1; i < maxi; i++ {
|
|
for j := 1; j < maxj; j++ {
|
|
estimated, err := estimateContractSize(seed, current, i, j, trace)
|
|
check(err)
|
|
e := (float64(actual) - float64(estimated)) / float64(actual)
|
|
eAbs := math.Abs(e)
|
|
if eAbs > maxVals[i][j] {
|
|
maxVals[i][j] = eAbs
|
|
}
|
|
if eAbs > maxAllVals[i][j] {
|
|
maxAllVals[i][j] = eAbs
|
|
}
|
|
vals[i][j] += e
|
|
allVals[i][j] += e
|
|
}
|
|
}
|
|
valCount[category]++
|
|
valCount[0]++
|
|
return true
|
|
}
|
|
|
|
func estimate() {
|
|
startTime := time.Now()
|
|
db, err := bolt.Open("/Volumes/tb4/turbo-geth-10/geth/chaindata", 0600, &bolt.Options{ReadOnly: true})
|
|
//db, err := bolt.Open("/Users/alexeyakhunov/Library/Ethereum/geth/chaindata", 0600, &bolt.Options{ReadOnly: true})
|
|
//db, err := bolt.Open("/home/akhounov/.ethereum/geth/chaindata", 0600, &bolt.Options{ReadOnly: true})
|
|
check(err)
|
|
defer db.Close()
|
|
maxi := 20
|
|
maxj := 50
|
|
//maxi := 2
|
|
//maxj := 10
|
|
trace := false
|
|
maxValMap := make(map[int][][]float64)
|
|
maxAllVals := make([][]float64, maxi)
|
|
for i := 1; i < maxi; i++ {
|
|
maxAllVals[i] = make([]float64, maxj)
|
|
}
|
|
maxValMap[0] = maxAllVals
|
|
valMap := make(map[int][][]float64)
|
|
allVals := make([][]float64, maxi)
|
|
for i := 1; i < maxi; i++ {
|
|
allVals[i] = make([]float64, maxj)
|
|
}
|
|
valMap[0] = allVals
|
|
valCount := make(map[int]int)
|
|
|
|
// Go through the current state
|
|
var addr common.Address
|
|
itemsByAddress := make(map[common.Address]int)
|
|
deleted := make(map[common.Address]bool) // Deleted contracts
|
|
numDeleted := 0
|
|
var current *llrb.LLRB
|
|
count := 0
|
|
contractCount := 0
|
|
err = db.View(func(tx *bolt.Tx) error {
|
|
a := tx.Bucket(dbutils.AccountsBucket)
|
|
b := tx.Bucket(dbutils.StorageBucket)
|
|
if b == nil {
|
|
return nil
|
|
}
|
|
c := b.Cursor()
|
|
for k, _ := c.First(); k != nil; k, _ = c.Next() {
|
|
copy(addr[:], k[:20])
|
|
del, ok := deleted[addr]
|
|
if !ok {
|
|
v, _ := a.Get(crypto.Keccak256(addr[:]))
|
|
del = v == nil
|
|
deleted[addr] = del
|
|
if del {
|
|
numDeleted++
|
|
}
|
|
}
|
|
if del {
|
|
continue
|
|
}
|
|
if _, ok := itemsByAddress[addr]; !ok {
|
|
if current != nil {
|
|
seed, err := storageRoot(db, addr)
|
|
check(err)
|
|
//if contractCount == 4 {
|
|
done := estimateContract(contractCount, current, seed, valMap, maxValMap, valCount, maxi, maxj, trace)
|
|
if done && trace {
|
|
return nil
|
|
}
|
|
//}
|
|
contractCount++
|
|
if contractCount%1000 == 0 {
|
|
fmt.Printf("Processed contracts: %d\n", contractCount)
|
|
}
|
|
}
|
|
current = llrb.New()
|
|
}
|
|
ki := &KeyItem{}
|
|
copy(ki.key[:], k[20:])
|
|
current.InsertNoReplace(ki)
|
|
itemsByAddress[addr]++
|
|
count++
|
|
if count%100000 == 0 {
|
|
fmt.Printf("Processed %d storage records, deleted contracts: %d\n", count, numDeleted)
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
check(err)
|
|
|
|
for category, vals := range valMap {
|
|
for i := 1; i < maxi; i++ {
|
|
for j := 1; j < maxj; j++ {
|
|
vals[i][j] /= float64(valCount[category])
|
|
}
|
|
}
|
|
}
|
|
|
|
fmt.Printf("Generating images...\n")
|
|
for category, vals := range valMap {
|
|
var maxe float64
|
|
var mine float64 = 100000000.0
|
|
for i := 1; i < maxi; i++ {
|
|
for j := 1; j < maxj; j++ {
|
|
a := math.Abs(vals[i][j])
|
|
if a > maxe {
|
|
maxe = a
|
|
}
|
|
if a < mine {
|
|
mine = a
|
|
}
|
|
}
|
|
}
|
|
if maxe > 1.0 {
|
|
maxe = 1.0
|
|
}
|
|
if maxe == mine {
|
|
maxe = mine + 1.0
|
|
}
|
|
// Initialize the graphic context on an RGBA image
|
|
imageWidth := 2000
|
|
imageHeight := 480
|
|
dest := image.NewRGBA(image.Rect(0, 0, imageWidth, imageHeight))
|
|
gc := draw2dimg.NewGraphicContext(dest)
|
|
|
|
// Set some properties
|
|
gc.SetFillColor(color.RGBA{0x44, 0xff, 0x44, 0xff})
|
|
gc.SetStrokeColor(color.RGBA{0x44, 0x44, 0x44, 0xff})
|
|
gc.SetLineWidth(1)
|
|
cellWidth := float64(imageWidth) / float64(maxj+1)
|
|
cellHeight := float64(imageHeight) / float64(maxi+1)
|
|
for i := 1; i < maxi; i++ {
|
|
fi := float64(i)
|
|
gc.SetFontData(draw2d.FontData{Name: "luxi", Family: draw2d.FontFamilyMono})
|
|
gc.SetFillColor(image.Black)
|
|
gc.SetFontSize(12)
|
|
gc.FillStringAt(fmt.Sprintf("%d", i), 5, (fi+0.5)*cellHeight)
|
|
}
|
|
for j := 1; j < maxj; j++ {
|
|
fj := float64(j)
|
|
gc.SetFontData(draw2d.FontData{Name: "luxi", Family: draw2d.FontFamilyMono})
|
|
gc.SetFillColor(image.Black)
|
|
gc.SetFontSize(12)
|
|
gc.FillStringAt(fmt.Sprintf("%d", j), fj*cellWidth+5, 0.5*cellHeight)
|
|
}
|
|
for i := 1; i < maxi; i++ {
|
|
for j := 1; j < maxj; j++ {
|
|
e := vals[i][j]
|
|
heat := math.Abs(e)
|
|
if heat > 1.0 {
|
|
heat = 1.0
|
|
}
|
|
heat = (heat - mine) / (maxe - mine)
|
|
red, green, blue := getHeatMapColor(heat)
|
|
txt := fmt.Sprintf("%.1f%%", e*100.0)
|
|
fi := float64(i)
|
|
fj := float64(j)
|
|
gc.BeginPath() // Initialize a new path
|
|
gc.MoveTo(fj*cellWidth, fi*cellHeight)
|
|
gc.LineTo(fj*cellWidth, (fi+1)*cellHeight)
|
|
gc.LineTo((fj+1)*cellWidth, (fi+1)*cellHeight)
|
|
gc.LineTo((fj+1)*cellWidth, fi*cellHeight)
|
|
gc.LineTo(fj*cellWidth, fi*cellHeight)
|
|
gc.Close()
|
|
gc.SetFillColor(color.RGBA{byte(255.0 * red), byte(255.0 * green), byte(255.0 * blue), 0xff})
|
|
gc.FillStroke()
|
|
gc.SetFontData(draw2d.FontData{Name: "luxi", Family: draw2d.FontFamilyMono})
|
|
gc.SetFillColor(image.Black)
|
|
gc.SetFontSize(8)
|
|
gc.FillStringAt(txt, fj*cellWidth+5, (fi+0.5)*cellHeight)
|
|
}
|
|
}
|
|
// Save to file
|
|
draw2dimg.SaveToPngFile(fmt.Sprintf("heat_%d.png", category), dest)
|
|
}
|
|
fmt.Printf("Estimation took %s\n", time.Since(startTime))
|
|
}
|