mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-31 16:21:21 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Never store Ethash caches to disk (#944)

Browse Source 
* always keep ethash caches in ram

* fix tests
This commit is contained in:
Igor Mandrigin 2020-08-20 12:12:51 +02:00 committed by GitHub
parent f827f86c11
commit 3f81472831
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 55 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/geth/main.go
View File

@ -70,9 +70,7 @@ var (
utils.ExternalSignerFlag,
utils.NoUSBFlag,
utils.SmartCardDaemonPathFlag,
utils.EthashCacheDirFlag,
utils.EthashCachesInMemoryFlag,
utils.EthashCachesOnDiskFlag,
utils.EthashCachesLockMmapFlag,
utils.EthashDatasetDirFlag,
utils.EthashDatasetsInMemoryFlag,

2
cmd/geth/usage.go
View File

@ -81,9 +81,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
{
Name: "ETHASH",
Flags: []cli.Flag{
utils.EthashCacheDirFlag,
utils.EthashCachesInMemoryFlag,
utils.EthashCachesOnDiskFlag,
utils.EthashCachesLockMmapFlag,
utils.EthashDatasetDirFlag,
utils.EthashDatasetsInMemoryFlag,

17
cmd/utils/flags.go
View File

@ -274,20 +274,11 @@ var (
Usage: "Enable the DBG (debug) protocol",
}
// Ethash settings
EthashCacheDirFlag = DirectoryFlag{
Name: "ethash.cachedir",
Usage: "Directory to store the ethash verification caches (default = inside the datadir)",
}
EthashCachesInMemoryFlag = cli.IntFlag{
Name: "ethash.cachesinmem",
Usage: "Number of recent ethash caches to keep in memory (16MB each)",
Value: eth.DefaultConfig.Ethash.CachesInMem,
}
EthashCachesOnDiskFlag = cli.IntFlag{
Name: "ethash.cachesondisk",
Usage: "Number of recent ethash caches to keep on disk (16MB each)",
Value: eth.DefaultConfig.Ethash.CachesOnDisk,
}
EthashCachesLockMmapFlag = cli.BoolFlag{
Name: "ethash.cacheslockmmap",
Usage: "Lock memory maps of recent ethash caches",
@ -1410,18 +1401,12 @@ func setTxPool(ctx *cli.Context, cfg *core.TxPoolConfig) {
}
func setEthash(ctx *cli.Context, cfg *eth.Config) {
if ctx.GlobalIsSet(EthashCacheDirFlag.Name) {
cfg.Ethash.CacheDir = ctx.GlobalString(EthashCacheDirFlag.Name)
}
if ctx.GlobalIsSet(EthashDatasetDirFlag.Name) {
cfg.Ethash.DatasetDir = ctx.GlobalString(EthashDatasetDirFlag.Name)
}
if ctx.GlobalIsSet(EthashCachesInMemoryFlag.Name) {
cfg.Ethash.CachesInMem = ctx.GlobalInt(EthashCachesInMemoryFlag.Name)
}
if ctx.GlobalIsSet(EthashCachesOnDiskFlag.Name) {
cfg.Ethash.CachesOnDisk = ctx.GlobalInt(EthashCachesOnDiskFlag.Name)
}
if ctx.GlobalIsSet(EthashCachesLockMmapFlag.Name) {
cfg.Ethash.CachesLockMmap = ctx.GlobalBool(EthashCachesLockMmapFlag.Name)
}
@ -1845,9 +1830,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readOnly bool) (chainConfig *
engine = ethash.NewFaker()
if !ctx.GlobalBool(FakePoWFlag.Name) {
engine = ethash.New(ethash.Config{
CacheDir: stack.ResolvePath(eth.DefaultConfig.Ethash.CacheDir),
CachesInMem: eth.DefaultConfig.Ethash.CachesInMem,
CachesOnDisk: eth.DefaultConfig.Ethash.CachesOnDisk,
CachesLockMmap: eth.DefaultConfig.Ethash.CachesLockMmap,
DatasetDir: stack.ResolvePath(eth.DefaultConfig.Ethash.DatasetDir),
DatasetsInMem: eth.DefaultConfig.Ethash.DatasetsInMem,

80
consensus/ethash/algorithm_test.go
View File

@ -20,15 +20,12 @@ import (
"bytes"
"encoding/binary"
"io/ioutil"
"math/big"
"os"
"reflect"
"sync"
"testing"
"github.com/ledgerwatch/turbo-geth/common"
"github.com/ledgerwatch/turbo-geth/common/hexutil"
"github.com/ledgerwatch/turbo-geth/core/types"
)
// prepare converts an ethash cache or dataset from a byte stream into the internal
@ -697,46 +694,49 @@ func TestHashimoto(t *testing.T) {
// Tests that caches generated on disk may be done concurrently.
func TestConcurrentDiskCacheGeneration(t *testing.T) {
// Create a temp folder to generate the caches into
cachedir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Failed to create temporary cache dir: %v", err)
}
defer os.RemoveAll(cachedir)
t.Skip("turbo-geth doesn't use disk cache")
/*
// Create a temp folder to generate the caches into
cachedir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("Failed to create temporary cache dir: %v", err)
}
defer os.RemoveAll(cachedir)
// Define a heavy enough block, one from mainnet should do
block := types.NewBlockWithHeader(&types.Header{
Number: big.NewInt(3311058),
ParentHash: common.HexToHash("0xd783efa4d392943503f28438ad5830b2d5964696ffc285f338585e9fe0a37a05"),
UncleHash: common.HexToHash("0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"),
Coinbase: common.HexToAddress("0xc0ea08a2d404d3172d2add29a45be56da40e2949"),
Root: common.HexToHash("0x77d14e10470b5850332524f8cd6f69ad21f070ce92dca33ab2858300242ef2f1"),
TxHash: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"),
ReceiptHash: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"),
Difficulty: big.NewInt(167925187834220),
GasLimit: 4015682,
GasUsed: 0,
Time: 1488928920,
Extra: []byte("www.bw.com"),
MixDigest: common.HexToHash("0x3e140b0784516af5e5ec6730f2fb20cca22f32be399b9e4ad77d32541f798cd0"),
Nonce: types.EncodeNonce(0xf400cd0006070c49),
})
// Simulate multiple processes sharing the same datadir
var pend sync.WaitGroup
// Define a heavy enough block, one from mainnet should do
block := types.NewBlockWithHeader(&types.Header{
Number: big.NewInt(3311058),
ParentHash: common.HexToHash("0xd783efa4d392943503f28438ad5830b2d5964696ffc285f338585e9fe0a37a05"),
UncleHash: common.HexToHash("0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"),
Coinbase: common.HexToAddress("0xc0ea08a2d404d3172d2add29a45be56da40e2949"),
Root: common.HexToHash("0x77d14e10470b5850332524f8cd6f69ad21f070ce92dca33ab2858300242ef2f1"),
TxHash: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"),
ReceiptHash: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"),
Difficulty: big.NewInt(167925187834220),
GasLimit: 4015682,
GasUsed: 0,
Time: 1488928920,
Extra: []byte("www.bw.com"),
MixDigest: common.HexToHash("0x3e140b0784516af5e5ec6730f2fb20cca22f32be399b9e4ad77d32541f798cd0"),
Nonce: types.EncodeNonce(0xf400cd0006070c49),
})
// Simulate multiple processes sharing the same datadir
var pend sync.WaitGroup
for i := 0; i < 3; i++ {
pend.Add(1)
for i := 0; i < 3; i++ {
pend.Add(1)
go func(idx int) {
defer pend.Done()
ethash := New(Config{cachedir, 0, 1, false, "", 0, 0, false, ModeNormal, nil}, nil, false)
defer ethash.Close()
if err := ethash.VerifySeal(nil, block.Header()); err != nil {
t.Errorf("proc %d: block verification failed: %v", idx, err)
}
}(i)
}
pend.Wait()
go func(idx int) {
defer pend.Done()
ethash := New(Config{cachedir, 0, 1, false, "", 0, 0, false, ModeNormal, nil}, nil, false)
defer ethash.Close()
if err := ethash.VerifySeal(nil, block.Header()); err != nil {
t.Errorf("proc %d: block verification failed: %v", idx, err)
}
}(i)
}
pend.Wait()
*/
}
// Benchmarks the cache generation performance.

13
consensus/ethash/ethash.go
View File

@ -43,6 +43,8 @@ import (
"github.com/hashicorp/golang-lru/simplelru"
)
const doNotStoreCachesOnDisk = ""
var ErrInvalidDumpMagic = errors.New("invalid dump magic")
var (
@ -50,7 +52,7 @@ var (
two256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0))
// sharedEthash is a full instance that can be shared between multiple users.
sharedEthash = New(Config{"", 3, 0, false, "", 1, 0, false, ModeNormal, nil}, nil, false)
sharedEthash = New(Config{3, false, "", 1, 0, false, ModeNormal, nil}, nil, false)
// algorithmRevision is the data structure version used for file naming.
algorithmRevision = 23
@ -403,9 +405,7 @@ const (
// Config are the configuration parameters of the ethash.
type Config struct {
CacheDir string
CachesInMem int
CachesOnDisk int
CachesLockMmap bool
DatasetDir string
DatasetsInMem int
@ -451,9 +451,6 @@ func New(config Config, notify []string, noverify bool) *Ethash {
config.Log.Warn("One ethash cache must always be in memory", "requested", config.CachesInMem)
config.CachesInMem = 1
}
if config.CacheDir != "" && config.CachesOnDisk > 0 {
config.Log.Info("Disk storage enabled for ethash caches", "dir", config.CacheDir, "count", config.CachesOnDisk)
}
if config.DatasetDir != "" && config.DatasetsOnDisk > 0 {
config.Log.Info("Disk storage enabled for ethash DAGs", "dir", config.DatasetDir, "count", config.DatasetsOnDisk)
}
@ -560,12 +557,12 @@ func (ethash *Ethash) cache(block uint64) *cache {
current := currentI.(*cache)
// Wait for generation finish.
current.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.CachesLockMmap, ethash.config.PowMode == ModeTest)
current.generate(doNotStoreCachesOnDisk, 0, ethash.config.CachesLockMmap, ethash.config.PowMode == ModeTest)
// If we need a new future cache, now's a good time to regenerate it.
if futureI != nil {
future := futureI.(*cache)
go future.generate(ethash.config.CacheDir, ethash.config.CachesOnDisk, ethash.config.CachesLockMmap, ethash.config.PowMode == ModeTest)
go future.generate(doNotStoreCachesOnDisk, 0, ethash.config.CachesLockMmap, ethash.config.PowMode == ModeTest)
}
return current
}

3
consensus/ethash/ethash_test.go
View File

@ -58,12 +58,13 @@ func TestTestMode(t *testing.T) {
// This test checks that cache lru logic doesn't crash under load.
// It reproduces https://github.com/ledgerwatch/turbo-geth/issues/14943
func TestCacheFileEvict(t *testing.T) {
// turbo-geth: all caches are in RAM, no files are actually used there
tmpdir, err := ioutil.TempDir("", "ethash-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}, nil, false)
e := New(Config{CachesInMem: 3, PowMode: ModeTest}, nil, false)
defer e.Close()
workers := 8

2
eth/backend.go
View File

@ -344,9 +344,7 @@ func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, co
return ethash.NewShared()
default:
engine := ethash.New(ethash.Config{
CacheDir: stack.ResolvePath(config.CacheDir),
CachesInMem: config.CachesInMem,
CachesOnDisk: config.CachesOnDisk,
CachesLockMmap: config.CachesLockMmap,
DatasetDir: config.DatasetDir,
DatasetsInMem: config.DatasetsInMem,

18
eth/config.go
View File

@ -50,24 +50,22 @@ var DefaultLightGPOConfig = gasprice.Config{
var DefaultConfig = Config{
SyncMode: downloader.StagedSync,
Ethash: ethash.Config{
CacheDir: "ethash",
CachesInMem: 2,
CachesOnDisk: 3,
CachesLockMmap: false,
DatasetsInMem: 1,
DatasetsOnDisk: 2,
DatasetsLockMmap: false,
},
NetworkID: 1,
LightPeers: 100,
UltraLightFraction: 75,
DatabaseCache: 512,
TrieCleanCache: 256,
NetworkID: 1,
LightPeers: 100,
UltraLightFraction: 75,
DatabaseCache: 512,
TrieCleanCache: 256,
TrieCleanCacheJournal: "triecache",
TrieCleanCacheRejournal: 60 * time.Minute,
TrieDirtyCache: 256,
TrieTimeout: 60 * time.Minute,
StorageMode: ethdb.DefaultStorageMode,
TrieDirtyCache: 256,
TrieTimeout: 60 * time.Minute,
StorageMode: ethdb.DefaultStorageMode,
Miner: miner.Config{
GasFloor: 8000000,
GasCeil: 8000000,