mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-23 04:03:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

log ETL temp files datasize (#873)

Browse Source
This commit is contained in:
Igor Mandrigin 2020-08-05 17:33:58 +02:00 committed by GitHub
parent f41959f030
commit fa89cdefd5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
common/etl/dataprovider.go
View File

@ -14,7 +14,7 @@ import (
type dataProvider interface {
Next(decoder Decoder) ([]byte, []byte, error)
Dispose() error
Dispose() (uint64, error)
}
type fileDataProvider struct {
@ -74,16 +74,20 @@ func (p *fileDataProvider) Next(decoder Decoder) ([]byte, []byte, error) {
return readElementFromDisk(decoder)
}
func (p *fileDataProvider) Dispose() error {
func (p *fileDataProvider) Dispose() (uint64, error) {
info, errStat := os.Stat(p.file.Name())
errClose := p.file.Close()
errRemove := os.Remove(p.file.Name())
if errClose != nil {
return errClose
return 0, errClose
}
if errRemove != nil {
return errRemove
return 0, errRemove
}
return nil
if errStat != nil {
return 0, errStat
}
return uint64(info.Size()), nil
}
func (p *fileDataProvider) String() string {
@ -119,8 +123,8 @@ func (p *memoryDataProvider) Next(decoder Decoder) ([]byte, []byte, error) {
return entry.key, entry.value, nil
}
func (p *memoryDataProvider) Dispose() error {
return nil
func (p *memoryDataProvider) Dispose() (uint64, error) {
return 0 /* doesn't take space on disk */, nil
}
func (p *memoryDataProvider) String() string {

6
common/etl/etl.go
View File

@ -6,6 +6,7 @@ import (
"io"
"time"
"github.com/c2h5oh/datasize"
"github.com/ledgerwatch/turbo-geth/common"
"github.com/ledgerwatch/turbo-geth/ethdb"
"github.com/ledgerwatch/turbo-geth/log"
@ -120,12 +121,15 @@ func extractBucketIntoFiles(
return collector.flushBuffer(nil, true)
}
func disposeProviders(providers []dataProvider) {
totalSize := uint64(0)
for _, p := range providers {
err := p.Dispose()
providerSize, err := p.Dispose()
if err != nil {
log.Warn("promoting hashed state, error while disposing provider", "provier", p, "err", err)
}
totalSize += providerSize
}
log.Info("etl: temp files removed successfully", "total size", datasize.ByteSize(totalSize).HumanReadable())
}
type bucketState struct {