mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2025-01-01 00:31:21 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

e3: auto-remove useless files (#779)

Browse Source
This commit is contained in:
Alex Sharov 2022-12-14 09:03:09 +07:00 committed by GitHub
parent c9c1b15547
commit b20afcad3c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 18 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
state/domain.go
View File

@ -28,7 +28,6 @@ import (
"path/filepath"
"regexp"
"strconv"
"strings"
"sync/atomic"
"time"
@ -126,7 +125,10 @@ func NewDomain(
if err != nil {
return nil, err
}
d.scanStateFiles(files)
uselessFiles := d.scanStateFiles(files)
for _, f := range uselessFiles {
_ = os.Remove(filepath.Join(d.dir, f))
}
if err = d.openFiles(); err != nil {
return nil, err
}
@ -142,10 +144,9 @@ func (d *Domain) GetAndResetStats() DomainStats {
return r
}
func (d *Domain) scanStateFiles(files []fs.DirEntry) {
func (d *Domain) scanStateFiles(files []fs.DirEntry) (uselessFiles []string) {
re := regexp.MustCompile("^" + d.filenameBase + ".([0-9]+)-([0-9]+).kv$")
var err error
var uselessFiles []string
for _, f := range files {
if !f.Type().IsRegular() {
continue
@ -226,10 +227,7 @@ func (d *Domain) scanStateFiles(files []fs.DirEntry) {
}
d.files.ReplaceOrInsert(item)
}
if len(uselessFiles) > 0 {
log.Info("[snapshots] history can delete", "files", strings.Join(uselessFiles, ","))
}
return uselessFiles
}
func (d *Domain) openFiles() error {

13
state/history.go
View File

@ -28,7 +28,6 @@ import (
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"time"
@ -92,17 +91,19 @@ func NewHistory(
if err != nil {
return nil, err
}
h.scanStateFiles(files)
uselessFiles := h.scanStateFiles(files)
for _, f := range uselessFiles {
_ = os.Remove(filepath.Join(h.dir, f))
}
if err = h.openFiles(); err != nil {
return nil, fmt.Errorf("NewHistory.openFiles: %s, %w", filenameBase, err)
}
return &h, nil
}
func (h *History) scanStateFiles(files []fs.DirEntry) {
func (h *History) scanStateFiles(files []fs.DirEntry) (uselessFiles []string) {
re := regexp.MustCompile("^" + h.filenameBase + ".([0-9]+)-([0-9]+).v$")
var err error
var uselessFiles []string
for _, f := range files {
if !f.Type().IsRegular() {
continue
@ -184,9 +185,7 @@ func (h *History) scanStateFiles(files []fs.DirEntry) {
}
h.files.ReplaceOrInsert(item)
}
if len(uselessFiles) > 0 {
log.Info("[snapshots] history can delete", "files", strings.Join(uselessFiles, ","))
}
return uselessFiles
}
func (h *History) openFiles() error {

13
state/inverted_index.go
View File

@ -28,7 +28,6 @@ import (
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"time"
@ -87,17 +86,19 @@ func NewInvertedIndex(
if err != nil {
return nil, fmt.Errorf("NewInvertedIndex: %s, %w", filenameBase, err)
}
ii.scanStateFiles(files)
uselessFiles := ii.scanStateFiles(files)
for _, f := range uselessFiles {
_ = os.Remove(filepath.Join(ii.dir, f))
}
if err = ii.openFiles(); err != nil {
return nil, fmt.Errorf("NewInvertedIndex: %s, %w", filenameBase, err)
}
return &ii, nil
}
func (ii *InvertedIndex) scanStateFiles(files []fs.DirEntry) {
func (ii *InvertedIndex) scanStateFiles(files []fs.DirEntry) (uselessFiles []string) {
re := regexp.MustCompile("^" + ii.filenameBase + ".([0-9]+)-([0-9]+).ef$")
var err error
var uselessFiles []string
for _, f := range files {
if !f.Type().IsRegular() {
continue
@ -179,9 +180,7 @@ func (ii *InvertedIndex) scanStateFiles(files []fs.DirEntry) {
}
ii.files.ReplaceOrInsert(item)
}
if len(uselessFiles) > 0 {
log.Info("[snapshots] history can delete", "files", strings.Join(uselessFiles, ","))
}
return uselessFiles
}
func (ii *InvertedIndex) missedIdxFiles() (l []*filesItem) {