mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-22 03:30:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

release: downloader: prohibit_new_downloads.lock check missed download (#9300)

Browse Source 
Cherry pick PR #9295

---------

Co-authored-by: Alex Sharov <AskAlexSharov@gmail.com>
This commit is contained in:
Andrew Ashikhmin 2024-01-24 12:41:31 +01:00 committed by GitHub
parent 4f6eda7694
commit 20999401b2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 53 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
erigon-lib/downloader/downloader.go
View File

@ -21,8 +21,6 @@ import (
"errors"
"fmt"
"net/url"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
@ -41,7 +39,6 @@ import (
"github.com/ledgerwatch/erigon-lib/common"
"github.com/ledgerwatch/erigon-lib/common/datadir"
"github.com/ledgerwatch/erigon-lib/common/dbg"
"github.com/ledgerwatch/erigon-lib/common/dir"
"github.com/ledgerwatch/erigon-lib/diagnostics"
"github.com/ledgerwatch/erigon-lib/downloader/downloadercfg"
"github.com/ledgerwatch/erigon-lib/downloader/snaptype"
@ -147,27 +144,6 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg, dirs datadir.Dirs, logger
return d, nil
}
const ProhibitNewDownloadsFileName = "prohibit_new_downloads.lock"
// Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast)
// After "download once" - Erigon will produce and seed new files
// Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts)
func (d *Downloader) prohibitNewDownloads() error {
fPath := filepath.Join(d.SnapDir(), ProhibitNewDownloadsFileName)
f, err := os.Create(fPath)
if err != nil {
return err
}
defer f.Close()
if err := f.Sync(); err != nil {
return err
}
return nil
}
func (d *Downloader) newDownloadsAreProhibited() bool {
return dir.FileExist(filepath.Join(d.SnapDir(), ProhibitNewDownloadsFileName))
}
func (d *Downloader) MainLoopInBackground(silent bool) {
d.wg.Add(1)
go func() {
@ -605,7 +581,7 @@ func (d *Downloader) AddMagnetLink(ctx context.Context, infoHash metainfo.Hash,
if d.alreadyHaveThisName(name) {
return nil
}
if d.newDownloadsAreProhibited() {
if d.torrentFiles.newDownloadsAreProhibited() {
return nil
}

2
erigon-lib/downloader/downloader_grpc_server.go
View File

@ -46,7 +46,7 @@ type GrpcServer struct {
}
func (s *GrpcServer) ProhibitNewDownloads(context.Context, *proto_downloader.ProhibitNewDownloadsRequest) (*emptypb.Empty, error) {
if err := s.d.prohibitNewDownloads(); err != nil {
if err := s.d.torrentFiles.prohibitNewDownloads(); err != nil {
return nil, err
}
return nil, nil

49
erigon-lib/downloader/torrent_files.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"github.com/anacrolix/torrent"
@ -28,8 +29,10 @@ func (tf *TorrentFiles) Exists(name string) bool {
}
func (tf *TorrentFiles) exists(name string) bool {
fPath := filepath.Join(tf.dir, name)
return dir2.FileExist(fPath + ".torrent")
if !strings.HasSuffix(name, ".torrent") {
name += ".torrent"
}
return dir2.FileExist(filepath.Join(tf.dir, name))
}
func (tf *TorrentFiles) Delete(name string) error {
tf.lock.Lock()
@ -38,8 +41,10 @@ func (tf *TorrentFiles) Delete(name string) error {
}
func (tf *TorrentFiles) delete(name string) error {
fPath := filepath.Join(tf.dir, name)
return os.Remove(fPath + ".torrent")
if !strings.HasSuffix(name, ".torrent") {
name += ".torrent"
}
return os.Remove(filepath.Join(tf.dir, name))
}
func (tf *TorrentFiles) Create(torrentFilePath string, res []byte) error {
@ -91,11 +96,10 @@ func (tf *TorrentFiles) createTorrentFromMetaInfo(fPath string, mi *metainfo.Met
return nil
}
func (tf *TorrentFiles) LoadByName(fName string) (*torrent.TorrentSpec, error) {
func (tf *TorrentFiles) LoadByName(name string) (*torrent.TorrentSpec, error) {
tf.lock.Lock()
defer tf.lock.Unlock()
fPath := filepath.Join(tf.dir, fName+".torrent")
return tf.load(fPath)
return tf.load(filepath.Join(tf.dir, name))
}
func (tf *TorrentFiles) LoadByPath(fPath string) (*torrent.TorrentSpec, error) {
@ -105,6 +109,9 @@ func (tf *TorrentFiles) LoadByPath(fPath string) (*torrent.TorrentSpec, error) {
}
func (tf *TorrentFiles) load(fPath string) (*torrent.TorrentSpec, error) {
if !strings.HasSuffix(fPath, ".torrent") {
fPath += ".torrent"
}
mi, err := metainfo.LoadFromFile(fPath)
if err != nil {
return nil, fmt.Errorf("LoadFromFile: %w, file=%s", err, fPath)
@ -112,3 +119,31 @@ func (tf *TorrentFiles) load(fPath string) (*torrent.TorrentSpec, error) {
mi.AnnounceList = Trackers
return torrent.TorrentSpecFromMetaInfoErr(mi)
}
const ProhibitNewDownloadsFileName = "prohibit_new_downloads.lock"
// Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast)
// After "download once" - Erigon will produce and seed new files
// Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts)
func (tf *TorrentFiles) prohibitNewDownloads() error {
tf.lock.Lock()
defer tf.lock.Unlock()
return CreateProhibitNewDownloadsFile(tf.dir)
}
func (tf *TorrentFiles) newDownloadsAreProhibited() bool {
tf.lock.Lock()
defer tf.lock.Unlock()
return dir2.FileExist(filepath.Join(tf.dir, ProhibitNewDownloadsFileName))
}
func CreateProhibitNewDownloadsFile(dir string) error {
fPath := filepath.Join(dir, ProhibitNewDownloadsFileName)
f, err := os.Create(fPath)
if err != nil {
return err
}
defer f.Close()
if err := f.Sync(); err != nil {
return err
}
return nil
}

3
erigon-lib/downloader/webseed.go
View File

@ -237,6 +237,9 @@ func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDi
if len(d.TorrentUrls()) == 0 {
return
}
if d.torrentFiles.newDownloadsAreProhibited() {
return
}
var addedNew int
e, ctx := errgroup.WithContext(ctx)
e.SetLimit(1024)

6
eth/backend.go
View File

@ -266,11 +266,15 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger
config.Sync.UseSnapshots = useSnapshots
config.Snapshot.Enabled = ethconfig.UseSnapshotsByChainName(config.Genesis.Config.ChainName) && useSnapshots
}
return nil
}); err != nil {
return nil, err
}
if !config.Sync.UseSnapshots {
if err := downloader.CreateProhibitNewDownloadsFile(dirs.Snap); err != nil {
return nil, err
}
}
ctx, ctxCancel := context.WithCancel(context.Background())

2
params/version.go
View File

@ -33,7 +33,7 @@ var (
const (
VersionMajor = 2 // Major version component of the current release
VersionMinor = 57 // Minor version component of the current release
VersionMicro = 0 // Patch version component of the current release
VersionMicro = 1 // Patch version component of the current release
VersionModifier = "" // Modifier component of the current release
VersionKeyCreated = "ErigonVersionCreated"
VersionKeyFinished = "ErigonVersionFinished"