Snapshots: rare nil pointer at fresh start (#3958)

README.md

@@ -362,6 +362,12 @@ Reserved for future use: **gRPC ports**: `9092` consensus engine, `9093` snapsho
 ### How to run local devnet?
 <code> 🔬 Detailed explanation is [here](/DEV_CHAIN.md).</code>
 
+### Docker permissions error
+
+Docker uses user erigon with UID/GID 1000 (for security reasons). You can see this user being created in the Dockerfile.
+Can fix by giving a host's user ownership of the folder, where the host's user UID/GID is the same as the docker's user UID/GID (1000).
+More details in [post](https://www.fullstaq.com/knowledge-hub/blogs/docker-and-the-host-filesystem-owner-matching-problem)
+
 Getting in touch
 ================
 

cmd/downloader/downloader/downloader.go

@@ -3,7 +3,6 @@ package downloader
 import (
 	"context"
 	"fmt"
-	"math"
 	"runtime"
 	"time"
 
@@ -118,10 +117,6 @@ func LoggingLoop(ctx context.Context, torrentClient *torrent.Client) {
 					"download", common2.ByteCount(uint64(stats.readBytesPerSec))+"/s",
 					"upload", common2.ByteCount(uint64(stats.writeBytesPerSec))+"/s",
 					"unique_peers", stats.peersCount,
-					"min_peers", stats.minPeers,
-					"max_peers", stats.maxPeers,
-					"min_seeds", stats.minSeeds,
-					"max_seeds", stats.maxSeeds,
 					"files", stats.torrentsCount,
 					"alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys))
 				continue
@@ -132,10 +127,6 @@ func LoggingLoop(ctx context.Context, torrentClient *torrent.Client) {
 				"download", common2.ByteCount(uint64(stats.readBytesPerSec))+"/s",
 				"upload", common2.ByteCount(uint64(stats.writeBytesPerSec))+"/s",
 				"unique_peers", stats.peersCount,
-				"min_peers", stats.minPeers,
-				"max_peers", stats.maxPeers,
-				"min_seeds", stats.minSeeds,
-				"max_seeds", stats.maxSeeds,
 				"files", stats.torrentsCount,
 				"alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys))
 			if stats.peersCount == 0 {
@@ -170,9 +161,6 @@ type AggStats struct {
 
 	bytesRead    int64
 	bytesWritten int64
-
-	minPeers, maxPeers int
-	minSeeds, maxSeeds int
 }
 
 func min(a, b int) int {
@@ -198,32 +186,7 @@ func CalcStats(prevStats AggStats, interval time.Duration, client *torrent.Clien
 	result.bytesRead += connStats.BytesReadUsefulIntendedData.Int64()
 	result.bytesWritten += connStats.BytesWrittenData.Int64()
 
-	result.minSeeds = math.MaxInt
-	result.minPeers = math.MaxInt
 	for _, t := range torrents {
-		stats := t.Stats()
-		if !t.Complete.Bool() {
-			result.minSeeds = min(result.minSeeds, stats.ConnectedSeeders)
-			result.maxSeeds = max(result.maxSeeds, stats.ConnectedSeeders)
-		}
-		result.minPeers = min(result.minPeers, stats.ActivePeers)
-		result.maxPeers = max(result.maxPeers, stats.ActivePeers)
-
-		/*
-			var completedPieces, partialPieces int
-			psrs := t.PieceStateRuns()
-			for _, r := range psrs {
-				if r.Complete {
-					completedPieces += r.Length
-				}
-				if r.Partial {
-					partialPieces += r.Length
-				}
-			}
-			aggCompletedPieces += completedPieces
-			aggPartialPieces += partialPieces
-			aggNumPieces = t.NumPieces()
-		*/
 		aggBytesCompleted += t.BytesCompleted()
 		aggLen += t.Length()
 
@@ -231,12 +194,6 @@ func CalcStats(prevStats AggStats, interval time.Duration, client *torrent.Clien
 			peers[peer.PeerID] = peer
 		}
 	}
-	if result.minSeeds == math.MaxInt {
-		result.minSeeds = 0
-	}
-	if result.minPeers == math.MaxInt {
-		result.minPeers = 0
-	}
 
 	result.readBytesPerSec += (result.bytesRead - prevStats.bytesRead) / int64(interval.Seconds())
 	result.writeBytesPerSec += (result.bytesWritten - prevStats.bytesWritten) / int64(interval.Seconds())
@@ -305,7 +262,6 @@ func ResolveAbsentTorrents(ctx context.Context, torrentClient *torrent.Client, p
 		}
 		t.AllowDataDownload()
 		t.AllowDataUpload()
-		t.DownloadAll()
 	}
 	if !silent {
 		ctxLocal, cancel := context.WithCancel(ctx)
@@ -318,6 +274,9 @@ func ResolveAbsentTorrents(ctx context.Context, torrentClient *torrent.Client, p
 		case <-ctx.Done():
 			return ctx.Err()
 		case <-t.GotInfo():
+			if !t.Complete.Bool() {
+				t.DownloadAll()
+			}
 			mi := t.Metainfo()
 			if err := CreateTorrentFileIfNotExists(snapshotDir, t.Info(), &mi); err != nil {
 				return err

cmd/downloader/downloader/grpc_server.go

@@ -43,7 +43,9 @@ func CreateTorrentFilesAndAdd(ctx context.Context, snapshotDir *dir.Rw, torrentC
 	for _, t := range torrentClient.Torrents() {
 		t.AllowDataDownload()
 		t.AllowDataUpload()
-		t.DownloadAll()
+		if !t.Complete.Bool() {
+			t.DownloadAll()
+		}
 	}
 	return nil
 }
@@ -78,7 +80,9 @@ func (s *GrpcServer) Download(ctx context.Context, request *proto_downloader.Dow
 	for _, t := range s.t.TorrentClient.Torrents() {
 		t.AllowDataDownload()
 		t.AllowDataUpload()
-		t.DownloadAll()
+		if !t.Complete.Bool() {
+			t.DownloadAll()
+		}
 	}
 	return &emptypb.Empty{}, nil
 }

cmd/downloader/downloader/torrentcfg/torrentcfg.go

@@ -37,9 +37,9 @@ func Default() *torrent.ClientConfig {
 	//torrentConfig.DisableWebseeds = true
 
 	// Increase default timeouts, because we often run on commodity networks
-	torrentConfig.MinDialTimeout = 6 * time.Second      // default: 3sec
+	torrentConfig.MinDialTimeout = 1 * time.Second      // default: 3sec
-	torrentConfig.NominalDialTimeout = 20 * time.Second // default: 20sec
+	torrentConfig.NominalDialTimeout = 10 * time.Second // default: 20sec
-	torrentConfig.HandshakesTimeout = 8 * time.Second   // default: 4sec
+	torrentConfig.HandshakesTimeout = 1 * time.Second   // default: 4sec
 
 	return torrentConfig
 }
@@ -49,9 +49,9 @@ func New(snapshotsDir *dir.Rw, verbosity lg.Level, natif nat.Interface, download
 	// We would-like to reduce amount of goroutines in Erigon, so reducing next params
 	torrentConfig.EstablishedConnsPerTorrent = connsPerFile // default: 50
 	torrentConfig.TorrentPeersHighWater = maxPeers          // default: 500
-	torrentConfig.TorrentPeersLowWater = 5                  // default: 50
+	torrentConfig.TorrentPeersLowWater = 50                 // default: 50
-	torrentConfig.HalfOpenConnsPerTorrent = 5               // default: 25
+	torrentConfig.HalfOpenConnsPerTorrent = 25              // default: 25
-	torrentConfig.TotalHalfOpenConns = 100                  // default: 100
+	torrentConfig.TotalHalfOpenConns = 200                  // default: 100
 
 	torrentConfig.ListenPort = port
 	torrentConfig.Seed = true