faster opening of snapshots and indices (#7757)

- mostly by MADV_SEQUENTIAL during opening 
- see https://github.com/ledgerwatch/erigon-lib/pull/1025

go.mod

@@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon
 go 1.19
 
 require (
-	github.com/ledgerwatch/erigon-lib v0.0.0-20230617020636-2aa24aa05de6
+	github.com/ledgerwatch/erigon-lib v0.0.0-20230619033850-4fbe694357f6
 	github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230605042354-196538d42475
 	github.com/ledgerwatch/log/v3 v3.8.0
 	github.com/ledgerwatch/secp256k1 v1.0.0

go.sum

@@ -417,8 +417,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
 github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
-github.com/ledgerwatch/erigon-lib v0.0.0-20230617020636-2aa24aa05de6 h1:GnKL1d8xQqVbf9nEt7T1qUR6TvtX5LTU+7al6VG5ugA=
+github.com/ledgerwatch/erigon-lib v0.0.0-20230619033850-4fbe694357f6 h1:x86x6UIBiz2V2XYBUGQ74QT6KaZ1RWU0T4oE/o3oyGo=
-github.com/ledgerwatch/erigon-lib v0.0.0-20230617020636-2aa24aa05de6/go.mod h1:iz1daifnfSn3P0Iwd21ioyjwdmFEOn8DKeynahoHeSc=
+github.com/ledgerwatch/erigon-lib v0.0.0-20230619033850-4fbe694357f6/go.mod h1:iz1daifnfSn3P0Iwd21ioyjwdmFEOn8DKeynahoHeSc=
 github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230605042354-196538d42475 h1:1BvWA6agTUS4RZUHx79f45HpvelMVv4iEddaURUYcC8=
 github.com/ledgerwatch/erigon-snapshot v1.2.1-0.20230605042354-196538d42475/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
 github.com/ledgerwatch/log/v3 v3.8.0 h1:gCpp7uGtIerEz1jKVPeDnbIopFPud9ZnCpBLlLBGqPU=

turbo/snapshotsync/freezeblocks/block_snapshots.go

@@ -556,20 +556,21 @@ Loop:
 
 		switch f.T {
 		case snaptype.Headers:
-			for _, sn := range s.Headers.segments {
+			var sn *HeaderSegment
-				if sn.seg == nil { // it's ok if some segment was not able to open
+			var exists bool
+			for _, sn2 := range s.Headers.segments {
+				if sn2.seg == nil { // it's ok if some segment was not able to open
 					continue
 				}
-				_, name := filepath.Split(sn.seg.FilePath())
+				if fName == sn2.seg.FileName() {
-				if fName == name {
+					sn = sn2
-					if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
+					exists = true
-						return err
+					break
-					}
-					continue Loop
 				}
 			}
-
+			if !exists {
-			sn := &HeaderSegment{ranges: Range{f.From, f.To}}
+				sn = &HeaderSegment{ranges: Range{f.From, f.To}}
+			}
 			if err := sn.reopenSeg(s.dir); err != nil {
 				if errors.Is(err, os.ErrNotExist) {
 					if optimistic {
@@ -586,27 +587,30 @@ Loop:
 				}
 			}
 
-			// it's possible to iterate over .seg file even if you don't have index
+			if !exists {
-			// then make segment available even if index open may fail
+				// it's possible to iterate over .seg file even if you don't have index
-			s.Headers.segments = append(s.Headers.segments, sn)
+				// then make segment available even if index open may fail
+				s.Headers.segments = append(s.Headers.segments, sn)
+			}
 			if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
 				return err
 			}
 		case snaptype.Bodies:
-			for _, sn := range s.Bodies.segments {
+			var sn *BodySegment
-				if sn.seg == nil {
+			var exists bool
+			for _, sn2 := range s.Bodies.segments {
+				if sn2.seg == nil { // it's ok if some segment was not able to open
 					continue
 				}
-				_, name := filepath.Split(sn.seg.FilePath())
+				if fName == sn2.seg.FileName() {
-				if fName == name {
+					sn = sn2
-					if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
+					exists = true
-						return err
+					break
-					}
-					continue Loop
 				}
 			}
-
+			if !exists {
-			sn := &BodySegment{ranges: Range{f.From, f.To}}
+				sn = &BodySegment{ranges: Range{f.From, f.To}}
+			}
 			if err := sn.reopenSeg(s.dir); err != nil {
 				if errors.Is(err, os.ErrNotExist) {
 					if optimistic {
@@ -622,25 +626,28 @@ Loop:
 					return err
 				}
 			}
-			s.Bodies.segments = append(s.Bodies.segments, sn)
+			if !exists {
+				s.Bodies.segments = append(s.Bodies.segments, sn)
+			}
 			if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
 				return err
 			}
 		case snaptype.Transactions:
-			for _, sn := range s.Txs.segments {
+			var sn *TxnSegment
-				if sn.Seg == nil {
+			var exists bool
+			for _, sn2 := range s.Txs.segments {
+				if sn2.Seg == nil { // it's ok if some segment was not able to open
 					continue
 				}
-				_, name := filepath.Split(sn.Seg.FilePath())
+				if fName == sn2.Seg.FileName() {
-				if fName == name {
+					sn = sn2
-					if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
+					exists = true
-						return err
+					break
-					}
-					continue Loop
 				}
 			}
-
+			if !exists {
-			sn := &TxnSegment{ranges: Range{f.From, f.To}}
+				sn = &TxnSegment{ranges: Range{f.From, f.To}}
+			}
 			if err := sn.reopenSeg(s.dir); err != nil {
 				if errors.Is(err, os.ErrNotExist) {
 					if optimistic {
@@ -656,7 +663,9 @@ Loop:
 					return err
 				}
 			}
-			s.Txs.segments = append(s.Txs.segments, sn)
+			if !exists {
+				s.Txs.segments = append(s.Txs.segments, sn)
+			}
 			if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
 				return err
 			}