From e60b38e283b7d29a6f194a6c034938aedc4e083f Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 1 Aug 2022 12:40:00 +0700 Subject: [PATCH] decompress: catch maxDepth underflow (#4882) * save * save --- go.mod | 4 ++-- go.sum | 8 ++++---- turbo/snapshotsync/block_snapshots.go | 18 +++++++++++++----- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 8222597dc..5dbd8bfee 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20220801040348-ad2344a6ccd1 + github.com/ledgerwatch/erigon-lib v0.0.0-20220801053710-127d1bac5b66 github.com/ledgerwatch/erigon-snapshot v1.0.0 github.com/ledgerwatch/log/v3 v3.4.1 github.com/ledgerwatch/secp256k1 v1.0.0 @@ -56,7 +56,7 @@ require ( github.com/stretchr/testify v1.8.0 github.com/tendermint/go-amino v0.14.1 github.com/tendermint/tendermint v0.31.11 - github.com/torquem-ch/mdbx-go v0.25.1 + github.com/torquem-ch/mdbx-go v0.25.2 github.com/ugorji/go/codec v1.1.13 github.com/ugorji/go/codec/codecgen v1.1.13 github.com/urfave/cli v1.22.9 diff --git a/go.sum b/go.sum index 199449b90..219087a67 100644 --- a/go.sum +++ b/go.sum @@ -390,8 +390,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20220801040348-ad2344a6ccd1 h1:fQ2SchdZmjtS3r865GPA4i5a9BAaL3DJKb6lKQBH7bM= -github.com/ledgerwatch/erigon-lib v0.0.0-20220801040348-ad2344a6ccd1/go.mod h1:H1yhcOcu+HDEU1i5Q+g0ezSoQszsJR+vbowF19a63yA= +github.com/ledgerwatch/erigon-lib v0.0.0-20220801053710-127d1bac5b66 h1:NEOW3KTAk7BPRfxsiebT++OTO9QIOXabwqmgYLSPnsU= +github.com/ledgerwatch/erigon-lib v0.0.0-20220801053710-127d1bac5b66/go.mod h1:JGetYkutIOckzCJ2TN2CsWlRibK1H0dsJUd3bn/pXCo= github.com/ledgerwatch/erigon-snapshot v1.0.0 h1:bp/7xoPdM5lK7LFdqEMH008RZmqxMZV0RUVEQiWs7v4= github.com/ledgerwatch/erigon-snapshot v1.0.0/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.4.1 h1:/xGwlVulXnsO9Uq+tzaExc8OWmXXHU0dnLalpbnY5Bc= @@ -623,8 +623,8 @@ github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDW github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/torquem-ch/mdbx-go v0.25.1 h1:4UxZdwTU+hPp7/czW2yb9qU1SwuwtPf86BXMYggWLZ4= -github.com/torquem-ch/mdbx-go v0.25.1/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E= +github.com/torquem-ch/mdbx-go v0.25.2 h1:Ub5M0krJRv4SJCh1GjNmQGTaDjWKH0YaSTH+S3DmYV4= +github.com/torquem-ch/mdbx-go v0.25.2/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E= github.com/ugorji/go v1.1.13/go.mod h1:jxau1n+/wyTGLQoCkjok9r5zFa/FxT6eI5HiHKQszjc= github.com/ugorji/go/codec v1.1.13 h1:013LbFhocBoIqgHeIHKlV4JWYhqogATYWZhIcH0WHn4= github.com/ugorji/go/codec v1.1.13/go.mod h1:oNVt3Dq+FO91WNQ/9JnHKQP2QJxTzoN7wCBFCq1OeuU= diff --git a/turbo/snapshotsync/block_snapshots.go b/turbo/snapshotsync/block_snapshots.go index 29871b92e..27dff7437 100644 --- a/turbo/snapshotsync/block_snapshots.go +++ b/turbo/snapshotsync/block_snapshots.go @@ -453,6 +453,9 @@ Loop: switch f.T { case snap.Headers: for _, sn := range s.Headers.segments { + if sn.seg == nil { // it's ok if some segment was not able to open + continue + } _, name := filepath.Split(sn.seg.FilePath()) if fName == name { if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil { @@ -487,6 +490,9 @@ Loop: } case snap.Bodies: for _, sn := range s.Bodies.segments { + if sn.seg == nil { + continue + } _, name := filepath.Split(sn.seg.FilePath()) if fName == name { if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil { @@ -518,6 +524,9 @@ Loop: } case snap.Transactions: for _, sn := range s.Txs.segments { + if sn.Seg == nil { + continue + } _, name := filepath.Split(sn.Seg.FilePath()) if fName == name { if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil { @@ -646,7 +655,7 @@ Loop3: s.Txs.segments[i] = nil } var i int - for i = 0; i < len(s.Headers.segments) && s.Headers.segments[i] != nil; i++ { + for i = 0; i < len(s.Headers.segments) && s.Headers.segments[i] != nil && s.Headers.segments[i].seg != nil; i++ { } tail := s.Headers.segments[i:] s.Headers.segments = s.Headers.segments[:i] @@ -657,7 +666,7 @@ Loop3: } } - for i = 0; i < len(s.Bodies.segments) && s.Bodies.segments[i] != nil; i++ { + for i = 0; i < len(s.Bodies.segments) && s.Bodies.segments[i] != nil && s.Bodies.segments[i].seg != nil; i++ { } tailB := s.Bodies.segments[i:] s.Bodies.segments = s.Bodies.segments[:i] @@ -668,7 +677,7 @@ Loop3: } } - for i = 0; i < len(s.Txs.segments) && s.Txs.segments[i] != nil; i++ { + for i = 0; i < len(s.Txs.segments) && s.Txs.segments[i] != nil && s.Txs.segments[i].Seg != nil; i++ { } tailC := s.Txs.segments[i:] s.Txs.segments = s.Txs.segments[:i] @@ -687,7 +696,6 @@ func (s *RoSnapshots) PrintDebug() { defer s.Bodies.lock.RUnlock() s.Txs.lock.RLock() defer s.Txs.lock.RUnlock() - fmt.Printf("sn: %d, %d\n", s.segmentsMax.Load(), s.idxMax.Load()) fmt.Println(" == Snapshots, Header") for _, sn := range s.Headers.segments { fmt.Printf("%d, %t\n", sn.ranges.from, sn.idxHeaderHash == nil) @@ -885,7 +893,7 @@ func EnforceSnapshotsInvariant(db kv.RwDB, dir string, allSnapshots *RoSnapshots return snList, err } if allSnapshots != nil { - if err := allSnapshots.ReopenList(snList, false); err != nil { + if err := allSnapshots.ReopenList(snList, true); err != nil { return snList, err } }