mirror of
https://gitlab.com/pulsechaincom/erigon-pulse.git
synced 2024-12-26 05:27:19 +00:00
e3: invalid txnum table (#828)
This commit is contained in:
Alex Sharov
GitHub
parent
63f3770aad
commit
16f4e2609b
4
go.mod
4
go.mod
@ -32,9 +32,9 @@ require (
|
||||
github.com/tidwall/btree v1.5.0
|
||||
go.uber.org/atomic v1.10.0
|
||||
golang.org/x/crypto v0.4.0
|
||||
golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15
|
||||
golang.org/x/exp v0.0.0-20230111222715-75897c7a292a
|
||||
golang.org/x/sync v0.1.0
|
||||
golang.org/x/sys v0.3.0
|
||||
golang.org/x/sys v0.4.0
|
||||
golang.org/x/time v0.3.0
|
||||
google.golang.org/grpc v1.51.0
|
||||
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0
|
||||
|
||||
7
go.sum
7
go.sum
@ -419,8 +419,8 @@ golang.org/x/crypto v0.0.0-20220516162934-403b01795ae8/go.mod h1:IxCIyHEi3zRg3s0
|
||||
golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
|
||||
golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 h1:5oN1Pz/eDhCpbMbLstvIPa0b/BEQo6g6nwV3pLjfM6w=
|
||||
golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
|
||||
golang.org/x/exp v0.0.0-20230111222715-75897c7a292a h1:/YWeLOBWYV5WAQORVPkZF3Pq9IppkcT72GKnWjNf5W8=
|
||||
golang.org/x/exp v0.0.0-20230111222715-75897c7a292a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@ -497,8 +497,9 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
|
||||
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
|
||||
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
||||
@ -32,6 +32,10 @@ import (
|
||||
|
||||
const MaxUint32 = 1<<32 - 1
|
||||
|
||||
type ToBitamp interface {
|
||||
ToBitamp() (*roaring64.Bitmap, error)
|
||||
}
|
||||
|
||||
var roaringPool = sync.Pool{
|
||||
New: func() any {
|
||||
return roaring.New()
|
||||
@ -192,6 +196,7 @@ func Get(db kv.Tx, bucket string, key []byte, from, to uint32) (*roaring.Bitmap,
|
||||
}
|
||||
chunks = append(chunks, bm)
|
||||
if binary.BigEndian.Uint32(k[len(k)-4:]) >= to {
|
||||
bm.RemoveRange(uint64(to), uint64(bm.Maximum()))
|
||||
break
|
||||
}
|
||||
}
|
||||
@ -349,6 +354,7 @@ func Get64(db kv.Tx, bucket string, key []byte, from, to uint64) (*roaring64.Bit
|
||||
}
|
||||
chunks = append(chunks, bm)
|
||||
if binary.BigEndian.Uint64(k[len(k)-8:]) >= to {
|
||||
bm.RemoveRange(to, bm.Maximum())
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@ -1058,16 +1058,23 @@ type HistoryContext struct {
|
||||
lr *recsplit.IndexReader
|
||||
locBm *bitmapdb.FixedSizeBitmaps
|
||||
|
||||
tx kv.Tx
|
||||
tx kv.Tx
|
||||
trace bool
|
||||
}
|
||||
|
||||
func (h *History) MakeContext() *HistoryContext {
|
||||
var hc = HistoryContext{h: h}
|
||||
hc.indexFiles = btree.NewG[ctxItem](32, ctxItemLess)
|
||||
var hc = HistoryContext{
|
||||
h: h,
|
||||
indexFiles: btree.NewG[ctxItem](32, ctxItemLess),
|
||||
trace: false,
|
||||
}
|
||||
h.InvertedIndex.files.Ascend(func(item *filesItem) bool {
|
||||
if item.index == nil {
|
||||
return false
|
||||
return true
|
||||
}
|
||||
//if item.startTxNum > h.endTxNumMinimax() { //after this number: not all filles are built yet (data still in DB)
|
||||
// return true
|
||||
//}
|
||||
hc.indexFiles.ReplaceOrInsert(ctxItem{
|
||||
startTxNum: item.startTxNum,
|
||||
endTxNum: item.endTxNum,
|
||||
@ -1079,8 +1086,11 @@ func (h *History) MakeContext() *HistoryContext {
|
||||
hc.historyFiles = btree.NewG[ctxItem](32, ctxItemLess)
|
||||
h.files.Ascend(func(item *filesItem) bool {
|
||||
if item.index == nil {
|
||||
return false
|
||||
return true
|
||||
}
|
||||
//if item.startTxNum > h.endTxNumMinimax() {
|
||||
// return true
|
||||
//}
|
||||
it := ctxItem{
|
||||
startTxNum: item.startTxNum,
|
||||
endTxNum: item.endTxNum,
|
||||
@ -1126,7 +1136,11 @@ func (hc *HistoryContext) GetNoState(key []byte, txNum uint64) ([]byte, bool, er
|
||||
eliasVal, _ := g.NextUncompressed()
|
||||
ef, _ := eliasfano32.ReadEliasFano(eliasVal)
|
||||
n, ok := ef.Search(txNum)
|
||||
|
||||
if hc.trace {
|
||||
n2, _ := ef.Search(n + 1)
|
||||
n3, _ := ef.Search(n - 1)
|
||||
fmt.Printf("hist: files: %s %d<-%d->%d->%d, %x\n", hc.h.filenameBase, n3, txNum, n, n2, key)
|
||||
}
|
||||
if ok {
|
||||
foundTxNum = n
|
||||
foundEndTxNum = item.endTxNum
|
||||
@ -1167,7 +1181,7 @@ func (hc *HistoryContext) GetNoState(key []byte, txNum uint64) ([]byte, bool, er
|
||||
search.startTxNum = foundStartTxNum
|
||||
search.endTxNum = foundEndTxNum
|
||||
if historyItem, ok = hc.historyFiles.Get(search); !ok {
|
||||
return nil, false, fmt.Errorf("no %s file found for [%x]", hc.h.filenameBase, key)
|
||||
return nil, false, fmt.Errorf("hist file not found: key=%x, %s.%d-%d", key, hc.h.filenameBase, foundStartTxNum/hc.h.aggregationStep, foundEndTxNum/hc.h.aggregationStep)
|
||||
}
|
||||
var txKey [8]byte
|
||||
binary.BigEndian.PutUint64(txKey[:], foundTxNum)
|
||||
@ -1272,6 +1286,13 @@ func (hc *HistoryContext) getNoStateFromDB(key []byte, txNum uint64, tx kv.Tx) (
|
||||
return nil, false, err
|
||||
}
|
||||
if foundTxNumVal != nil {
|
||||
if hc.trace {
|
||||
_, vv, _ := indexCursor.NextDup()
|
||||
indexCursor.Prev()
|
||||
_, prevV, _ := indexCursor.Prev()
|
||||
fmt.Printf("hist: db: %s, %d<-%d->%d->%d, %x\n", hc.h.filenameBase, u64or0(prevV), txNum, u64or0(foundTxNumVal), u64or0(vv), key)
|
||||
}
|
||||
|
||||
var historyKeysCursor kv.CursorDupSort
|
||||
if historyKeysCursor, err = tx.CursorDupSort(hc.h.indexKeysTable); err != nil {
|
||||
return nil, false, err
|
||||
@ -1774,3 +1795,10 @@ func (hs *HistoryStep) Clone() *HistoryStep {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func u64or0(in []byte) (v uint64) {
|
||||
if len(in) > 0 {
|
||||
v = binary.BigEndian.Uint64(in)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
@ -580,12 +580,12 @@ func (it *InvertedIterator) next() uint64 {
|
||||
it.advance()
|
||||
return n
|
||||
}
|
||||
func (it *InvertedIterator) ToBitamp() *roaring64.Bitmap {
|
||||
func (it *InvertedIterator) ToBitamp() (*roaring64.Bitmap, error) {
|
||||
bm := bitmapdb.NewBitmap64()
|
||||
for it.HasNext() {
|
||||
bm.Add(it.next())
|
||||
}
|
||||
return bm
|
||||
return bm, nil
|
||||
}
|
||||
func (it *InvertedIterator) ToArray() (res []uint64) {
|
||||
for it.HasNext() {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user