mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2025-01-01 00:31:21 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

erigon22: optimize index.add (#571)

Browse Source 
* save

* save
This commit is contained in:
Alex Sharov 2022-08-09 10:28:29 +07:00 committed by GitHub
parent 2be46669d5
commit 95e94b2eb5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 12 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
state/history.go
View File

@ -565,8 +565,6 @@ func (hc *HistoryContext) GetNoState(key []byte, txNum uint64) ([]byte, bool, ui
return true return true
}) })
if found { if found {
var txKey [8]byte
binary.BigEndian.PutUint64(txKey[:], foundTxNum)
var historyItem *ctxItem var historyItem *ctxItem
var ok bool var ok bool
var search ctxItem var search ctxItem
@ -575,6 +573,8 @@ func (hc *HistoryContext) GetNoState(key []byte, txNum uint64) ([]byte, bool, ui
if historyItem, ok = hc.historyFiles.Get(&search); !ok { if historyItem, ok = hc.historyFiles.Get(&search); !ok {
return nil, false, 0, fmt.Errorf("no %s file found for [%x]", hc.h.filenameBase, key) return nil, false, 0, fmt.Errorf("no %s file found for [%x]", hc.h.filenameBase, key)
} }
var txKey [8]byte
binary.BigEndian.PutUint64(txKey[:], foundTxNum)
offset := historyItem.reader.Lookup2(txKey[:], key) offset := historyItem.reader.Lookup2(txKey[:], key)
//fmt.Printf("offset = %d, txKey=[%x], key=[%x]\n", offset, txKey[:], key) //fmt.Printf("offset = %d, txKey=[%x], key=[%x]\n", offset, txKey[:], key)
g := historyItem.getter g := historyItem.getter

14
state/inverted_index.go
View File

@ -46,6 +46,7 @@ type InvertedIndex struct {
indexTable string // Needs to be table with DupSort indexTable string // Needs to be table with DupSort
tx kv.RwTx tx kv.RwTx
txNum uint64 txNum uint64
txNumBytes [8]byte
files *btree.BTreeG[*filesItem] files *btree.BTreeG[*filesItem]
} }
@ -121,6 +122,7 @@ func (ii *InvertedIndex) openFiles() error {
ii.files.Ascend(func(item *filesItem) bool { ii.files.Ascend(func(item *filesItem) bool {
datPath := filepath.Join(ii.dir, fmt.Sprintf("%s.%d-%d.ef", ii.filenameBase, item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep)) datPath := filepath.Join(ii.dir, fmt.Sprintf("%s.%d-%d.ef", ii.filenameBase, item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep))
if item.decompressor, err = compress.NewDecompressor(datPath); err != nil { if item.decompressor, err = compress.NewDecompressor(datPath); err != nil {
log.Debug("InvertedIndex.openFiles: %w, %s", err, datPath)
return false return false
} }
idxPath := filepath.Join(ii.dir, fmt.Sprintf("%s.%d-%d.efi", ii.filenameBase, item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep)) idxPath := filepath.Join(ii.dir, fmt.Sprintf("%s.%d-%d.efi", ii.filenameBase, item.startTxNum/ii.aggregationStep, item.endTxNum/ii.aggregationStep))
@ -130,6 +132,7 @@ func (ii *InvertedIndex) openFiles() error {
} }
} }
if item.index, err = recsplit.OpenIndex(idxPath); err != nil { if item.index, err = recsplit.OpenIndex(idxPath); err != nil {
log.Debug("InvertedIndex.openFiles: %w, %s", err, datPath)
return false return false
} }
totalKeys += item.index.KeyCount() totalKeys += item.index.KeyCount()
@ -163,15 +166,14 @@ func (ii *InvertedIndex) SetTx(tx kv.RwTx) {
func (ii *InvertedIndex) SetTxNum(txNum uint64) { func (ii *InvertedIndex) SetTxNum(txNum uint64) {
ii.txNum = txNum ii.txNum = txNum
binary.BigEndian.PutUint64(ii.txNumBytes[:], ii.txNum)
} }
func (ii *InvertedIndex) add(key, indexKey []byte) error { func (ii *InvertedIndex) add(key, indexKey []byte) error {
var txKey [8]byte if err := ii.tx.Put(ii.indexKeysTable, ii.txNumBytes[:], key); err != nil {
binary.BigEndian.PutUint64(txKey[:], ii.txNum)
if err := ii.tx.Put(ii.indexKeysTable, txKey[:], key); err != nil {
return err return err
} }
if err := ii.tx.Put(ii.indexTable, indexKey, txKey[:]); err != nil { if err := ii.tx.Put(ii.indexTable, indexKey, ii.txNumBytes[:]); err != nil {
return err return err
} }
return nil return nil
@ -258,6 +260,10 @@ func (it *InvertedIterator) advanceInDb() {
} }
} else { } else {
_, v, err = it.cursor.NextDup() _, v, err = it.cursor.NextDup()
if err != nil {
// TODO pass error properly around
panic(err)
}
} }
for ; err == nil && v != nil; _, v, err = it.cursor.NextDup() { for ; err == nil && v != nil; _, v, err = it.cursor.NextDup() {
n := binary.BigEndian.Uint64(v) n := binary.BigEndian.Uint64(v)