mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2025-01-01 00:31:21 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Compress: fix compress bytes share (#154)

Browse Source 
* fix compress bytes share

* fix compress bytes share
This commit is contained in:
Alex Sharov 2021-11-10 19:50:36 +07:00 committed by GitHub
parent dad84cccc9
commit 3a17502353
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 4 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
compress/compress.go
View File

@ -131,7 +131,7 @@ func (db *DictionaryBuilder) Pop() interface{} {
} }
func (db *DictionaryBuilder) processWord(chars []byte, score uint64) { func (db *DictionaryBuilder) processWord(chars []byte, score uint64) {
heap.Push(db, &Pattern{word: chars, score: score}) heap.Push(db, &Pattern{word: common.Copy(chars), score: score})
if db.Len() > db.limit { if db.Len() > db.limit {
// Remove the element with smallest score // Remove the element with smallest score
heap.Pop(db) heap.Pop(db)
@ -146,9 +146,6 @@ func (db *DictionaryBuilder) loadFunc(k, v []byte, table etl.CurrentTableReader,
if db.lastWord != nil { if db.lastWord != nil {
db.processWord(db.lastWord, db.lastWordScore) db.processWord(db.lastWord, db.lastWordScore)
} }
if cap(db.lastWord) < len(k) {
db.lastWord = make([]byte, 0, len(k))
}
db.lastWord = append(db.lastWord[:0], k...) db.lastWord = append(db.lastWord[:0], k...)
db.lastWordScore = score db.lastWordScore = score
} }
@ -162,7 +159,7 @@ func (db *DictionaryBuilder) finish() {
} }
func (db *DictionaryBuilder) ForEach(f func(score uint64, word []byte)) { func (db *DictionaryBuilder) ForEach(f func(score uint64, word []byte)) {
for i := len(db.items); i > 0; i-- { for i := db.Len(); i > 0; i-- {
f(db.items[i-1].score, db.items[i-1].word) f(db.items[i-1].score, db.items[i-1].word)
} }
} }
@ -1191,7 +1188,7 @@ func (da *DictAggregator) aggLoadFunc(k, v []byte, table etl.CurrentTableReader,
return err return err
} }
} }
da.lastWord = common.Copy(k) da.lastWord = append(da.lastWord[:0], k...)
da.lastWordScore = score da.lastWordScore = score
} }
return nil return nil

2
compress/decompress.go
View File

@ -201,7 +201,7 @@ func (g *Getter) HasNext() bool {
// After extracting next word, it moves to the beginning of the next one // After extracting next word, it moves to the beginning of the next one
func (g *Getter) Next(buf []byte) ([]byte, uint64) { func (g *Getter) Next(buf []byte) ([]byte, uint64) {
l := g.nextPos(true) l := g.nextPos(true)
l-- l-- // because when create huffman tree we do ++ , because 0 is terminator
if l == 0 { if l == 0 {
return buf, g.dataP return buf, g.dataP
} }