mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-28 14:47:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

compress: less allocs (#361)

Browse Source
This commit is contained in:
Alex Sharov 2022-03-12 15:33:01 +07:00 committed by GitHub
parent d3e05fb352
commit c0fcdabf91
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
compress/parallel_compress.go
View File

@ -788,7 +788,7 @@ func reducedict(trace bool, logPrefix, segmentFilePath, tmpDir string, datFile *
// No error channels for now
func processSuperstring(superstringCh chan []byte, dictCollector *etl.Collector, minPatternScore uint64, completion *sync.WaitGroup) {
defer completion.Done()
var dictVal [8]byte
dictVal := make([]byte, 8)
dictKey := make([]byte, maxPatternLen)
var lcp, sa, inv []int32
for superstring := range superstringCh {
@ -799,11 +799,14 @@ func processSuperstring(superstringCh chan []byte, dictCollector *etl.Collector,
}
//log.Info("Superstring", "len", len(superstring))
//start := time.Now()
sais.Sais(superstring, sa)
if err := sais.Sais(superstring, sa); err != nil {
panic(err)
}
//log.Info("Suffix array built", "in", time.Since(start))
// filter out suffixes that start with odd positions
n := len(sa) / 2
filtered := make([]int32, n)
filtered := sa[:n]
//filtered := make([]int32, n)
var j int
for i := 0; i < len(sa); i++ {
if sa[i]&1 == 0 {
@ -937,8 +940,8 @@ func processSuperstring(superstringCh chan []byte, dictCollector *etl.Collector,
for s := 0; s < l; s++ {
dictKey[s] = superstring[(int(filtered[i])+s)*2+1]
}
binary.BigEndian.PutUint64(dictVal[:], score)
if err := dictCollector.Collect(dictKey, dictVal[:]); err != nil {
binary.BigEndian.PutUint64(dictVal, score)
if err := dictCollector.Collect(dictKey, dictVal); err != nil {
log.Error("processSuperstring", "collect", err)
}
prevSkipped = false

4
patricia/patricia.go
View File

@ -574,7 +574,9 @@ func (mf2 *MatchFinder2) FindLongestMatches(data []byte) []Match {
} else {
mf2.sa = mf2.sa[:n]
}
sais.Sais(data, mf2.sa)
if err := sais.Sais(data, mf2.sa); err != nil {
panic(err)
}
if cap(mf2.inv) < n {
mf2.inv = make([]int32, n)
} else {