mirror/erigon-pulse
1
0
Fork 0
mirror of https://gitlab.com/pulsechaincom/erigon-pulse.git synced 2024-12-24 20:47:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

compression/rle: delete RLE compression (#16468)

Browse Source
This commit is contained in:
Felix Lange 2018-04-09 13:47:06 +02:00 committed by GitHub
parent 315b9b18df
commit 0fac705ed0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 2 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
compression/rle/read_write.go
View File

@ -1,101 +0,0 @@
// Copyright 2014 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Package rle implements the run-length encoding used for Ethereum data.
package rle
import (
"bytes"
"errors"
"github.com/ethereum/go-ethereum/crypto"
)
const (
token byte = 0xfe
emptyShaToken = 0xfd
emptyListShaToken = 0xfe
tokenToken = 0xff
)
var empty = crypto.Keccak256([]byte(""))
var emptyList = crypto.Keccak256([]byte{0x80})
func Decompress(dat []byte) ([]byte, error) {
buf := new(bytes.Buffer)
for i := 0; i < len(dat); i++ {
if dat[i] == token {
if i+1 < len(dat) {
switch dat[i+1] {
case emptyShaToken:
buf.Write(empty)
case emptyListShaToken:
buf.Write(emptyList)
case tokenToken:
buf.WriteByte(token)
default:
buf.Write(make([]byte, int(dat[i+1]-2)))
}
i++
} else {
return nil, errors.New("error reading bytes. token encountered without proceeding bytes")
}
} else {
buf.WriteByte(dat[i])
}
}
return buf.Bytes(), nil
}
func compressChunk(dat []byte) (ret []byte, n int) {
switch {
case dat[0] == token:
return []byte{token, tokenToken}, 1
case len(dat) > 1 && dat[0] == 0x0 && dat[1] == 0x0:
j := 0
for j <= 254 && j < len(dat) {
if dat[j] != 0 {
break
}
j++
}
return []byte{token, byte(j + 2)}, j
case len(dat) >= 32:
if dat[0] == empty[0] && bytes.Equal(dat[:32], empty) {
return []byte{token, emptyShaToken}, 32
} else if dat[0] == emptyList[0] && bytes.Equal(dat[:32], emptyList) {
return []byte{token, emptyListShaToken}, 32
}
fallthrough
default:
return dat[:1], 1
}
}
func Compress(dat []byte) []byte {
buf := new(bytes.Buffer)
i := 0
for i < len(dat) {
b, n := compressChunk(dat[i:])
buf.Write(b)
i += n
}
return buf.Bytes()
}

50
compression/rle/read_write_test.go
View File

@ -1,50 +0,0 @@
// Copyright 2014 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package rle
import (
"testing"
checker "gopkg.in/check.v1"
)
func Test(t *testing.T) { checker.TestingT(t) }
type CompressionRleSuite struct{}
var _ = checker.Suite(&CompressionRleSuite{})
func (s *CompressionRleSuite) TestDecompressSimple(c *checker.C) {
exp := []byte{0xc5, 0xd2, 0x46, 0x1, 0x86, 0xf7, 0x23, 0x3c, 0x92, 0x7e, 0x7d, 0xb2, 0xdc, 0xc7, 0x3, 0xc0, 0xe5, 0x0, 0xb6, 0x53, 0xca, 0x82, 0x27, 0x3b, 0x7b, 0xfa, 0xd8, 0x4, 0x5d, 0x85, 0xa4, 0x70}
res, err := Decompress([]byte{token, 0xfd})
c.Assert(err, checker.IsNil)
c.Assert(res, checker.DeepEquals, exp)
exp = []byte{0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x1, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21}
res, err = Decompress([]byte{token, 0xfe})
c.Assert(err, checker.IsNil)
c.Assert(res, checker.DeepEquals, exp)
res, err = Decompress([]byte{token, 0xff})
c.Assert(err, checker.IsNil)
c.Assert(res, checker.DeepEquals, []byte{token})
res, err = Decompress([]byte{token, 12})
c.Assert(err, checker.IsNil)
c.Assert(res, checker.DeepEquals, make([]byte, 10))
}

6
ethdb/database.go
View File

@ -91,9 +91,6 @@ func (db *LDBDatabase) Path() string {
// Put puts the given key / value to the queue
func (db *LDBDatabase) Put(key []byte, value []byte) error {
// Generate the data to write to disk, update the meter and write
//value = rle.Compress(value)
return db.db.Put(key, value, nil)
}
@ -103,18 +100,15 @@ func (db *LDBDatabase) Has(key []byte) (bool, error) {
// Get returns the given key if it's present.
func (db *LDBDatabase) Get(key []byte) ([]byte, error) {
// Retrieve the key and increment the miss counter if not found
dat, err := db.db.Get(key, nil)
if err != nil {
return nil, err
}
return dat, nil
//return rle.Decompress(dat)
}
// Delete deletes the key from the queue and database
func (db *LDBDatabase) Delete(key []byte) error {
// Execute the actual operation
return db.db.Delete(key, nil)
}

15
swarm/storage/database.go
View File

@ -22,7 +22,6 @@ package storage
import (
"fmt"
"github.com/ethereum/go-ethereum/compression/rle"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt"
@ -31,8 +30,7 @@ import (
const openFileLimit = 128
type LDBDatabase struct {
db *leveldb.DB
comp bool
db *leveldb.DB
}
func NewLDBDatabase(file string) (*LDBDatabase, error) {
@ -42,16 +40,12 @@ func NewLDBDatabase(file string) (*LDBDatabase, error) {
return nil, err
}
database := &LDBDatabase{db: db, comp: false}
database := &LDBDatabase{db: db}
return database, nil
}
func (self *LDBDatabase) Put(key []byte, value []byte) {
if self.comp {
value = rle.Compress(value)
}
err := self.db.Put(key, value, nil)
if err != nil {
fmt.Println("Error put", err)
@ -63,11 +57,6 @@ func (self *LDBDatabase) Get(key []byte) ([]byte, error) {
if err != nil {
return nil, err
}
if self.comp {
return rle.Decompress(dat)
}
return dat, nil
}