erigon-pulse/common/dbutils/composite_keys.go
Alex Sharov daa359c363
Mgr schedule iterator (#566)
* db based version of PrefixByCumulativeWitnessSize

* db based version of PrefixByCumulativeWitnessSize

* retain all in Trie by default

* fix WitnessLen logic in calcTrie roots

* Rename IntermediateTrieWitnessLenBucket to IntermediateWitnessLenBucket

* handle corner cases in WL

* Use correct incarnation for IH bucket

* use name WitnessSize

* save progress towards db-only witness estimation

* results from trie and from db are still different

* less recursion

* correct incarnation in CumulativeSearch

* reuse results from previous Tick, separate concepts of parent and startKey

* experiment: if not including trie structure to WitnessSize will reduce cumulative error

* tool to generate all IH and tool to calculate assessment of cumulative error

* tool to generate all IH

* Calculate totalWitnessSize based on DB data - then schedule will not overrun state during MGR cycle

* better stats

* Calculate totalWitnessSize based on DB data - then schedule will not overrun state during MGR cycle

* Calculate totalWitnessSize based on DB data - then schedule will not overrun state during MGR cycle

* calculate ticks size distribution

* estimate cumulative error

* fix linter

* resetIH from scratch if needed

* cleanup

* fix test

* fix test
2020-05-28 12:33:05 +01:00

201 lines
6.5 KiB
Go

package dbutils
import (
"bytes"
"encoding/binary"
"github.com/ledgerwatch/turbo-geth/common"
)
// EncodeBlockNumber encodes a block number as big endian uint64
func EncodeBlockNumber(number uint64) []byte {
enc := make([]byte, 8)
binary.BigEndian.PutUint64(enc, number)
return enc
}
// headerKey = headerPrefix + num (uint64 big endian) + hash
func HeaderKey(number uint64, hash common.Hash) []byte {
return append(EncodeBlockNumber(number), hash.Bytes()...)
}
func IsHeaderKey(k []byte) bool {
l := common.BlockNumberLength + common.HashLength
if len(k) != l {
return false
}
return !IsHeaderHashKey(k) && !IsHeaderTDKey(k)
}
// headerTDKey = headerPrefix + num (uint64 big endian) + hash + headerTDSuffix
func HeaderTDKey(number uint64, hash common.Hash) []byte {
return append(HeaderKey(number, hash), HeaderTDSuffix...)
}
func IsHeaderTDKey(k []byte) bool {
l := common.BlockNumberLength + common.HashLength + 1
return len(k) == l && bytes.Equal(k[l-1:], HeaderTDSuffix)
}
// headerHashKey = headerPrefix + num (uint64 big endian) + headerHashSuffix
func HeaderHashKey(number uint64) []byte {
return append(EncodeBlockNumber(number), HeaderHashSuffix...)
}
func IsHeaderHashKey(k []byte) bool {
l := common.BlockNumberLength + 1
return len(k) == l && bytes.Equal(k[l-1:], HeaderHashSuffix)
}
// headerNumberKey = headerNumberPrefix + hash
func HeaderNumberKey(hash common.Hash) []byte {
return append(HeaderNumberPrefix, hash.Bytes()...)
}
// blockBodyKey = blockBodyPrefix + num (uint64 big endian) + hash
func BlockBodyKey(number uint64, hash common.Hash) []byte {
return append(EncodeBlockNumber(number), hash.Bytes()...)
}
// blockReceiptsKey = blockReceiptsPrefix + num (uint64 big endian) + hash
func BlockReceiptsKey(number uint64, hash common.Hash) []byte {
return append(EncodeBlockNumber(number), hash.Bytes()...)
}
// txLookupKey = txLookupPrefix + hash
func TxLookupKey(hash common.Hash) []byte {
return append(TxLookupPrefix, hash.Bytes()...)
}
// bloomBitsKey = bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash
func BloomBitsKey(bit uint, section uint64, hash common.Hash) []byte {
key := append(make([]byte, 10), hash.Bytes()...)
binary.BigEndian.PutUint16(key[0:], uint16(bit))
binary.BigEndian.PutUint64(key[2:], section)
return key
}
// preimageKey = preimagePrefix + hash
func PreimageKey(hash common.Hash) []byte {
return append(PreimagePrefix, hash.Bytes()...)
}
// configKey = configPrefix + hash
func ConfigKey(hash common.Hash) []byte {
return append(ConfigPrefix, hash.Bytes()...)
}
// AddrHash + KeyHash
// Only for trie
func GenerateCompositeTrieKey(addressHash common.Hash, seckey common.Hash) []byte {
compositeKey := make([]byte, 0, common.HashLength+common.HashLength)
compositeKey = append(compositeKey, addressHash[:]...)
compositeKey = append(compositeKey, seckey[:]...)
return compositeKey
}
// AddrHash + incarnation + KeyHash
// For contract storage
func GenerateCompositeStorageKey(addressHash common.Hash, incarnation uint64, seckey common.Hash) []byte {
compositeKey := make([]byte, 0, common.HashLength+8+common.HashLength)
compositeKey = append(compositeKey, GenerateStoragePrefix(addressHash[:], incarnation)...)
compositeKey = append(compositeKey, seckey[:]...)
return compositeKey
}
func ParseCompositeStorageKey(compositeKey []byte) (common.Hash, uint64, common.Hash) {
prefixLen := common.HashLength + common.IncarnationLength
addrHash, inc := ParseStoragePrefix(compositeKey[:prefixLen])
var key common.Hash
copy(key[:], compositeKey[prefixLen:prefixLen+common.HashLength])
return addrHash, inc, key
}
// AddrHash + incarnation + KeyHash
// For contract storage (for plain state)
func PlainGenerateCompositeStorageKey(address common.Address, incarnation uint64, key common.Hash) []byte {
compositeKey := make([]byte, 0, common.AddressLength+8+common.HashLength)
compositeKey = append(compositeKey, PlainGenerateStoragePrefix(address, incarnation)...)
compositeKey = append(compositeKey, key[:]...)
return compositeKey
}
func PlainParseCompositeStorageKey(compositeKey []byte) (common.Address, uint64, common.Hash) {
prefixLen := common.AddressLength + common.IncarnationLength
addr, inc := PlainParseStoragePrefix(compositeKey[:prefixLen])
var key common.Hash
copy(key[:], compositeKey[prefixLen:prefixLen+common.HashLength])
return addr, inc, key
}
// AddrHash + incarnation + StorageHashPrefix
func GenerateCompositeStoragePrefix(addressHash []byte, incarnation uint64, storageHashPrefix []byte) []byte {
key := make([]byte, common.HashLength+8+len(storageHashPrefix))
copy(key, addressHash)
binary.BigEndian.PutUint64(key[common.HashLength:], ^incarnation)
copy(key[common.HashLength+8:], storageHashPrefix)
return key
}
// address hash + incarnation prefix
func GenerateStoragePrefix(addressHash []byte, incarnation uint64) []byte {
prefix := make([]byte, common.HashLength+8)
copy(prefix, addressHash)
binary.BigEndian.PutUint64(prefix[common.HashLength:], ^incarnation)
return prefix
}
// address hash + incarnation prefix (for plain state)
func PlainGenerateStoragePrefix(address common.Address, incarnation uint64) []byte {
prefix := make([]byte, common.AddressLength+8)
copy(prefix, address[:])
binary.BigEndian.PutUint64(prefix[common.AddressLength:], ^incarnation)
return prefix
}
func PlainParseStoragePrefix(prefix []byte) (common.Address, uint64) {
var addr common.Address
copy(addr[:], prefix[:common.AddressLength])
inc := ^binary.BigEndian.Uint64(prefix[common.AddressLength : common.AddressLength+common.IncarnationLength])
return addr, inc
}
func ParseStoragePrefix(prefix []byte) (common.Hash, uint64) {
var addrHash common.Hash
copy(addrHash[:], prefix[:common.HashLength])
inc := ^binary.BigEndian.Uint64(prefix[common.HashLength : common.HashLength+common.IncarnationLength])
return addrHash, inc
}
func DecodeIncarnation(buf []byte) uint64 {
incarnation := binary.BigEndian.Uint64(buf)
return incarnation ^ ^uint64(0)
}
func EncodeIncarnation(incarnation uint64, buf []byte) {
binary.BigEndian.PutUint64(buf, ^incarnation)
}
func RemoveIncarnationFromKey(key []byte, out *[]byte) {
tmp := (*out)[:0]
if len(key) <= common.HashLength {
tmp = append(tmp, key...)
} else {
tmp = append(tmp, key[:common.HashLength]...)
tmp = append(tmp, key[common.HashLength+8:]...)
}
*out = tmp
}
// Key + blockNum
func CompositeKeySuffix(key []byte, timestamp uint64) (composite, encodedTS []byte) {
encodedTS = EncodeTimestamp(timestamp)
composite = make([]byte, len(key)+len(encodedTS))
copy(composite, key)
copy(composite[len(key):], encodedTS)
return composite, encodedTS
}