mirror of
https://gitlab.com/pulsechaincom/erigon-pulse.git
synced 2024-12-22 03:30:37 +00:00
THIN_HISTORY - Reinstate incarnation encoding in ChangeSets, but remove from index keys (#443)
* Cutting incarnation out of history and changeset keys * Rollback the remove of incarnation only in changesets * Fix linter * Remove debug printing * Reinstate encoding of incarnations in the ChangeSets * Fix linter * Fix linter * Fix DefaultIncarnation * Fixes * Fix lint * Fix to history writing * Fix linter
This commit is contained in:
ledgerwatch
GitHub
parent
9e65f886ea
commit
fbe7a2cb06
@ -11,8 +11,9 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultIncarnation = uint64(1)
|
||||
storageEnodingIndexSize = 4
|
||||
storageEnodingStartElem = uint32(4)
|
||||
storageEnodingStartElem = 4
|
||||
storageEnodingLengthOfNumOfElements = 4
|
||||
storageEnodingLengthOfDict = 2
|
||||
storageEnodingLengthOfNumTypeOfElements = 2
|
||||
@ -40,6 +41,7 @@ numOfUint32Values uint16
|
||||
[len(val0), len(val0)+len(val1), ..., len(val0)+len(val1)+...+len(val_{numOfUint8Values-1})] []uint8
|
||||
[len(valnumOfUint8Values), len(val0)+len(val1), ..., len(val0)+len(val1)+...+len(val_{numOfUint16Values-1})] []uint16
|
||||
[len(valnumOfUint16Values), len(val0)+len(val1), ..., len(val0)+len(val1)+...+len(val_{numOfUint32Values-1})] []uint32
|
||||
[elementNum:incarnation] - optional [uint32:uint64...]
|
||||
*/
|
||||
|
||||
func EncodeStorage(s *ChangeSet) ([]byte, error) {
|
||||
@ -54,30 +56,32 @@ func EncodeStorage(s *ChangeSet) ([]byte, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
addrHashesMap := make(map[common.Hash]uint32)
|
||||
addrHashList := make([]byte, 0)
|
||||
addrHashesMap := make([]uint32, n)
|
||||
var addrHashList []byte
|
||||
notDefaultIncarnationList := make([]byte, 0)
|
||||
|
||||
//collect information about unique addHashes and non default incarnations
|
||||
nextIDAddrHash := uint32(0)
|
||||
var addrHash common.Hash
|
||||
for i := 0; i < n; i++ {
|
||||
//copy addrHash
|
||||
copy(
|
||||
addrHash[:],
|
||||
s.Changes[i].Key[0:common.HashLength],
|
||||
)
|
||||
|
||||
//fill addrHashesMap and addrHashList
|
||||
if _, ok := addrHashesMap[addrHash]; !ok {
|
||||
addrHashesMap[addrHash] = nextIDAddrHash
|
||||
nextIDAddrHash++
|
||||
addrHashList = append(addrHashList, addrHash[:]...)
|
||||
var addrID uint32
|
||||
var addrIdxToIncarnation [12]byte
|
||||
for i, change := range s.Changes {
|
||||
if i == 0 || !bytes.Equal(change.Key[:common.HashLength], s.Changes[i-1].Key[:common.HashLength]) {
|
||||
if i > 0 {
|
||||
addrID++
|
||||
}
|
||||
addrHashList = append(addrHashList, change.Key[:common.HashLength]...)
|
||||
incarnation := ^binary.BigEndian.Uint64(change.Key[common.HashLength : common.HashLength+common.IncarnationLength])
|
||||
if incarnation != DefaultIncarnation {
|
||||
binary.BigEndian.PutUint32(addrIdxToIncarnation[:4], addrID)
|
||||
binary.BigEndian.PutUint64(addrIdxToIncarnation[4:12], ^incarnation)
|
||||
notDefaultIncarnationList = append(notDefaultIncarnationList, addrIdxToIncarnation[:]...)
|
||||
}
|
||||
}
|
||||
addrHashesMap[i] = addrID
|
||||
}
|
||||
|
||||
//write numOfUniqAddrHashes
|
||||
numOfUniqAddrHashes := make([]byte, storageEnodingLengthOfDict)
|
||||
binary.BigEndian.PutUint16(numOfUniqAddrHashes, uint16(len(addrHashesMap)))
|
||||
binary.BigEndian.PutUint16(numOfUniqAddrHashes, uint16(addrID+1))
|
||||
if _, err := buf.Write(numOfUniqAddrHashes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -87,60 +91,62 @@ func EncodeStorage(s *ChangeSet) ([]byte, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
lenOfAddr := getNumOfBytesByLen(len(addrHashesMap))
|
||||
values := new(bytes.Buffer)
|
||||
lengthes := make([]byte, storageEnodingLengthOfNumTypeOfElements*3)
|
||||
lenOfAddr := getNumOfBytesByLen(int(addrID + 1))
|
||||
numOfUint8 := uint16(0)
|
||||
numOfUint16 := uint16(0)
|
||||
numOfUint32 := uint16(0)
|
||||
|
||||
keys := new(bytes.Buffer)
|
||||
lengthOfValues := uint32(0)
|
||||
row := make([]byte, lenOfAddr+common.HashLength)
|
||||
for i := 0; i < len(s.Changes); i++ {
|
||||
for i, change := range s.Changes {
|
||||
writeKeyRow(
|
||||
addrHashesMap[common.BytesToHash(s.Changes[i].Key[0:common.HashLength])],
|
||||
addrHashesMap[i],
|
||||
row[0:lenOfAddr],
|
||||
)
|
||||
copy(row[lenOfAddr:lenOfAddr+common.HashLength], common.CopyBytes(s.Changes[i].Key[common.IncarnationLength+common.HashLength:common.IncarnationLength+2*common.HashLength]))
|
||||
keys.Write(row)
|
||||
|
||||
lengthOfValues += uint32(len(s.Changes[i].Value))
|
||||
copy(row[lenOfAddr:lenOfAddr+common.HashLength], change.Key[common.IncarnationLength+common.HashLength:common.IncarnationLength+2*common.HashLength])
|
||||
buf.Write(row)
|
||||
}
|
||||
// Remember position to fix up the lengths later
|
||||
lengthPos := buf.Len()
|
||||
uint16b := make([]byte, 2)
|
||||
// Reserve 3 positions for lengths
|
||||
buf.Write(uint16b)
|
||||
buf.Write(uint16b)
|
||||
buf.Write(uint16b)
|
||||
uint32b := make([]byte, 4)
|
||||
for _, change := range s.Changes {
|
||||
lengthOfValues += uint32(len(change.Value))
|
||||
switch {
|
||||
case lengthOfValues <= 255:
|
||||
numOfUint8++
|
||||
lengthes = append(lengthes, uint8(lengthOfValues))
|
||||
buf.WriteByte(uint8(lengthOfValues))
|
||||
case lengthOfValues <= 65535:
|
||||
numOfUint16++
|
||||
uint16b := make([]byte, 2)
|
||||
binary.BigEndian.PutUint16(uint16b, uint16(lengthOfValues))
|
||||
lengthes = append(lengthes, uint16b...)
|
||||
buf.Write(uint16b)
|
||||
default:
|
||||
numOfUint32++
|
||||
uint32b := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(uint32b, lengthOfValues)
|
||||
lengthes = append(lengthes, uint32b...)
|
||||
buf.Write(uint32b)
|
||||
}
|
||||
values.Write(s.Changes[i].Value)
|
||||
}
|
||||
|
||||
binary.BigEndian.PutUint16(lengthes[0:storageEnodingLengthOfNumTypeOfElements], numOfUint8)
|
||||
binary.BigEndian.PutUint16(lengthes[storageEnodingLengthOfNumTypeOfElements:2*storageEnodingLengthOfNumTypeOfElements], numOfUint16)
|
||||
binary.BigEndian.PutUint16(lengthes[2*storageEnodingLengthOfNumTypeOfElements:3*storageEnodingLengthOfNumTypeOfElements], numOfUint32)
|
||||
if _, err := buf.Write(keys.Bytes()); err != nil {
|
||||
return nil, err
|
||||
for _, change := range s.Changes {
|
||||
buf.Write(change.Value)
|
||||
}
|
||||
|
||||
if _, err := buf.Write(lengthes); err != nil {
|
||||
return nil, err
|
||||
if len(notDefaultIncarnationList) > 0 {
|
||||
if _, err := buf.Write(notDefaultIncarnationList); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := buf.Write(values.Bytes()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
byt := buf.Bytes()
|
||||
return byt, nil
|
||||
b := buf.Bytes()
|
||||
// Fix up the lengths
|
||||
binary.BigEndian.PutUint16(b[lengthPos:], numOfUint8)
|
||||
binary.BigEndian.PutUint16(b[lengthPos+2:], numOfUint16)
|
||||
binary.BigEndian.PutUint16(b[lengthPos+4:], numOfUint32)
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func DecodeStorage(b []byte) (*ChangeSet, error) {
|
||||
@ -153,50 +159,65 @@ func DecodeStorage(b []byte) (*ChangeSet, error) {
|
||||
return h, fmt.Errorf("decode: input too short (%d bytes)", len(b))
|
||||
}
|
||||
|
||||
//numOfElements uint32
|
||||
numOfElements := binary.BigEndian.Uint32(b[0:storageEnodingLengthOfNumOfElements])
|
||||
numOfElements := int(binary.BigEndian.Uint32(b[0:storageEnodingLengthOfNumOfElements]))
|
||||
h.Changes = make([]Change, numOfElements)
|
||||
|
||||
if numOfElements == 0 {
|
||||
return h, nil
|
||||
}
|
||||
|
||||
dictLen := binary.BigEndian.Uint16(b[storageEnodingLengthOfNumOfElements : storageEnodingLengthOfNumOfElements+storageEnodingLengthOfDict])
|
||||
addMap := make(map[uint32]common.Hash)
|
||||
for i := 0; i < int(dictLen); i++ {
|
||||
dictLen := int(binary.BigEndian.Uint16(b[storageEnodingLengthOfNumOfElements : storageEnodingLengthOfNumOfElements+storageEnodingLengthOfDict]))
|
||||
addMap := make(map[uint32][]byte)
|
||||
for i := 0; i < dictLen; i++ {
|
||||
elemStart := storageEnodingLengthOfNumOfElements + storageEnodingLengthOfDict + i*common.HashLength
|
||||
addMap[uint32(i)] = common.BytesToHash(b[elemStart : elemStart+common.HashLength])
|
||||
addMap[uint32(i)] = b[elemStart : elemStart+common.HashLength]
|
||||
}
|
||||
|
||||
lenOfValsPos := storageEnodingStartElem +
|
||||
2 + uint32(dictLen)*common.HashLength +
|
||||
numOfElements*uint32(getNumOfBytesByLen(int(dictLen))+common.HashLength)
|
||||
2 + dictLen*common.HashLength +
|
||||
numOfElements*(getNumOfBytesByLen(dictLen)+common.HashLength)
|
||||
|
||||
numOfUint8 := int(binary.BigEndian.Uint16(b[lenOfValsPos : lenOfValsPos+2]))
|
||||
numOfUint16 := int(binary.BigEndian.Uint16(b[lenOfValsPos+2 : lenOfValsPos+4]))
|
||||
numOfUint32 := int(binary.BigEndian.Uint16(b[lenOfValsPos+4 : lenOfValsPos+6]))
|
||||
lenOfValsPos = lenOfValsPos + 3*storageEnodingLengthOfNumTypeOfElements
|
||||
valuesPos := lenOfValsPos + uint32(numOfUint8) + uint32(numOfUint16*2) + uint32(numOfUint32*4)
|
||||
valuesPos := lenOfValsPos + numOfUint8 + numOfUint16*2 + numOfUint32*4
|
||||
|
||||
elementStart := storageEnodingStartElem + storageEnodingLengthOfDict + uint32(dictLen)*common.HashLength
|
||||
key := make([]byte, common.HashLength*2+common.IncarnationLength)
|
||||
incarnationPosition := lenOfValsPos + calculateIncarnationPos3(b[lenOfValsPos:], numOfUint8, numOfUint16, numOfUint32)
|
||||
notDefaultIncarnation := make(map[uint32]uint64)
|
||||
|
||||
lenOfAddHash := uint32(getNumOfBytesByLen(len(addMap)))
|
||||
//lastValLen:=0
|
||||
for i := uint32(0); i < numOfElements; i++ {
|
||||
if len(b) > incarnationPosition {
|
||||
if len(b[incarnationPosition:])%12 != 0 {
|
||||
return h, fmt.Errorf("decode: incarnatin part is incorrect(%d bytes)", len(b[incarnationPosition:]))
|
||||
}
|
||||
for i := incarnationPosition; i < len(b); i += 12 {
|
||||
id := binary.BigEndian.Uint32(b[i : i+4])
|
||||
inc := ^binary.BigEndian.Uint64(b[i+4 : i+12])
|
||||
notDefaultIncarnation[id] = inc
|
||||
}
|
||||
}
|
||||
|
||||
elementStart := storageEnodingStartElem + storageEnodingLengthOfDict + dictLen*common.HashLength
|
||||
|
||||
lenOfAddHash := getNumOfBytesByLen(len(addMap))
|
||||
for i := 0; i < numOfElements; i++ {
|
||||
//copy addrHash
|
||||
key := make([]byte, common.HashLength*2+common.IncarnationLength)
|
||||
elem := elementStart + i*(lenOfAddHash+common.HashLength)
|
||||
copy(
|
||||
key[0:common.HashLength],
|
||||
readFromMap(addMap, b[elem:elem+lenOfAddHash]).Bytes(),
|
||||
)
|
||||
addrIdx := getUint32(b[elem : elem+lenOfAddHash])
|
||||
copy(key[:common.HashLength], addMap[addrIdx])
|
||||
if inc, ok := notDefaultIncarnation[addrIdx]; ok {
|
||||
binary.BigEndian.PutUint64(key[common.HashLength:], ^inc)
|
||||
} else {
|
||||
binary.BigEndian.PutUint64(key[common.HashLength:], ^DefaultIncarnation)
|
||||
}
|
||||
//copy key hash
|
||||
copy(
|
||||
key[common.HashLength+common.IncarnationLength:2*common.HashLength+common.IncarnationLength],
|
||||
common.CopyBytes(b[elem+lenOfAddHash:elem+lenOfAddHash+common.HashLength]),
|
||||
)
|
||||
binary.BigEndian.PutUint64(key[common.HashLength:common.HashLength+common.IncarnationLength], ^uint64(1))
|
||||
h.Changes[i].Key = common.CopyBytes(key)
|
||||
|
||||
h.Changes[i].Key = key
|
||||
h.Changes[i].Value = findVal(b[lenOfValsPos:valuesPos], b[valuesPos:], i, numOfUint8, numOfUint16, numOfUint32)
|
||||
}
|
||||
return h, nil
|
||||
@ -216,41 +237,57 @@ func getNumOfBytesByLen(n int) int {
|
||||
}
|
||||
}
|
||||
|
||||
func findVal(lenOfVals []byte, values []byte, i uint32, numOfUint8, numOfUint16, numOfUint32 int) []byte {
|
||||
lenOfValStart := uint32(0)
|
||||
lenOfValEnd := uint32(0)
|
||||
func calculateIncarnationPos3(b []byte, numOfUint8, numOfUint16, numOfUint32 int) int {
|
||||
res := 0
|
||||
end := 0
|
||||
switch {
|
||||
case i < uint32(numOfUint8):
|
||||
lenOfValEnd = uint32(lenOfVals[i])
|
||||
case numOfUint32 > 0:
|
||||
end = numOfUint8 + numOfUint16*2 + numOfUint32*4
|
||||
res = int(binary.BigEndian.Uint32(b[end-4:end])) + end
|
||||
case numOfUint16 > 0:
|
||||
end = numOfUint8 + numOfUint16*2
|
||||
res = int(binary.BigEndian.Uint16(b[end-2:end])) + end
|
||||
case numOfUint8 > 0:
|
||||
end = numOfUint8
|
||||
res = int(b[end-1]) + end
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func findVal(lenOfVals []byte, values []byte, i int, numOfUint8, numOfUint16, numOfUint32 int) []byte {
|
||||
lenOfValStart := 0
|
||||
lenOfValEnd := 0
|
||||
switch {
|
||||
case i < numOfUint8:
|
||||
lenOfValEnd = int(lenOfVals[i])
|
||||
if i > 0 {
|
||||
lenOfValStart = uint32(lenOfVals[i-1])
|
||||
lenOfValStart = int(lenOfVals[i-1])
|
||||
}
|
||||
return common.CopyBytes(values[lenOfValStart:lenOfValEnd])
|
||||
case i < uint32(numOfUint8)+uint32(numOfUint16):
|
||||
one := uint32(numOfUint8) + (i-uint32(numOfUint8))*2
|
||||
lenOfValEnd = uint32(binary.BigEndian.Uint16(lenOfVals[one : one+2]))
|
||||
if i-1 < uint32(numOfUint8) {
|
||||
lenOfValStart = uint32(lenOfVals[i-1])
|
||||
case i < numOfUint8+numOfUint16:
|
||||
one := (i-numOfUint8)*2 + numOfUint8
|
||||
lenOfValEnd = int(binary.BigEndian.Uint16(lenOfVals[one : one+2]))
|
||||
if i-1 < numOfUint8 {
|
||||
lenOfValStart = int(lenOfVals[i-1])
|
||||
} else {
|
||||
one = uint32(numOfUint8) + (i-1-uint32(numOfUint8))*2
|
||||
lenOfValStart = uint32(binary.BigEndian.Uint16(lenOfVals[one : one+2]))
|
||||
one = (i-1)*2 - numOfUint8
|
||||
lenOfValStart = int(binary.BigEndian.Uint16(lenOfVals[one : one+2]))
|
||||
}
|
||||
return common.CopyBytes(values[lenOfValStart:lenOfValEnd])
|
||||
case i < uint32(numOfUint8)+uint32(numOfUint16)+uint32(numOfUint32):
|
||||
fmt.Println(i, uint32(numOfUint8), numOfUint16, numOfUint32)
|
||||
one := uint32(numOfUint8) + uint32(numOfUint16)*2 + (i-uint32(numOfUint8)-uint32(numOfUint16))*4
|
||||
lenOfValEnd = binary.BigEndian.Uint32(lenOfVals[one : one+4])
|
||||
if i-1 < uint32(numOfUint8)+uint32(numOfUint16) {
|
||||
one = uint32(numOfUint8) + (i-1-uint32(numOfUint8))*2
|
||||
lenOfValStart = uint32(binary.BigEndian.Uint16(lenOfVals[one : one+2]))
|
||||
case i < numOfUint8+numOfUint16+numOfUint32:
|
||||
one := numOfUint8 + numOfUint16*2 + (i-numOfUint8-numOfUint16)*4
|
||||
lenOfValEnd = int(binary.BigEndian.Uint32(lenOfVals[one : one+4]))
|
||||
if i-1 < numOfUint8+numOfUint16 {
|
||||
one = (i-1)*2 - numOfUint8
|
||||
lenOfValStart = int(binary.BigEndian.Uint16(lenOfVals[one : one+2]))
|
||||
} else {
|
||||
one := uint32(numOfUint8) + uint32(numOfUint16)*2 + (i-1-uint32(numOfUint8)-uint32(numOfUint16))*4
|
||||
lenOfValStart = binary.BigEndian.Uint32(lenOfVals[one : one+4])
|
||||
one = numOfUint8 + numOfUint16*2 + (i-1-numOfUint8-numOfUint16)*4
|
||||
lenOfValStart = int(binary.BigEndian.Uint32(lenOfVals[one : one+4]))
|
||||
}
|
||||
return common.CopyBytes(values[lenOfValStart:lenOfValEnd])
|
||||
default:
|
||||
panic("findval err")
|
||||
}
|
||||
return common.CopyBytes(values[lenOfValStart:lenOfValEnd])
|
||||
}
|
||||
|
||||
func writeKeyRow(id uint32, row []byte) {
|
||||
@ -267,16 +304,17 @@ func writeKeyRow(id uint32, row []byte) {
|
||||
panic("wrong size of row")
|
||||
}
|
||||
}
|
||||
func readFromMap(m map[uint32]common.Hash, row []byte) common.Hash {
|
||||
|
||||
func getUint32(row []byte) uint32 {
|
||||
switch len(row) {
|
||||
case 1:
|
||||
return m[uint32(row[0])]
|
||||
return uint32(row[0])
|
||||
case 2:
|
||||
return m[uint32(binary.BigEndian.Uint16(row))]
|
||||
return uint32(binary.BigEndian.Uint16(row))
|
||||
case 4:
|
||||
return m[binary.BigEndian.Uint32(row)]
|
||||
return binary.BigEndian.Uint32(row)
|
||||
case 8:
|
||||
return m[uint32(binary.BigEndian.Uint64(row))]
|
||||
return uint32(binary.BigEndian.Uint64(row))
|
||||
default:
|
||||
panic("wrong")
|
||||
}
|
||||
@ -292,50 +330,65 @@ func (b StorageChangeSetBytes) Walk(f func(k, v []byte) error) error {
|
||||
return fmt.Errorf("decode: input too short (%d bytes)", len(b))
|
||||
}
|
||||
|
||||
numOfItems := binary.BigEndian.Uint32(b[0:4])
|
||||
numOfItems := int(binary.BigEndian.Uint32(b[0:4]))
|
||||
|
||||
if numOfItems == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
numOfUniqueItems := binary.BigEndian.Uint16(b[storageEnodingLengthOfNumOfElements : storageEnodingLengthOfNumOfElements+storageEnodingLengthOfDict])
|
||||
numOfUniqueItems := int(binary.BigEndian.Uint16(b[storageEnodingLengthOfNumOfElements : storageEnodingLengthOfNumOfElements+storageEnodingLengthOfDict]))
|
||||
lenOfValsPos := storageEnodingStartElem +
|
||||
storageEnodingLengthOfDict +
|
||||
uint32(numOfUniqueItems)*common.HashLength +
|
||||
numOfItems*uint32(getNumOfBytesByLen(int(numOfUniqueItems))+common.HashLength)
|
||||
numOfUniqueItems*common.HashLength +
|
||||
numOfItems*(getNumOfBytesByLen(numOfUniqueItems)+common.HashLength)
|
||||
|
||||
numOfUint8 := int(binary.BigEndian.Uint16(b[lenOfValsPos : lenOfValsPos+storageEnodingLengthOfNumTypeOfElements]))
|
||||
numOfUint16 := int(binary.BigEndian.Uint16(b[lenOfValsPos+storageEnodingLengthOfNumTypeOfElements : lenOfValsPos+storageEnodingLengthOfNumTypeOfElements*2]))
|
||||
numOfUint32 := int(binary.BigEndian.Uint16(b[lenOfValsPos+storageEnodingLengthOfNumTypeOfElements*2 : lenOfValsPos+storageEnodingLengthOfNumTypeOfElements*3]))
|
||||
|
||||
lenOfValsPos += storageEnodingLengthOfNumTypeOfElements * 3
|
||||
valuesPos := lenOfValsPos +
|
||||
uint32(numOfUint8) +
|
||||
uint32(numOfUint16*2) +
|
||||
uint32(numOfUint32*4)
|
||||
valuesPos := lenOfValsPos + numOfUint8 + numOfUint16*2 + numOfUint32*4
|
||||
|
||||
addrHashMap := make(map[uint32]common.Hash, numOfUniqueItems)
|
||||
incarnationPosition := lenOfValsPos + calculateIncarnationPos3(b[lenOfValsPos:], numOfUint8, numOfUint16, numOfUint32)
|
||||
notDefaultIncarnation := make(map[uint32]uint64)
|
||||
|
||||
if len(b) > incarnationPosition {
|
||||
if len(b[incarnationPosition:])%12 != 0 {
|
||||
return fmt.Errorf("decode: incarnatin part is incorrect(%d bytes)", len(b[incarnationPosition:]))
|
||||
}
|
||||
for i := incarnationPosition; i < len(b); i += 12 {
|
||||
id := binary.BigEndian.Uint32(b[i : i+4])
|
||||
inc := ^binary.BigEndian.Uint64(b[i+4 : i+12])
|
||||
notDefaultIncarnation[id] = inc
|
||||
}
|
||||
}
|
||||
|
||||
addrHashMap := make(map[uint32][]byte, numOfUniqueItems)
|
||||
for i := uint32(0); i < uint32(numOfUniqueItems); i++ {
|
||||
elemStart := storageEnodingStartElem + storageEnodingLengthOfDict + i*(common.HashLength)
|
||||
addrHashMap[i] = common.BytesToHash(b[elemStart : elemStart+common.HashLength])
|
||||
addrHashMap[i] = b[elemStart : elemStart+common.HashLength]
|
||||
}
|
||||
|
||||
key := make([]byte, common.HashLength*2+common.IncarnationLength)
|
||||
elemLength := uint32(getNumOfBytesByLen(int(numOfUniqueItems)))
|
||||
for i := uint32(0); i < numOfItems; i++ {
|
||||
elemLength := getNumOfBytesByLen(numOfUniqueItems)
|
||||
for i := 0; i < numOfItems; i++ {
|
||||
elemStart := storageEnodingStartElem +
|
||||
storageEnodingLengthOfDict +
|
||||
uint32(numOfUniqueItems)*(common.HashLength) +
|
||||
numOfUniqueItems*(common.HashLength) +
|
||||
i*(elemLength+common.HashLength)
|
||||
|
||||
//copy addrHash
|
||||
copy(key[0:common.HashLength], readFromMap(addrHashMap, b[elemStart:elemStart+elemLength]).Bytes())
|
||||
addrIdx := getUint32(b[elemStart : elemStart+elemLength])
|
||||
copy(key[:common.HashLength], addrHashMap[addrIdx])
|
||||
if inc, ok := notDefaultIncarnation[addrIdx]; ok {
|
||||
binary.BigEndian.PutUint64(key[common.HashLength:common.HashLength+common.IncarnationLength], ^inc)
|
||||
} else {
|
||||
binary.BigEndian.PutUint64(key[common.HashLength:common.HashLength+common.IncarnationLength], ^DefaultIncarnation)
|
||||
}
|
||||
//copy key hash
|
||||
copy(
|
||||
key[common.HashLength+common.IncarnationLength:2*common.HashLength+common.IncarnationLength],
|
||||
b[elemStart+elemLength:elemStart+elemLength+common.HashLength],
|
||||
)
|
||||
binary.BigEndian.PutUint64(key[common.HashLength:common.HashLength+common.IncarnationLength], ^uint64(1))
|
||||
err := f(common.CopyBytes(key), findVal(b[lenOfValsPos:valuesPos], b[valuesPos:], i, numOfUint8, numOfUint16, numOfUint32))
|
||||
if err != nil {
|
||||
return err
|
||||
@ -352,52 +405,50 @@ func (b StorageChangeSetBytes) Find(addrHash []byte, keyHash []byte) ([]byte, er
|
||||
return nil, fmt.Errorf("decode: input too short (%d bytes)", len(b))
|
||||
}
|
||||
|
||||
numOfItems := binary.BigEndian.Uint32(b[0:storageEnodingLengthOfNumOfElements])
|
||||
numOfItems := int(binary.BigEndian.Uint32(b[:storageEnodingLengthOfNumOfElements]))
|
||||
if numOfItems == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
numOfUniqueItems := binary.BigEndian.Uint16(b[storageEnodingLengthOfNumOfElements : storageEnodingLengthOfNumOfElements+storageEnodingLengthOfDict])
|
||||
numOfUniqueItems := int(binary.BigEndian.Uint16(b[storageEnodingLengthOfNumOfElements:]))
|
||||
var addHashID uint32
|
||||
found := false
|
||||
|
||||
var elemStart uint32
|
||||
elemStart := storageEnodingLengthOfNumOfElements + storageEnodingLengthOfDict
|
||||
//todo[boris] here should be binary search
|
||||
for i := uint32(0); i < uint32(numOfUniqueItems); i++ {
|
||||
elemStart = storageEnodingLengthOfNumOfElements + storageEnodingLengthOfDict + i*common.HashLength
|
||||
for i := 0; i < numOfUniqueItems; i++ {
|
||||
if bytes.Equal(addrHash, b[elemStart:elemStart+common.HashLength]) {
|
||||
found = true
|
||||
addHashID = i
|
||||
addHashID = uint32(i)
|
||||
break
|
||||
}
|
||||
elemStart += common.HashLength
|
||||
}
|
||||
if !found {
|
||||
fmt.Printf("did not find addrHash %x\n", addrHash)
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
elemLength := getNumOfBytesByLen(numOfUniqueItems)
|
||||
lenOfValsPos := storageEnodingStartElem +
|
||||
storageEnodingLengthOfDict +
|
||||
uint32(numOfUniqueItems)*common.HashLength +
|
||||
numOfItems*uint32(getNumOfBytesByLen(int(numOfUniqueItems))+common.HashLength)
|
||||
numOfUniqueItems*common.HashLength +
|
||||
numOfItems*(elemLength+common.HashLength)
|
||||
|
||||
numOfUint8 := int(binary.BigEndian.Uint16(b[lenOfValsPos : lenOfValsPos+storageEnodingLengthOfNumTypeOfElements]))
|
||||
numOfUint16 := int(binary.BigEndian.Uint16(b[lenOfValsPos+storageEnodingLengthOfNumTypeOfElements : lenOfValsPos+storageEnodingLengthOfNumTypeOfElements*2]))
|
||||
numOfUint32 := int(binary.BigEndian.Uint16(b[lenOfValsPos+storageEnodingLengthOfNumTypeOfElements*2 : lenOfValsPos+storageEnodingLengthOfNumTypeOfElements*3]))
|
||||
numOfUint8 := int(binary.BigEndian.Uint16(b[lenOfValsPos:]))
|
||||
numOfUint16 := int(binary.BigEndian.Uint16(b[lenOfValsPos+storageEnodingLengthOfNumTypeOfElements:]))
|
||||
numOfUint32 := int(binary.BigEndian.Uint16(b[lenOfValsPos+storageEnodingLengthOfNumTypeOfElements*2:]))
|
||||
|
||||
lenOfValsPos += storageEnodingLengthOfNumTypeOfElements * 3
|
||||
valuesPos := lenOfValsPos +
|
||||
uint32(numOfUint8) +
|
||||
uint32(numOfUint16*2) +
|
||||
uint32(numOfUint32*4)
|
||||
valuesPos := lenOfValsPos + numOfUint8 + numOfUint16*2 + numOfUint32*4
|
||||
|
||||
//here should be binary search too
|
||||
elemLength := uint32(getNumOfBytesByLen(int(numOfUniqueItems)))
|
||||
encodedAddHashID := make([]byte, elemLength)
|
||||
writeKeyRow(addHashID, encodedAddHashID)
|
||||
for i := uint32(0); i < numOfItems; i++ {
|
||||
for i := 0; i < numOfItems; i++ {
|
||||
elemStart := storageEnodingStartElem +
|
||||
storageEnodingLengthOfDict +
|
||||
uint32(numOfUniqueItems)*(common.HashLength) +
|
||||
numOfUniqueItems*(common.HashLength) +
|
||||
i*(elemLength+common.HashLength)
|
||||
|
||||
if !bytes.Equal(encodedAddHashID, b[elemStart:elemStart+elemLength]) {
|
||||
|
||||
@ -3,6 +3,7 @@ package changeset
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
@ -17,7 +18,7 @@ const (
|
||||
defaultIncarnation = 1
|
||||
)
|
||||
|
||||
func TestEncodingStorageWithoutNotDefaultIncarnation(t *testing.T) {
|
||||
func TestEncodingStorageWithoutNotDefaultIncarnation1(t *testing.T) {
|
||||
f := func(t *testing.T, numOfElements int) {
|
||||
// empty StorageChangeSset first
|
||||
ch := NewStorageChangeSet()
|
||||
@ -84,7 +85,7 @@ func TestEncodingStorageWithtRandomIncarnation(t *testing.T) {
|
||||
addrHash, _ := common.HashData([]byte("addrHash" + strconv.Itoa(i)))
|
||||
key, _ := common.HashData([]byte("key" + strconv.Itoa(i)))
|
||||
val, _ := common.HashData([]byte("val" + strconv.Itoa(i)))
|
||||
err = ch.Add(dbutils.GenerateCompositeStorageKey(addrHash, defaultIncarnation, key), val.Bytes())
|
||||
err = ch.Add(dbutils.GenerateCompositeStorageKey(addrHash, rand.Uint64(), key), val.Bytes())
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@ -150,10 +151,10 @@ func TestEncodingStorageWithoutNotDefaultIncarnationWalk(t *testing.T) {
|
||||
i := 0
|
||||
err = StorageChangeSetBytes(b).Walk(func(k, v []byte) error {
|
||||
if !bytes.Equal(k, ch.Changes[i].Key) {
|
||||
t.Error(i, "key was incorrect", k, ch.Changes[i].Key)
|
||||
t.Errorf("%d key was incorrect %x %x", i, k, ch.Changes[i].Key)
|
||||
}
|
||||
if !bytes.Equal(v, ch.Changes[i].Value) {
|
||||
t.Error(i, "val is incorrect", v, ch.Changes[i].Value)
|
||||
t.Errorf("%d val is incorrect %x %x", i, v, ch.Changes[i].Value)
|
||||
}
|
||||
i++
|
||||
return nil
|
||||
@ -202,7 +203,7 @@ func TestEncodingStorageWithoutNotDefaultIncarnationFind(t *testing.T) {
|
||||
t.Error(err, i)
|
||||
}
|
||||
if !bytes.Equal(val, v.Value) {
|
||||
t.Error("not equal for ", v, val)
|
||||
t.Errorf("not equal for %x %x", v, val)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -110,7 +110,7 @@ func GenerateCompositeStorageKey(addressHash common.Hash, incarnation uint64, se
|
||||
func GenerateStoragePrefix(addressHash common.Hash, incarnation uint64) []byte {
|
||||
prefix := make([]byte, common.HashLength+8)
|
||||
copy(prefix, addressHash[:])
|
||||
binary.BigEndian.PutUint64(prefix[common.HashLength:], incarnation^^uint64(0))
|
||||
binary.BigEndian.PutUint64(prefix[common.HashLength:], ^incarnation)
|
||||
return prefix
|
||||
}
|
||||
|
||||
|
||||
@ -859,6 +859,7 @@ func (tds *TrieDbState) GetBlockNr() uint64 {
|
||||
}
|
||||
|
||||
func (tds *TrieDbState) UnwindTo(blockNr uint64) error {
|
||||
//fmt.Printf("Unwind from block %d to block %d\n", tds.blockNr, blockNr)
|
||||
tds.StartNewBuffer()
|
||||
b := tds.currentBuffer
|
||||
|
||||
|
||||
@ -187,13 +187,16 @@ func (dsw *DbStateWriter) WriteHistory() error {
|
||||
}
|
||||
if debug.IsThinHistory() {
|
||||
for _, change := range storageChanges.Changes {
|
||||
value, err1 := dsw.tds.db.Get(dbutils.StorageHistoryBucket, change.Key)
|
||||
keyNoInc := make([]byte, len(change.Key)-common.IncarnationLength)
|
||||
copy(keyNoInc, change.Key[:common.HashLength])
|
||||
copy(keyNoInc[common.HashLength:], change.Key[common.HashLength+common.IncarnationLength:])
|
||||
value, err1 := dsw.tds.db.Get(dbutils.StorageHistoryBucket, keyNoInc)
|
||||
if err1 != nil && err1 != ethdb.ErrKeyNotFound {
|
||||
return fmt.Errorf("db.Get failed: %w", err1)
|
||||
}
|
||||
index := dbutils.WrapHistoryIndex(value)
|
||||
index.Append(dsw.tds.blockNr)
|
||||
if err := dsw.tds.db.Put(dbutils.StorageHistoryBucket, change.Key, *index); err != nil {
|
||||
if err := dsw.tds.db.Put(dbutils.StorageHistoryBucket, keyNoInc, *index); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@ -324,7 +324,7 @@ func TestMutationCommitThinHistory(t *testing.T) {
|
||||
|
||||
resultHash := common.BytesToHash(res)
|
||||
if resultHash != v {
|
||||
t.Fatal("incorrect storage history for ", addrHash.String(), v, resultHash)
|
||||
t.Fatalf("incorrect storage history for %x %x %x", addrHash.String(), v, resultHash)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -507,6 +507,9 @@ func (db *BoltDatabase) walkAsOfThinStorage(startkey []byte, fixedbits uint, tim
|
||||
if csB == nil {
|
||||
return fmt.Errorf("storageChangeBucket not found")
|
||||
}
|
||||
startkeyNoInc := make([]byte, len(startkey)-common.IncarnationLength)
|
||||
copy(startkeyNoInc, startkey[:common.HashLength])
|
||||
copy(startkeyNoInc[common.HashLength:], startkey[common.HashLength+common.IncarnationLength:])
|
||||
//for storage
|
||||
mainCursor := newSplitCursor(
|
||||
b,
|
||||
@ -518,8 +521,8 @@ func (db *BoltDatabase) walkAsOfThinStorage(startkey []byte, fixedbits uint, tim
|
||||
//for historic data
|
||||
historyCursor := newSplitCursor(
|
||||
hB,
|
||||
startkey,
|
||||
fixedbits,
|
||||
startkeyNoInc,
|
||||
fixedbits-8*common.IncarnationLength,
|
||||
common.HashLength, /* part1end */
|
||||
common.HashLength+common.IncarnationLength, /* part2start */
|
||||
)
|
||||
@ -912,7 +915,15 @@ func BoltDBFindByHistory(tx *bolt.Tx, hBucket []byte, key []byte, timestamp uint
|
||||
if hB == nil {
|
||||
return nil, ErrKeyNotFound
|
||||
}
|
||||
v, _ := hB.Get(key)
|
||||
var v []byte
|
||||
if bytes.Equal(dbutils.StorageHistoryBucket, hBucket) {
|
||||
keyNoInc := make([]byte, len(key)-common.IncarnationLength)
|
||||
copy(keyNoInc, key[:common.HashLength])
|
||||
copy(keyNoInc[common.HashLength:], key[common.HashLength+common.IncarnationLength:])
|
||||
v, _ = hB.Get(keyNoInc)
|
||||
} else {
|
||||
v, _ = hB.Get(key)
|
||||
}
|
||||
index := dbutils.WrapHistoryIndex(v)
|
||||
|
||||
changeSetBlock, ok := index.Search(timestamp)
|
||||
@ -933,9 +944,9 @@ func BoltDBFindByHistory(tx *bolt.Tx, hBucket []byte, key []byte, timestamp uint
|
||||
err error
|
||||
)
|
||||
switch {
|
||||
case debug.IsThinHistory() && bytes.Equal(dbutils.AccountsHistoryBucket, hBucket):
|
||||
case bytes.Equal(dbutils.AccountsHistoryBucket, hBucket):
|
||||
data, err = changeset.AccountChangeSetBytes(changeSetData).FindLast(key)
|
||||
case debug.IsThinHistory() && bytes.Equal(dbutils.StorageHistoryBucket, hBucket):
|
||||
case bytes.Equal(dbutils.StorageHistoryBucket, hBucket):
|
||||
data, err = changeset.StorageChangeSetBytes(changeSetData).Find(key[:common.HashLength], key[common.HashLength+common.IncarnationLength:])
|
||||
default:
|
||||
data, err = changeset.FindLast(changeSetData, key)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user