erigon-pulse/common/changeset/storage_changeset.go

package changeset

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"github.com/ledgerwatch/turbo-geth/common"
	"sort"
)

const (
	DefaultIncarnation      = ^uint64(1)
	storageEnodingIndexSize = 4
	storageEnodingStartElem = uint32(4)
	storageEnodingRowSize   = uint32(common.HashLength * 3)
)

func NewStorageChangeSet() *ChangeSet {
	return &ChangeSet{
		Changes: make([]Change, 0),
		keyLen:  2*common.HashLength + common.IncarnationLength,
	}
}

/*
Storage ChangeSet is serialized in the following manner in order to facilitate binary search:
4:[32:32:32]:[4:8]
numOfElements:[addrHash:keyHash:value]:[elementNum:incarnation]

1. The number of keys N (uint32, 4 bytes).
2. Contiguous array of [addrHash(32b)+keyHash(32b)+value(32b)] (N*(32+32+32) bytes).
3. Contiguous array of not default incarnations, like: index of change(uint64) + incarnation(uint64)

uint32 integers are serialized as big-endian.
*/
func EncodeStorage(s *ChangeSet) ([]byte, error) {
	sort.Sort(s)
	buf := new(bytes.Buffer)
	intArr := make([]byte, 4)
	n := s.Len()
	binary.BigEndian.PutUint32(intArr, uint32(n))
	_, err := buf.Write(intArr)
	if err != nil {
		return nil, err
	}

	notDefaultIncarnationList := make([]struct {
		ID  uint8
		Inc uint64
	}, 0)
	change := make([]byte, 32+32+32)
	for i := 0; i < n; i++ {
		//copy addrHash
		copy(
			change[0:common.HashLength],
			s.Changes[i].Key[0:common.HashLength],
		)
		//copy key
		copy(
			change[common.HashLength:2*common.HashLength],
			s.Changes[i].Key[common.HashLength+common.IncarnationLength:2*common.HashLength+common.IncarnationLength],
		)
		//copy value
		copy(
			change[2*common.HashLength:3*common.HashLength],
			s.Changes[i].Value,
		)

		_, err = buf.Write(change)
		if err != nil {
			return nil, err
		}

		incarnation := binary.BigEndian.Uint64(s.Changes[i].Key[common.HashLength : common.HashLength+common.IncarnationLength])
		if incarnation != DefaultIncarnation {
			notDefaultIncarnationList = append(notDefaultIncarnationList, struct {
				ID  uint8
				Inc uint64
			}{ID: uint8(i), Inc: incarnation})
		}
	}
	if len(notDefaultIncarnationList) > 0 {
		b := make([]byte, storageEnodingIndexSize+common.IncarnationLength)
		for _, v := range notDefaultIncarnationList {
			binary.BigEndian.PutUint32(b[0:storageEnodingIndexSize], uint32(v.ID))
			binary.BigEndian.PutUint64(b[storageEnodingIndexSize:storageEnodingIndexSize+common.IncarnationLength], v.Inc)
			_, err = buf.Write(b)
			if err != nil {
				return nil, err
			}
		}
	}

	byt := buf.Bytes()
	return byt, nil
}

func DecodeStorage(b []byte) (*ChangeSet, error) {
	h := NewStorageChangeSet()
	if len(b) == 0 {
		h.Changes = make([]Change, 0)
		return h, nil
	}

	if len(b) < 4 {
		return h, fmt.Errorf("decode: input too short (%d bytes)", len(b))
	}

	numOfElements := binary.BigEndian.Uint32(b[0:4])
	h.Changes = make([]Change, numOfElements)

	if numOfElements == 0 {
		return h, nil
	}

	incarnationPosition := storageEnodingStartElem + numOfElements*(3*common.HashLength)
	if uint32(len(b)) < incarnationPosition {
		fmt.Println("DecodeStorage")
		return nil, fmt.Errorf("decode: input too short (%d bytes, expected at least %d bytes)", len(b), incarnationPosition)
	}

	//parse not default incarnations
	incarnationsLength := len(b[incarnationPosition:])
	notDefaultIncarnation := make(map[uint32]uint64)
	var (
		id  uint32
		inc uint64
		ok  bool
	)

	if incarnationsLength > 0 {
		if incarnationsLength%(storageEnodingIndexSize+common.IncarnationLength) != 0 {
			return h, fmt.Errorf("decode: incarnatin part is incorrect(%d bytes)", len(b[incarnationPosition:]))
		}
		numOfIncarnations := incarnationsLength / (storageEnodingIndexSize + common.IncarnationLength)
		for i := 0; i < numOfIncarnations; i++ {
			id = binary.BigEndian.Uint32(b[incarnationPosition : incarnationPosition+4])
			inc = binary.BigEndian.Uint64(b[incarnationPosition+4 : incarnationPosition+4+8])
			notDefaultIncarnation[id] = inc
			incarnationPosition += (storageEnodingIndexSize + common.IncarnationLength)
		}
	}

	elementStart := 4
	key := make([]byte, common.HashLength*2+common.IncarnationLength)
	for i := uint32(0); i < numOfElements; i++ {
		//copy addrHash
		copy(key[0:common.HashLength], b[elementStart:elementStart+common.HashLength])
		//copy key hash
		copy(
			key[common.HashLength+common.IncarnationLength:2*common.HashLength+common.IncarnationLength],
			b[elementStart+common.HashLength:elementStart+2*common.HashLength],
		)
		//set incarnation
		if inc, ok = notDefaultIncarnation[i]; ok {
			binary.BigEndian.PutUint64(key[common.HashLength:common.HashLength+common.IncarnationLength], inc)
		} else {
			binary.BigEndian.PutUint64(key[common.HashLength:common.HashLength+common.IncarnationLength], DefaultIncarnation)
		}
		h.Changes[i].Key = common.CopyBytes(key)
		h.Changes[i].Value = common.CopyBytes(b[elementStart+2*common.HashLength : elementStart+3*common.HashLength])
		//shift element
		elementStart += 3 * common.HashLength
	}

	return h, nil
}

type StorageChangeSetBytes []byte

func (b StorageChangeSetBytes) Walk(f func(k, v []byte) error) error {
	if len(b) == 0 {
		return nil
	}
	if len(b) < 4 {
		return fmt.Errorf("decode: input too short (%d bytes)", len(b))
	}

	numOfItems := binary.BigEndian.Uint32(b[0:4])

	if numOfItems == 0 {
		return nil
	}

	incarnationPosition := storageEnodingStartElem + numOfItems*(3*common.HashLength)
	if uint32(len(b)) < incarnationPosition {
		return fmt.Errorf("decode: input too short (%d bytes, expected at least %d bytes)", len(b), incarnationPosition)
	}
	incarnationsLength := len(b[incarnationPosition:])
	notDefaultIncarnation := make(map[uint32]uint64)
	var (
		id  uint32
		inc uint64
		ok  bool
	)

	if incarnationsLength > 0 {
		if incarnationsLength%(storageEnodingIndexSize+common.IncarnationLength) != 0 {
			return fmt.Errorf("decode: incarnatin part is incorrect(%d bytes)", len(b[incarnationPosition:]))
		}
		numOfIncarnations := incarnationsLength / (storageEnodingIndexSize + common.IncarnationLength)
		for i := 0; i < numOfIncarnations; i++ {
			id = binary.BigEndian.Uint32(b[incarnationPosition : incarnationPosition+storageEnodingIndexSize])
			inc = binary.BigEndian.Uint64(b[incarnationPosition+storageEnodingIndexSize : incarnationPosition+storageEnodingIndexSize+common.IncarnationLength])
			notDefaultIncarnation[id] = inc
			incarnationPosition += (storageEnodingIndexSize + common.IncarnationLength)
		}
	}

	key := make([]byte, common.HashLength*2+common.IncarnationLength)
	for i := uint32(0); i < numOfItems; i++ {
		//copy addrHash
		copy(key[0:common.HashLength], b[storageEnodingStartElem+storageEnodingRowSize*i:storageEnodingStartElem+storageEnodingRowSize*i+common.HashLength])
		//copy key hash
		copy(
			key[common.HashLength+common.IncarnationLength:2*common.HashLength+common.IncarnationLength],
			b[storageEnodingStartElem+storageEnodingRowSize*i+common.HashLength:storageEnodingStartElem+storageEnodingRowSize*i+2*common.HashLength],
		)
		//set incarnation
		if inc, ok = notDefaultIncarnation[i]; ok {
			binary.BigEndian.PutUint64(key[common.HashLength:common.HashLength+common.IncarnationLength], inc)
		} else {
			binary.BigEndian.PutUint64(key[common.HashLength:common.HashLength+common.IncarnationLength], DefaultIncarnation)
		}

		err := f(common.CopyBytes(key), common.CopyBytes(b[storageEnodingStartElem+storageEnodingRowSize*i+2*common.HashLength:storageEnodingStartElem+storageEnodingRowSize*i+3*common.HashLength]))
		if err != nil {
			return err
		}
	}
	return nil
}

func (b StorageChangeSetBytes) FindLast(k []byte) ([]byte, error) {
	if len(b) == 0 {
		return nil, nil
	}
	if len(b) < 4 {
		return nil, fmt.Errorf("decode: input too short (%d bytes)", len(b))
	}

	numOfItems := binary.BigEndian.Uint32(b[0:4])

	if numOfItems == 0 {
		return nil, nil
	}

	incarnationPosition := storageEnodingStartElem + numOfItems*(3*common.HashLength)
	if uint32(len(b)) < incarnationPosition {
		return nil, fmt.Errorf("decode: input too short (%d bytes, expected at least %d bytes)", len(b), incarnationPosition)
	}

	incarnationsLength := len(b[incarnationPosition:])
	notDefaultIncarnation := make(map[uint32]uint64)
	var (
		id  uint32
		inc uint64
		ok  bool
	)

	if incarnationsLength > 0 {
		if incarnationsLength%(storageEnodingIndexSize+common.IncarnationLength) != 0 {
			return nil, fmt.Errorf("decode: incarnatin part is incorrect(%d bytes)", len(b[incarnationPosition:]))
		}
		numOfIncarnations := incarnationsLength / (storageEnodingIndexSize + common.IncarnationLength)
		for i := 0; i < numOfIncarnations; i++ {
			id = binary.BigEndian.Uint32(b[incarnationPosition : incarnationPosition+storageEnodingIndexSize])
			inc = binary.BigEndian.Uint64(b[incarnationPosition+storageEnodingIndexSize : incarnationPosition+storageEnodingIndexSize+common.IncarnationLength])
			notDefaultIncarnation[id] = inc
			incarnationPosition += (storageEnodingIndexSize + common.IncarnationLength)
		}
	}

	key := make([]byte, common.HashLength*2+common.IncarnationLength)
	for i := numOfItems - 1; int(i) >= 0; i-- {
		//copy addrHash
		copy(key[0:common.HashLength], b[storageEnodingStartElem+storageEnodingRowSize*i:storageEnodingStartElem+storageEnodingRowSize*i+common.HashLength])
		//copy key hash
		copy(
			key[common.HashLength+common.IncarnationLength:2*common.HashLength+common.IncarnationLength],
			b[storageEnodingStartElem+storageEnodingRowSize*i+common.HashLength:storageEnodingStartElem+storageEnodingRowSize*i+2*common.HashLength],
		)
		//set incarnation
		if inc, ok = notDefaultIncarnation[i]; ok {
			binary.BigEndian.PutUint64(key[common.HashLength:common.HashLength+common.IncarnationLength], inc)
		} else {
			binary.BigEndian.PutUint64(key[common.HashLength:common.HashLength+common.IncarnationLength], DefaultIncarnation)
		}

		if bytes.Equal(key, k) {
			return common.CopyBytes(b[storageEnodingStartElem+storageEnodingRowSize*i+2*common.HashLength : storageEnodingStartElem+storageEnodingRowSize*i+3*common.HashLength]), nil
		}
	}
	return nil, errors.New("not found")
}