Revert "Remove unused tables, soft rename" (#2205)

This commit is contained in:
Artem Vorotnikov 2021-06-20 17:08:48 +03:00 committed by GitHub
parent 989a8305f2
commit 504c31e97a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 137 additions and 49 deletions

View File

@ -41,12 +41,13 @@ Physical layout:
[acc2_hash] | [acc2_value] [acc2_hash] | [acc2_value]
... ...
*/ */
const PlainStateBucket = "PlainState" const PlainStateBucket = "PLAIN-CST2"
const PlainStateBucketOld1 = "PLAIN-CST"
//PlainContractCodeBucket - //PlainContractCodeBucket -
//key - address+incarnation //key - address+incarnation
//value - code hash //value - code hash
const PlainContractCodeBucket = "PlainCodeHash" const PlainContractCodeBucket = "PLAIN-contractCode"
/* /*
AccountChangeSetBucket and StorageChangeSetBucket - of block N store values of state before block N changed them. AccountChangeSetBucket and StorageChangeSetBucket - of block N store values of state before block N changed them.
@ -72,8 +73,8 @@ StorageChangeSetBucket:
key - blockNum_u64 + address + incarnation_u64 key - blockNum_u64 + address + incarnation_u64
value - plain_storage_key + value value - plain_storage_key + value
*/ */
const AccountChangeSetBucket = "AccountChangeSet" const AccountChangeSetBucket = "PLAIN-ACS"
const StorageChangeSetBucket = "StorageChangeSet" const StorageChangeSetBucket = "PLAIN-SCS"
const ( const (
@ -83,8 +84,9 @@ const (
// Contains Storage: // Contains Storage:
//key - address hash + incarnation + storage key hash //key - address hash + incarnation + storage key hash
//value - storage value(common.hash) //value - storage value(common.hash)
HashedAccountsBucket = "HashedAccount" HashedAccountsBucket = "hashed_accounts"
HashedStorageBucket = "HashedStorage" HashedStorageBucket = "hashed_storage"
CurrentStateBucketOld2 = "CST2"
) )
/* /*
@ -119,23 +121,23 @@ StorageHistoryBucket
key - address + storage_key + shard_id_u64 key - address + storage_key + shard_id_u64
value - roaring bitmap - list of block where it changed value - roaring bitmap - list of block where it changed
*/ */
var AccountsHistoryBucket = "AccountHistory" var AccountsHistoryBucket = "hAT"
var StorageHistoryBucket = "StorageHistory" var StorageHistoryBucket = "hST"
var ( var (
//key - contract code hash //key - contract code hash
//value - contract code //value - contract code
CodeBucket = "Code" CodeBucket = "CODE"
//key - addressHash+incarnation //key - addressHash+incarnation
//value - code hash //value - code hash
ContractCodeBucket = "HashedCodeHash" ContractCodeBucket = "contractCode"
// IncarnationMapBucket for deleted accounts // IncarnationMapBucket for deleted accounts
//key - address //key - address
//value - incarnation of account when it was last deleted //value - incarnation of account when it was last deleted
IncarnationMapBucket = "IncarnationMap" IncarnationMapBucket = "incarnationMap"
//TEVMCodeBucket - //TEVMCodeBucket -
//key - contract code hash //key - contract code hash
@ -183,31 +185,31 @@ Invariants:
- TrieAccount records with length=1 can satisfy (hasBranch==0&&hasHash==0) condition - TrieAccount records with length=1 can satisfy (hasBranch==0&&hasHash==0) condition
- Other records in TrieAccount and TrieStorage must (hasTree!=0 || hasHash!=0) - Other records in TrieAccount and TrieStorage must (hasTree!=0 || hasHash!=0)
*/ */
const TrieOfAccountsBucket = "TrieAccount" const TrieOfAccountsBucket = "trie_account"
const TrieOfStorageBucket = "TrieStorage" const TrieOfStorageBucket = "trie_storage"
const IntermediateTrieHashBucketOld2 = "iTh2" const IntermediateTrieHashBucketOld2 = "iTh2"
const ( const (
// DatabaseInfoBucket is used to store information about data layout. // DatabaseInfoBucket is used to store information about data layout.
DatabaseInfoBucket = "DbInfo" DatabaseInfoBucket = "DBINFO"
SnapshotInfoBucket = "SnapshotInfo" SnapshotInfoBucket = "SNINFO"
BittorrentInfoBucket = "BittorrentInfo" BittorrentInfoBucket = "BTINFO"
HeadersSnapshotInfoBucket = "HeadersSnapshotInfo" HeadersSnapshotInfoBucket = "hSNINFO"
BodiesSnapshotInfoBucket = "BodiesSnapshotInfo" BodiesSnapshotInfoBucket = "bSNINFO"
StateSnapshotInfoBucket = "StateSnapshotInfo" StateSnapshotInfoBucket = "sSNINFO"
// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
HeaderPrefixOld = "h" // block_num_u64 + hash -> header HeaderPrefixOld = "h" // block_num_u64 + hash -> header
HeaderNumberBucket = "HeaderNumber" // headerNumberPrefix + hash -> num (uint64 big endian) HeaderNumberBucket = "H" // headerNumberPrefix + hash -> num (uint64 big endian)
HeaderCanonicalBucket = "CanonicalHeader" // block_num_u64 -> header hash HeaderCanonicalBucket = "canonical_headers" // block_num_u64 -> header hash
HeadersBucket = "Header" // block_num_u64 + hash -> header (RLP) HeadersBucket = "headers" // block_num_u64 + hash -> header (RLP)
HeaderTDBucket = "HeadersTotalDifficulty" // block_num_u64 + hash -> td (RLP) HeaderTDBucket = "header_to_td" // block_num_u64 + hash -> td (RLP)
BlockBodyPrefix = "BlockBody" // block_num_u64 + hash -> block body BlockBodyPrefix = "b" // block_num_u64 + hash -> block body
EthTx = "BlockTransaction" // tbl_sequence_u64 -> rlp(tx) EthTx = "eth_tx" // tbl_sequence_u64 -> rlp(tx)
BlockReceiptsPrefix = "Receipt" // block_num_u64 -> canonical block receipts (non-canonical are not stored) BlockReceiptsPrefix = "r" // block_num_u64 -> canonical block receipts (non-canonical are not stored)
Log = "TransactionLog" // block_num_u64 + txId -> logs of transaction Log = "log" // block_num_u64 + txId -> logs of transaction
// Stores bitmap indices - in which block numbers saw logs of given 'address' or 'topic' // Stores bitmap indices - in which block numbers saw logs of given 'address' or 'topic'
// [addr or topic] + [2 bytes inverted shard number] -> bitmap(blockN) // [addr or topic] + [2 bytes inverted shard number] -> bitmap(blockN)
@ -217,41 +219,43 @@ const (
// if last existing shard size merge it with delta // if last existing shard size merge it with delta
// if serialized size of delta > ShardLimit - break down to multiple shards // if serialized size of delta > ShardLimit - break down to multiple shards
// shard number - it's biggest value in bitmap // shard number - it's biggest value in bitmap
LogTopicIndex = "LogTopicIndex" LogTopicIndex = "log_topic_index"
LogAddressIndex = "LogAddressIndex" LogAddressIndex = "log_address_index"
// CallTraceSet is the name of the table that contain the mapping of block number to the set (sorted) of all accounts // CallTraceSet is the name of the table that contain the mapping of block number to the set (sorted) of all accounts
// touched by call traces. It is DupSort-ed table // touched by call traces. It is DupSort-ed table
// 8-byte BE block number -> account address -> two bits (one for "from", another for "to") // 8-byte BE block number -> account address -> two bits (one for "from", another for "to")
CallTraceSet = "CallTraceSet" CallTraceSet = "call_trace_set"
// Indices for call traces - have the same format as LogTopicIndex and LogAddressIndex // Indices for call traces - have the same format as LogTopicIndex and LogAddressIndex
// Store bitmap indices - in which block number we saw calls from (CallFromIndex) or to (CallToIndex) some addresses // Store bitmap indices - in which block number we saw calls from (CallFromIndex) or to (CallToIndex) some addresses
CallFromIndex = "CallFromIndex" CallFromIndex = "call_from_index"
CallToIndex = "CallToIndex" CallToIndex = "call_to_index"
TxLookupPrefix = "BlockTransactionLookup" // hash -> transaction/receipt lookup metadata TxLookupPrefix = "l" // hash -> transaction/receipt lookup metadata
BloomBitsPrefix = "BloomBits" // bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits BloomBitsPrefix = "B" // bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits
ConfigPrefix = "Config" // config prefix for the db ConfigPrefix = "ethereum-config-" // config prefix for the db
// Chain index prefixes (use `i` + single byte to avoid mixing data types). // Chain index prefixes (use `i` + single byte to avoid mixing data types).
BloomBitsIndexPrefix = "BloomBitsIndex" // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress BloomBitsIndexPrefix = "iB" // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress
// Progress of sync stages: stageName -> stageData // Progress of sync stages: stageName -> stageData
SyncStageProgress = "SyncStage" SyncStageProgress = "SSP2"
SyncStageProgressOld1 = "SSP"
// Position to where to unwind sync stages: stageName -> stageData // Position to where to unwind sync stages: stageName -> stageData
SyncStageUnwind = "SyncStageUnwind" SyncStageUnwind = "SSU2"
SyncStageUnwindOld1 = "SSU"
CliqueBucket = "Clique" CliqueBucket = "clique-"
CliqueSeparateBucket = "CliqueSeparate" CliqueSeparateBucket = "clique-snapshots-"
CliqueSnapshotBucket = "CliqueSnapshot" CliqueSnapshotBucket = "snap"
CliqueLastSnapshotBucket = "CliqueLastSnapshot" CliqueLastSnapshotBucket = "lastSnap"
// this bucket stored in separated database // this bucket stored in separated database
InodesBucket = "Inode" InodesBucket = "inodes"
// Transaction senders - stored separately from the block bodies // Transaction senders - stored separately from the block bodies
Senders = "TxSender" // block_num_u64 + blockHash -> sendersList (no serialization format, every 20 bytes is new sender) Senders = "txSenders" // block_num_u64 + blockHash -> sendersList (no serialization format, every 20 bytes is new sender)
// headBlockKey tracks the latest know full block's hash. // headBlockKey tracks the latest know full block's hash.
HeadBlockKey = "LastBlock" HeadBlockKey = "LastBlock"
@ -259,12 +263,63 @@ const (
// migrationName -> serialized SyncStageProgress and SyncStageUnwind buckets // migrationName -> serialized SyncStageProgress and SyncStageUnwind buckets
// it stores stages progress to understand in which context was executed migration // it stores stages progress to understand in which context was executed migration
// in case of bug-report developer can ask content of this bucket // in case of bug-report developer can ask content of this bucket
Migrations = "Migration" Migrations = "migrations"
Sequence = "Sequence" // tbl_name -> seq_u64 Sequence = "sequence" // tbl_name -> seq_u64
HeadHeaderKey = "LastHeader" HeadHeaderKey = "LastHeader"
) )
var Rename = map[string]string{
PlainStateBucket: "PlainState",
PlainContractCodeBucket: "PlainCodeHash",
AccountChangeSetBucket: "AccountChangeSet",
StorageChangeSetBucket: "StorageChangeSet",
HashedAccountsBucket: "HashedAccount",
HashedStorageBucket: "HashedStorage",
AccountsHistoryBucket: "AccountHistory",
StorageHistoryBucket: "StorageHistory",
CodeBucket: "Code",
ContractCodeBucket: "HashedCodeHash",
IncarnationMapBucket: "IncarnationMap",
TrieOfAccountsBucket: "TrieAccount",
TrieOfStorageBucket: "TrieStorage",
DatabaseInfoBucket: "DbInfo",
SnapshotInfoBucket: "SnapshotInfo",
BittorrentInfoBucket: "BittorrentInfo",
HeadersSnapshotInfoBucket: "HeadersSnapshotInfo",
BodiesSnapshotInfoBucket: "BodiesSnapshotInfo",
StateSnapshotInfoBucket: "StateSnapshotInfo",
HeaderNumberBucket: "HeaderNumber",
HeaderCanonicalBucket: "CanonicalHeader",
HeadersBucket: "Header",
HeaderTDBucket: "HeadersTotalDifficulty",
BlockBodyPrefix: "BlockBody",
EthTx: "BlockTransaction",
BlockReceiptsPrefix: "Receipt",
Log: "TransactionLog",
LogTopicIndex: "LogTopicIndex",
LogAddressIndex: "LogAddressIndex",
CallTraceSet: "CallTraceSet",
CallFromIndex: "CallFromIndex",
CallToIndex: "CallToIndex",
TxLookupPrefix: "BlockTransactionLookup",
BloomBitsPrefix: "BloomBits",
ConfigPrefix: "Config",
BloomBitsIndexPrefix: "BloomBitsIndex",
SyncStageProgress: "SyncStage",
SyncStageUnwind: "SyncStageUnwind",
CliqueBucket: "Clique",
CliqueSeparateBucket: "CliqueSeparate",
CliqueSnapshotBucket: "CliqueSnapshot",
CliqueLastSnapshotBucket: "CliqueLastSnapshot",
InodesBucket: "Inode",
Senders: "TxSender",
HeadBlockKey: "LastBlock",
Migrations: "Migration",
Sequence: "Sequence",
HeadHeaderKey: "LastHeader",
}
// Keys // Keys
var ( var (
//StorageModeHistory - does node save history. //StorageModeHistory - does node save history.
@ -346,6 +401,10 @@ var Buckets = []string{
// DeprecatedBuckets - list of buckets which can be programmatically deleted - for example after migration // DeprecatedBuckets - list of buckets which can be programmatically deleted - for example after migration
var DeprecatedBuckets = []string{ var DeprecatedBuckets = []string{
IntermediateTrieHashBucketOld2, IntermediateTrieHashBucketOld2,
CurrentStateBucketOld2,
SyncStageProgressOld1,
SyncStageUnwindOld1,
PlainStateBucketOld1,
HeaderPrefixOld, HeaderPrefixOld,
CliqueBucket, CliqueBucket,
} }
@ -403,6 +462,12 @@ type BucketConfigItem struct {
} }
var BucketsConfigs = BucketsCfg{ var BucketsConfigs = BucketsCfg{
CurrentStateBucketOld2: {
Flags: DupSort,
AutoDupSortKeysConversion: true,
DupFromLen: 72,
DupToLen: 40,
},
HashedStorageBucket: { HashedStorageBucket: {
Flags: DupSort, Flags: DupSort,
AutoDupSortKeysConversion: true, AutoDupSortKeysConversion: true,

View File

@ -621,6 +621,26 @@ func (tx *MdbxTx) CreateBucket(name string) error {
return nil return nil
} }
// if bucket with this name not found - check renamed one
rename := dbutils.Rename[name]
dbi, err = tx.tx.OpenDBI(rename, mdbx.DBAccede, nil, dcmp)
if err != nil && !mdbx.IsNotFound(err) {
return fmt.Errorf("create bucket: %s, %w", name, err)
}
if err == nil {
cnfCopy.DBI = dbutils.DBI(dbi)
var flags uint
flags, err = tx.tx.Flags(dbi)
if err != nil {
return err
}
cnfCopy.Flags = dbutils.BucketFlags(flags)
tx.db.buckets[name] = cnfCopy
return nil
}
// if bucket doesn't exists - create it // if bucket doesn't exists - create it
var flags = tx.db.buckets[name].Flags var flags = tx.db.buckets[name].Flags
@ -637,8 +657,11 @@ func (tx *MdbxTx) CreateBucket(name string) error {
return fmt.Errorf("some not supported flag provided for bucket") return fmt.Errorf("some not supported flag provided for bucket")
} }
dbi, err = tx.tx.OpenDBI(name, nativeFlags, nil, dcmp) if rename != "" {
dbi, err = tx.tx.OpenDBI(rename, nativeFlags, nil, dcmp)
} else {
dbi, err = tx.tx.OpenDBI(name, nativeFlags, nil, dcmp)
}
if err != nil { if err != nil {
return fmt.Errorf("create bucket: %s, %w", name, err) return fmt.Errorf("create bucket: %s, %w", name, err)
} }