From 504c31e97a094d2887bdc7d5a55f271d95f27214 Mon Sep 17 00:00:00 2001 From: Artem Vorotnikov Date: Sun, 20 Jun 2021 17:08:48 +0300 Subject: [PATCH] Revert "Remove unused tables, soft rename" (#2205) --- common/dbutils/bucket.go | 159 +++++++++++++++++++++++++++------------ ethdb/kv/kv_mdbx.go | 27 ++++++- 2 files changed, 137 insertions(+), 49 deletions(-) diff --git a/common/dbutils/bucket.go b/common/dbutils/bucket.go index 8af164fdc..5f5a2517e 100644 --- a/common/dbutils/bucket.go +++ b/common/dbutils/bucket.go @@ -41,12 +41,13 @@ Physical layout: [acc2_hash] | [acc2_value] ... */ -const PlainStateBucket = "PlainState" +const PlainStateBucket = "PLAIN-CST2" +const PlainStateBucketOld1 = "PLAIN-CST" //PlainContractCodeBucket - //key - address+incarnation //value - code hash -const PlainContractCodeBucket = "PlainCodeHash" +const PlainContractCodeBucket = "PLAIN-contractCode" /* AccountChangeSetBucket and StorageChangeSetBucket - of block N store values of state before block N changed them. @@ -72,8 +73,8 @@ StorageChangeSetBucket: key - blockNum_u64 + address + incarnation_u64 value - plain_storage_key + value */ -const AccountChangeSetBucket = "AccountChangeSet" -const StorageChangeSetBucket = "StorageChangeSet" +const AccountChangeSetBucket = "PLAIN-ACS" +const StorageChangeSetBucket = "PLAIN-SCS" const ( @@ -83,8 +84,9 @@ const ( // Contains Storage: //key - address hash + incarnation + storage key hash //value - storage value(common.hash) - HashedAccountsBucket = "HashedAccount" - HashedStorageBucket = "HashedStorage" + HashedAccountsBucket = "hashed_accounts" + HashedStorageBucket = "hashed_storage" + CurrentStateBucketOld2 = "CST2" ) /* @@ -119,23 +121,23 @@ StorageHistoryBucket key - address + storage_key + shard_id_u64 value - roaring bitmap - list of block where it changed */ -var AccountsHistoryBucket = "AccountHistory" -var StorageHistoryBucket = "StorageHistory" +var AccountsHistoryBucket = "hAT" +var StorageHistoryBucket = "hST" var ( //key - contract code hash //value - contract code - CodeBucket = "Code" + CodeBucket = "CODE" //key - addressHash+incarnation //value - code hash - ContractCodeBucket = "HashedCodeHash" + ContractCodeBucket = "contractCode" // IncarnationMapBucket for deleted accounts //key - address //value - incarnation of account when it was last deleted - IncarnationMapBucket = "IncarnationMap" + IncarnationMapBucket = "incarnationMap" //TEVMCodeBucket - //key - contract code hash @@ -183,31 +185,31 @@ Invariants: - TrieAccount records with length=1 can satisfy (hasBranch==0&&hasHash==0) condition - Other records in TrieAccount and TrieStorage must (hasTree!=0 || hasHash!=0) */ -const TrieOfAccountsBucket = "TrieAccount" -const TrieOfStorageBucket = "TrieStorage" +const TrieOfAccountsBucket = "trie_account" +const TrieOfStorageBucket = "trie_storage" const IntermediateTrieHashBucketOld2 = "iTh2" const ( // DatabaseInfoBucket is used to store information about data layout. - DatabaseInfoBucket = "DbInfo" - SnapshotInfoBucket = "SnapshotInfo" - BittorrentInfoBucket = "BittorrentInfo" - HeadersSnapshotInfoBucket = "HeadersSnapshotInfo" - BodiesSnapshotInfoBucket = "BodiesSnapshotInfo" - StateSnapshotInfoBucket = "StateSnapshotInfo" + DatabaseInfoBucket = "DBINFO" + SnapshotInfoBucket = "SNINFO" + BittorrentInfoBucket = "BTINFO" + HeadersSnapshotInfoBucket = "hSNINFO" + BodiesSnapshotInfoBucket = "bSNINFO" + StateSnapshotInfoBucket = "sSNINFO" // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). - HeaderPrefixOld = "h" // block_num_u64 + hash -> header - HeaderNumberBucket = "HeaderNumber" // headerNumberPrefix + hash -> num (uint64 big endian) + HeaderPrefixOld = "h" // block_num_u64 + hash -> header + HeaderNumberBucket = "H" // headerNumberPrefix + hash -> num (uint64 big endian) - HeaderCanonicalBucket = "CanonicalHeader" // block_num_u64 -> header hash - HeadersBucket = "Header" // block_num_u64 + hash -> header (RLP) - HeaderTDBucket = "HeadersTotalDifficulty" // block_num_u64 + hash -> td (RLP) + HeaderCanonicalBucket = "canonical_headers" // block_num_u64 -> header hash + HeadersBucket = "headers" // block_num_u64 + hash -> header (RLP) + HeaderTDBucket = "header_to_td" // block_num_u64 + hash -> td (RLP) - BlockBodyPrefix = "BlockBody" // block_num_u64 + hash -> block body - EthTx = "BlockTransaction" // tbl_sequence_u64 -> rlp(tx) - BlockReceiptsPrefix = "Receipt" // block_num_u64 -> canonical block receipts (non-canonical are not stored) - Log = "TransactionLog" // block_num_u64 + txId -> logs of transaction + BlockBodyPrefix = "b" // block_num_u64 + hash -> block body + EthTx = "eth_tx" // tbl_sequence_u64 -> rlp(tx) + BlockReceiptsPrefix = "r" // block_num_u64 -> canonical block receipts (non-canonical are not stored) + Log = "log" // block_num_u64 + txId -> logs of transaction // Stores bitmap indices - in which block numbers saw logs of given 'address' or 'topic' // [addr or topic] + [2 bytes inverted shard number] -> bitmap(blockN) @@ -217,41 +219,43 @@ const ( // if last existing shard size merge it with delta // if serialized size of delta > ShardLimit - break down to multiple shards // shard number - it's biggest value in bitmap - LogTopicIndex = "LogTopicIndex" - LogAddressIndex = "LogAddressIndex" + LogTopicIndex = "log_topic_index" + LogAddressIndex = "log_address_index" // CallTraceSet is the name of the table that contain the mapping of block number to the set (sorted) of all accounts // touched by call traces. It is DupSort-ed table // 8-byte BE block number -> account address -> two bits (one for "from", another for "to") - CallTraceSet = "CallTraceSet" + CallTraceSet = "call_trace_set" // Indices for call traces - have the same format as LogTopicIndex and LogAddressIndex // Store bitmap indices - in which block number we saw calls from (CallFromIndex) or to (CallToIndex) some addresses - CallFromIndex = "CallFromIndex" - CallToIndex = "CallToIndex" + CallFromIndex = "call_from_index" + CallToIndex = "call_to_index" - TxLookupPrefix = "BlockTransactionLookup" // hash -> transaction/receipt lookup metadata - BloomBitsPrefix = "BloomBits" // bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits + TxLookupPrefix = "l" // hash -> transaction/receipt lookup metadata + BloomBitsPrefix = "B" // bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits - ConfigPrefix = "Config" // config prefix for the db + ConfigPrefix = "ethereum-config-" // config prefix for the db // Chain index prefixes (use `i` + single byte to avoid mixing data types). - BloomBitsIndexPrefix = "BloomBitsIndex" // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress + BloomBitsIndexPrefix = "iB" // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress // Progress of sync stages: stageName -> stageData - SyncStageProgress = "SyncStage" + SyncStageProgress = "SSP2" + SyncStageProgressOld1 = "SSP" // Position to where to unwind sync stages: stageName -> stageData - SyncStageUnwind = "SyncStageUnwind" + SyncStageUnwind = "SSU2" + SyncStageUnwindOld1 = "SSU" - CliqueBucket = "Clique" - CliqueSeparateBucket = "CliqueSeparate" - CliqueSnapshotBucket = "CliqueSnapshot" - CliqueLastSnapshotBucket = "CliqueLastSnapshot" + CliqueBucket = "clique-" + CliqueSeparateBucket = "clique-snapshots-" + CliqueSnapshotBucket = "snap" + CliqueLastSnapshotBucket = "lastSnap" // this bucket stored in separated database - InodesBucket = "Inode" + InodesBucket = "inodes" // Transaction senders - stored separately from the block bodies - Senders = "TxSender" // block_num_u64 + blockHash -> sendersList (no serialization format, every 20 bytes is new sender) + Senders = "txSenders" // block_num_u64 + blockHash -> sendersList (no serialization format, every 20 bytes is new sender) // headBlockKey tracks the latest know full block's hash. HeadBlockKey = "LastBlock" @@ -259,12 +263,63 @@ const ( // migrationName -> serialized SyncStageProgress and SyncStageUnwind buckets // it stores stages progress to understand in which context was executed migration // in case of bug-report developer can ask content of this bucket - Migrations = "Migration" + Migrations = "migrations" - Sequence = "Sequence" // tbl_name -> seq_u64 + Sequence = "sequence" // tbl_name -> seq_u64 HeadHeaderKey = "LastHeader" ) +var Rename = map[string]string{ + PlainStateBucket: "PlainState", + PlainContractCodeBucket: "PlainCodeHash", + AccountChangeSetBucket: "AccountChangeSet", + StorageChangeSetBucket: "StorageChangeSet", + HashedAccountsBucket: "HashedAccount", + HashedStorageBucket: "HashedStorage", + AccountsHistoryBucket: "AccountHistory", + StorageHistoryBucket: "StorageHistory", + CodeBucket: "Code", + ContractCodeBucket: "HashedCodeHash", + IncarnationMapBucket: "IncarnationMap", + TrieOfAccountsBucket: "TrieAccount", + TrieOfStorageBucket: "TrieStorage", + DatabaseInfoBucket: "DbInfo", + SnapshotInfoBucket: "SnapshotInfo", + BittorrentInfoBucket: "BittorrentInfo", + HeadersSnapshotInfoBucket: "HeadersSnapshotInfo", + BodiesSnapshotInfoBucket: "BodiesSnapshotInfo", + StateSnapshotInfoBucket: "StateSnapshotInfo", + HeaderNumberBucket: "HeaderNumber", + HeaderCanonicalBucket: "CanonicalHeader", + HeadersBucket: "Header", + HeaderTDBucket: "HeadersTotalDifficulty", + BlockBodyPrefix: "BlockBody", + EthTx: "BlockTransaction", + BlockReceiptsPrefix: "Receipt", + Log: "TransactionLog", + LogTopicIndex: "LogTopicIndex", + LogAddressIndex: "LogAddressIndex", + CallTraceSet: "CallTraceSet", + CallFromIndex: "CallFromIndex", + CallToIndex: "CallToIndex", + TxLookupPrefix: "BlockTransactionLookup", + BloomBitsPrefix: "BloomBits", + ConfigPrefix: "Config", + BloomBitsIndexPrefix: "BloomBitsIndex", + SyncStageProgress: "SyncStage", + SyncStageUnwind: "SyncStageUnwind", + CliqueBucket: "Clique", + CliqueSeparateBucket: "CliqueSeparate", + CliqueSnapshotBucket: "CliqueSnapshot", + CliqueLastSnapshotBucket: "CliqueLastSnapshot", + InodesBucket: "Inode", + Senders: "TxSender", + HeadBlockKey: "LastBlock", + Migrations: "Migration", + Sequence: "Sequence", + HeadHeaderKey: "LastHeader", +} + // Keys var ( //StorageModeHistory - does node save history. @@ -346,6 +401,10 @@ var Buckets = []string{ // DeprecatedBuckets - list of buckets which can be programmatically deleted - for example after migration var DeprecatedBuckets = []string{ IntermediateTrieHashBucketOld2, + CurrentStateBucketOld2, + SyncStageProgressOld1, + SyncStageUnwindOld1, + PlainStateBucketOld1, HeaderPrefixOld, CliqueBucket, } @@ -403,6 +462,12 @@ type BucketConfigItem struct { } var BucketsConfigs = BucketsCfg{ + CurrentStateBucketOld2: { + Flags: DupSort, + AutoDupSortKeysConversion: true, + DupFromLen: 72, + DupToLen: 40, + }, HashedStorageBucket: { Flags: DupSort, AutoDupSortKeysConversion: true, diff --git a/ethdb/kv/kv_mdbx.go b/ethdb/kv/kv_mdbx.go index 1503d64c3..2cfc5aa5f 100644 --- a/ethdb/kv/kv_mdbx.go +++ b/ethdb/kv/kv_mdbx.go @@ -621,6 +621,26 @@ func (tx *MdbxTx) CreateBucket(name string) error { return nil } + // if bucket with this name not found - check renamed one + rename := dbutils.Rename[name] + + dbi, err = tx.tx.OpenDBI(rename, mdbx.DBAccede, nil, dcmp) + if err != nil && !mdbx.IsNotFound(err) { + return fmt.Errorf("create bucket: %s, %w", name, err) + } + if err == nil { + cnfCopy.DBI = dbutils.DBI(dbi) + var flags uint + flags, err = tx.tx.Flags(dbi) + if err != nil { + return err + } + cnfCopy.Flags = dbutils.BucketFlags(flags) + + tx.db.buckets[name] = cnfCopy + return nil + } + // if bucket doesn't exists - create it var flags = tx.db.buckets[name].Flags @@ -637,8 +657,11 @@ func (tx *MdbxTx) CreateBucket(name string) error { return fmt.Errorf("some not supported flag provided for bucket") } - dbi, err = tx.tx.OpenDBI(name, nativeFlags, nil, dcmp) - + if rename != "" { + dbi, err = tx.tx.OpenDBI(rename, nativeFlags, nil, dcmp) + } else { + dbi, err = tx.tx.OpenDBI(name, nativeFlags, nil, dcmp) + } if err != nil { return fmt.Errorf("create bucket: %s, %w", name, err) }