From 243a89140dd113cb40a592f2bf055d0df5f9a34e Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 22 Dec 2023 14:50:41 +0800 Subject: [PATCH 1/3] core/rawdb: improve state scheme checking (#28724) This pull request improves the condition to check if path state scheme is in use. Originally, root node presence was used as the indicator if path scheme is used or not. However due to fact that root node will be deleted during the initial snap sync, this condition is no longer useful. If PersistentStateID is present, it shows that we've already configured for path scheme. --- core/rawdb/accessors_trie.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go index 14b3a96ba..869991a38 100644 --- a/core/rawdb/accessors_trie.go +++ b/core/rawdb/accessors_trie.go @@ -294,6 +294,11 @@ func ReadStateScheme(db ethdb.Reader) string { if len(blob) != 0 { return PathScheme } + // The root node might be deleted during the initial snap sync, check + // the persistent state id then. + if id := ReadPersistentStateID(db); id != 0 { + return PathScheme + } // In a hash-based scheme, the genesis state is consistently stored // on the disk. To assess the scheme of the persistent state, it // suffices to inspect the scheme of the genesis state. From 27672cbdb252348e0742e0ca94b7762246bcbbea Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 28 Feb 2024 20:23:52 +0800 Subject: [PATCH 2/3] core, triedb/pathdb: calculate the size for batch pre-allocation (#29106) * core, triedb/pathdb: calculate the size for batch pre-allocation * triedb/pathdb: address comment --- core/rawdb/schema.go | 26 +++++++++++++------------- trie/triedb/pathdb/nodebuffer.go | 15 ++++++++++++++- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 3c99d1db6..c1ea97e16 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -114,8 +114,8 @@ var ( dirtyAccountsKey = []byte("dacc") // dirtyAccountsPrefix + block hash -> dirty accounts // Path-based storage scheme of merkle patricia trie. - trieNodeAccountPrefix = []byte("A") // trieNodeAccountPrefix + hexPath -> trie node - trieNodeStoragePrefix = []byte("O") // trieNodeStoragePrefix + accountHash + hexPath -> trie node + TrieNodeAccountPrefix = []byte("A") // TrieNodeAccountPrefix + hexPath -> trie node + TrieNodeStoragePrefix = []byte("O") // TrieNodeStoragePrefix + accountHash + hexPath -> trie node stateIDPrefix = []byte("L") // stateIDPrefix + state root -> state id PreimagePrefix = []byte("secure-key-") // PreimagePrefix + hash -> preimage @@ -250,12 +250,12 @@ func genesisStateSpecKey(hash common.Hash) []byte { // accountTrieNodeKey = trieNodeAccountPrefix + nodePath. func accountTrieNodeKey(path []byte) []byte { - return append(trieNodeAccountPrefix, path...) + return append(TrieNodeAccountPrefix, path...) } -// storageTrieNodeKey = trieNodeStoragePrefix + accountHash + nodePath. +// storageTrieNodeKey = TrieNodeStoragePrefix + accountHash + nodePath. func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte { - return append(append(trieNodeStoragePrefix, accountHash.Bytes()...), path...) + return append(append(TrieNodeStoragePrefix, accountHash.Bytes()...), path...) } func snapshotConsortiumKey(hash common.Hash) []byte { @@ -277,16 +277,16 @@ func IsLegacyTrieNode(key []byte, val []byte) bool { // account trie node in path-based state scheme, and returns the resolved // node path if so. func ResolveAccountTrieNodeKey(key []byte) (bool, []byte) { - if !bytes.HasPrefix(key, trieNodeAccountPrefix) { + if !bytes.HasPrefix(key, TrieNodeAccountPrefix) { return false, nil } // The remaining key should only consist a hex node path // whose length is in the range 0 to 64 (64 is excluded // since leaves are always wrapped with shortNode). - if len(key) >= len(trieNodeAccountPrefix)+common.HashLength*2 { + if len(key) >= len(TrieNodeAccountPrefix)+common.HashLength*2 { return false, nil } - return true, key[len(trieNodeAccountPrefix):] + return true, key[len(TrieNodeAccountPrefix):] } // IsAccountTrieNode reports whether a provided database entry is an account @@ -300,20 +300,20 @@ func IsAccountTrieNode(key []byte) bool { // trie node in path-based state scheme, and returns the resolved account hash // and node path if so. func ResolveStorageTrieNode(key []byte) (bool, common.Hash, []byte) { - if !bytes.HasPrefix(key, trieNodeStoragePrefix) { + if !bytes.HasPrefix(key, TrieNodeStoragePrefix) { return false, common.Hash{}, nil } // The remaining key consists of 2 parts: // - 32 bytes account hash // - hex node path whose length is in the range 0 to 64 - if len(key) < len(trieNodeStoragePrefix)+common.HashLength { + if len(key) < len(TrieNodeStoragePrefix)+common.HashLength { return false, common.Hash{}, nil } - if len(key) >= len(trieNodeStoragePrefix)+common.HashLength+common.HashLength*2 { + if len(key) >= len(TrieNodeStoragePrefix)+common.HashLength+common.HashLength*2 { return false, common.Hash{}, nil } - accountHash := common.BytesToHash(key[len(trieNodeStoragePrefix) : len(trieNodeStoragePrefix)+common.HashLength]) - return true, accountHash, key[len(trieNodeStoragePrefix)+common.HashLength:] + accountHash := common.BytesToHash(key[len(TrieNodeStoragePrefix) : len(TrieNodeStoragePrefix)+common.HashLength]) + return true, accountHash, key[len(TrieNodeStoragePrefix)+common.HashLength:] } // IsStorageTrieNode reports whether a provided database entry is a storage diff --git a/trie/triedb/pathdb/nodebuffer.go b/trie/triedb/pathdb/nodebuffer.go index b024986d3..0a334baba 100644 --- a/trie/triedb/pathdb/nodebuffer.go +++ b/trie/triedb/pathdb/nodebuffer.go @@ -205,6 +205,19 @@ func (b *nodebuffer) setSize(size int, db ethdb.KeyValueStore, clean *fastcache. return b.flush(db, clean, id, false) } +// allocBatch returns a database batch with pre-allocated buffer. +func (b *nodebuffer) allocBatch(db ethdb.KeyValueStore) ethdb.Batch { + var metasize int + for owner, nodes := range b.nodes { + if owner == (common.Hash{}) { + metasize += len(nodes) * len(rawdb.TrieNodeAccountPrefix) // database key prefix + } else { + metasize += len(nodes) * (len(rawdb.TrieNodeStoragePrefix) + common.HashLength) // database key prefix + owner + } + } + return db.NewBatchWithSize((metasize + int(b.size)) * 11 / 10) // extra 10% for potential pebble internal stuff +} + // flush persists the in-memory dirty trie node into the disk if the configured // memory threshold is reached. Note, all data must be written atomically. func (b *nodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64, force bool) error { @@ -218,7 +231,7 @@ func (b *nodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id ui } var ( start = time.Now() - batch = db.NewBatchWithSize(int(b.size)) + batch = b.allocBatch(db) ) nodes := writeNodes(batch, b.nodes, clean) rawdb.WritePersistentStateID(batch, id) From e714852a462dbbd598f7cdfd903b9569cb7ee3ff Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 28 Feb 2024 20:40:28 +0800 Subject: [PATCH 3/3] triedb/pathdb: fix panic in recoverable (#29107) * triedb/pathdb: fix panic in recoverable * triedb/pathdb: add todo * triedb/pathdb: rename * triedb/pathdb: rename --- core/rawdb/schema_test.go | 4 ++-- trie/triedb/pathdb/database.go | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/core/rawdb/schema_test.go b/core/rawdb/schema_test.go index 06b017d41..11d036756 100644 --- a/core/rawdb/schema_test.go +++ b/core/rawdb/schema_test.go @@ -99,7 +99,7 @@ func TestResolveAccountTrieNodeKey(t *testing.T) { }, { name: "storage prefixed", - inputKey: append(trieNodeStoragePrefix, bytes4...), + inputKey: append(TrieNodeStoragePrefix, bytes4...), expectedCheck: false, expectedKey: nil, }, @@ -175,7 +175,7 @@ func TestResolveStorageTrieNode(t *testing.T) { }, { name: "storage prefixed hash 20 length 4", - inputKey: append(append(trieNodeStoragePrefix, bytes20...), bytes4...), + inputKey: append(append(TrieNodeStoragePrefix, bytes20...), bytes4...), expectedCheck: false, expectedHash: common.Hash{}, expectedKey: nil, diff --git a/trie/triedb/pathdb/database.go b/trie/triedb/pathdb/database.go index 873418c9f..ea5c50ab6 100644 --- a/trie/triedb/pathdb/database.go +++ b/trie/triedb/pathdb/database.go @@ -380,18 +380,23 @@ func (db *Database) Recoverable(root common.Hash) bool { if *id >= dl.stateID() { return false } - + // This is a temporary workaround for the unavailability of the freezer in + // dev mode. As a consequence, the Pathdb loses the ability for deep reorg + // in certain cases. + // TODO(rjl493456442): Implement the in-memory ancient store. + if db.freezer == nil { + return false + } // Ensure the requested state is a canonical state and all state // histories in range [id+1, disklayer.ID] are present and complete. - parent := root return checkHistories(db.freezer, *id+1, dl.stateID()-*id, func(m *meta) error { - if m.parent != parent { + if m.parent != root { return errors.New("unexpected state history") } if len(m.incomplete) > 0 { return errors.New("incomplete state history") } - parent = m.root + root = m.root return nil }) == nil }