Skip to content

Commit 3f266bf

Browse files
RealUnclekeefel
RealUncle
authored and
keefel
committed
implement the framework of fast node
1 parent 476d520 commit 3f266bf

18 files changed

+458
-66
lines changed

‎cmd/geth/main.go

+1
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ var (
114114
utils.WhitelistFlag,
115115
utils.BloomFilterSizeFlag,
116116
utils.TriesInMemoryFlag,
117+
utils.AllowInsecureNoTriesFlag,
117118
utils.CacheFlag,
118119
utils.CacheDatabaseFlag,
119120
utils.CacheTrieFlag,

‎cmd/geth/snapshot.go

+74-2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package main
1818

1919
import (
2020
"bytes"
21+
"encoding/json"
2122
"errors"
2223
"fmt"
2324
"os"
@@ -29,13 +30,16 @@ import (
2930

3031
"github.com/ethereum/go-ethereum/cmd/utils"
3132
"github.com/ethereum/go-ethereum/common"
33+
"github.com/ethereum/go-ethereum/core"
3234
"github.com/ethereum/go-ethereum/core/rawdb"
3335
"github.com/ethereum/go-ethereum/core/state"
3436
"github.com/ethereum/go-ethereum/core/state/pruner"
3537
"github.com/ethereum/go-ethereum/core/state/snapshot"
3638
"github.com/ethereum/go-ethereum/crypto"
39+
"github.com/ethereum/go-ethereum/eth/ethconfig"
3740
"github.com/ethereum/go-ethereum/ethdb"
3841
"github.com/ethereum/go-ethereum/log"
42+
"github.com/ethereum/go-ethereum/metrics"
3943
"github.com/ethereum/go-ethereum/node"
4044
"github.com/ethereum/go-ethereum/rlp"
4145
"github.com/ethereum/go-ethereum/trie"
@@ -129,6 +133,32 @@ geth snapshot verify-state <state-root>
129133
will traverse the whole accounts and storages set based on the specified
130134
snapshot and recalculate the root hash of state for verification.
131135
In other words, this command does the snapshot to trie conversion.
136+
`,
137+
},
138+
{
139+
Name: "insecure-prune-all",
140+
Usage: "Prune all trie state data except genesis block, it will break storage for fullnode, only suitable for fast node " +
141+
"who do not need trie storage at all",
142+
ArgsUsage: "<genesisPath>",
143+
Action: utils.MigrateFlags(pruneAllState),
144+
Category: "MISCELLANEOUS COMMANDS",
145+
Flags: []cli.Flag{
146+
utils.DataDirFlag,
147+
utils.AncientFlag,
148+
utils.RopstenFlag,
149+
utils.RinkebyFlag,
150+
utils.GoerliFlag,
151+
},
152+
Description: `
153+
will prune all historical trie state data except genesis block.
154+
All trie nodes will be deleted from the database.
155+
156+
It expects the genesis file as argument.
157+
158+
WARNING: It's necessary to delete the trie clean cache after the pruning.
159+
If you specify another directory for the trie clean cache via "--cache.trie.journal"
160+
during the use of Geth, please also specify it here for correct deletion. Otherwise
161+
the trie clean cache with default directory will be deleted.
132162
`,
133163
},
134164
{
@@ -195,7 +225,7 @@ func accessDb(ctx *cli.Context, stack *node.Node) (ethdb.Database, error) {
195225
}
196226
headHeader := headBlock.Header()
197227
//Make sure the MPT and snapshot matches before pruning, otherwise the node can not start.
198-
snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, TriesInMemory, headBlock.Root(), false, false, false)
228+
snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, TriesInMemory, headBlock.Root(), false, false, false, false)
199229
if err != nil {
200230
log.Error("snaptree error", "err", err)
201231
return nil, err // The relevant snapshot(s) might not exist
@@ -362,6 +392,48 @@ func pruneState(ctx *cli.Context) error {
362392
return nil
363393
}
364394

395+
func pruneAllState(ctx *cli.Context) error {
396+
stack, _ := makeConfigNode(ctx)
397+
defer stack.Close()
398+
399+
genesisPath := ctx.Args().First()
400+
if len(genesisPath) == 0 {
401+
utils.Fatalf("Must supply path to genesis JSON file")
402+
}
403+
file, err := os.Open(genesisPath)
404+
if err != nil {
405+
utils.Fatalf("Failed to read genesis file: %v", err)
406+
}
407+
defer file.Close()
408+
409+
g := new(core.Genesis)
410+
if err := json.NewDecoder(file).Decode(g); err != nil {
411+
cfg := gethConfig{
412+
Eth: ethconfig.Defaults,
413+
Node: defaultNodeConfig(),
414+
Metrics: metrics.DefaultConfig,
415+
}
416+
417+
// Load config file.
418+
if err := loadConfig(genesisPath, &cfg); err != nil {
419+
utils.Fatalf("%v", err)
420+
}
421+
g = cfg.Eth.Genesis
422+
}
423+
424+
chaindb := utils.MakeChainDatabase(ctx, stack, false, false)
425+
pruner, err := pruner.NewAllPruner(chaindb)
426+
if err != nil {
427+
log.Error("Failed to open snapshot tree", "err", err)
428+
return err
429+
}
430+
if err = pruner.PruneAll(g); err != nil {
431+
log.Error("Failed to prune state", "err", err)
432+
return err
433+
}
434+
return nil
435+
}
436+
365437
func verifyState(ctx *cli.Context) error {
366438
stack, _ := makeConfigNode(ctx)
367439
defer stack.Close()
@@ -372,7 +444,7 @@ func verifyState(ctx *cli.Context) error {
372444
log.Error("Failed to load head block")
373445
return errors.New("no head block")
374446
}
375-
snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, 128, headBlock.Root(), false, false, false)
447+
snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, 128, headBlock.Root(), false, false, false, false)
376448
if err != nil {
377449
log.Error("Failed to open snapshot tree", "err", err)
378450
return err

‎cmd/utils/flags.go

+7
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,10 @@ var (
260260
Usage: "The layer of tries trees that keep in memory",
261261
Value: 128,
262262
}
263+
AllowInsecureNoTriesFlag = cli.BoolTFlag{
264+
Name: "allow-insecure-no-tries",
265+
Usage: `Disable the tries state root verification, the state consistency is no longer 100% guaranteed, diffsync is not allowed if enabled. Do not enable it unless you know exactly what the consequence it will cause.`,
266+
}
263267
OverrideBerlinFlag = cli.Uint64Flag{
264268
Name: "override.berlin",
265269
Usage: "Manually specify Berlin fork-block, overriding the bundled setting",
@@ -1659,6 +1663,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
16591663
if ctx.GlobalIsSet(TriesInMemoryFlag.Name) {
16601664
cfg.TriesInMemory = ctx.GlobalUint64(TriesInMemoryFlag.Name)
16611665
}
1666+
if ctx.GlobalIsSet(AllowInsecureNoTriesFlag.Name) {
1667+
cfg.NoTries = ctx.GlobalBool(AllowInsecureNoTriesFlag.Name)
1668+
}
16621669
if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheSnapshotFlag.Name) {
16631670
cfg.SnapshotCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheSnapshotFlag.Name) / 100
16641671
}

‎core/block_validator.go

-6
Original file line numberDiff line numberDiff line change
@@ -64,12 +64,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
6464
}
6565

6666
validateFuns := []func() error{
67-
func() error {
68-
if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) {
69-
return ErrKnownBlock
70-
}
71-
return nil
72-
},
7367
func() error {
7468
if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash {
7569
return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash)

‎core/blockchain.go

+12-1
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ type CacheConfig struct {
137137
SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory
138138
Preimages bool // Whether to store preimage of trie key to the disk
139139
TriesInMemory uint64 // How many tries keeps in memory
140+
NoTries bool // Insecure settings. Do not have any tries in databases if enabled.
140141

141142
SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it
142143
}
@@ -275,6 +276,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
275276
Cache: cacheConfig.TrieCleanLimit,
276277
Journal: cacheConfig.TrieCleanJournal,
277278
Preimages: cacheConfig.Preimages,
279+
NoTries: cacheConfig.NoTries,
278280
}),
279281
triesInMemory: cacheConfig.TriesInMemory,
280282
quit: make(chan struct{}),
@@ -429,7 +431,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
429431
log.Warn("Enabling snapshot recovery", "chainhead", head.NumberU64(), "diskbase", *layer)
430432
recover = true
431433
}
432-
bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, int(bc.cacheConfig.TriesInMemory), head.Root(), !bc.cacheConfig.SnapshotWait, true, recover)
434+
bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, int(bc.cacheConfig.TriesInMemory), head.Root(), !bc.cacheConfig.SnapshotWait, true, recover, bc.stateCache.NoTries())
433435
}
434436
// do options before start any routine
435437
for _, option := range options {
@@ -1056,6 +1058,9 @@ func (bc *BlockChain) HasFastBlock(hash common.Hash, number uint64) bool {
10561058

10571059
// HasState checks if state trie is fully present in the database or not.
10581060
func (bc *BlockChain) HasState(hash common.Hash) bool {
1061+
if bc.stateCache.NoTries() {
1062+
return bc.snaps != nil && bc.snaps.Snapshot(hash) != nil
1063+
}
10591064
_, err := bc.stateCache.OpenTrie(hash)
10601065
return err == nil
10611066
}
@@ -1068,6 +1073,9 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool {
10681073
if block == nil {
10691074
return false
10701075
}
1076+
if bc.stateCache.NoTries() {
1077+
return bc.snaps != nil && bc.snaps.Snapshot(block.Root()) != nil
1078+
}
10711079
return bc.HasState(block.Root())
10721080
}
10731081

@@ -2054,6 +2062,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er
20542062
if err != nil {
20552063
return it.index, err
20562064
}
2065+
if statedb.NoTrie() {
2066+
statedb.SetCurrentRoot(block.Root())
2067+
}
20572068
bc.updateHighestVerifiedHeader(block.Header())
20582069

20592070
// Enable prefetching to pull in trie node paths while processing transactions

‎core/state/database.go

+23-1
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ type Database interface {
7474

7575
// Purge cache
7676
Purge()
77+
78+
// NoTries returns whether the database has tries storage.
79+
NoTries() bool
7780
}
7881

7982
// Trie is a Ethereum Merkle Patricia trie.
@@ -134,10 +137,12 @@ func NewDatabase(db ethdb.Database) Database {
134137
func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database {
135138
csc, _ := lru.New(codeSizeCacheSize)
136139
cc, _ := lru.New(codeCacheSize)
140+
noTries := config != nil && config.NoTries
137141
return &cachingDB{
138142
db: trie.NewDatabaseWithConfig(db, config),
139143
codeSizeCache: csc,
140144
codeCache: cc,
145+
noTries: noTries,
141146
}
142147
}
143148

@@ -146,15 +151,19 @@ func NewDatabaseWithConfigAndCache(db ethdb.Database, config *trie.Config) Datab
146151
cc, _ := lru.New(codeCacheSize)
147152
atc, _ := lru.New(accountTrieCacheSize)
148153
stc, _ := lru.New(storageTrieCacheSize)
154+
noTries := config != nil && config.NoTries
149155

150156
database := &cachingDB{
151157
db: trie.NewDatabaseWithConfig(db, config),
152158
codeSizeCache: csc,
153159
codeCache: cc,
154160
accountTrieCache: atc,
155161
storageTrieCache: stc,
162+
noTries: noTries,
163+
}
164+
if !noTries {
165+
go database.purgeLoop()
156166
}
157-
go database.purgeLoop()
158167
return database
159168
}
160169

@@ -164,6 +173,7 @@ type cachingDB struct {
164173
codeCache *lru.Cache
165174
accountTrieCache *lru.Cache
166175
storageTrieCache *lru.Cache
176+
noTries bool
167177
}
168178

169179
type triePair struct {
@@ -187,6 +197,9 @@ func (db *cachingDB) purgeLoop() {
187197

188198
// OpenTrie opens the main account trie at a specific root hash.
189199
func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
200+
if db.noTries {
201+
return trie.NewEmptyTrie(), nil
202+
}
190203
if db.accountTrieCache != nil {
191204
if tr, exist := db.accountTrieCache.Get(root); exist {
192205
return tr.(Trie).(*trie.SecureTrie).Copy(), nil
@@ -201,6 +214,9 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
201214

202215
// OpenStorageTrie opens the storage trie of an account.
203216
func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
217+
if db.noTries {
218+
return trie.NewEmptyTrie(), nil
219+
}
204220
if db.storageTrieCache != nil {
205221
if tries, exist := db.storageTrieCache.Get(addrHash); exist {
206222
triesPairs := tries.([3]*triePair)
@@ -246,6 +262,10 @@ func (db *cachingDB) CacheStorage(addrHash common.Hash, root common.Hash, t Trie
246262
}
247263
}
248264

265+
func (db *cachingDB) NoTries() bool {
266+
return db.noTries
267+
}
268+
249269
func (db *cachingDB) Purge() {
250270
if db.storageTrieCache != nil {
251271
db.storageTrieCache.Purge()
@@ -260,6 +280,8 @@ func (db *cachingDB) CopyTrie(t Trie) Trie {
260280
switch t := t.(type) {
261281
case *trie.SecureTrie:
262282
return t.Copy()
283+
case *trie.EmptyTrie:
284+
return t.Copy()
263285
default:
264286
panic(fmt.Errorf("unknown trie type %T", t))
265287
}

0 commit comments

Comments
 (0)