implement the framework of fast node

Author: RealUncle

cmd/geth/main.go

@@ -115,6 +115,7 @@ var (
 		utils.WhitelistFlag,
 		utils.BloomFilterSizeFlag,
 		utils.TriesInMemoryFlag,
+		utils.AllowInsecureNoTriesFlag,
 		utils.CacheFlag,
 		utils.CacheDatabaseFlag,
 		utils.CacheTrieFlag,

cmd/geth/snapshot.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -29,13 +30,16 @@ import (
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/state/pruner"
 	"github.com/ethereum/go-ethereum/core/state/snapshot"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
@@ -129,6 +133,32 @@ geth snapshot verify-state <state-root>
 will traverse the whole accounts and storages set based on the specified
 snapshot and recalculate the root hash of state for verification.
 In other words, this command does the snapshot to trie conversion.
+`,
+			},
+			{
+				Name: "insecure-prune-all",
+				Usage: "Prune all trie state data except genesis block, it will break storage for fullnode, only suitable for fast node " +
+					"who do not need trie storage at all",
+				ArgsUsage: "<genesisPath>",
+				Action:    utils.MigrateFlags(pruneAllState),
+				Category:  "MISCELLANEOUS COMMANDS",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.AncientFlag,
+					utils.RopstenFlag,
+					utils.RinkebyFlag,
+					utils.GoerliFlag,
+				},
+				Description: `
+will prune all historical trie state data except genesis block.
+All trie nodes will be deleted from the database. 
+
+It expects the genesis file as argument.
+
+WARNING: It's necessary to delete the trie clean cache after the pruning.
+If you specify another directory for the trie clean cache via "--cache.trie.journal"
+during the use of Geth, please also specify it here for correct deletion. Otherwise
+the trie clean cache with default directory will be deleted.
 `,
 			},
 			{
@@ -195,7 +225,7 @@ func accessDb(ctx *cli.Context, stack *node.Node) (ethdb.Database, error) {
 	}
 	headHeader := headBlock.Header()
 	//Make sure the MPT and snapshot matches before pruning, otherwise the node can not start.
-	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, TriesInMemory, headBlock.Root(), false, false, false)
+	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, TriesInMemory, headBlock.Root(), false, false, false, false)
 	if err != nil {
 		log.Error("snaptree error", "err", err)
 		return nil, err // The relevant snapshot(s) might not exist
@@ -362,6 +392,48 @@ func pruneState(ctx *cli.Context) error {
 	return nil
 }
 
+func pruneAllState(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	genesisPath := ctx.Args().First()
+	if len(genesisPath) == 0 {
+		utils.Fatalf("Must supply path to genesis JSON file")
+	}
+	file, err := os.Open(genesisPath)
+	if err != nil {
+		utils.Fatalf("Failed to read genesis file: %v", err)
+	}
+	defer file.Close()
+
+	g := new(core.Genesis)
+	if err := json.NewDecoder(file).Decode(g); err != nil {
+		cfg := gethConfig{
+			Eth:     ethconfig.Defaults,
+			Node:    defaultNodeConfig(),
+			Metrics: metrics.DefaultConfig,
+		}
+
+		// Load config file.
+		if err := loadConfig(genesisPath, &cfg); err != nil {
+			utils.Fatalf("%v", err)
+		}
+		g = cfg.Eth.Genesis
+	}
+
+	chaindb := utils.MakeChainDatabase(ctx, stack, false, false)
+	pruner, err := pruner.NewAllPruner(chaindb)
+	if err != nil {
+		log.Error("Failed to open snapshot tree", "err", err)
+		return err
+	}
+	if err = pruner.PruneAll(g); err != nil {
+		log.Error("Failed to prune state", "err", err)
+		return err
+	}
+	return nil
+}
+
 func verifyState(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
@@ -372,7 +444,7 @@ func verifyState(ctx *cli.Context) error {
 		log.Error("Failed to load head block")
 		return errors.New("no head block")
 	}
-	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, 128, headBlock.Root(), false, false, false)
+	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, 128, headBlock.Root(), false, false, false, false)
 	if err != nil {
 		log.Error("Failed to open snapshot tree", "err", err)
 		return err

cmd/utils/flags.go

@@ -264,6 +264,10 @@ var (
 		Usage: "The layer of tries trees that keep in memory",
 		Value: 128,
 	}
+	AllowInsecureNoTriesFlag = cli.BoolTFlag{
+		Name:  "allow-insecure-no-tries",
+		Usage: `Disable the tries state root verification, the state consistency is no longer 100% guaranteed, diffsync is not allowed if enabled. Do not enable it unless you know exactly what the consequence it will cause.`,
+	}
 	OverrideBerlinFlag = cli.Uint64Flag{
 		Name:  "override.berlin",
 		Usage: "Manually specify Berlin fork-block, overriding the bundled setting",
@@ -1666,6 +1670,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.GlobalIsSet(TriesInMemoryFlag.Name) {
 		cfg.TriesInMemory = ctx.GlobalUint64(TriesInMemoryFlag.Name)
 	}
+	if ctx.GlobalIsSet(AllowInsecureNoTriesFlag.Name) {
+		cfg.NoTries = ctx.GlobalBool(AllowInsecureNoTriesFlag.Name)
+	}
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheSnapshotFlag.Name) {
 		cfg.SnapshotCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheSnapshotFlag.Name) / 100
 	}

core/block_validator.go

@@ -70,12 +70,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	}
 
 	validateFuns := []func() error{
-		func() error {
-			if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) {
-				return ErrKnownBlock
-			}
-			return nil
-		},
 		func() error {
 			if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash {
 				return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash)

core/blockchain.go

@@ -141,6 +141,7 @@ type CacheConfig struct {
 	SnapshotLimit      int           // Memory allowance (MB) to use for caching snapshot entries in memory
 	Preimages          bool          // Whether to store preimage of trie key to the disk
 	TriesInMemory      uint64        // How many tries keeps in memory
+	NoTries            bool          // Insecure settings. Do not have any tries in databases if enabled.
 
 	SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it
 }
@@ -284,6 +285,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 			Cache:     cacheConfig.TrieCleanLimit,
 			Journal:   cacheConfig.TrieCleanJournal,
 			Preimages: cacheConfig.Preimages,
+			NoTries:   cacheConfig.NoTries,
 		}),
 		triesInMemory:         cacheConfig.TriesInMemory,
 		quit:                  make(chan struct{}),
@@ -439,7 +441,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 			log.Warn("Enabling snapshot recovery", "chainhead", head.NumberU64(), "diskbase", *layer)
 			recover = true
 		}
-		bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, int(bc.cacheConfig.TriesInMemory), head.Root(), !bc.cacheConfig.SnapshotWait, true, recover)
+		bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, int(bc.cacheConfig.TriesInMemory), head.Root(), !bc.cacheConfig.SnapshotWait, true, recover, bc.stateCache.NoTries())
 	}
 	// do options before start any routine
 	for _, option := range options {
@@ -1097,6 +1099,9 @@ func (bc *BlockChain) HasState(hash common.Hash) bool {
 			return true
 		}
 	}
+	if bc.stateCache.NoTries() {
+		return bc.snaps != nil && bc.snaps.Snapshot(hash) != nil
+	}
 	_, err := bc.stateCache.OpenTrie(hash)
 	return err == nil
 }
@@ -1109,6 +1114,9 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool {
 	if block == nil {
 		return false
 	}
+	if bc.stateCache.NoTries() {
+		return bc.snaps != nil && bc.snaps.Snapshot(block.Root()) != nil
+	}
 	return bc.HasState(block.Root())
 }
 
@@ -2105,6 +2113,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er
 		if err != nil {
 			return it.index, err
 		}
+		if statedb.NoTrie() {
+			statedb.SetCurrentRoot(block.Root())
+		}
 		bc.updateHighestVerifiedHeader(block.Header())
 
 		// Enable prefetching to pull in trie node paths while processing transactions

core/state/database.go

@@ -74,6 +74,9 @@ type Database interface {
 
 	// Purge cache
 	Purge()
+
+	// NoTries returns whether the database has tries storage.
+	NoTries() bool
 }
 
 // Trie is a Ethereum Merkle Patricia trie.
@@ -134,10 +137,12 @@ func NewDatabase(db ethdb.Database) Database {
 func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database {
 	csc, _ := lru.New(codeSizeCacheSize)
 	cc, _ := lru.New(codeCacheSize)
+	noTries := config != nil && config.NoTries
 	return &cachingDB{
 		db:            trie.NewDatabaseWithConfig(db, config),
 		codeSizeCache: csc,
 		codeCache:     cc,
+		noTries:       noTries,
 	}
 }
 
@@ -146,15 +151,19 @@ func NewDatabaseWithConfigAndCache(db ethdb.Database, config *trie.Config) Datab
 	cc, _ := lru.New(codeCacheSize)
 	atc, _ := lru.New(accountTrieCacheSize)
 	stc, _ := lru.New(storageTrieCacheSize)
+	noTries := config != nil && config.NoTries
 
 	database := &cachingDB{
 		db:               trie.NewDatabaseWithConfig(db, config),
 		codeSizeCache:    csc,
 		codeCache:        cc,
 		accountTrieCache: atc,
 		storageTrieCache: stc,
+		noTries:          noTries,
+	}
+	if !noTries {
+		go database.purgeLoop()
 	}
-	go database.purgeLoop()
 	return database
 }
 
@@ -164,6 +173,7 @@ type cachingDB struct {
 	codeCache        *lru.Cache
 	accountTrieCache *lru.Cache
 	storageTrieCache *lru.Cache
+	noTries          bool
 }
 
 type triePair struct {
@@ -187,6 +197,9 @@ func (db *cachingDB) purgeLoop() {
 
 // OpenTrie opens the main account trie at a specific root hash.
 func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
+	if db.noTries {
+		return trie.NewEmptyTrie(), nil
+	}
 	if db.accountTrieCache != nil {
 		if tr, exist := db.accountTrieCache.Get(root); exist {
 			return tr.(Trie).(*trie.SecureTrie).Copy(), nil
@@ -201,6 +214,9 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
 
 // OpenStorageTrie opens the storage trie of an account.
 func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
+	if db.noTries {
+		return trie.NewEmptyTrie(), nil
+	}
 	if db.storageTrieCache != nil {
 		if tries, exist := db.storageTrieCache.Get(addrHash); exist {
 			triesPairs := tries.([3]*triePair)
@@ -246,6 +262,10 @@ func (db *cachingDB) CacheStorage(addrHash common.Hash, root common.Hash, t Trie
 	}
 }
 
+func (db *cachingDB) NoTries() bool {
+	return db.noTries
+}
+
 func (db *cachingDB) Purge() {
 	if db.storageTrieCache != nil {
 		db.storageTrieCache.Purge()
@@ -263,6 +283,8 @@ func (db *cachingDB) CopyTrie(t Trie) Trie {
 	switch t := t.(type) {
 	case *trie.SecureTrie:
 		return t.Copy()
+	case *trie.EmptyTrie:
+		return t.Copy()
 	default:
 		panic(fmt.Errorf("unknown trie type %T", t))
 	}