fix: change version db config

chore: update trivals

cmd/geth/main.go

@@ -361,6 +361,8 @@ func geth(ctx *cli.Context) error {
 	defer stack.Close()
 
 	startNode(ctx, stack, backend, false)
+	// TODO:: debug code , will be deleted in the future
+	debug.StartPProf("127.0.0.1:7060", !ctx.IsSet("metrics.addr"))
 	stack.Wait()
 	return nil
 }

cmd/utils/cmd.go

@@ -343,7 +343,7 @@ func missingBlocks(chain *core.BlockChain, blocks []*types.Block) []*types.Block
 			continue
 		}
 		// If we're above the chain head, state availability is a must
-		if !chain.HasBlockAndState(block.Hash(), block.NumberU64()) {
+		if !chain.HasBlockAndState(block.Hash(), block.Number().Int64()) {
 			return blocks[i:]
 		}
 	}

core/block_validator.go

@@ -25,6 +25,7 @@ import (
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/trie"
 )
@@ -40,6 +41,12 @@ func EnableRemoteVerifyManager(remoteValidator *remoteVerifyManager) BlockValida
 	}
 }
 
+var (
+	validateBloomTimer   = metrics.NewRegisteredTimer("validate/bloom/time", nil)
+	validateReceiptTimer = metrics.NewRegisteredTimer("validate/receipt/time", nil)
+	validateRootTimer    = metrics.NewRegisteredTimer("validate/root/time", nil)
+)
+
 // BlockValidator is responsible for validating block headers, uncles and
 // processed state.
 //
@@ -96,7 +103,7 @@ func ValidateListsInBody(block *types.Block) error {
 // validated at this point.
 func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	// Check whether the block is already imported.
-	if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) {
+	if v.bc.HasBlockAndState(block.Hash(), block.Number().Int64()) {
 		return ErrKnownBlock
 	}
 	if v.bc.isCachedBadBlock(block) {
@@ -142,7 +149,7 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 			return nil
 		},
 		func() error {
-			if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
+			if !v.bc.HasBlockAndState(block.ParentHash(), block.Number().Int64()-1) {
 				if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
 					return consensus.ErrUnknownAncestor
 				}
@@ -184,6 +191,10 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD
 	// For valid blocks this should always validate to true.
 	validateFuns := []func() error{
 		func() error {
+			defer func(start time.Time) {
+				validateBloomTimer.UpdateSince(start)
+			}(time.Now())
+
 			rbloom := types.CreateBloom(receipts)
 			if rbloom != header.Bloom {
 				return fmt.Errorf("invalid bloom (remote: %x  local: %x)", header.Bloom, rbloom)
@@ -191,6 +202,9 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD
 			return nil
 		},
 		func() error {
+			defer func(start time.Time) {
+				validateReceiptTimer.UpdateSince(start)
+			}(time.Now())
 			receiptSha := types.DeriveSha(receipts, trie.NewStackTrie(nil))
 			if receiptSha != header.ReceiptHash {
 				return fmt.Errorf("invalid receipt root hash (remote: %x local: %x)", header.ReceiptHash, receiptSha)
@@ -209,6 +223,9 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD
 		})
 	} else {
 		validateFuns = append(validateFuns, func() error {
+			defer func(start time.Time) {
+				validateRootTimer.UpdateSince(start)
+			}(time.Now())
 			if root := statedb.IntermediateRoot(v.config.IsEIP158(header.Number)); header.Root != root {
 				return fmt.Errorf("invalid merkle root (remote: %x local: %x) dberr: %w", header.Root, root, statedb.Error())
 			}

core/blockchain.go

@@ -74,6 +74,7 @@ var (
 	blockInsertMgaspsGauge = metrics.NewRegisteredGauge("chain/insert/mgasps", nil)
 
 	chainInfoGauge = metrics.NewRegisteredGaugeInfo("chain/info", nil)
+	mGasPsGauge    = metrics.NewRegisteredGauge("chain/process/gas", nil)
 
 	accountReadTimer   = metrics.NewRegisteredTimer("chain/account/reads", nil)
 	accountHashTimer   = metrics.NewRegisteredTimer("chain/account/hashes", nil)
@@ -91,10 +92,13 @@ var (
 
 	triedbCommitTimer = metrics.NewRegisteredTimer("chain/triedb/commits", nil)
 
-	blockInsertTimer     = metrics.NewRegisteredTimer("chain/inserts", nil)
+	blockInsertTimer          = metrics.NewRegisteredTimer("chain/inserts", nil)
-	blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil)
+	blockValidationTimer      = metrics.NewRegisteredTimer("chain/validation", nil)
-	blockExecutionTimer  = metrics.NewRegisteredTimer("chain/execution", nil)
+	blockExecutionTimer       = metrics.NewRegisteredTimer("chain/execution", nil)
-	blockWriteTimer      = metrics.NewRegisteredTimer("chain/write", nil)
+	blockWriteTimer           = metrics.NewRegisteredTimer("chain/write", nil)
+	blockValidationTotalTimer = metrics.NewRegisteredTimer("chain/total/validation", nil)
+	blockExecutionTotalTimer  = metrics.NewRegisteredTimer("chain/total/execution", nil)
+	blockWriteTotalTimer      = metrics.NewRegisteredTimer("chain/total/write", nil)
 
 	blockReorgMeter     = metrics.NewRegisteredMeter("chain/reorg/executes", nil)
 	blockReorgAddMeter  = metrics.NewRegisteredMeter("chain/reorg/add", nil)
@@ -342,6 +346,15 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 	// Open trie database with provided config
 	triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig())
 
+	if triedb.Scheme() == rawdb.VersionScheme {
+		vdb := triedb.VersaDB()
+		ver, root := vdb.LatestStoreDiskVersionInfo()
+		if ver == -1 {
+			rawdb.WriteCanonicalHash(db, common.Hash{}, 0)
+		}
+		log.Info("version db latest version info", "version", ver, "root", root.String())
+	}
+
 	// Setup the genesis block, commit the provided genesis specification
 	// to database if the genesis block is not present yet, or load the
 	// stored one from database.
@@ -424,78 +437,138 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 	// Make sure the state associated with the block is available, or log out
 	// if there is no available state, waiting for state sync.
 	head := bc.CurrentBlock()
-	if !bc.HasState(head.Root) {
+	if bc.triedb.Scheme() != rawdb.VersionScheme {
-		if head.Number.Uint64() == 0 {
+		if !bc.HasState(head.Number.Int64(), head.Root) {
-			// The genesis state is missing, which is only possible in the path-based
+			if head.Number.Uint64() == 0 {
-			// scheme. This situation occurs when the initial state sync is not finished
+				// The genesis state is missing, which is only possible in the path-based
-			// yet, or the chain head is rewound below the pivot point. In both scenarios,
+				// scheme. This situation occurs when the initial state sync is not finished
-			// there is no possible recovery approach except for rerunning a snap sync.
+				// yet, or the chain head is rewound below the pivot point. In both scenarios,
-			// Do nothing here until the state syncer picks it up.
+				// there is no possible recovery approach except for rerunning a snap sync.
-			log.Info("Genesis state is missing, wait state sync")
+				// Do nothing here until the state syncer picks it up.
-		} else {
+				log.Info("Genesis state is missing, wait state sync")
-			// Head state is missing, before the state recovery, find out the
-			// disk layer point of snapshot(if it's enabled). Make sure the
-			// rewound point is lower than disk layer.
-			var diskRoot common.Hash
-			if bc.cacheConfig.SnapshotLimit > 0 {
-				diskRoot = rawdb.ReadSnapshotRoot(bc.db)
-			}
-			if bc.triedb.Scheme() == rawdb.PathScheme && !bc.NoTries() {
-				recoverable, _ := bc.triedb.Recoverable(diskRoot)
-				if !bc.HasState(diskRoot) && !recoverable {
-					diskRoot = bc.triedb.Head()
-				}
-			}
-			if diskRoot != (common.Hash{}) {
-				log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash(), "diskRoot", diskRoot)
-
-				snapDisk, err := bc.setHeadBeyondRoot(head.Number.Uint64(), 0, diskRoot, true)
-				if err != nil {
-					return nil, err
-				}
-				// Chain rewound, persist old snapshot number to indicate recovery procedure
-				if snapDisk != 0 {
-					rawdb.WriteSnapshotRecoveryNumber(bc.db, snapDisk)
-				}
 			} else {
-				log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash())
+				// Head state is missing, before the state recovery, find out the
-				if _, err := bc.setHeadBeyondRoot(head.Number.Uint64(), 0, common.Hash{}, true); err != nil {
+				// disk layer point of snapshot(if it's enabled). Make sure the
+				// rewound point is lower than disk layer.
+				var diskRoot common.Hash
+				if bc.cacheConfig.SnapshotLimit > 0 {
+					diskRoot = rawdb.ReadSnapshotRoot(bc.db)
+				}
+				if bc.triedb.Scheme() == rawdb.PathScheme && !bc.NoTries() {
+					recoverable, _ := bc.triedb.Recoverable(diskRoot)
+					if !bc.HasState(0, diskRoot) && !recoverable {
+						diskRoot = bc.triedb.Head()
+					}
+				}
+				if diskRoot != (common.Hash{}) {
+					log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash(), "diskRoot", diskRoot)
+
+					snapDisk, err := bc.setHeadBeyondRoot(head.Number.Uint64(), 0, diskRoot, true)
+					if err != nil {
+						return nil, err
+					}
+					// Chain rewound, persist old snapshot number to indicate recovery procedure
+					if snapDisk != 0 {
+						rawdb.WriteSnapshotRecoveryNumber(bc.db, snapDisk)
+					}
+				} else {
+					log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash())
+					if _, err := bc.setHeadBeyondRoot(head.Number.Uint64(), 0, common.Hash{}, true); err != nil {
+						return nil, err
+					}
+				}
+			}
+		}
+	} else {
+		log.Warn("versa db no recovery, rewind in load state")
+	}
+	// Ensure that a previous crash in SetHead doesn't leave extra ancients
+	if bc.triedb.Scheme() != rawdb.VersionScheme {
+		if frozen, err := bc.db.BlockStore().ItemAmountInAncient(); err == nil && frozen > 0 {
+			frozen, err = bc.db.BlockStore().Ancients()
+			if err != nil {
+				return nil, err
+			}
+			var (
+				needRewind bool
+				low        uint64
+			)
+			// The head full block may be rolled back to a very low height due to
+			// blockchain repair. If the head full block is even lower than the ancient
+			// chain, truncate the ancient store.
+			fullBlock := bc.CurrentBlock()
+			if fullBlock != nil && fullBlock.Hash() != bc.genesisBlock.Hash() && fullBlock.Number.Uint64() < frozen-1 {
+				needRewind = true
+				low = fullBlock.Number.Uint64()
+			}
+			// In snap sync, it may happen that ancient data has been written to the
+			// ancient store, but the LastFastBlock has not been updated, truncate the
+			// extra data here.
+			snapBlock := bc.CurrentSnapBlock()
+			if snapBlock != nil && snapBlock.Number.Uint64() < frozen-1 {
+				needRewind = true
+				if snapBlock.Number.Uint64() < low || low == 0 {
+					low = snapBlock.Number.Uint64()
+				}
+			}
+			if needRewind {
+				log.Error("Truncating ancient chain", "from", bc.CurrentHeader().Number.Uint64(), "to", low)
+				if err := bc.SetHead(low); err != nil {
 					return nil, err
 				}
 			}
 		}
 	}
 	// Ensure that a previous crash in SetHead doesn't leave extra ancients
-	if frozen, err := bc.db.BlockStore().ItemAmountInAncient(); err == nil && frozen > 0 {
+	if bc.triedb.Scheme() != rawdb.VersionScheme {
-		frozen, err = bc.db.BlockStore().Ancients()
+		if frozen, err := bc.db.BlockStore().ItemAmountInAncient(); err == nil && frozen > 0 {
+			frozen, err = bc.db.BlockStore().Ancients()
+			if err != nil {
+				return nil, err
+			}
+			var (
+				needRewind bool
+				low        uint64
+			)
+			// The head full block may be rolled back to a very low height due to
+			// blockchain repair. If the head full block is even lower than the ancient
+			// chain, truncate the ancient store.
+			fullBlock := bc.CurrentBlock()
+			if fullBlock != nil && fullBlock.Hash() != bc.genesisBlock.Hash() && fullBlock.Number.Uint64() < frozen-1 {
+				needRewind = true
+				low = fullBlock.Number.Uint64()
+			}
+			// In snap sync, it may happen that ancient data has been written to the
+			// ancient store, but the LastFastBlock has not been updated, truncate the
+			// extra data here.
+			snapBlock := bc.CurrentSnapBlock()
+			if snapBlock != nil && snapBlock.Number.Uint64() < frozen-1 {
+				needRewind = true
+				if snapBlock.Number.Uint64() < low || low == 0 {
+					low = snapBlock.Number.Uint64()
+				}
+			}
+			if needRewind {
+				log.Error("Truncating ancient chain", "from", bc.CurrentHeader().Number.Uint64(), "to", low)
+				if err := bc.SetHead(low); err != nil {
+					return nil, err
+				}
+			}
+		}
+	} else {
+		//TODO:: need consider the offline and inline prune block
+		frozen, err := bc.db.BlockStore().Ancients()
+		if err != nil {
+			return nil, err
+		}
+		items, err := bc.db.BlockStore().ItemAmountInAncient()
 		if err != nil {
 			return nil, err
 		}
-		var (
-			needRewind bool
-			low        uint64
-		)
-		// The head full block may be rolled back to a very low height due to
-		// blockchain repair. If the head full block is even lower than the ancient
-		// chain, truncate the ancient store.
 		fullBlock := bc.CurrentBlock()
-		if fullBlock != nil && fullBlock.Hash() != bc.genesisBlock.Hash() && fullBlock.Number.Uint64() < frozen-1 {
+		log.Info("version mode rewind ancient store", "target", fullBlock.Number.Uint64(), "old head", frozen, "items", items, "offset", bc.db.BlockStore().AncientOffSet())
-			needRewind = true
+		if frozen >= fullBlock.Number.Uint64() {
-			low = fullBlock.Number.Uint64()
+			if _, err = bc.db.BlockStore().TruncateTail(fullBlock.Number.Uint64()); err != nil {
-		}
-		// In snap sync, it may happen that ancient data has been written to the
-		// ancient store, but the LastFastBlock has not been updated, truncate the
-		// extra data here.
-		snapBlock := bc.CurrentSnapBlock()
-		if snapBlock != nil && snapBlock.Number.Uint64() < frozen-1 {
-			needRewind = true
-			if snapBlock.Number.Uint64() < low || low == 0 {
-				low = snapBlock.Number.Uint64()
-			}
-		}
-		if needRewind {
-			log.Error("Truncating ancient chain", "from", bc.CurrentHeader().Number.Uint64(), "to", low)
-			if err := bc.SetHead(low); err != nil {
 				return nil, err
 			}
 		}
@@ -702,20 +775,61 @@ func (bc *BlockChain) getFinalizedNumber(header *types.Header) uint64 {
 // loadLastState loads the last known chain state from the database. This method
 // assumes that the chain manager mutex is held.
 func (bc *BlockChain) loadLastState() error {
-	// Restore the last known head block
+	// TODO:: before versa db support recovery, only rewind
-	head := rawdb.ReadHeadBlockHash(bc.db)
+	var headBlock *types.Block
-	if head == (common.Hash{}) {
+	if bc.triedb.Scheme() == rawdb.VersionScheme {
-		// Corrupt or empty database, init from scratch
+		head := rawdb.ReadHeadBlockHash(bc.db)
-		log.Warn("Empty database, resetting chain")
+		headBlock = bc.GetBlockByHash(head)
-		return bc.Reset()
+
-	}
+		versa := bc.triedb.VersaDB()
-	// Make sure the entire head block is available
+		archiveVersion, archiveRoot := versa.LatestStoreDiskVersionInfo()
-	headBlock := bc.GetBlockByHash(head)
+		// first start
-	if headBlock == nil {
+		if archiveVersion == -1 {
-		// Corrupt or empty database, init from scratch
+			archiveVersion = 0
-		log.Warn("Head block missing, resetting chain", "hash", head)
+			archiveRoot = bc.genesisBlock.Root()
-		return bc.Reset()
+		}
+
+		if int64(headBlock.NumberU64()) < archiveVersion {
+			log.Crit("versa db disk version large than header block", "head number", headBlock.NumberU64(), "versa archive number", archiveVersion)
+		}
+		log.Info("begin rewind versa db head", "target_number", archiveVersion, "target_root", archiveRoot.String(), "head_number", headBlock.NumberU64(), "head_root", headBlock.Root().String())
+		for {
+			if int64(headBlock.NumberU64()) == archiveVersion && archiveRoot.Cmp(headBlock.Root()) == 0 {
+				rawdb.WriteCanonicalHash(bc.db, headBlock.Hash(), headBlock.NumberU64())
+				rawdb.WriteHeadHeaderHash(bc.db, headBlock.Hash())
+				rawdb.WriteHeadBlockHash(bc.db, headBlock.Hash())
+				rawdb.WriteHeadFastBlockHash(bc.db, headBlock.Hash())
+				log.Info("reset versa db head block", "number", headBlock.NumberU64(), "hash", headBlock.Hash())
+				break
+			} else if int64(headBlock.NumberU64()) == archiveVersion {
+				log.Crit("rewinding meet same number", "target_number", archiveVersion, "target_root", archiveRoot.String(), "head_number", headBlock.NumberU64(), "head_root", headBlock.Root().String())
+			} else if archiveRoot.Cmp(headBlock.Root()) == 0 {
+				log.Info("rewinding meet same root", "target_number", archiveVersion, "target_root", archiveRoot.String(), "head_number", headBlock.NumberU64(), "head_root", headBlock.Root().String())
+			}
+
+			log.Info("rewinding", "target_number", archiveVersion, "target_root", archiveRoot.String(), "head_number", headBlock.NumberU64(), "head_root", headBlock.Root().String())
+			headBlock = rawdb.ReadBlock(bc.db, headBlock.ParentHash(), headBlock.NumberU64()-1)
+			if headBlock == nil {
+				panic("versa db rewind head is nil")
+			}
+		}
+	} else {
+		// Restore the last known head block
+		head := rawdb.ReadHeadBlockHash(bc.db)
+		if head == (common.Hash{}) {
+			// Corrupt or empty database, init from scratch
+			log.Warn("Empty database, resetting chain")
+			return bc.Reset()
+		}
+		// Make sure the entire head block is available
+		headBlock = bc.GetBlockByHash(head)
+		if headBlock == nil {
+			// Corrupt or empty database, init from scratch
+			log.Warn("Head block missing, resetting chain", "hash", head)
+			return bc.Reset()
+		}
 	}
+	log.Info("load state head block", "number", headBlock.NumberU64())
 
 	// Everything seems to be fine, set as the head block
 	bc.currentBlock.Store(headBlock.Header())
@@ -884,7 +998,7 @@ func (bc *BlockChain) rewindHashHead(head *types.Header, root common.Hash) (*typ
 		}
 		// If the associated state is not reachable, continue searching
 		// backwards until an available state is found.
-		if !bc.HasState(head.Root) {
+		if !bc.HasState(head.Number.Int64(), head.Root) {
 			// If the chain is gapped in the middle, return the genesis
 			// block as the new chain head.
 			parent := bc.GetHeader(head.ParentHash, head.Number.Uint64()-1)
@@ -924,7 +1038,7 @@ func (bc *BlockChain) rewindPathHead(head *types.Header, root common.Hash) (*typ
 
 		// noState represents if the target state requested for search
 		// is unavailable and impossible to be recovered.
-		noState = !bc.HasState(root) && !bc.stateRecoverable(root)
+		noState = !bc.HasState(head.Number.Int64(), root) && !bc.stateRecoverable(root)
 
 		start  = time.Now() // Timestamp the rewinding is restarted
 		logged = time.Now() // Timestamp last progress log was printed
@@ -945,13 +1059,13 @@ func (bc *BlockChain) rewindPathHead(head *types.Header, root common.Hash) (*typ
 		// If the root threshold hasn't been crossed but the available
 		// state is reached, quickly determine if the target state is
 		// possible to be reached or not.
-		if !beyondRoot && noState && bc.HasState(head.Root) {
+		if !beyondRoot && noState && bc.HasState(head.Number.Int64(), head.Root) {
 			beyondRoot = true
 			log.Info("Disable the search for unattainable state", "root", root)
 		}
 		// Check if the associated state is available or recoverable if
 		// the requested root has already been crossed.
-		if beyondRoot && (bc.HasState(head.Root) || bc.stateRecoverable(head.Root)) {
+		if beyondRoot && (bc.HasState(head.Number.Int64(), head.Root) || bc.stateRecoverable(head.Root)) {
 			break
 		}
 		// If pivot block is reached, return the genesis block as the
@@ -978,7 +1092,7 @@ func (bc *BlockChain) rewindPathHead(head *types.Header, root common.Hash) (*typ
 		}
 	}
 	// Recover if the target state if it's not available yet.
-	if !bc.HasState(head.Root) {
+	if !bc.HasState(head.Number.Int64(), head.Root) {
 		if err := bc.triedb.Recover(head.Root); err != nil {
 			log.Crit("Failed to rollback state", "err", err)
 		}
@@ -1073,7 +1187,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
 			// the pivot point. In this scenario, there is no possible recovery
 			// approach except for rerunning a snap sync. Do nothing here until the
 			// state syncer picks it up.
-			if !bc.HasState(newHeadBlock.Root) {
+			if !bc.HasState(newHeadBlock.Number.Int64(), newHeadBlock.Root) {
 				if newHeadBlock.Number.Uint64() != 0 {
 					log.Crit("Chain is stateless at a non-genesis block")
 				}
@@ -1185,7 +1299,7 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error {
 			return err
 		}
 	}
-	if !bc.NoTries() && !bc.HasState(root) {
+	if !bc.NoTries() && !bc.HasState(0, root) {
 		return fmt.Errorf("non existent state [%x..]", root[:4])
 	}
 	// If all checks out, manually set the head block.
@@ -2242,13 +2356,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 			parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1)
 		}
 
-		if bc.stateCache.Scheme() != rawdb.VersionScheme {
+		bc.stateCache.SetVersion(int64(block.NumberU64()) - 1)
-			if block.NumberU64() == 2000001 {
-				log.Crit("exit.... path mode, 200w blocks")
-			}
-		}
-
-		bc.stateCache.SetVersion(int64(block.NumberU64()))
 		statedb, err := state.NewWithSharedPool(parent.Root, bc.stateCache, bc.snaps)
 		if err != nil {
 			bc.stateCache.Release()
@@ -2260,18 +2368,18 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 		//statedb.StartPrefetcher("chain")
 		interruptCh := make(chan struct{})
 		// For diff sync, it may fallback to full sync, so we still do prefetch
-		//if len(block.Transactions()) >= prefetchTxNumber {
+		if len(block.Transactions()) >= prefetchTxNumber {
-		//	// do Prefetch in a separate goroutine to avoid blocking the critical path
+			// do Prefetch in a separate goroutine to avoid blocking the critical path
-		//
+
-		//	// 1.do state prefetch for snapshot cache
+			//1.do state prefetch for snapshot cache
-		//	throwaway := statedb.CopyDoPrefetch()
+			throwaway := statedb.CopyDoPrefetch()
-		//	go bc.prefetcher.Prefetch(block, throwaway, &bc.vmConfig, interruptCh)
+			go bc.prefetcher.Prefetch(block, throwaway, &bc.vmConfig, interruptCh)
-		//
+
-		//	// 2.do trie prefetch for MPT trie node cache
+			//	// 2.do trie prefetch for MPT trie node cache
-		//	// it is for the big state trie tree, prefetch based on transaction's From/To address.
+			//	// it is for the big state trie tree, prefetch based on transaction's From/To address.
-		//	// trie prefetcher is thread safe now, ok to prefetch in a separate routine
+			//	// trie prefetcher is thread safe now, ok to prefetch in a separate routine
-		//	go throwaway.TriePrefetchInAdvance(block, signer)
+			//	go throwaway.TriePrefetchInAdvance(block, signer)
-		//}
+		}
 
 		// Process block using the parent state as reference point
 		if bc.pipeCommit {
@@ -2287,6 +2395,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 			statedb.StopPrefetcher()
 			return it.index, err
 		}
+		blockExecutionTotalTimer.UpdateSince(pstart)
+
 		ptime := time.Since(pstart)
 
 		// Validate the state using the default validator
@@ -2297,6 +2407,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 			statedb.StopPrefetcher()
 			return it.index, err
 		}
+		blockValidationTotalTimer.UpdateSince(vstart)
+
 		vtime := time.Since(vstart)
 		proctime := time.Since(start) // processing + validation
 
@@ -2332,6 +2444,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 			return it.index, err
 		}
 		bc.stateCache.Release()
+		blockWriteTotalTimer.UpdateSince(wstart)
 
 		bc.cacheReceipts(block.Hash(), receipts, block)
 
@@ -2511,7 +2624,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i
 		numbers []uint64
 	)
 	parent := it.previous()
-	for parent != nil && !bc.HasState(parent.Root) {
+	for parent != nil && !bc.HasState(parent.Number.Int64(), parent.Root) {
 		if bc.stateRecoverable(parent.Root) {
 			if err := bc.triedb.Recover(parent.Root); err != nil {
 				return 0, err
@@ -2578,7 +2691,7 @@ func (bc *BlockChain) recoverAncestors(block *types.Block) (common.Hash, error)
 		numbers []uint64
 		parent  = block
 	)
-	for parent != nil && !bc.HasState(parent.Root()) {
+	for parent != nil && !bc.HasState(parent.Number().Int64(), parent.Root()) {
 		if bc.stateRecoverable(parent.Root()) {
 			if err := bc.triedb.Recover(parent.Root()); err != nil {
 				return common.Hash{}, err
@@ -2849,7 +2962,7 @@ func (bc *BlockChain) SetCanonical(head *types.Block) (common.Hash, error) {
 	defer bc.chainmu.Unlock()
 
 	// Re-execute the reorged chain in case the head state is missing.
-	if !bc.HasState(head.Root()) {
+	if !bc.HasState(head.Number().Int64(), head.Root()) {
 		if latestValidHash, err := bc.recoverAncestors(head); err != nil {
 			return latestValidHash, err
 		}

core/blockchain_insert.go

@@ -64,6 +64,7 @@ func (st *insertStats) report(chain []*types.Block, index int, snapDiffItems, sn
 			"blocks", st.processed, "txs", txs, "blobs", blobs, "mgas", float64(st.usedGas) / 1000000,
 			"elapsed", common.PrettyDuration(elapsed), "mgasps", mgasps,
 		}
+		mGasPsGauge.Update(int64(mgasps))
 		blockInsertMgaspsGauge.Update(int64(mgasps))
 		if timestamp := time.Unix(int64(end.Time()), 0); time.Since(timestamp) > time.Minute {
 			context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)

core/blockchain_reader.go

@@ -18,6 +18,7 @@ package core
 
 import (
 	"errors"
+	"fmt"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -338,7 +339,7 @@ func (bc *BlockChain) GetTd(hash common.Hash, number uint64) *big.Int {
 }
 
 // HasState checks if state trie is fully present in the database or not.
-func (bc *BlockChain) HasState(hash common.Hash) bool {
+func (bc *BlockChain) HasState(number int64, hash common.Hash) bool {
 	if bc.NoTries() {
 		return bc.snaps != nil && bc.snaps.Snapshot(hash) != nil
 	}
@@ -348,18 +349,24 @@ func (bc *BlockChain) HasState(hash common.Hash) bool {
 			return true
 		}
 	}
-	return bc.stateCache.HasState(hash)
+	return bc.stateCache.HasState(number, hash)
 }
 
 // HasBlockAndState checks if a block and associated state trie is fully present
 // in the database or not, caching it if present.
-func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool {
+func (bc *BlockChain) HasBlockAndState(hash common.Hash, number int64) bool {
 	// Check first that the block itself is known
-	block := bc.GetBlock(hash, number)
+	var root common.Hash
-	if block == nil {
+	if number < 0 {
-		return false
+		root = types.EmptyRootHash
+	} else {
+		block := bc.GetBlock(hash, uint64(number))
+		if block == nil {
+			return false
+		}
+		root = block.Root()
 	}
-	return bc.HasState(block.Root())
+	return bc.HasState(number, root)
 }
 
 // stateRecoverable checks if the specified state is recoverable.
@@ -390,13 +397,19 @@ func (bc *BlockChain) ContractCodeWithPrefix(hash common.Hash) ([]byte, error) {
 
 // State returns a new mutable state based on the current HEAD block.
 func (bc *BlockChain) State() (*state.StateDB, error) {
-	return bc.StateAt(bc.CurrentBlock().Root)
+	return bc.StateAt(bc.CurrentBlock().Number.Int64(), bc.CurrentBlock().Root)
 }
 
 // StateAt returns a new mutable state based on a particular point in time.
-func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) {
+func (bc *BlockChain) StateAt(number int64, root common.Hash) (*state.StateDB, error) {
 	// new state db with no need commit mode
-	stateDb, err := state.New(root, state.NewDatabaseWithNodeDB(bc.db, bc.triedb, false), bc.snaps)
+	has := bc.HasState(number, root)
+	if !has {
+		return nil, fmt.Errorf(fmt.Sprintf("do not has state, verison: %d, root: %s", number, root.String()))
+	}
+	sdb := state.NewDatabaseWithNodeDB(bc.db, bc.triedb, false)
+	sdb.SetVersion(number)
+	stateDb, err := state.New(root, sdb, bc.snaps)
 	if err != nil {
 		return nil, err
 	}

core/genesis.go

@@ -158,7 +158,7 @@ func flushAlloc(ga *types.GenesisAlloc, db ethdb.Database, triedb *triedb.Databa
 		triedbConfig.NoTries = false
 	}
 	cachingdb := state.NewDatabaseWithNodeDB(db, triedb, true)
-	cachingdb.SetVersion(0)
+	cachingdb.SetVersion(-1)
 	defer cachingdb.Release()
 	statedb, err := state.New(types.EmptyRootHash, cachingdb, nil)
 	if err != nil {
@@ -274,6 +274,8 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g
 		log.Info("genesis block hash", "hash", block.Hash())
 		return genesis.Config, block.Hash(), nil
 	}
+	log.Info("init genesis", "stored root", stored.String())
+	
 	// The genesis block is present(perhaps in ancient database) while the
 	// state database is not initialized yet. It can happen that the node
 	// is initialized with an external ancient store. Commit genesis state

core/state/caching_versa_db.go

@@ -60,6 +60,7 @@ func (cv *cachingVersaDB) Copy() Database {
 	cp.triedb = cv.triedb
 	cp.versionDB = cv.versionDB
 	cp.codeDB = cv.codeDB
+	cp.version = cv.version
 	cp.mode = versa.S_RW // it is important
 
 	// TODO:: maybe add lock for cv.root
@@ -109,8 +110,8 @@ func (cv *cachingVersaDB) CopyTrie(tr Trie) Trie {
 	return nil
 }
 
-func (cv *cachingVersaDB) HasState(root common.Hash) bool {
+func (cv *cachingVersaDB) HasState(version int64, root common.Hash) bool {
-	return cv.versionDB.HasState(root)
+	return cv.versionDB.HasState(version, root)
 }
 
 func (cv *cachingVersaDB) OpenTrie(root common.Hash) (Trie, error) {
@@ -214,7 +215,7 @@ func (cv *cachingVersaDB) Flush() error {
 }
 
 func (cv *cachingVersaDB) SetVersion(version int64) {
-	cv.version = version - 1
+	cv.version = version
 	//cv.debug = NewDebugVersionState(cv.codeDB, cv.versionDB)
 	//cv.debug.Version = version
 }

core/state/database.go

@@ -87,7 +87,7 @@ type Database interface {
 	Copy() Database
 
 	// HasState returns the state data whether in the triedb.
-	HasState(root common.Hash) bool
+	HasState(version int64, root common.Hash) bool
 
 	// HasTreeExpired used for caching versa db, whether the state where the opened tree resides has been closed
 	HasTreeExpired(tr Trie) bool
@@ -402,7 +402,7 @@ func (db *cachingDB) Copy() Database {
 	return db
 }
 
-func (db *cachingDB) HasState(root common.Hash) bool {
+func (db *cachingDB) HasState(_ int64, root common.Hash) bool {
 	_, err := db.OpenTrie(root)
 	return err == nil
 }

core/state/metrics.go

@@ -34,4 +34,7 @@ var (
 	slotDeletionCount    = metrics.NewRegisteredMeter("state/delete/storage/slot", nil)
 	slotDeletionSize     = metrics.NewRegisteredMeter("state/delete/storage/size", nil)
 	slotDeletionSkip     = metrics.NewRegisteredGauge("state/delete/storage/skip", nil)
+
+	accountIntermediateRootTimer = metrics.NewRegisteredTimer("state/account/intermediate/root/time", nil)
+	storageIntermediateRootTimer = metrics.NewRegisteredTimer("state/storage/intermediate/root/time", nil)
 )

core/state/state_object.go

@@ -235,6 +235,14 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash {
 		return common.Hash{}
 	}
 	// If no live objects are available, attempt to use snapshots
+
+	defer func(start time.Time) {
+		stateDBGetTimer.UpdateSince(start)
+		stateDBGetQPS.Mark(1)
+		stateDBGetStorageTimer.UpdateSince(start)
+		stateDBGetStorageQPS.Mark(1)
+	}(time.Now())
+
 	var (
 		enc   []byte
 		err   error
@@ -390,6 +398,10 @@ func (s *stateObject) updateTrie() (Trie, error) {
 		s.db.setError(err)
 		return nil, err
 	}
+	if len(s.pendingStorage) == 0 {
+		return s.trie, nil
+	}
+
 	// Insert all the pending storage updates into the trie
 	usedStorage := make([][]byte, 0, len(s.pendingStorage))
 	dirtyStorage := make(map[common.Hash][]byte)
@@ -507,6 +519,11 @@ func (s *stateObject) updateRoot() {
 // The returned set can be nil if nothing to commit. This function assumes all
 // storage mutations have already been flushed into trie by updateRoot.
 func (s *stateObject) commit() (*trienode.NodeSet, error) {
+	if s.IsContractAccount() && s.trie == nil && s.db.db.GetVersion() != 0 {
+		panic(fmt.Sprintf("not open contract account, owner: %s, r_version: %d, c_version: %d, root: %s",
+			s.address.String(), s.db.db.GetVersion(), s.version, s.Root().String()))
+	}
+
 	// Short circuit if trie is not even loaded, don't bother with committing anything
 	if s.trie == nil {
 		s.origin = s.data.Copy()

core/state/statedb.go

@@ -54,6 +54,16 @@ type revision struct {
 	journalIndex int
 }
 
+var (
+	stateDBGetTimer = metrics.NewRegisteredTimer("statedb/get/time", nil)
+	stateDBGetQPS   = metrics.NewRegisteredMeter("statedb/get/qps", nil)
+
+	stateDBGetAccountTimer = metrics.NewRegisteredTimer("statedb/account/get/time", nil)
+	stateDBGetAccountQPS   = metrics.NewRegisteredMeter("statedb/account/get/qps", nil)
+	stateDBGetStorageTimer = metrics.NewRegisteredTimer("statedb/storage/get/time", nil)
+	stateDBGetStorageQPS   = metrics.NewRegisteredMeter("statedb/storage/get/qps", nil)
+)
+
 // StateDB structs within the ethereum protocol are used to store anything
 // within the merkle trie. StateDBs take care of caching and storing
 // nested states. It's the general query interface to retrieve:
@@ -733,6 +743,14 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject {
 	if obj := s.stateObjects[addr]; obj != nil {
 		return obj
 	}
+
+	defer func(start time.Time) {
+		stateDBGetTimer.UpdateSince(start)
+		stateDBGetQPS.Mark(1)
+		stateDBGetAccountTimer.UpdateSince(start)
+		stateDBGetAccountQPS.Mark(1)
+	}(time.Now())
+
 	// If no live objects are available, attempt to use snapshots
 	var data *types.StateAccount
 	if s.snap != nil {
@@ -1177,6 +1195,10 @@ func (s *StateDB) populateSnapStorage(obj *stateObject) bool {
 }
 
 func (s *StateDB) AccountsIntermediateRoot() {
+	defer func(start time.Time) {
+		storageIntermediateRootTimer.UpdateSince(start)
+	}(time.Now())
+
 	tasks := make(chan func())
 	finishCh := make(chan struct{})
 	defer close(finishCh)
@@ -1204,7 +1226,6 @@ func (s *StateDB) AccountsIntermediateRoot() {
 			wg.Add(1)
 			tasks <- func() {
 				obj.updateRoot()
-
 				// Cache the data until commit. Note, this update mechanism is not symmetric
 				// to the deletion, because whereas it is enough to track account updates
 				// at commit time, deletions need tracking at transaction boundary level to
@@ -1221,6 +1242,9 @@ func (s *StateDB) AccountsIntermediateRoot() {
 }
 
 func (s *StateDB) StateIntermediateRoot() common.Hash {
+	defer func(start time.Time) {
+		accountIntermediateRootTimer.UpdateSince(start)
+	}(time.Now())
 	// If there was a trie prefetcher operating, it gets aborted and irrevocably
 	// modified after we start retrieving tries. Remove it from the statedb after
 	// this round of use.
@@ -1654,9 +1678,9 @@ func (s *StateDB) Commit(block uint64, failPostCommitFunc func(), postCommitFunc
 					if err := s.db.Flush(); err != nil {
 						return err
 					}
-					if err := s.db.Release(); err != nil {
+					//if err := s.db.Release(); err != nil {
-						return err
+					//	return err
-					}
+					//}
 					s.originalRoot = root
 					if metrics.EnabledExpensive {
 						s.TrieDBCommits += time.Since(start)

core/state_prefetcher.go

@@ -49,7 +49,6 @@ func NewStatePrefetcher(config *params.ChainConfig, bc *BlockChain, engine conse
 // the transaction messages using the statedb, but any changes are discarded. The
 // only goal is to pre-cache transaction signatures and state trie nodes.
 func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, cfg *vm.Config, interruptCh <-chan struct{}) {
-	panic("prefetcher not support")
 	var (
 		header = block.Header()
 		signer = types.MakeSigner(p.config, header.Number, header.Time)

core/txpool/blobpool/blobpool.go

@@ -366,9 +366,9 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres
 	// Initialize the state with head block, or fallback to empty one in
 	// case the head state is not available (might occur when node is not
 	// fully synced).
-	state, err := p.chain.StateAt(head.Root)
+	state, err := p.chain.StateAt(head.Number.Int64(), head.Root)
 	if err != nil {
-		state, err = p.chain.StateAt(types.EmptyRootHash)
+		state, err = p.chain.StateAt(-1, types.EmptyRootHash)
 	}
 	if err != nil {
 		return err
@@ -793,7 +793,7 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) {
 		resettimeHist.Update(time.Since(start).Nanoseconds())
 	}(time.Now())
 
-	statedb, err := p.chain.StateAt(newHead.Root)
+	statedb, err := p.chain.StateAt(newHead.Number.Int64(), newHead.Root)
 	if err != nil {
 		log.Error("Failed to reset blobpool state", "err", err)
 		return

core/txpool/blobpool/interface.go

@@ -40,5 +40,5 @@ type BlockChain interface {
 	GetBlock(hash common.Hash, number uint64) *types.Block
 
 	// StateAt returns a state database for a given root hash (generally the head).
-	StateAt(root common.Hash) (*state.StateDB, error)
+	StateAt(number int64, root common.Hash) (*state.StateDB, error)
 }

core/txpool/legacypool/legacypool.go

@@ -120,7 +120,7 @@ type BlockChain interface {
 	GetBlock(hash common.Hash, number uint64) *types.Block
 
 	// StateAt returns a state database for a given root hash (generally the head).
-	StateAt(root common.Hash) (*state.StateDB, error)
+	StateAt(number int64, root common.Hash) (*state.StateDB, error)
 }
 
 // Config are the configuration parameters of the transaction pool.
@@ -311,9 +311,9 @@ func (pool *LegacyPool) Init(gasTip uint64, head *types.Header, reserve txpool.A
 	// Initialize the state with head block, or fallback to empty one in
 	// case the head state is not available (might occur when node is not
 	// fully synced).
-	statedb, err := pool.chain.StateAt(head.Root)
+	statedb, err := pool.chain.StateAt(head.Number.Int64(), head.Root)
 	if err != nil {
-		statedb, err = pool.chain.StateAt(types.EmptyRootHash)
+		statedb, err = pool.chain.StateAt(-1, types.EmptyRootHash)
 	}
 	if err != nil {
 		return err
@@ -1492,7 +1492,7 @@ func (pool *LegacyPool) reset(oldHead, newHead *types.Header) {
 	if newHead == nil {
 		newHead = pool.chain.CurrentBlock() // Special case during testing
 	}
-	statedb, err := pool.chain.StateAt(newHead.Root)
+	statedb, err := pool.chain.StateAt(newHead.Number.Int64(), newHead.Root)
 	if err != nil {
 		log.Error("Failed to reset txpool state", "err", err)
 		return

eth/api_backend.go

@@ -204,7 +204,7 @@ func (b *EthAPIBackend) StateAndHeaderByNumber(ctx context.Context, number rpc.B
 	if header == nil {
 		return nil, nil, errors.New("header not found")
 	}
-	stateDb, err := b.eth.BlockChain().StateAt(header.Root)
+	stateDb, err := b.eth.BlockChain().StateAt(header.Number.Int64(), header.Root)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -226,7 +226,7 @@ func (b *EthAPIBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockN
 		if blockNrOrHash.RequireCanonical && b.eth.blockchain.GetCanonicalHash(header.Number.Uint64()) != hash {
 			return nil, nil, errors.New("hash is not currently canonical")
 		}
-		stateDb, err := b.eth.BlockChain().StateAt(header.Root)
+		stateDb, err := b.eth.BlockChain().StateAt(header.Number.Int64(), header.Root)
 		if err != nil {
 			return nil, nil, err
 		}

eth/api_debug.go

@@ -81,7 +81,7 @@ func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) {
 	if header == nil {
 		return state.Dump{}, fmt.Errorf("block #%d not found", blockNr)
 	}
-	stateDb, err := api.eth.BlockChain().StateAt(header.Root)
+	stateDb, err := api.eth.BlockChain().StateAt(header.Number.Int64(), header.Root)
 	if err != nil {
 		return state.Dump{}, err
 	}
@@ -167,7 +167,7 @@ func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hex
 			if header == nil {
 				return state.Dump{}, fmt.Errorf("block #%d not found", number)
 			}
-			stateDb, err = api.eth.BlockChain().StateAt(header.Root)
+			stateDb, err = api.eth.BlockChain().StateAt(header.Number.Int64(), header.Root)
 			if err != nil {
 				return state.Dump{}, err
 			}
@@ -177,7 +177,7 @@ func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hex
 		if block == nil {
 			return state.Dump{}, fmt.Errorf("block %s not found", hash.Hex())
 		}
-		stateDb, err = api.eth.BlockChain().StateAt(block.Root())
+		stateDb, err = api.eth.BlockChain().StateAt(block.Number().Int64(), block.Root())
 		if err != nil {
 			return state.Dump{}, err
 		}

eth/handler.go

@@ -220,13 +220,13 @@ func newHandler(config *handlerConfig) (*handler, error) {
 				}
 				h.snapSync.Store(true)
 				log.Warn("Switch sync mode from full sync to snap sync", "reason", "snap sync incomplete")
-			} else if !h.chain.NoTries() && !h.chain.HasState(fullBlock.Root) {
+			} else if !h.chain.NoTries() && !h.chain.HasState(fullBlock.Number.Int64(), fullBlock.Root) {
 				h.snapSync.Store(true)
 				log.Warn("Switch sync mode from full sync to snap sync", "reason", "head state missing")
 			}
 		} else {
 			head := h.chain.CurrentBlock()
-			if head.Number.Uint64() > 0 && h.chain.HasState(head.Root) {
+			if head.Number.Uint64() > 0 && h.chain.HasState(head.Number.Int64(), head.Root) {
 				// Print warning log if database is not empty to run snap sync.
 				log.Warn("Switch sync mode from snap sync to full sync", "reason", "snap sync complete")
 			} else {

eth/state_accessor.go

@@ -55,7 +55,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u
 		// The state is available in live database, create a reference
 		// on top to prevent garbage collection and return a release
 		// function to deref it.
-		if statedb, err = eth.blockchain.StateAt(block.Root()); err == nil {
+		if statedb, err = eth.blockchain.StateAt(block.Number().Int64(), block.Root()); err == nil {
 			eth.blockchain.TrieDB().Reference(block.Root(), common.Hash{})
 			return statedb, func() {
 				eth.blockchain.TrieDB().Dereference(block.Root())
@@ -185,7 +185,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u
 
 func (eth *Ethereum) pathState(block *types.Block) (*state.StateDB, func(), error) {
 	// Check if the requested state is available in the live chain.
-	statedb, err := eth.blockchain.StateAt(block.Root())
+	statedb, err := eth.blockchain.StateAt(block.Number().Int64(), block.Root())
 	if err == nil {
 		return statedb, noopReleaser, nil
 	}

eth/sync.go

@@ -17,6 +17,7 @@
 package eth
 
 import (
+	"fmt"
 	"math/big"
 	"time"
 
@@ -191,33 +192,40 @@ func peerToSyncOp(mode downloader.SyncMode, p *eth.Peer) *chainSyncOp {
 }
 
 func (cs *chainSyncer) modeAndLocalHead() (downloader.SyncMode, *big.Int) {
-	// If we're in snap sync mode, return that directly
-	if cs.handler.snapSync.Load() {
-		block := cs.handler.chain.CurrentSnapBlock()
-		td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64())
-		return downloader.SnapSync, td
-	}
-	// We are probably in full sync, but we might have rewound to before the
-	// snap sync pivot, check if we should re-enable snap sync.
 	head := cs.handler.chain.CurrentBlock()
-	if pivot := rawdb.ReadLastPivotNumber(cs.handler.database); pivot != nil {
+	if cs.handler.chain.TrieDB().Scheme() != rawdb.VersionScheme {
-		if head.Number.Uint64() < *pivot {
+		// If we're in snap sync mode, return that directly
-			if rawdb.ReadAncientType(cs.handler.database) == rawdb.PruneFreezerType {
+		if cs.handler.snapSync.Load() {
-				log.Crit("Current rewound to before the fast sync pivot, can't enable pruneancient mode", "current block number", head.Number.Uint64(), "pivot", *pivot)
-			}
 			block := cs.handler.chain.CurrentSnapBlock()
 			td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64())
 			return downloader.SnapSync, td
 		}
-	}
+		// We are probably in full sync, but we might have rewound to before the
-	// We are in a full sync, but the associated head state is missing. To complete
+		// snap sync pivot, check if we should re-enable snap sync.
-	// the head state, forcefully rerun the snap sync. Note it doesn't mean the
+		head = cs.handler.chain.CurrentBlock()
-	// persistent state is corrupted, just mismatch with the head block.
+		if pivot := rawdb.ReadLastPivotNumber(cs.handler.database); pivot != nil {
-	if !cs.handler.chain.NoTries() && !cs.handler.chain.HasState(head.Root) {
+			if head.Number.Uint64() < *pivot {
-		block := cs.handler.chain.CurrentSnapBlock()
+				if rawdb.ReadAncientType(cs.handler.database) == rawdb.PruneFreezerType {
-		td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64())
+					log.Crit("Current rewound to before the fast sync pivot, can't enable pruneancient mode", "current block number", head.Number.Uint64(), "pivot", *pivot)
-		log.Info("Reenabled snap sync as chain is stateless")
+				}
-		return downloader.SnapSync, td
+				block := cs.handler.chain.CurrentSnapBlock()
+				td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64())
+				return downloader.SnapSync, td
+			}
+		}
+		// We are in a full sync, but the associated head state is missing. To complete
+		// the head state, forcefully rerun the snap sync. Note it doesn't mean the
+		// persistent state is corrupted, just mismatch with the head block.
+		if !cs.handler.chain.NoTries() && !cs.handler.chain.HasState(head.Number.Int64(), head.Root) {
+			block := cs.handler.chain.CurrentSnapBlock()
+			td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64())
+			log.Info("Reenabled snap sync as chain is stateless")
+			return downloader.SnapSync, td
+		}
+	} else {
+		if !cs.handler.chain.HasState(head.Number.Int64(), head.Root) {
+			panic(fmt.Sprintf("version db not support snap sync, version: %d, root: %s", head.Number.Int64(), head.Root))
+		}
 	}
 	// Nope, we're really full syncing
 	td := cs.handler.chain.GetTd(head.Hash(), head.Number.Uint64())

go.mod

@@ -42,7 +42,7 @@ require (
 	github.com/gorilla/websocket v1.5.1
 	github.com/graph-gophers/graphql-go v1.3.0
 	github.com/hashicorp/go-bexpr v0.1.10
-	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d
+	github.com/hashicorp/golang-lru v1.0.2
 	github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4
 	github.com/holiman/bloomfilter/v2 v2.0.3
 	github.com/holiman/uint256 v1.3.0
@@ -252,7 +252,6 @@ require (
 	github.com/spf13/afero v1.10.0 // indirect
 	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
 	github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e // indirect
-	github.com/tidwall/btree v1.7.0 // indirect
 	github.com/tidwall/gjson v1.10.2 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect

go.sum

@@ -598,8 +598,8 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs=
+github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
-github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4=
 github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
@@ -1152,8 +1152,6 @@ github.com/tendermint/iavl v0.12.0 h1:xcaFAr+ycqCj7WN1RzL2EfcBioRDOHcU1oWcg83K02
 github.com/tendermint/iavl v0.12.0/go.mod h1:EoKMMv++tDOL5qKKVnoIqtVPshRrEPeJ0WsgDOLAauM=
 github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e h1:cR8/SYRgyQCt5cNCMniB/ZScMkhI9nk8U5C7SbISXjo=
 github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e/go.mod h1:Tu4lItkATkonrYuvtVjG0/rhy15qrNGNTjPdaphtZ/8=
-github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
-github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
 github.com/tidwall/gjson v1.10.2 h1:APbLGOM0rrEkd8WBw9C24nllro4ajFuJu0Sc9hRz8Bo=
 github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=

internal/ethapi/api.go

@@ -1367,11 +1367,11 @@ func (s *BlockChainAPI) needToReplay(ctx context.Context, block *types.Block, ac
 	if err != nil {
 		return false, fmt.Errorf("block not found for block number (%d): %v", block.NumberU64()-1, err)
 	}
-	parentState, err := s.b.Chain().StateAt(parent.Root())
+	parentState, err := s.b.Chain().StateAt(parent.Number().Int64(), parent.Root())
 	if err != nil {
 		return false, fmt.Errorf("statedb not found for block number (%d): %v", block.NumberU64()-1, err)
 	}
-	currentState, err := s.b.Chain().StateAt(block.Root())
+	currentState, err := s.b.Chain().StateAt(parent.Number().Int64(), block.Root())
 	if err != nil {
 		return false, fmt.Errorf("statedb not found for block number (%d): %v", block.NumberU64(), err)
 	}
@@ -1397,7 +1397,7 @@ func (s *BlockChainAPI) replay(ctx context.Context, block *types.Block, accounts
 	if err != nil {
 		return nil, nil, fmt.Errorf("block not found for block number (%d): %v", block.NumberU64()-1, err)
 	}
-	statedb, err := s.b.Chain().StateAt(parent.Root())
+	statedb, err := s.b.Chain().StateAt(parent.Number().Int64(), parent.Root())
 	if err != nil {
 		return nil, nil, fmt.Errorf("state not found for block number (%d): %v", block.NumberU64()-1, err)
 	}

miner/miner.go

@@ -245,7 +245,7 @@ func (miner *Miner) Pending() (*types.Block, *state.StateDB) {
 	if block == nil {
 		return nil, nil
 	}
-	stateDb, err := miner.worker.chain.StateAt(block.Root)
+	stateDb, err := miner.worker.chain.StateAt(block.Number.Int64(), block.Root)
 	if err != nil {
 		return nil, nil
 	}

miner/worker.go

@@ -690,7 +690,7 @@ func (w *worker) makeEnv(parent *types.Header, header *types.Header, coinbase co
 	prevEnv *environment) (*environment, error) {
 	// Retrieve the parent state to execute on top and start a prefetcher for
 	// the miner to speed block sealing up a bit
-	state, err := w.chain.StateAt(parent.Root)
+	state, err := w.chain.StateAt(parent.Number.Int64(), parent.Root)
 	if err != nil {
 		return nil, err
 	}

triedb/versiondb/versadb.go

@@ -29,11 +29,15 @@ func New(config *Config) *VersionDB {
 	if config != nil {
 		path = config.Path
 		cfg = &versa.VersaDBConfig{
-			FlushInterval:  config.FlushInterval,
+			FlushInterval:  6000,
-			MaxStatesInMem: config.MaxStatesInMem,
+			MaxStatesInMem: 128,
 		}
+		_ = cfg
 	}
-	db, err := versa.NewVersaDB(path, cfg)
+	db, err := versa.NewVersaDB(path, &versa.VersaDBConfig{
+		FlushInterval:  6000,
+		MaxStatesInMem: 128,
+	})
 	if err != nil {
 		log.Crit("failed to new version db", "error", err)
 	}
@@ -49,7 +53,8 @@ func (v *VersionDB) Scheme() string {
 }
 
 func (v *VersionDB) Initialized(genesisRoot common.Hash) bool {
-	return v.db.HasState(genesisRoot)
+	version, _ := v.db.LatestStoreDiskVersionInfo()
+	return version >= 0
 }
 
 func (v *VersionDB) Size() (common.StorageSize, common.StorageSize, common.StorageSize) {