feat: support rewind for versa db

fix: rewind ancient store

.nancy-ignore

@@ -1 +1,2 @@
 CVE-2024-34478 # "CWE-754: Improper Check for Unusual or Exceptional Conditions." This vulnerability is BTC only, BSC does not have the issue.
+CVE-2024-6104 # "CWE-532: Information Exposure Through Log Files" This is caused by the vulnerabilities go-retryablehttp@v0.7.4, it is only used in cmd devp2p, impact is limited. will upgrade to v0.7.7 later

CHANGELOG.md

@@ -1,4 +1,57 @@
 # Changelog
+
+## v1.4.12
+
+### BUGFIX
+* [\#2557](https://github.com/bnb-chain/bsc/pull/2557) fix: fix state inspect error after pruned state
+* [\#2562](https://github.com/bnb-chain/bsc/pull/2562) fix: delete unexpected block
+* [\#2566](https://github.com/bnb-chain/bsc/pull/2566) core: avoid to cache block before wroten into db
+* [\#2567](https://github.com/bnb-chain/bsc/pull/2567) fix: fix statedb copy
+* [\#2574](https://github.com/bnb-chain/bsc/pull/2574) core: adapt highestVerifiedHeader to FastFinality
+* [\#2542](https://github.com/bnb-chain/bsc/pull/2542) fix: pruneancient freeze from the previous position when the first time
+* [\#2564](https://github.com/bnb-chain/bsc/pull/2564) fix: the bug of blobsidecars and downloader with multi-database
+* [\#2582](https://github.com/bnb-chain/bsc/pull/2582) fix: remove delete and dangling side chains in prunefreezer
+
+### FEATURE
+* [\#2513](https://github.com/bnb-chain/bsc/pull/2513) cmd/jsutils: add a tool to get performance between a range of blocks
+* [\#2569](https://github.com/bnb-chain/bsc/pull/2569) cmd/jsutils: add a tool to get slash count
+* [\#2583](https://github.com/bnb-chain/bsc/pull/2583) cmd/jsutill: add log about validator name
+
+### IMPROVEMENT
+* [\#2546](https://github.com/bnb-chain/bsc/pull/2546) go.mod: update missing dependency
+* [\#2559](https://github.com/bnb-chain/bsc/pull/2559) nancy: ignore go-retryablehttp@v0.7.4 in .nancy-ignore
+* [\#2556](https://github.com/bnb-chain/bsc/pull/2556) chore: update greenfield cometbft version
+* [\#2561](https://github.com/bnb-chain/bsc/pull/2561) tests: fix unstable test
+* [\#2572](https://github.com/bnb-chain/bsc/pull/2572) core: clearup testflag for Cancun and Haber
+* [\#2573](https://github.com/bnb-chain/bsc/pull/2573) cmd/utils: support use NetworkId to distinguish chapel when do syncing
+* [\#2538](https://github.com/bnb-chain/bsc/pull/2538) feat: enhance bid comparison and reply bidding results && detail logs
+* [\#2568](https://github.com/bnb-chain/bsc/pull/2568) core/vote: not vote if too late for next in turn validator
+* [\#2576](https://github.com/bnb-chain/bsc/pull/2576) miner/worker: broadcast block immediately once sealed
+* [\#2580](https://github.com/bnb-chain/bsc/pull/2580) freezer: Opt freezer env checking
+
+## v1.4.11
+
+### BUGFIX
+* [\#2534](https://github.com/bnb-chain/bsc/pull/2534) fix: nil pointer when clear simulating bid
+* [\#2535](https://github.com/bnb-chain/bsc/pull/2535) upgrade: add HaberFix hardfork
+
+
+## v1.4.10
+### FEATURE
+NA
+
+### IMPROVEMENT
+* [\#2512](https://github.com/bnb-chain/bsc/pull/2512) feat: add mev helper params and func
+* [\#2508](https://github.com/bnb-chain/bsc/pull/2508) perf: speedup pbss trienode read
+* [\#2509](https://github.com/bnb-chain/bsc/pull/2509) perf: optimize chain commit performance for multi-database
+* [\#2451](https://github.com/bnb-chain/bsc/pull/2451) core/forkchoice: improve stability when inturn block not generate
+
+### BUGFIX
+* [\#2518](https://github.com/bnb-chain/bsc/pull/2518) fix: remove zero gasprice check for BSC
+* [\#2519](https://github.com/bnb-chain/bsc/pull/2519) UT: random failure of TestSnapSyncWithBlobs
+* [\#2515](https://github.com/bnb-chain/bsc/pull/2515) fix getBlobSidecars by ethclient
+* [\#2525](https://github.com/bnb-chain/bsc/pull/2525) fix: ensure empty withdrawals after cancun before broadcast
+
 ## v1.4.9
 ### FEATURE
 * [\#2463](https://github.com/bnb-chain/bsc/pull/2463)  utils: add check_blobtx.js

README.md

@@ -149,8 +149,6 @@ unzip testnet.zip
 #### 3. Download snapshot
 Download latest chaindata snapshot from [here](https://github.com/bnb-chain/bsc-snapshots). Follow the guide to structure your files.
 
-Note: If you encounter difficulties downloading the chaindata snapshot and prefer to synchronize from the genesis block on the Chapel testnet, remember to include the additional flag `--chapel` when initially launching Geth.
-
 #### 4. Start a full node
 ```shell
 ./geth --config ./config.toml --datadir ./node  --cache 8000 --rpc.allow-unprotected-txs --history.transactions 0

cmd/faucet/faucet.go

@@ -642,7 +642,7 @@ func (f *faucet) loop() {
 			f.lock.RUnlock()
 		}
 	}()
-	// Wait for various events and assing to the appropriate background threads
+	// Wait for various events and assign to the appropriate background threads
 	for {
 		select {
 		case head := <-heads:

cmd/geth/chaincmd.go

@@ -62,10 +62,9 @@ var (
 		ArgsUsage: "<genesisPath>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.CachePreimagesFlag,
-			utils.OverrideCancun,
-			utils.OverrideHaber,
 			utils.OverrideBohr,
 			utils.OverrideVerkle,
+			utils.MultiDataBaseFlag,
 		}, utils.DatabaseFlags),
 		Description: `
 The init command initializes a new genesis block and definition for the network.
@@ -254,14 +253,6 @@ func initGenesis(ctx *cli.Context) error {
 	defer stack.Close()
 
 	var overrides core.ChainOverrides
-	if ctx.IsSet(utils.OverrideCancun.Name) {
-		v := ctx.Uint64(utils.OverrideCancun.Name)
-		overrides.OverrideCancun = &v
-	}
-	if ctx.IsSet(utils.OverrideHaber.Name) {
-		v := ctx.Uint64(utils.OverrideHaber.Name)
-		overrides.OverrideHaber = &v
-	}
 	if ctx.IsSet(utils.OverrideBohr.Name) {
 		v := ctx.Uint64(utils.OverrideBohr.Name)
 		overrides.OverrideBohr = &v
@@ -769,7 +760,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth
 		arg := ctx.Args().First()
 		if hashish(arg) {
 			hash := common.HexToHash(arg)
-			if number := rawdb.ReadHeaderNumber(db.BlockStore(), hash); number != nil {
+			if number := rawdb.ReadHeaderNumber(db, hash); number != nil {
 				header = rawdb.ReadHeader(db, hash, *number)
 			} else {
 				return nil, nil, common.Hash{}, fmt.Errorf("block %x not found", hash)
@@ -842,7 +833,7 @@ func dump(ctx *cli.Context) error {
 	triedb := utils.MakeTrieDatabase(ctx, stack, db, true, true, false) // always enable preimage lookup
 	defer triedb.Close()
 
-	state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb), nil)
+	state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb, false), nil)
 	if err != nil {
 		return err
 	}

cmd/geth/config.go

@@ -185,14 +185,6 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 		params.RialtoGenesisHash = common.HexToHash(v)
 	}
 
-	if ctx.IsSet(utils.OverrideCancun.Name) {
-		v := ctx.Uint64(utils.OverrideCancun.Name)
-		cfg.Eth.OverrideCancun = &v
-	}
-	if ctx.IsSet(utils.OverrideHaber.Name) {
-		v := ctx.Uint64(utils.OverrideHaber.Name)
-		cfg.Eth.OverrideHaber = &v
-	}
 	if ctx.IsSet(utils.OverrideBohr.Name) {
 		v := ctx.Uint64(utils.OverrideBohr.Name)
 		cfg.Eth.OverrideBohr = &v

cmd/geth/dbcmd.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"math"
@@ -34,6 +35,7 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/console/prompt"
 	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/state/snapshot"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -91,6 +93,9 @@ Remove blockchain and state databases`,
 			dbHbss2PbssCmd,
 			dbTrieGetCmd,
 			dbTrieDeleteCmd,
+			getVersionDBState,
+			getHashDBState,
+			diffDebugStateDB,
 		},
 	}
 	dbInspectCmd = &cli.Command{
@@ -286,8 +291,141 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Description: `This commands will read current offset from kvdb, which is the current offset and starting BlockNumber
 of ancientStore, will also displays the reserved number of blocks in ancientStore `,
 	}
+	getVersionDBState = &cli.Command{
+		Action: getDebugVersionState,
+		Name:   "get-debug-version-state",
+		Flags: []cli.Flag{
+			utils.VersionStateDirFlag,
+			utils.BlockNumber,
+		},
+	}
+	getHashDBState = &cli.Command{
+		Action: getDebugHashState,
+		Name:   "get-debug-hash-state",
+		Flags: []cli.Flag{
+			utils.HashStateDirFlag,
+			utils.BlockNumber,
+		},
+	}
+
+	diffDebugStateDB = &cli.Command{
+		Action: diffDebugState,
+		Name:   "diff-debug-state",
+		Flags: []cli.Flag{
+			utils.VersionStateDirFlag,
+			utils.HashStateDirFlag,
+			utils.BlockNumber,
+		},
+	}
 )
 
+func diffDebugState(ctx *cli.Context) error {
+	if !ctx.IsSet(utils.VersionStateDirFlag.Name) {
+		return fmt.Errorf("please set `--versionstatedir` flag")
+	}
+	if !ctx.IsSet(utils.BlockNumber.Name) {
+		return fmt.Errorf("please set `--block` flag")
+	}
+	if !ctx.IsSet(utils.HashStateDirFlag.Name) {
+		return fmt.Errorf("please set `--hashstatedir` flag")
+	}
+	verDir := ctx.String(utils.VersionStateDirFlag.Name)
+	hasDir := ctx.String(utils.HashStateDirFlag.Name)
+	block := ctx.Int64(utils.BlockNumber.Name)
+
+	vdb, err := rawdb.Open(rawdb.OpenOptions{
+		ReadOnly:  true,
+		Type:      "pebble",
+		Directory: verDir,
+	})
+	if err != nil {
+		return err
+	}
+	verData, err := vdb.Get(state.DebugVersionStateKey(block))
+	if err != nil {
+		return err
+	}
+
+	verDebugState := &state.DebugVersionState{}
+	err = json.Unmarshal(verData, verDebugState)
+	if err != nil {
+		return nil
+	}
+
+	hdb, err := rawdb.Open(rawdb.OpenOptions{
+		ReadOnly:  true,
+		Type:      "pebble",
+		Directory: hasDir,
+	})
+	if err != nil {
+		return err
+	}
+	hashData, err := hdb.Get(state.DebugHashStateKey(block))
+	if err != nil {
+		return err
+	}
+	hasDebugState := &state.DebugHashState{}
+	err = json.Unmarshal(hashData, hasDebugState)
+	if err != nil {
+		return err
+	}
+
+	res := state.GenerateDebugStateDiff(verDebugState, hasDebugState)
+	fmt.Println(res)
+
+	return nil
+}
+
+func getDebugVersionState(ctx *cli.Context) error {
+	if !ctx.IsSet(utils.VersionStateDirFlag.Name) {
+		return fmt.Errorf("please set `--versionstatedir` flag")
+	}
+	if !ctx.IsSet(utils.BlockNumber.Name) {
+		return fmt.Errorf("please set `--block` flag")
+	}
+	dir := ctx.String(utils.VersionStateDirFlag.Name)
+	block := ctx.Int64(utils.BlockNumber.Name)
+	db, err := rawdb.Open(rawdb.OpenOptions{
+		ReadOnly:  true,
+		Type:      "pebble",
+		Directory: dir,
+	})
+	if err != nil {
+		return err
+	}
+	data, err := db.Get(state.DebugVersionStateKey(block))
+	if err != nil {
+		return err
+	}
+	fmt.Println(string(data))
+	return nil
+}
+
+func getDebugHashState(ctx *cli.Context) error {
+	if !ctx.IsSet(utils.HashStateDirFlag.Name) {
+		return fmt.Errorf("please set `--hashstatedir` flag")
+	}
+	if !ctx.IsSet(utils.BlockNumber.Name) {
+		return fmt.Errorf("please set `--block` flag")
+	}
+	dir := ctx.String(utils.HashStateDirFlag.Name)
+	block := ctx.Int64(utils.BlockNumber.Name)
+	db, err := rawdb.Open(rawdb.OpenOptions{
+		ReadOnly:  true,
+		Type:      "pebble",
+		Directory: dir,
+	})
+	if err != nil {
+		return err
+	}
+	data, err := db.Get(state.DebugHashStateKey(block))
+	if err != nil {
+		return err
+	}
+	fmt.Println(string(data))
+	return nil
+}
+
 func removeDB(ctx *cli.Context) error {
 	stack, config := makeConfigNode(ctx)
 
@@ -397,8 +535,8 @@ func inspectTrie(ctx *cli.Context) error {
 	var headerBlockHash common.Hash
 	if ctx.NArg() >= 1 {
 		if ctx.Args().Get(0) == "latest" {
-			headerHash := rawdb.ReadHeadHeaderHash(db.BlockStore())
-			blockNumber = *(rawdb.ReadHeaderNumber(db.BlockStore(), headerHash))
+			headerHash := rawdb.ReadHeadHeaderHash(db)
+			blockNumber = *(rawdb.ReadHeaderNumber(db, headerHash))
 		} else if ctx.Args().Get(0) == "snapshot" {
 			trieRootHash = rawdb.ReadSnapshotRoot(db)
 			blockNumber = math.MaxUint64
@@ -1212,7 +1350,7 @@ func showMetaData(ctx *cli.Context) error {
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error accessing ancients: %v", err)
 	}
-	data := rawdb.ReadChainMetadataFromMultiDatabase(db)
+	data := rawdb.ReadChainMetadata(db)
 	data = append(data, []string{"frozen", fmt.Sprintf("%d items", ancients)})
 	data = append(data, []string{"snapshotGenerator", snapshot.ParseGeneratorStatus(rawdb.ReadSnapshotGenerator(db))})
 	if b := rawdb.ReadHeadBlock(db); b != nil {
@@ -1255,7 +1393,7 @@ func hbss2pbss(ctx *cli.Context) error {
 	defer stack.Close()
 
 	db := utils.MakeChainDatabase(ctx, stack, false, false)
-	db.Sync()
+	db.BlockStore().Sync()
 	stateDiskDb := db.StateStore()
 	defer db.Close()
 
@@ -1273,8 +1411,8 @@ func hbss2pbss(ctx *cli.Context) error {
 		log.Info("hbss2pbss triedb", "scheme", triedb.Scheme())
 		defer triedb.Close()
 
-		headerHash := rawdb.ReadHeadHeaderHash(db.BlockStore())
-		blockNumber := rawdb.ReadHeaderNumber(db.BlockStore(), headerHash)
+		headerHash := rawdb.ReadHeadHeaderHash(db)
+		blockNumber := rawdb.ReadHeaderNumber(db, headerHash)
 		if blockNumber == nil {
 			log.Error("read header number failed.")
 			return fmt.Errorf("read header number failed")

cmd/geth/main.go

@@ -72,8 +72,6 @@ var (
 		utils.USBFlag,
 		utils.SmartCardDaemonPathFlag,
 		utils.RialtoHash,
-		utils.OverrideCancun,
-		utils.OverrideHaber,
 		utils.OverrideBohr,
 		utils.OverrideVerkle,
 		utils.OverrideFullImmutabilityThreshold,
@@ -127,6 +125,7 @@ var (
 		utils.CacheSnapshotFlag,
 		// utils.CacheNoPrefetchFlag,
 		utils.CachePreimagesFlag,
+		utils.MultiDataBaseFlag,
 		utils.PersistDiffFlag,
 		utils.DiffBlockFlag,
 		utils.PruneAncientDataFlag,
@@ -336,9 +335,6 @@ func prepare(ctx *cli.Context) {
   5. Networking is disabled; there is no listen-address, the maximum number of peers is set
      to 0, and discovery is disabled.
 `)
-
-	case !ctx.IsSet(utils.NetworkIdFlag.Name):
-		log.Info("Starting Geth on BSC mainnet...")
 	}
 	// If we're a full node on mainnet without --cache specified, bump default cache allowance
 	if !ctx.IsSet(utils.CacheFlag.Name) && !ctx.IsSet(utils.NetworkIdFlag.Name) {

cmd/geth/pruneblock_test.go

@@ -155,6 +155,12 @@ func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemai
 	triedb := triedb.NewDatabase(db, nil)
 	defer triedb.Close()
 
+	if err = db.SetupFreezerEnv(&ethdb.FreezerEnv{
+		ChainCfg:         gspec.Config,
+		BlobExtraReserve: params.DefaultExtraReserveForBlobRequests,
+	}); err != nil {
+		t.Fatalf("Failed to create chain: %v", err)
+	}
 	genesis := gspec.MustCommit(db, triedb)
 	// Initialize a fresh chain with only a genesis block
 	blockchain, err := core.NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil)

cmd/jsutils/README.md

@@ -24,4 +24,24 @@ testnet validators version
 ### 2.Get Transaction Count
 ```bash
 node gettxcount.js --rpc ${url} --startNum ${start} --endNum ${end} --miner ${miner} (optional)
-```
+```
+
+### 3. Get Performance
+```bash
+node get_perf.js --rpc ${url} --startNum ${start} --endNum ${end}
+```
+output as following
+```bash
+Get the performance between [ 19470 , 19670 )
+txCountPerBlock = 3142.81 txCountTotal = 628562 BlockCount = 200 avgBlockTime = 3.005 inturnBlocksRatio = 0.975
+txCountPerSecond = 1045.8602329450914 avgGasUsedPerBlock = 250.02062627 avgGasUsedPerSecond =  83.20153952412646
+```
+
+### 4. Get validators slash count
+```bash
+use the latest block 
+node getslashcount.js --Rpc ${ArchiveRpc} 
+use a block number
+node getslashcount.js --Rpc ${ArchiveRpc} --Num ${blockNum}
+```
+

cmd/jsutils/get_perf.js

@@ -0,0 +1,58 @@
+import { ethers } from "ethers";
+import program from "commander";
+
+program.option("--rpc <rpc>", "Rpc");
+program.option("--startNum <startNum>", "start num")
+program.option("--endNum <endNum>", "end num")
+program.parse(process.argv);
+
+const provider = new ethers.JsonRpcProvider(program.rpc)
+
+const main = async () => {
+    let txCountTotal = 0;
+    let gasUsedTotal = 0;
+    let inturnBlocks = 0;
+    for (let i = program.startNum; i < program.endNum; i++) {
+        let txCount = await provider.send("eth_getBlockTransactionCountByNumber", [
+            ethers.toQuantity(i)]);
+        txCountTotal += ethers.toNumber(txCount)
+
+        let header = await provider.send("eth_getHeaderByNumber", [
+            ethers.toQuantity(i)]);
+        let gasUsed = eval(eval(header.gasUsed).toString(10))
+        gasUsedTotal += gasUsed
+        let difficulty = eval(eval(header.difficulty).toString(10))
+        if (difficulty == 2) {
+            inturnBlocks += 1
+        }
+        let timestamp = eval(eval(header.timestamp).toString(10))
+        console.log("BlockNumber =", i, "mod =", i%4, "miner =", header.miner , "difficulty =", difficulty, "txCount =", ethers.toNumber(txCount), "gasUsed", gasUsed, "timestamp", timestamp)
+    }
+
+    let blockCount = program.endNum - program.startNum
+    let txCountPerBlock = txCountTotal/blockCount
+
+    let startHeader = await provider.send("eth_getHeaderByNumber", [
+        ethers.toQuantity(program.startNum)]);
+    let startTime = eval(eval(startHeader.timestamp).toString(10))
+    let endHeader = await provider.send("eth_getHeaderByNumber", [
+        ethers.toQuantity(program.endNum)]);
+    let endTime = eval(eval(endHeader.timestamp).toString(10))
+    let timeCost = endTime - startTime
+    let avgBlockTime = timeCost/blockCount
+    let inturnBlocksRatio = inturnBlocks/blockCount
+    let tps = txCountTotal/timeCost
+    let M = 1000000
+    let avgGasUsedPerBlock = gasUsedTotal/blockCount/M
+    let avgGasUsedPerSecond = gasUsedTotal/timeCost/M
+
+    console.log("Get the performance between [", program.startNum, ",", program.endNum, ")");
+    console.log("txCountPerBlock =", txCountPerBlock, "txCountTotal =", txCountTotal, "BlockCount =", blockCount, "avgBlockTime =", avgBlockTime, "inturnBlocksRatio =", inturnBlocksRatio);
+    console.log("txCountPerSecond =", tps, "avgGasUsedPerBlock =", avgGasUsedPerBlock, "avgGasUsedPerSecond =", avgGasUsedPerSecond);
+};
+
+main().then(() => process.exit(0))
+    .catch((error) => {
+        console.error(error);
+        process.exit(1);
+    });

cmd/jsutils/getslashcount.js

@@ -0,0 +1,119 @@
+import { ethers } from "ethers";
+import program from "commander";
+
+program.option("--Rpc <Rpc>", "Rpc");
+program.option("--Num <Num>", "num", 0)
+program.parse(process.argv);
+
+const provider = new ethers.JsonRpcProvider(program.Rpc);
+
+const slashAbi = [
+    "function getSlashIndicator(address validatorAddr) external view returns (uint256, uint256)"
+]
+const validatorSetAbi = [
+    "function getLivingValidators() external view returns (address[], bytes[])"
+]
+const stakeHubAbi = [
+    "function getValidatorDescription(address validatorAddr) external view returns (tuple(string, string, string, string))",
+    "function consensusToOperator(address consensusAddr) public view returns (address)"
+]
+const addrValidatorSet = '0x0000000000000000000000000000000000001000';
+const validatorSet = new ethers.Contract(addrValidatorSet, validatorSetAbi, provider);
+
+const addrSlash = '0x0000000000000000000000000000000000001001';
+const slashIndicator = new ethers.Contract(addrSlash, slashAbi,  provider)
+
+const addrStakeHub = '0x0000000000000000000000000000000000002002';
+const stakeHub = new ethers.Contract(addrStakeHub, stakeHubAbi, provider)
+
+const validatorMap = new Map([
+    //BSC
+    ["0x37e9627A91DD13e453246856D58797Ad6583D762", "LegendII"],
+    ["0xB4647b856CB9C3856d559C885Bed8B43e0846a47", "CertiK"],
+    ["0x75B851a27D7101438F45fce31816501193239A83", "Figment"],
+    ["0x502aECFE253E6AA0e8D2A06E12438FFeD0Fe16a0", "BscScan"],
+    ["0xCa503a7eD99eca485da2E875aedf7758472c378C", "InfStones"],
+    ["0x5009317FD4F6F8FeEa9dAe41E5F0a4737BB7A7D5", "NodeReal"],
+    ["0x1cFDBd2dFf70C6e2e30df5012726F87731F38164", "Tranchess"],
+    ["0xF8de5e61322302b2c6e0a525cC842F10332811bf", "Namelix"],
+    ["0xCcB42A9b8d6C46468900527Bc741938E78AB4577", "Turing"],
+    ["0x9f1b7FAE54BE07F4FEE34Eb1aaCb39A1F7B6FC92", "TWStaking"],
+    ["0x7E1FdF03Eb3aC35BF0256694D7fBe6B6d7b3E0c8","LegendIII"],
+    ["0x7b501c7944185130DD4aD73293e8Aa84eFfDcee7","MathW"],
+    ["0x58567F7A51a58708C8B40ec592A38bA64C0697De","Legend"],
+    ["0x460A252B4fEEFA821d3351731220627D7B7d1F3d","Defibit"],
+    ["0x8A239732871AdC8829EA2f47e94087C5FBad47b6","The48Club"],
+    ["0xD3b0d838cCCEAe7ebF1781D11D1bB741DB7Fe1A7","BNBEve"],
+    ["0xF8B99643fAfC79d9404DE68E48C4D49a3936f787","Avengers"],
+    ["0x4e5acf9684652BEa56F2f01b7101a225Ee33d23f","HashKey"],
+    ["0x9bb56C2B4DBE5a06d79911C9899B6f817696ACFc","Feynman"],
+    ["0xbdcc079BBb23C1D9a6F36AA31309676C258aBAC7","Fuji"],
+    ["0x38944092685a336CB6B9ea58836436709a2adC89","Shannon"],
+    ["0xfC1004C0f296Ec3Df4F6762E9EabfcF20EB304a2","Aoraki"],
+    ["0xa0884bb00E5F23fE2427f0E5eC9E51F812848563","Coda"],
+    ["0xe7776De78740f28a96412eE5cbbB8f90896b11A5","Ankr"],
+    ["0xA2D969E82524001Cb6a2357dBF5922B04aD2FCD8","Pexmons"],
+    ["0x5cf810AB8C718ac065b45f892A5BAdAB2B2946B9","Zen"],
+    ["0x4d15D9BCd0c2f33E7510c0de8b42697CA558234a","LegendVII"],
+    ["0x1579ca96EBd49A0B173f86C372436ab1AD393380","LegendV"],
+    ["0xd1F72d433f362922f6565FC77c25e095B29141c8","LegendVI"],
+    ["0xf9814D93b4d904AaA855cBD4266D6Eb0Ec1Aa478","Legend8"],
+    ["0x025a4e09Ea947b8d695f53ddFDD48ddB8F9B06b7","Ciscox"],
+    ["0xE9436F6F30b4B01b57F2780B2898f3820EbD7B98","LegendIV"],
+    ["0xC2d534F079444E6E7Ff9DabB3FD8a26c607932c8","Axion"],
+    ["0x9F7110Ba7EdFda83Fc71BeA6BA3c0591117b440D","LegendIX"],
+    ["0xB997Bf1E3b96919fBA592c1F61CE507E165Ec030","Seoraksan"],
+    ["0x286C1b674d48cFF67b4096b6c1dc22e769581E91","Sigm8"],
+    ["0x73A26778ef9509a6E94b55310eE7233795a9EB25","Coinlix"],
+    ["0x18c44f4FBEde9826C7f257d500A65a3D5A8edebc","Nozti"],
+    ["0xA100FCd08cE722Dc68Ddc3b54237070Cb186f118","Tiollo"],
+    ["0x0F28847cfdbf7508B13Ebb9cEb94B2f1B32E9503","Raptas"],
+    ["0xfD85346c8C991baC16b9c9157e6bdfDACE1cD7d7","Glorin"],
+    ["0x978F05CED39A4EaFa6E8FD045Fe2dd6Da836c7DF","NovaX"],
+    ["0xd849d1dF66bFF1c2739B4399425755C2E0fAbbAb","Nexa"],
+    ["0xA015d9e9206859c13201BB3D6B324d6634276534","Star"],
+    ["0x5ADde0151BfAB27f329e5112c1AeDeed7f0D3692","Veri"],
+    //Chapel
+    ["0x08265dA01E1A65d62b903c7B34c08cB389bF3D99","Ararat"],
+    ["0x7f5f2cF1aec83bF0c74DF566a41aa7ed65EA84Ea","Kita"],
+    ["0x53387F3321FD69d1E030BB921230dFb188826AFF","Fuji"],
+    ["0x76D76ee8823dE52A1A431884c2ca930C5e72bff3","Seoraksan"],
+    ["0xd447b49CD040D20BC21e49ffEa6487F5638e4346","Everest"],
+    ["0x1a3d9D7A717D64e6088aC937d5aAcDD3E20ca963","Elbrus"],
+    ["0x40D3256EB0BaBE89f0ea54EDAa398513136612f5","Bloxroute"],
+    ["0xF9a1Db0d6f22Bd78ffAECCbc8F47c83Df9FBdbCf","Test"]
+]);
+
+
+const main = async () => {
+    let blockNum = ethers.getNumber(program.Num)
+    if (blockNum === 0) {
+       blockNum = await provider.getBlockNumber()
+    }
+    let block = await provider.getBlock(blockNum)
+    console.log("At block", blockNum, "time", block.date)
+    const data = await validatorSet.getLivingValidators({blockTag:blockNum})
+    let totalSlash = 0
+    for (let i = 0; i < data[0].length; i++) {
+        let addr = data[0][i];
+        var val
+        if (!validatorMap.has(addr)) {
+            let opAddr = await stakeHub.consensusToOperator(addr, {blockTag:blockNum})
+            let value = await stakeHub.getValidatorDescription(opAddr, {blockTag:blockNum})
+            val = value[0]
+            console.log(addr, val)
+        } else {
+            val = validatorMap.get(addr)
+        }
+        let info = await slashIndicator.getSlashIndicator(addr, {blockTag:blockNum})
+        let count = ethers.toNumber(info[1])
+        totalSlash += count
+        console.log("Slash:", count, addr, val)
+    }
+    console.log("Total slash count", totalSlash)
+};
+main().then(() => process.exit(0))
+    .catch((error) => {
+        console.error(error);
+        process.exit(1);
+    });

cmd/utils/flags.go

@@ -305,16 +305,6 @@ var (
 		Usage:    "Manually specify the Rialto Genesis Hash, to trigger builtin network logic",
 		Category: flags.EthCategory,
 	}
-	OverrideCancun = &cli.Uint64Flag{
-		Name:     "override.cancun",
-		Usage:    "Manually specify the Cancun fork timestamp, overriding the bundled setting",
-		Category: flags.EthCategory,
-	}
-	OverrideHaber = &cli.Uint64Flag{
-		Name:     "override.haber",
-		Usage:    "Manually specify the Haber fork timestamp, overriding the bundled setting",
-		Category: flags.EthCategory,
-	}
 	OverrideBohr = &cli.Uint64Flag{
 		Name:     "override.bohr",
 		Usage:    "Manually specify the Bohr fork timestamp, overriding the bundled setting",
@@ -363,7 +353,7 @@ var (
 	}
 	StateSchemeFlag = &cli.StringFlag{
 		Name:     "state.scheme",
-		Usage:    "Scheme to use for storing ethereum state ('hash' or 'path')",
+		Usage:    "Scheme to use for storing ethereum state ('hash', 'path', 'version')",
 		Category: flags.StateCategory,
 	}
 	PathDBSyncFlag = &cli.BoolFlag{
@@ -1145,6 +1135,25 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server.
 		Value:    params.DefaultExtraReserveForBlobRequests,
 		Category: flags.MiscCategory,
 	}
+
+	BlockNumber = &cli.Int64Flag{
+		Name:  "block",
+		Value: int64(0),
+	}
+
+	VersionStateDirFlag = &flags.DirectoryFlag{
+		Name:     "versionstatedir",
+		Usage:    "Data directory for the version databases and keystore",
+		Value:    flags.DirectoryString(node.DefaultDataDir()),
+		Category: flags.EthCategory,
+	}
+
+	HashStateDirFlag = &flags.DirectoryFlag{
+		Name:     "hashstatedir",
+		Usage:    "Data directory for the version databases and keystore",
+		Value:    flags.DirectoryString(node.DefaultDataDir()),
+		Category: flags.EthCategory,
+	}
 )
 
 var (
@@ -1163,7 +1172,6 @@ var (
 		DBEngineFlag,
 		StateSchemeFlag,
 		HttpHeaderFlag,
-		MultiDataBaseFlag,
 	}
 )
 
@@ -1964,11 +1972,17 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.IsSet(StateHistoryFlag.Name) {
 		cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name)
 	}
-	scheme, err := ParseCLIAndConfigStateScheme(ctx.String(StateSchemeFlag.Name), cfg.StateScheme)
-	if err != nil {
-		Fatalf("%v", err)
+	if ctx.String(StateSchemeFlag.Name) != rawdb.VersionScheme {
+		scheme, err := ParseCLIAndConfigStateScheme(ctx.String(StateSchemeFlag.Name), cfg.StateScheme)
+		if err != nil {
+			Fatalf("%v", err)
+		}
+		cfg.StateScheme = scheme
+	} else {
+		// TODO:: compatible with cli line and configuration file, currently only supports cli.
+		cfg.StateScheme = rawdb.VersionScheme
 	}
-	cfg.StateScheme = scheme
+
 	// Parse transaction history flag, if user is still using legacy config
 	// file with 'TxLookupLimit' configured, copy the value to 'TransactionHistory'.
 	if cfg.TransactionHistory == ethconfig.Defaults.TransactionHistory && cfg.TxLookupLimit != ethconfig.Defaults.TxLookupLimit {
@@ -2083,7 +2097,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		}
 		cfg.Genesis = core.DefaultBSCGenesisBlock()
 		SetDNSDiscoveryDefaults(cfg, params.BSCGenesisHash)
-	case ctx.Bool(ChapelFlag.Name):
+	case ctx.Bool(ChapelFlag.Name) || cfg.NetworkId == 97:
 		if !ctx.IsSet(NetworkIdFlag.Name) {
 			cfg.NetworkId = 97
 		}

cmd/utils/history_test.go

@@ -83,7 +83,7 @@ func TestHistoryImportAndExport(t *testing.T) {
 		t.Fatalf("unable to initialize chain: %v", err)
 	}
 	if _, err := chain.InsertChain(blocks); err != nil {
-		t.Fatalf("error insterting chain: %v", err)
+		t.Fatalf("error inserting chain: %v", err)
 	}
 
 	// Make temp directory for era files.

consensus/parlia/parlia.go

@@ -307,6 +307,10 @@ func New(
 	return c
 }
 
+func (p *Parlia) Period() uint64 {
+	return p.config.Period
+}
+
 func (p *Parlia) IsSystemTransaction(tx *types.Transaction, header *types.Header) (bool, error) {
 	// deploy a contract
 	if tx.To() == nil {

core/blockchain.go

@@ -196,6 +196,10 @@ func (c *CacheConfig) triedbConfig() *triedb.Config {
 			JournalFile:     c.JournalFile,
 		}
 	}
+	// TODO:: support other versa db config items, currently use the default config
+	if c.StateScheme == rawdb.VersionScheme {
+		config.IsVersion = true
+	}
 	return config
 }
 
@@ -301,6 +305,7 @@ type BlockChain struct {
 	diffLayerFreezerBlockLimit uint64
 
 	wg            sync.WaitGroup
+	dbWg          sync.WaitGroup
 	quit          chan struct{} // shutdown signal, closed in Stop.
 	stopping      atomic.Bool   // false if chain is running, true when stopped
 	procInterrupt atomic.Bool   // interrupt signaler for block processing
@@ -383,7 +388,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 	}
 	bc.flushInterval.Store(int64(cacheConfig.TrieTimeLimit))
 	bc.forker = NewForkChoice(bc, shouldPreserve)
-	bc.stateCache = state.NewDatabaseWithNodeDB(bc.db, bc.triedb)
+	bc.stateCache = state.NewDatabaseWithNodeDB(bc.db, bc.triedb, true)
 	bc.validator = NewBlockValidator(chainConfig, bc, engine)
 	bc.prefetcher = NewStatePrefetcher(chainConfig, bc, engine)
 	bc.processor = NewStateProcessor(chainConfig, bc, engine)
@@ -419,78 +424,101 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 	// Make sure the state associated with the block is available, or log out
 	// if there is no available state, waiting for state sync.
 	head := bc.CurrentBlock()
-	if !bc.HasState(head.Root) {
-		if head.Number.Uint64() == 0 {
-			// The genesis state is missing, which is only possible in the path-based
-			// scheme. This situation occurs when the initial state sync is not finished
-			// yet, or the chain head is rewound below the pivot point. In both scenarios,
-			// there is no possible recovery approach except for rerunning a snap sync.
-			// Do nothing here until the state syncer picks it up.
-			log.Info("Genesis state is missing, wait state sync")
-		} else {
-			// Head state is missing, before the state recovery, find out the
-			// disk layer point of snapshot(if it's enabled). Make sure the
-			// rewound point is lower than disk layer.
-			var diskRoot common.Hash
-			if bc.cacheConfig.SnapshotLimit > 0 {
-				diskRoot = rawdb.ReadSnapshotRoot(bc.db)
-			}
-			if bc.triedb.Scheme() == rawdb.PathScheme && !bc.NoTries() {
-				recoverable, _ := bc.triedb.Recoverable(diskRoot)
-				if !bc.HasState(diskRoot) && !recoverable {
-					diskRoot = bc.triedb.Head()
-				}
-			}
-			if diskRoot != (common.Hash{}) {
-				log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash(), "diskRoot", diskRoot)
-
-				snapDisk, err := bc.setHeadBeyondRoot(head.Number.Uint64(), 0, diskRoot, true)
-				if err != nil {
-					return nil, err
-				}
-				// Chain rewound, persist old snapshot number to indicate recovery procedure
-				if snapDisk != 0 {
-					rawdb.WriteSnapshotRecoveryNumber(bc.db, snapDisk)
-				}
+	if bc.triedb.Scheme() != rawdb.VersionScheme {
+		if !bc.HasState(head.Root) {
+			if head.Number.Uint64() == 0 {
+				// The genesis state is missing, which is only possible in the path-based
+				// scheme. This situation occurs when the initial state sync is not finished
+				// yet, or the chain head is rewound below the pivot point. In both scenarios,
+				// there is no possible recovery approach except for rerunning a snap sync.
+				// Do nothing here until the state syncer picks it up.
+				log.Info("Genesis state is missing, wait state sync")
 			} else {
-				log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash())
-				if _, err := bc.setHeadBeyondRoot(head.Number.Uint64(), 0, common.Hash{}, true); err != nil {
+				// Head state is missing, before the state recovery, find out the
+				// disk layer point of snapshot(if it's enabled). Make sure the
+				// rewound point is lower than disk layer.
+				var diskRoot common.Hash
+				if bc.cacheConfig.SnapshotLimit > 0 {
+					diskRoot = rawdb.ReadSnapshotRoot(bc.db)
+				}
+				if bc.triedb.Scheme() == rawdb.PathScheme && !bc.NoTries() {
+					recoverable, _ := bc.triedb.Recoverable(diskRoot)
+					if !bc.HasState(diskRoot) && !recoverable {
+						diskRoot = bc.triedb.Head()
+					}
+				}
+				if diskRoot != (common.Hash{}) {
+					log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash(), "diskRoot", diskRoot)
+
+					snapDisk, err := bc.setHeadBeyondRoot(head.Number.Uint64(), 0, diskRoot, true)
+					if err != nil {
+						return nil, err
+					}
+					// Chain rewound, persist old snapshot number to indicate recovery procedure
+					if snapDisk != 0 {
+						rawdb.WriteSnapshotRecoveryNumber(bc.db, snapDisk)
+					}
+				} else {
+					log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash())
+					if _, err := bc.setHeadBeyondRoot(head.Number.Uint64(), 0, common.Hash{}, true); err != nil {
+						return nil, err
+					}
+				}
+			}
+		}
+	} else {
+		log.Warn("versa db no recovery, rewind in load state")
+	}
+	// Ensure that a previous crash in SetHead doesn't leave extra ancients
+	if bc.triedb.Scheme() != rawdb.VersionScheme {
+		if frozen, err := bc.db.BlockStore().ItemAmountInAncient(); err == nil && frozen > 0 {
+			frozen, err = bc.db.BlockStore().Ancients()
+			if err != nil {
+				return nil, err
+			}
+			var (
+				needRewind bool
+				low        uint64
+			)
+			// The head full block may be rolled back to a very low height due to
+			// blockchain repair. If the head full block is even lower than the ancient
+			// chain, truncate the ancient store.
+			fullBlock := bc.CurrentBlock()
+			if fullBlock != nil && fullBlock.Hash() != bc.genesisBlock.Hash() && fullBlock.Number.Uint64() < frozen-1 {
+				needRewind = true
+				low = fullBlock.Number.Uint64()
+			}
+			// In snap sync, it may happen that ancient data has been written to the
+			// ancient store, but the LastFastBlock has not been updated, truncate the
+			// extra data here.
+			snapBlock := bc.CurrentSnapBlock()
+			if snapBlock != nil && snapBlock.Number.Uint64() < frozen-1 {
+				needRewind = true
+				if snapBlock.Number.Uint64() < low || low == 0 {
+					low = snapBlock.Number.Uint64()
+				}
+			}
+			if needRewind {
+				log.Error("Truncating ancient chain", "from", bc.CurrentHeader().Number.Uint64(), "to", low)
+				if err := bc.SetHead(low); err != nil {
 					return nil, err
 				}
 			}
 		}
-	}
-	// Ensure that a previous crash in SetHead doesn't leave extra ancients
-	if frozen, err := bc.db.ItemAmountInAncient(); err == nil && frozen > 0 {
-		frozen, err = bc.db.Ancients()
+	} else {
+		//TODO:: need consider the offline and inline prune block
+		frozen, err := bc.db.BlockStore().Ancients()
+		if err != nil {
+			return nil, err
+		}
+		items, err := bc.db.BlockStore().ItemAmountInAncient()
 		if err != nil {
 			return nil, err
 		}
-		var (
-			needRewind bool
-			low        uint64
-		)
-		// The head full block may be rolled back to a very low height due to
-		// blockchain repair. If the head full block is even lower than the ancient
-		// chain, truncate the ancient store.
 		fullBlock := bc.CurrentBlock()
-		if fullBlock != nil && fullBlock.Hash() != bc.genesisBlock.Hash() && fullBlock.Number.Uint64() < frozen-1 {
-			needRewind = true
-			low = fullBlock.Number.Uint64()
-		}
-		// In snap sync, it may happen that ancient data has been written to the
-		// ancient store, but the LastFastBlock has not been updated, truncate the
-		// extra data here.
-		snapBlock := bc.CurrentSnapBlock()
-		if snapBlock != nil && snapBlock.Number.Uint64() < frozen-1 {
-			needRewind = true
-			if snapBlock.Number.Uint64() < low || low == 0 {
-				low = snapBlock.Number.Uint64()
-			}
-		}
-		if needRewind {
-			log.Error("Truncating ancient chain", "from", bc.CurrentHeader().Number.Uint64(), "to", low)
-			if err := bc.SetHead(low); err != nil {
+		log.Info("version mode rewind ancient store", "target", fullBlock.Number.Uint64(), "old head", frozen, "items", items, "offset", bc.db.BlockStore().AncientOffSet())
+		if frozen >= fullBlock.Number.Uint64() {
+			if _, err = bc.db.BlockStore().TruncateTail(fullBlock.Number.Uint64()); err != nil {
 				return nil, err
 			}
 		}
@@ -656,20 +684,13 @@ func (bc *BlockChain) cacheDiffLayer(diffLayer *types.DiffLayer, diffLayerCh cha
 	}
 }
 
-func (bc *BlockChain) cacheBlock(hash common.Hash, block *types.Block) {
-	bc.blockCache.Add(hash, block)
-	if bc.chainConfig.IsCancun(block.Number(), block.Time()) {
-		bc.sidecarsCache.Add(hash, block.Sidecars())
-	}
-}
-
 // empty returns an indicator whether the blockchain is empty.
 // Note, it's a special case that we connect a non-empty ancient
 // database with an empty node, so that we can plugin the ancient
 // into node seamlessly.
 func (bc *BlockChain) empty() bool {
 	genesis := bc.genesisBlock.Hash()
-	for _, hash := range []common.Hash{rawdb.ReadHeadBlockHash(bc.db.BlockStore()), rawdb.ReadHeadHeaderHash(bc.db.BlockStore()), rawdb.ReadHeadFastBlockHash(bc.db)} {
+	for _, hash := range []common.Hash{rawdb.ReadHeadBlockHash(bc.db), rawdb.ReadHeadHeaderHash(bc.db), rawdb.ReadHeadFastBlockHash(bc.db)} {
 		if hash != genesis {
 			return false
 		}
@@ -704,20 +725,54 @@ func (bc *BlockChain) getFinalizedNumber(header *types.Header) uint64 {
 // loadLastState loads the last known chain state from the database. This method
 // assumes that the chain manager mutex is held.
 func (bc *BlockChain) loadLastState() error {
-	// Restore the last known head block
-	head := rawdb.ReadHeadBlockHash(bc.db.BlockStore())
-	if head == (common.Hash{}) {
-		// Corrupt or empty database, init from scratch
-		log.Warn("Empty database, resetting chain")
-		return bc.Reset()
-	}
-	// Make sure the entire head block is available
-	headBlock := bc.GetBlockByHash(head)
-	if headBlock == nil {
-		// Corrupt or empty database, init from scratch
-		log.Warn("Head block missing, resetting chain", "hash", head)
-		return bc.Reset()
+	// TODO:: before versa db support recovery, only rewind
+	var headBlock *types.Block
+	if bc.triedb.Scheme() == rawdb.VersionScheme {
+		head := rawdb.ReadHeadBlockHash(bc.db)
+		headBlock = bc.GetBlockByHash(head)
+
+		versa := bc.triedb.VersaDB()
+		archiveVersion, _ := versa.LatestStoreDiskVersionInfo()
+		// empty chain
+		if archiveVersion == -1 {
+			archiveVersion = 0
+		}
+
+		if int64(headBlock.NumberU64()) < archiveVersion {
+			log.Crit("versa db disk version large than header block", "head number", headBlock.NumberU64(), "versa archive number", archiveVersion)
+		}
+		log.Info("begin rewind versa db head", "target", archiveVersion)
+		for {
+			if int64(headBlock.NumberU64()) == archiveVersion {
+				rawdb.WriteCanonicalHash(bc.db, headBlock.Hash(), headBlock.NumberU64())
+				rawdb.WriteHeadHeaderHash(bc.db, headBlock.Hash())
+				rawdb.WriteHeadBlockHash(bc.db, headBlock.Hash())
+				rawdb.WriteHeadFastBlockHash(bc.db, headBlock.Hash())
+				log.Info("reset versa db head block", "number", headBlock.NumberU64(), "hash", headBlock.Hash())
+				break
+			}
+			headBlock = rawdb.ReadBlock(bc.db, headBlock.ParentHash(), headBlock.NumberU64()-1)
+			if headBlock == nil {
+				panic("versa db rewind head is nil")
+			}
+		}
+	} else {
+		// Restore the last known head block
+		head := rawdb.ReadHeadBlockHash(bc.db)
+		if head == (common.Hash{}) {
+			// Corrupt or empty database, init from scratch
+			log.Warn("Empty database, resetting chain")
+			return bc.Reset()
+		}
+		// Make sure the entire head block is available
+		headBlock = bc.GetBlockByHash(head)
+		if headBlock == nil {
+			// Corrupt or empty database, init from scratch
+			log.Warn("Head block missing, resetting chain", "hash", head)
+			return bc.Reset()
+		}
 	}
+	log.Info("load state head block", "number", headBlock.NumberU64())
 
 	// Everything seems to be fine, set as the head block
 	bc.currentBlock.Store(headBlock.Header())
@@ -727,7 +782,7 @@ func (bc *BlockChain) loadLastState() error {
 
 	// Restore the last known head header
 	headHeader := headBlock.Header()
-	if head := rawdb.ReadHeadHeaderHash(bc.db.BlockStore()); head != (common.Hash{}) {
+	if head := rawdb.ReadHeadHeaderHash(bc.db); head != (common.Hash{}) {
 		if header := bc.GetHeaderByHash(head); header != nil {
 			headHeader = header
 		}
@@ -1106,7 +1161,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
 		// intent afterwards is full block importing, delete the chain segment
 		// between the stateful-block and the sethead target.
 		var wipe bool
-		frozen, _ := bc.db.Ancients()
+		frozen, _ := bc.db.BlockStore().Ancients()
 		if headNumber+1 < frozen {
 			wipe = pivot == nil || headNumber >= *pivot
 		}
@@ -1115,11 +1170,11 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
 	// Rewind the header chain, deleting all block bodies until then
 	delFn := func(db ethdb.KeyValueWriter, hash common.Hash, num uint64) {
 		// Ignore the error here since light client won't hit this path
-		frozen, _ := bc.db.Ancients()
+		frozen, _ := bc.db.BlockStore().Ancients()
 		if num+1 <= frozen {
 			// Truncate all relative data(header, total difficulty, body, receipt
 			// and canonical hash) from ancient store.
-			if _, err := bc.db.TruncateHead(num); err != nil {
+			if _, err := bc.db.BlockStore().TruncateHead(num); err != nil {
 				log.Crit("Failed to truncate ancient data", "number", num, "err", err)
 			}
 			// Remove the hash <-> number mapping from the active store.
@@ -1137,7 +1192,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
 	// If SetHead was only called as a chain reparation method, try to skip
 	// touching the header chain altogether, unless the freezer is broken
 	if repair {
-		if target, force := updateFn(bc.db, bc.CurrentBlock()); force {
+		if target, force := updateFn(bc.db.BlockStore(), bc.CurrentBlock()); force {
 			bc.hc.SetHead(target.Number.Uint64(), updateFn, delFn)
 		}
 	} else {
@@ -1171,6 +1226,10 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
 // SnapSyncCommitHead sets the current head block to the one defined by the hash
 // irrelevant what the chain contents were prior.
 func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error {
+	// TODO:: temporarily not support for snapsync
+	if bc.triedb.Scheme() == rawdb.VersionScheme {
+		panic("version db not support snap sync")
+	}
 	// Make sure that both the block as well at its state trie exists
 	block := bc.GetBlockByHash(hash)
 	if block == nil {
@@ -1298,19 +1357,33 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error {
 //
 // Note, this function assumes that the `mu` mutex is held!
 func (bc *BlockChain) writeHeadBlock(block *types.Block) {
-	// Add the block to the canonical chain number scheme and mark as the head
-	rawdb.WriteCanonicalHash(bc.db.BlockStore(), block.Hash(), block.NumberU64())
-	rawdb.WriteHeadHeaderHash(bc.db.BlockStore(), block.Hash())
-	rawdb.WriteHeadBlockHash(bc.db.BlockStore(), block.Hash())
+	bc.dbWg.Add(2)
+	defer bc.dbWg.Wait()
+	go func() {
+		defer bc.dbWg.Done()
+		// Add the block to the canonical chain number scheme and mark as the head
+		blockBatch := bc.db.BlockStore().NewBatch()
+		rawdb.WriteCanonicalHash(blockBatch, block.Hash(), block.NumberU64())
+		rawdb.WriteHeadHeaderHash(blockBatch, block.Hash())
+		rawdb.WriteHeadBlockHash(blockBatch, block.Hash())
+		rawdb.WriteHeadFastBlockHash(blockBatch, block.Hash())
+		// Flush the whole batch into the disk, exit the node if failed
+		if err := blockBatch.Write(); err != nil {
+			log.Crit("Failed to update chain indexes and markers in block db", "err", err)
+		}
+	}()
+	go func() {
+		defer bc.dbWg.Done()
 
-	batch := bc.db.NewBatch()
-	rawdb.WriteHeadFastBlockHash(batch, block.Hash())
-	rawdb.WriteTxLookupEntriesByBlock(batch, block)
+		batch := bc.db.NewBatch()
+		rawdb.WriteTxLookupEntriesByBlock(batch, block)
+
+		// Flush the whole batch into the disk, exit the node if failed
+		if err := batch.Write(); err != nil {
+			log.Crit("Failed to update chain indexes in chain db", "err", err)
+		}
+	}()
 
-	// Flush the whole batch into the disk, exit the node if failed
-	if err := batch.Write(); err != nil {
-		log.Crit("Failed to update chain indexes and markers", "err", err)
-	}
 	// Update all in-memory chain markers in the last step
 	bc.hc.SetCurrentHeader(block.Header())
 
@@ -1368,48 +1441,50 @@ func (bc *BlockChain) Stop() {
 		}
 		bc.snaps.Release()
 	}
-	if bc.triedb.Scheme() == rawdb.PathScheme {
-		// Ensure that the in-memory trie nodes are journaled to disk properly.
-		if err := bc.triedb.Journal(bc.CurrentBlock().Root); err != nil {
-			log.Info("Failed to journal in-memory trie nodes", "err", err)
-		}
-	} else {
-		// Ensure the state of a recent block is also stored to disk before exiting.
-		// We're writing three different states to catch different restart scenarios:
-		//  - HEAD:     So we don't need to reprocess any blocks in the general case
-		//  - HEAD-1:   So we don't do large reorgs if our HEAD becomes an uncle
-		//  - HEAD-127: So we have a hard limit on the number of blocks reexecuted
-		if !bc.cacheConfig.TrieDirtyDisabled {
-			triedb := bc.triedb
-			var once sync.Once
-			for _, offset := range []uint64{0, 1, TriesInMemory - 1} {
-				if number := bc.CurrentBlock().Number.Uint64(); number > offset {
-					recent := bc.GetBlockByNumber(number - offset)
-					log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
-					if err := triedb.Commit(recent.Root(), true); err != nil {
-						log.Error("Failed to commit recent state trie", "err", err)
-					} else {
-						rawdb.WriteSafePointBlockNumber(bc.db, recent.NumberU64())
-						once.Do(func() {
-							rawdb.WriteHeadBlockHash(bc.db.BlockStore(), recent.Hash())
-						})
+	if bc.triedb.Scheme() != rawdb.VersionScheme {
+		if bc.triedb.Scheme() == rawdb.PathScheme {
+			// Ensure that the in-memory trie nodes are journaled to disk properly.
+			if err := bc.triedb.Journal(bc.CurrentBlock().Root); err != nil {
+				log.Info("Failed to journal in-memory trie nodes", "err", err)
+			}
+		} else {
+			// Ensure the state of a recent block is also stored to disk before exiting.
+			// We're writing three different states to catch different restart scenarios:
+			//  - HEAD:     So we don't need to reprocess any blocks in the general case
+			//  - HEAD-1:   So we don't do large reorgs if our HEAD becomes an uncle
+			//  - HEAD-127: So we have a hard limit on the number of blocks reexecuted
+			if !bc.cacheConfig.TrieDirtyDisabled {
+				triedb := bc.triedb
+				var once sync.Once
+				for _, offset := range []uint64{0, 1, TriesInMemory - 1} {
+					if number := bc.CurrentBlock().Number.Uint64(); number > offset {
+						recent := bc.GetBlockByNumber(number - offset)
+						log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
+						if err := triedb.Commit(recent.Root(), true); err != nil {
+							log.Error("Failed to commit recent state trie", "err", err)
+						} else {
+							rawdb.WriteSafePointBlockNumber(bc.db, recent.NumberU64())
+							once.Do(func() {
+								rawdb.WriteHeadBlockHash(bc.db.BlockStore(), recent.Hash())
+							})
+						}
 					}
 				}
-			}
 
-			if snapBase != (common.Hash{}) {
-				log.Info("Writing snapshot state to disk", "root", snapBase)
-				if err := triedb.Commit(snapBase, true); err != nil {
-					log.Error("Failed to commit recent state trie", "err", err)
-				} else {
-					rawdb.WriteSafePointBlockNumber(bc.db, bc.CurrentBlock().Number.Uint64())
+				if snapBase != (common.Hash{}) {
+					log.Info("Writing snapshot state to disk", "root", snapBase)
+					if err := triedb.Commit(snapBase, true); err != nil {
+						log.Error("Failed to commit recent state trie", "err", err)
+					} else {
+						rawdb.WriteSafePointBlockNumber(bc.db, bc.CurrentBlock().Number.Uint64())
+					}
+				}
+				for !bc.triegc.Empty() {
+					triedb.Dereference(bc.triegc.PopItem())
+				}
+				if _, size, _, _ := triedb.Size(); size != 0 {
+					log.Error("Dangling trie nodes after full cleanup")
 				}
-			}
-			for !bc.triegc.Empty() {
-				triedb.Dereference(bc.triegc.PopItem())
-			}
-			if _, size, _, _ := triedb.Size(); size != 0 {
-				log.Error("Dangling trie nodes after full cleanup")
 			}
 		}
 	}
@@ -1531,7 +1606,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 			} else if !reorg {
 				return false
 			}
-			rawdb.WriteHeadFastBlockHash(bc.db, head.Hash())
+			rawdb.WriteHeadFastBlockHash(bc.db.BlockStore(), head.Hash())
 			bc.currentSnapBlock.Store(head.Header())
 			headFastBlockGauge.Update(int64(head.NumberU64()))
 			return true
@@ -1548,9 +1623,9 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 
 		// Ensure genesis is in ancients.
 		if first.NumberU64() == 1 {
-			if frozen, _ := bc.db.Ancients(); frozen == 0 {
+			if frozen, _ := bc.db.BlockStore().Ancients(); frozen == 0 {
 				td := bc.genesisBlock.Difficulty()
-				writeSize, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []types.Receipts{nil}, td)
+				writeSize, err := rawdb.WriteAncientBlocks(bc.db.BlockStore(), []*types.Block{bc.genesisBlock}, []types.Receipts{nil}, td)
 				if err != nil {
 					log.Error("Error writing genesis to ancients", "err", err)
 					return 0, err
@@ -1568,7 +1643,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 
 		// Write all chain data to ancients.
 		td := bc.GetTd(first.Hash(), first.NumberU64())
-		writeSize, err := rawdb.WriteAncientBlocksWithBlobs(bc.db, blockChain, receiptChain, td)
+		writeSize, err := rawdb.WriteAncientBlocksWithBlobs(bc.db.BlockStore(), blockChain, receiptChain, td)
 		if err != nil {
 			log.Error("Error importing chain data to ancients", "err", err)
 			return 0, err
@@ -1576,7 +1651,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 		size += writeSize
 
 		// Sync the ancient store explicitly to ensure all data has been flushed to disk.
-		if err := bc.db.Sync(); err != nil {
+		if err := bc.db.BlockStore().Sync(); err != nil {
 			return 0, err
 		}
 		// Update the current snap block because all block data is now present in DB.
@@ -1584,7 +1659,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 		if !updateHead(blockChain[len(blockChain)-1]) {
 			// We end up here if the header chain has reorg'ed, and the blocks/receipts
 			// don't match the canonical chain.
-			if _, err := bc.db.TruncateHead(previousSnapBlock + 1); err != nil {
+			if _, err := bc.db.BlockStore().TruncateHead(previousSnapBlock + 1); err != nil {
 				log.Error("Can't truncate ancient store after failed insert", "err", err)
 			}
 			return 0, errSideChainReceipts
@@ -1604,7 +1679,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 			rawdb.DeleteBlockWithoutNumber(blockBatch, block.Hash(), block.NumberU64())
 		}
 		// Delete side chain hash-to-number mappings.
-		for _, nh := range rawdb.ReadAllHashesInRange(bc.db, first.NumberU64(), last.NumberU64()) {
+		for _, nh := range rawdb.ReadAllHashesInRange(bc.db.BlockStore(), first.NumberU64(), last.NumberU64()) {
 			if _, canon := canonHashes[nh.Hash]; !canon {
 				rawdb.DeleteHeader(blockBatch, nh.Hash, nh.Number)
 			}
@@ -1774,7 +1849,6 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 	wg := sync.WaitGroup{}
 	wg.Add(1)
 	go func() {
-		rawdb.WritePreimages(bc.db, state.Preimages())
 		blockBatch := bc.db.BlockStore().NewBatch()
 		rawdb.WriteTd(blockBatch, block.Hash(), block.NumberU64(), externTd)
 		rawdb.WriteBlock(blockBatch, block)
@@ -1783,10 +1857,20 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 		if bc.chainConfig.IsCancun(block.Number(), block.Time()) {
 			rawdb.WriteBlobSidecars(blockBatch, block.Hash(), block.NumberU64(), block.Sidecars())
 		}
-		rawdb.WritePreimages(blockBatch, state.Preimages())
+		if bc.db.StateStore() != nil {
+			rawdb.WritePreimages(bc.db.StateStore(), state.Preimages())
+		} else {
+			rawdb.WritePreimages(blockBatch, state.Preimages())
+		}
 		if err := blockBatch.Write(); err != nil {
 			log.Crit("Failed to write block into disk", "err", err)
 		}
+		bc.hc.tdCache.Add(block.Hash(), externTd)
+		bc.blockCache.Add(block.Hash(), block)
+		bc.receiptsCache.Add(block.Hash(), receipts)
+		if bc.chainConfig.IsCancun(block.Number(), block.Time()) {
+			bc.sidecarsCache.Add(block.Hash(), block.Sidecars())
+		}
 		wg.Done()
 	}()
 
@@ -1796,7 +1880,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 
 		// If node is running in path mode, skip explicit gc operation
 		// which is unnecessary in this mode.
-		if bc.triedb.Scheme() == rawdb.PathScheme {
+		if bc.triedb.Scheme() != rawdb.HashScheme {
 			return nil
 		}
 
@@ -2215,28 +2299,36 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 			parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1)
 		}
 
+		if bc.stateCache.Scheme() != rawdb.VersionScheme {
+			if block.NumberU64() == 2000001 {
+				log.Crit("exit.... path mode, 200w blocks")
+			}
+		}
+
+		bc.stateCache.SetVersion(int64(block.NumberU64()))
 		statedb, err := state.NewWithSharedPool(parent.Root, bc.stateCache, bc.snaps)
 		if err != nil {
+			bc.stateCache.Release()
 			return it.index, err
 		}
 		bc.updateHighestVerifiedHeader(block.Header())
 
 		// Enable prefetching to pull in trie node paths while processing transactions
-		statedb.StartPrefetcher("chain")
+		//statedb.StartPrefetcher("chain")
 		interruptCh := make(chan struct{})
 		// For diff sync, it may fallback to full sync, so we still do prefetch
-		if len(block.Transactions()) >= prefetchTxNumber {
-			// do Prefetch in a separate goroutine to avoid blocking the critical path
-
-			// 1.do state prefetch for snapshot cache
-			throwaway := statedb.CopyDoPrefetch()
-			go bc.prefetcher.Prefetch(block, throwaway, &bc.vmConfig, interruptCh)
-
-			// 2.do trie prefetch for MPT trie node cache
-			// it is for the big state trie tree, prefetch based on transaction's From/To address.
-			// trie prefetcher is thread safe now, ok to prefetch in a separate routine
-			go throwaway.TriePrefetchInAdvance(block, signer)
-		}
+		//if len(block.Transactions()) >= prefetchTxNumber {
+		//	// do Prefetch in a separate goroutine to avoid blocking the critical path
+		//
+		//	// 1.do state prefetch for snapshot cache
+		//	throwaway := statedb.CopyDoPrefetch()
+		//	go bc.prefetcher.Prefetch(block, throwaway, &bc.vmConfig, interruptCh)
+		//
+		//	// 2.do trie prefetch for MPT trie node cache
+		//	// it is for the big state trie tree, prefetch based on transaction's From/To address.
+		//	// trie prefetcher is thread safe now, ok to prefetch in a separate routine
+		//	go throwaway.TriePrefetchInAdvance(block, signer)
+		//}
 
 		// Process block using the parent state as reference point
 		if bc.pipeCommit {
@@ -2247,6 +2339,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 		statedb, receipts, logs, usedGas, err := bc.processor.Process(block, statedb, bc.vmConfig)
 		close(interruptCh) // state prefetch can be stopped
 		if err != nil {
+			bc.stateCache.Release()
 			bc.reportBlock(block, receipts, err)
 			statedb.StopPrefetcher()
 			return it.index, err
@@ -2256,7 +2349,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 		// Validate the state using the default validator
 		vstart := time.Now()
 		if err := bc.validator.ValidateState(block, statedb, receipts, usedGas); err != nil {
-			log.Error("validate state failed", "error", err)
+			bc.stateCache.Release()
 			bc.reportBlock(block, receipts, err)
 			statedb.StopPrefetcher()
 			return it.index, err
@@ -2264,8 +2357,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 		vtime := time.Since(vstart)
 		proctime := time.Since(start) // processing + validation
 
-		bc.cacheBlock(block.Hash(), block)
-
 		// Update the metrics touched during block processing and validation
 		accountReadTimer.Update(statedb.AccountReads)                   // Account reads are complete(in processing)
 		storageReadTimer.Update(statedb.StorageReads)                   // Storage reads are complete(in processing)
@@ -2294,8 +2385,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 			status, err = bc.writeBlockAndSetHead(block, receipts, logs, statedb, false)
 		}
 		if err != nil {
+			bc.stateCache.Release()
 			return it.index, err
 		}
+		bc.stateCache.Release()
 
 		bc.cacheReceipts(block.Hash(), receipts, block)
 
@@ -2378,26 +2471,11 @@ func (bc *BlockChain) updateHighestVerifiedHeader(header *types.Header) {
 	if header == nil || header.Number == nil {
 		return
 	}
-	currentHeader := bc.highestVerifiedHeader.Load()
-	if currentHeader == nil {
+	currentBlock := bc.CurrentBlock()
+	reorg, err := bc.forker.ReorgNeededWithFastFinality(currentBlock, header)
+	if err == nil && reorg {
 		bc.highestVerifiedHeader.Store(types.CopyHeader(header))
-		return
-	}
-
-	newParentTD := bc.GetTd(header.ParentHash, header.Number.Uint64()-1)
-	if newParentTD == nil {
-		newParentTD = big.NewInt(0)
-	}
-	oldParentTD := bc.GetTd(currentHeader.ParentHash, currentHeader.Number.Uint64()-1)
-	if oldParentTD == nil {
-		oldParentTD = big.NewInt(0)
-	}
-	newTD := big.NewInt(0).Add(newParentTD, header.Difficulty)
-	oldTD := big.NewInt(0).Add(oldParentTD, currentHeader.Difficulty)
-
-	if newTD.Cmp(oldTD) > 0 {
-		bc.highestVerifiedHeader.Store(types.CopyHeader(header))
-		return
+		log.Trace("updateHighestVerifiedHeader", "number", header.Number.Uint64(), "hash", header.Hash())
 	}
 }
 

core/blockchain_reader.go

@@ -231,7 +231,7 @@ func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
 	if receipts, ok := bc.receiptsCache.Get(hash); ok {
 		return receipts
 	}
-	number := rawdb.ReadHeaderNumber(bc.db.BlockStore(), hash)
+	number := rawdb.ReadHeaderNumber(bc.db, hash)
 	if number == nil {
 		return nil
 	}
@@ -348,8 +348,7 @@ func (bc *BlockChain) HasState(hash common.Hash) bool {
 			return true
 		}
 	}
-	_, err := bc.stateCache.OpenTrie(hash)
-	return err == nil
+	return bc.stateCache.HasState(hash)
 }
 
 // HasBlockAndState checks if a block and associated state trie is fully present
@@ -396,7 +395,8 @@ func (bc *BlockChain) State() (*state.StateDB, error) {
 
 // StateAt returns a new mutable state based on a particular point in time.
 func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) {
-	stateDb, err := state.New(root, bc.stateCache, bc.snaps)
+	// new state db with no need commit mode
+	stateDb, err := state.New(root, state.NewDatabaseWithNodeDB(bc.db, bc.triedb, false), bc.snaps)
 	if err != nil {
 		return nil, err
 	}
@@ -514,7 +514,7 @@ func (bc *BlockChain) SubscribeFinalizedHeaderEvent(ch chan<- FinalizedHeaderEve
 
 // AncientTail retrieves the tail the ancients blocks
 func (bc *BlockChain) AncientTail() (uint64, error) {
-	tail, err := bc.db.Tail()
+	tail, err := bc.db.BlockStore().Tail()
 	if err != nil {
 		return 0, err
 	}

core/blockchain_repair_test.go

@@ -26,6 +26,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/ethdb"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core/rawdb"
@@ -1795,6 +1797,13 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
 		config.SnapshotWait = true
 	}
 	config.TriesInMemory = 128
+
+	if err = db.SetupFreezerEnv(&ethdb.FreezerEnv{
+		ChainCfg:         gspec.Config,
+		BlobExtraReserve: params.DefaultExtraReserveForBlobRequests,
+	}); err != nil {
+		t.Fatalf("Failed to create chain: %v", err)
+	}
 	chain, err := NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to create chain: %v", err)

core/blockchain_sethead_test.go

@@ -27,6 +27,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/ethdb"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core/rawdb"
@@ -1998,6 +2000,13 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme
 		config.SnapshotWait = true
 	}
 	config.TriesInMemory = 128
+
+	if err = db.SetupFreezerEnv(&ethdb.FreezerEnv{
+		ChainCfg:         gspec.Config,
+		BlobExtraReserve: params.DefaultExtraReserveForBlobRequests,
+	}); err != nil {
+		t.Fatalf("Failed to create chain: %v", err)
+	}
 	chain, err := NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to create chain: %v", err)

core/chain_indexer.go

@@ -227,8 +227,8 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainH
 				// Reorg to the common ancestor if needed (might not exist in light sync mode, skip reorg then)
 				// TODO(karalabe, zsfelfoldi): This seems a bit brittle, can we detect this case explicitly?
 
-				if rawdb.ReadCanonicalHash(c.chainDb.BlockStore(), prevHeader.Number.Uint64()) != prevHash {
-					if h := rawdb.FindCommonAncestor(c.chainDb.BlockStore(), prevHeader, header); h != nil {
+				if rawdb.ReadCanonicalHash(c.chainDb, prevHeader.Number.Uint64()) != prevHash {
+					if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, header); h != nil {
 						c.newHead(h.Number.Uint64(), true)
 					}
 				}

core/chain_makers.go

@@ -401,7 +401,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 	defer triedb.Close()
 
 	for i := 0; i < n; i++ {
-		statedb, err := state.New(parent.Root(), state.NewDatabaseWithNodeDB(db, triedb), nil)
+		statedb, err := state.New(parent.Root(), state.NewDatabaseWithNodeDB(db, triedb, true), nil)
 		if err != nil {
 			panic(err)
 		}

core/forkchoice.go

@@ -86,9 +86,16 @@ func (f *ForkChoice) ReorgNeeded(current *types.Header, extern *types.Header) (b
 		localTD  = f.chain.GetTd(current.Hash(), current.Number.Uint64())
 		externTd = f.chain.GetTd(extern.Hash(), extern.Number.Uint64())
 	)
-	if localTD == nil || externTd == nil {
+	if localTD == nil {
 		return false, errors.New("missing td")
 	}
+	if externTd == nil {
+		ptd := f.chain.GetTd(extern.ParentHash, extern.Number.Uint64()-1)
+		if ptd == nil {
+			return false, consensus.ErrUnknownAncestor
+		}
+		externTd = new(big.Int).Add(ptd, extern.Difficulty)
+	}
 	// Accept the new header as the chain head if the transition
 	// is already triggered. We assume all the headers after the
 	// transition come from the trusted consensus layer.
@@ -114,7 +121,9 @@ func (f *ForkChoice) ReorgNeeded(current *types.Header, extern *types.Header) (b
 		if f.preserve != nil {
 			currentPreserve, externPreserve = f.preserve(current), f.preserve(extern)
 		}
-		reorg = !currentPreserve && (externPreserve || f.rand.Float64() < 0.5)
+		reorg = !currentPreserve && (externPreserve ||
+			extern.Time < current.Time ||
+			extern.Time == current.Time && f.rand.Float64() < 0.5)
 	}
 	return reorg, nil
 }

core/genesis.go

@@ -126,7 +126,10 @@ func hashAlloc(ga *types.GenesisAlloc, isVerkle bool) (common.Hash, error) {
 	}
 	// Create an ephemeral in-memory database for computing hash,
 	// all the derived states will be discarded to not pollute disk.
-	db := state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), config)
+	db := state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), config, true)
+	log.Info("genesis calc root hash use hash mode triedb")
+	db.SetVersion(0)
+	defer db.Release()
 	statedb, err := state.New(types.EmptyRootHash, db, nil)
 	if err != nil {
 		return common.Hash{}, err
@@ -154,7 +157,10 @@ func flushAlloc(ga *types.GenesisAlloc, db ethdb.Database, triedb *triedb.Databa
 	if triedbConfig != nil {
 		triedbConfig.NoTries = false
 	}
-	statedb, err := state.New(types.EmptyRootHash, state.NewDatabaseWithNodeDB(db, triedb), nil)
+	cachingdb := state.NewDatabaseWithNodeDB(db, triedb, true)
+	cachingdb.SetVersion(0)
+	defer cachingdb.Release()
+	statedb, err := state.New(types.EmptyRootHash, cachingdb, nil)
 	if err != nil {
 		return err
 	}
@@ -174,7 +180,7 @@ func flushAlloc(ga *types.GenesisAlloc, db ethdb.Database, triedb *triedb.Databa
 		return err
 	}
 	// Commit newly generated states into disk if it's not empty.
-	if root != types.EmptyRootHash {
+	if root != types.EmptyRootHash && triedb.Scheme() != rawdb.VersionScheme {
 		if err := triedb.Commit(root, true); err != nil {
 			return err
 		}
@@ -216,8 +222,6 @@ func (e *GenesisMismatchError) Error() string {
 // ChainOverrides contains the changes to chain config
 // Typically, these modifications involve hardforks that are not enabled on the BSC mainnet, intended for testing purposes.
 type ChainOverrides struct {
-	OverrideCancun *uint64
-	OverrideHaber  *uint64
 	OverrideBohr   *uint64
 	OverrideVerkle *uint64
 }
@@ -245,12 +249,6 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g
 	}
 	applyOverrides := func(config *params.ChainConfig) {
 		if config != nil {
-			if overrides != nil && overrides.OverrideCancun != nil {
-				config.CancunTime = overrides.OverrideCancun
-			}
-			if overrides != nil && overrides.OverrideHaber != nil {
-				config.HaberTime = overrides.OverrideHaber
-			}
 			if overrides != nil && overrides.OverrideBohr != nil {
 				config.BohrTime = overrides.OverrideBohr
 			}
@@ -498,7 +496,7 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Blo
 	rawdb.WriteReceipts(db.BlockStore(), block.Hash(), block.NumberU64(), nil)
 	rawdb.WriteCanonicalHash(db.BlockStore(), block.Hash(), block.NumberU64())
 	rawdb.WriteHeadBlockHash(db.BlockStore(), block.Hash())
-	rawdb.WriteHeadFastBlockHash(db, block.Hash())
+	rawdb.WriteHeadFastBlockHash(db.BlockStore(), block.Hash())
 	rawdb.WriteHeadHeaderHash(db.BlockStore(), block.Hash())
 	rawdb.WriteChainConfig(db, block.Hash(), config)
 	return block, nil

core/headerchain.go

@@ -97,7 +97,7 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c
 		return nil, ErrNoGenesis
 	}
 	hc.currentHeader.Store(hc.genesisHeader)
-	if head := rawdb.ReadHeadBlockHash(chainDb.BlockStore()); head != (common.Hash{}) {
+	if head := rawdb.ReadHeadBlockHash(chainDb); head != (common.Hash{}) {
 		if chead := hc.GetHeaderByHash(head); chead != nil {
 			hc.currentHeader.Store(chead)
 		}
@@ -144,7 +144,7 @@ func (hc *HeaderChain) GetBlockNumber(hash common.Hash) *uint64 {
 	if cached, ok := hc.numberCache.Get(hash); ok {
 		return &cached
 	}
-	number := rawdb.ReadHeaderNumber(hc.chainDb.BlockStore(), hash)
+	number := rawdb.ReadHeaderNumber(hc.chainDb, hash)
 	if number != nil {
 		hc.numberCache.Add(hash, *number)
 	}
@@ -668,7 +668,7 @@ func (hc *HeaderChain) setHead(headBlock uint64, headTime uint64, updateFn Updat
 		// first then remove the relative data from the database.
 		//
 		// Update head first(head fast block, head full block) before deleting the data.
-		markerBatch := hc.chainDb.NewBatch()
+		markerBatch := hc.chainDb.BlockStore().NewBatch()
 		if updateFn != nil {
 			newHead, force := updateFn(markerBatch, parent)
 			if force && ((headTime > 0 && newHead.Time < headTime) || (headTime == 0 && newHead.Number.Uint64() < headBlock)) {
@@ -677,7 +677,7 @@ func (hc *HeaderChain) setHead(headBlock uint64, headTime uint64, updateFn Updat
 			}
 		}
 		// Update head header then.
-		rawdb.WriteHeadHeaderHash(hc.chainDb.BlockStore(), parentHash)
+		rawdb.WriteHeadHeaderHash(markerBatch, parentHash)
 		if err := markerBatch.Write(); err != nil {
 			log.Crit("Failed to update chain markers", "error", err)
 		}
@@ -691,7 +691,7 @@ func (hc *HeaderChain) setHead(headBlock uint64, headTime uint64, updateFn Updat
 		// we don't end up with dangling daps in the database
 		var nums []uint64
 		if origin {
-			for n := num + 1; len(rawdb.ReadAllHashes(hc.chainDb, n)) > 0; n++ {
+			for n := num + 1; len(rawdb.ReadAllHashes(hc.chainDb.BlockStore(), n)) > 0; n++ {
 				nums = append([]uint64{n}, nums...) // suboptimal, but we don't really expect this path
 			}
 			origin = false
@@ -701,7 +701,7 @@ func (hc *HeaderChain) setHead(headBlock uint64, headTime uint64, updateFn Updat
 		// Remove the related data from the database on all sidechains
 		for _, num := range nums {
 			// Gather all the side fork hashes
-			hashes := rawdb.ReadAllHashes(hc.chainDb, num)
+			hashes := rawdb.ReadAllHashes(hc.chainDb.BlockStore(), num)
 			if len(hashes) == 0 {
 				// No hashes in the database whatsoever, probably frozen already
 				hashes = append(hashes, hdr.Hash())

core/rawdb/accessors_chain.go

@@ -34,6 +34,15 @@ import (
 	"golang.org/x/exp/slices"
 )
 
+// Support Multi-Database Based on Data Pattern, the Chaindata will be divided into three stores: BlockStore, StateStore, and ChainStore,
+// according to data schema and read/write behavior. When using the following data interfaces, you should take note of the following:
+//
+// 1) Block-Related Data: For CanonicalHash, Header, Body, Td, Receipts, and BlobSidecars, the Write, Delete, and Iterator
+// 	operations should carefully ensure that the database being used is BlockStore.
+// 2) Meta-Related Data: For HeaderNumber, HeadHeaderHash, HeadBlockHash, HeadFastBlockHash, and FinalizedBlockHash, the
+// 	Write and Delete operations should carefully ensure that the database being used is BlockStore.
+// 3) Ancient Data: When using a multi-database, Ancient data will use the BlockStore.
+
 // ReadCanonicalHash retrieves the hash assigned to a canonical block number.
 func ReadCanonicalHash(db ethdb.Reader, number uint64) common.Hash {
 	var data []byte
@@ -144,8 +153,8 @@ func ReadAllCanonicalHashes(db ethdb.Iteratee, from uint64, to uint64, limit int
 }
 
 // ReadHeaderNumber returns the header number assigned to a hash.
-func ReadHeaderNumber(db ethdb.KeyValueReader, hash common.Hash) *uint64 {
-	data, _ := db.Get(headerNumberKey(hash))
+func ReadHeaderNumber(db ethdb.MultiDatabaseReader, hash common.Hash) *uint64 {
+	data, _ := db.BlockStoreReader().Get(headerNumberKey(hash))
 	if len(data) != 8 {
 		return nil
 	}
@@ -170,8 +179,8 @@ func DeleteHeaderNumber(db ethdb.KeyValueWriter, hash common.Hash) {
 }
 
 // ReadHeadHeaderHash retrieves the hash of the current canonical head header.
-func ReadHeadHeaderHash(db ethdb.KeyValueReader) common.Hash {
-	data, _ := db.Get(headHeaderKey)
+func ReadHeadHeaderHash(db ethdb.MultiDatabaseReader) common.Hash {
+	data, _ := db.BlockStoreReader().Get(headHeaderKey)
 	if len(data) == 0 {
 		return common.Hash{}
 	}
@@ -186,8 +195,8 @@ func WriteHeadHeaderHash(db ethdb.KeyValueWriter, hash common.Hash) {
 }
 
 // ReadHeadBlockHash retrieves the hash of the current canonical head block.
-func ReadHeadBlockHash(db ethdb.KeyValueReader) common.Hash {
-	data, _ := db.Get(headBlockKey)
+func ReadHeadBlockHash(db ethdb.MultiDatabaseReader) common.Hash {
+	data, _ := db.BlockStoreReader().Get(headBlockKey)
 	if len(data) == 0 {
 		return common.Hash{}
 	}
@@ -202,8 +211,8 @@ func WriteHeadBlockHash(db ethdb.KeyValueWriter, hash common.Hash) {
 }
 
 // ReadHeadFastBlockHash retrieves the hash of the current fast-sync head block.
-func ReadHeadFastBlockHash(db ethdb.KeyValueReader) common.Hash {
-	data, _ := db.Get(headFastBlockKey)
+func ReadHeadFastBlockHash(db ethdb.MultiDatabaseReader) common.Hash {
+	data, _ := db.BlockStoreReader().Get(headFastBlockKey)
 	if len(data) == 0 {
 		return common.Hash{}
 	}
@@ -218,8 +227,8 @@ func WriteHeadFastBlockHash(db ethdb.KeyValueWriter, hash common.Hash) {
 }
 
 // ReadFinalizedBlockHash retrieves the hash of the finalized block.
-func ReadFinalizedBlockHash(db ethdb.KeyValueReader) common.Hash {
-	data, _ := db.Get(headFinalizedBlockKey)
+func ReadFinalizedBlockHash(db ethdb.MultiDatabaseReader) common.Hash {
+	data, _ := db.BlockStoreReader().Get(headFinalizedBlockKey)
 	if len(data) == 0 {
 		return common.Hash{}
 	}
@@ -297,7 +306,7 @@ func ReadHeaderRange(db ethdb.Reader, number uint64, count uint64) []rlp.RawValu
 		// It's ok to request block 0, 1 item
 		count = number + 1
 	}
-	limit, _ := db.Ancients()
+	limit, _ := db.BlockStoreReader().Ancients()
 	// First read live blocks
 	if i >= limit {
 		// If we need to read live blocks, we need to figure out the hash first
@@ -317,7 +326,7 @@ func ReadHeaderRange(db ethdb.Reader, number uint64, count uint64) []rlp.RawValu
 		return rlpHeaders
 	}
 	// read remaining from ancients, cap at 2M
-	data, err := db.AncientRange(ChainFreezerHeaderTable, i+1-count, count, 2*1024*1024)
+	data, err := db.BlockStoreReader().AncientRange(ChainFreezerHeaderTable, i+1-count, count, 2*1024*1024)
 	if err != nil {
 		log.Error("Failed to read headers from freezer", "err", err)
 		return rlpHeaders
@@ -468,7 +477,7 @@ func ReadCanonicalBodyRLP(db ethdb.Reader, number uint64) rlp.RawValue {
 		// Block is not in ancients, read from leveldb by hash and number.
 		// Note: ReadCanonicalHash cannot be used here because it also
 		// calls ReadAncients internally.
-		hash, _ := db.Get(headerHashKey(number))
+		hash, _ := db.BlockStoreReader().Get(headerHashKey(number))
 		data, _ = db.BlockStoreReader().Get(blockBodyKey(number, common.BytesToHash(hash)))
 		return nil
 	})
@@ -516,6 +525,13 @@ func WriteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64, body *t
 	WriteBodyRLP(db, hash, number, data)
 }
 
+// DeleteBody removes all block body data associated with a hash.
+func DeleteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
+	if err := db.Delete(blockBodyKey(number, hash)); err != nil {
+		log.Crit("Failed to delete block body", "err", err)
+	}
+}
+
 func WriteDiffLayer(db ethdb.KeyValueWriter, hash common.Hash, layer *types.DiffLayer) {
 	data, err := rlp.EncodeToBytes(layer)
 	if err != nil {
@@ -554,13 +570,6 @@ func DeleteDiffLayer(db ethdb.KeyValueWriter, blockHash common.Hash) {
 	}
 }
 
-// DeleteBody removes all block body data associated with a hash.
-func DeleteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
-	if err := db.Delete(blockBodyKey(number, hash)); err != nil {
-		log.Crit("Failed to delete block body", "err", err)
-	}
-}
-
 // ReadTdRLP retrieves a block's total difficulty corresponding to the hash in RLP encoding.
 func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue {
 	var data []byte
@@ -884,7 +893,7 @@ func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts
 // ReadBlobSidecarsRLP retrieves all the transaction blobs belonging to a block in RLP encoding.
 func ReadBlobSidecarsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue {
 	var data []byte
-	db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
+	db.BlockStoreReader().ReadAncients(func(reader ethdb.AncientReaderOp) error {
 		// Check if the data is in ancients
 		if isCanon(reader, number, hash) {
 			data, _ = reader.Ancient(ChainFreezerBlobSidecarTable, number)
@@ -1093,24 +1102,24 @@ func FindCommonAncestor(db ethdb.Reader, a, b *types.Header) *types.Header {
 
 // ReadHeadHeader returns the current canonical head header.
 func ReadHeadHeader(db ethdb.Reader) *types.Header {
-	headHeaderHash := ReadHeadHeaderHash(db.BlockStoreReader())
+	headHeaderHash := ReadHeadHeaderHash(db)
 	if headHeaderHash == (common.Hash{}) {
 		return nil
 	}
-	headHeaderNumber := ReadHeaderNumber(db.BlockStoreReader(), headHeaderHash)
+	headHeaderNumber := ReadHeaderNumber(db, headHeaderHash)
 	if headHeaderNumber == nil {
 		return nil
 	}
-	return ReadHeader(db.BlockStoreReader(), headHeaderHash, *headHeaderNumber)
+	return ReadHeader(db, headHeaderHash, *headHeaderNumber)
 }
 
 // ReadHeadBlock returns the current canonical head block.
 func ReadHeadBlock(db ethdb.Reader) *types.Block {
-	headBlockHash := ReadHeadBlockHash(db.BlockStoreReader())
+	headBlockHash := ReadHeadBlockHash(db)
 	if headBlockHash == (common.Hash{}) {
 		return nil
 	}
-	headBlockNumber := ReadHeaderNumber(db.BlockStoreReader(), headBlockHash)
+	headBlockNumber := ReadHeaderNumber(db, headBlockHash)
 	if headBlockNumber == nil {
 		return nil
 	}

core/rawdb/accessors_indexes.go

@@ -42,7 +42,7 @@ func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) *uint64 {
 	}
 	// Database v4-v5 tx lookup format just stores the hash
 	if len(data) == common.HashLength {
-		return ReadHeaderNumber(db.BlockStoreReader(), common.BytesToHash(data))
+		return ReadHeaderNumber(db, common.BytesToHash(data))
 	}
 	// Finally try database v3 tx lookup format
 	var entry LegacyTxLookupEntry

core/rawdb/accessors_trie.go

@@ -46,6 +46,8 @@ const HashScheme = "hash"
 // on extra state diffs to survive deep reorg.
 const PathScheme = "path"
 
+const VersionScheme = "version"
+
 // hasher is used to compute the sha256 hash of the provided data.
 type hasher struct{ sha crypto.KeccakState }
 
@@ -314,7 +316,7 @@ func ReadStateScheme(db ethdb.Reader) string {
 // ValidateStateScheme used to check state scheme whether is valid.
 // Valid state scheme: hash and path.
 func ValidateStateScheme(stateScheme string) bool {
-	if stateScheme == HashScheme || stateScheme == PathScheme {
+	if stateScheme == HashScheme || stateScheme == PathScheme || stateScheme == VersionScheme {
 		return true
 	}
 	return false

core/rawdb/ancient_utils.go

@@ -18,6 +18,8 @@ package rawdb
 
 import (
 	"fmt"
+	"io"
+	"os"
 	"path/filepath"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -98,6 +100,18 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
 			if err != nil {
 				return nil, err
 			}
+
+			file, err := os.Open(filepath.Join(datadir, StateFreezerName))
+			if err != nil {
+				return nil, err
+			}
+			defer file.Close()
+			// if state freezer folder has been pruned, there is no need for inspection
+			_, err = file.Readdirnames(1)
+			if err == io.EOF {
+				continue
+			}
+
 			f, err := NewStateFreezer(datadir, true, 0)
 			if err != nil {
 				return nil, err

core/rawdb/chain_freezer.go

@@ -58,7 +58,8 @@ type chainFreezer struct {
 	wg      sync.WaitGroup
 	trigger chan chan struct{} // Manual blocking freeze trigger, test determinism
 
-	freezeEnv atomic.Value
+	freezeEnv    atomic.Value
+	waitEnvTimes int
 
 	multiDatabase bool
 }
@@ -91,7 +92,7 @@ func (f *chainFreezer) Close() error {
 
 // readHeadNumber returns the number of chain head block. 0 is returned if the
 // block is unknown or not available yet.
-func (f *chainFreezer) readHeadNumber(db ethdb.KeyValueReader) uint64 {
+func (f *chainFreezer) readHeadNumber(db ethdb.Reader) uint64 {
 	hash := ReadHeadBlockHash(db)
 	if hash == (common.Hash{}) {
 		log.Error("Head block is not reachable")
@@ -107,7 +108,7 @@ func (f *chainFreezer) readHeadNumber(db ethdb.KeyValueReader) uint64 {
 
 // readFinalizedNumber returns the number of finalized block. 0 is returned
 // if the block is unknown or not available yet.
-func (f *chainFreezer) readFinalizedNumber(db ethdb.KeyValueReader) uint64 {
+func (f *chainFreezer) readFinalizedNumber(db ethdb.Reader) uint64 {
 	hash := ReadFinalizedBlockHash(db)
 	if hash == (common.Hash{}) {
 		return 0
@@ -122,7 +123,7 @@ func (f *chainFreezer) readFinalizedNumber(db ethdb.KeyValueReader) uint64 {
 
 // freezeThreshold returns the threshold for chain freezing. It's determined
 // by formula: max(finality, HEAD-params.FullImmutabilityThreshold).
-func (f *chainFreezer) freezeThreshold(db ethdb.KeyValueReader) (uint64, error) {
+func (f *chainFreezer) freezeThreshold(db ethdb.Reader) (uint64, error) {
 	var (
 		head      = f.readHeadNumber(db)
 		final     = f.readFinalizedNumber(db)
@@ -178,19 +179,6 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
 			}
 		}
 
-		// check freezer env first, it must wait a while when the env is necessary
-		err := f.checkFreezerEnv()
-		if err == missFreezerEnvErr {
-			log.Warn("Freezer need related env, may wait for a while", "err", err)
-			backoff = true
-			continue
-		}
-		if err != nil {
-			log.Error("Freezer check FreezerEnv err", "err", err)
-			backoff = true
-			continue
-		}
-
 		var (
 			frozen    uint64
 			threshold uint64
@@ -200,6 +188,7 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
 			hash   common.Hash
 			number *uint64
 			head   *types.Header
+			err    error
 		)
 
 		// use finalized block as the chain freeze indicator was used for multiDatabase feature, if multiDatabase is false, keep 9W blocks in db
@@ -282,6 +271,18 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
 				last = first + freezerBatchLimit
 			}
 		}
+
+		// check env first before chain freeze, it must wait when the env is necessary
+		if err := f.checkFreezerEnv(); err != nil {
+			f.waitEnvTimes++
+			if f.waitEnvTimes%30 == 0 {
+				log.Warn("Freezer need related env, may wait for a while, and it's not a issue when non-import block", "err", err)
+				return
+			}
+			backoff = true
+			continue
+		}
+
 		// Seems we have data ready to be frozen, process in usable batches
 		var (
 			start = time.Now()
@@ -544,14 +545,7 @@ func (f *chainFreezer) checkFreezerEnv() error {
 	if exist {
 		return nil
 	}
-	blobFrozen, err := f.TableAncients(ChainFreezerBlobSidecarTable)
-	if err != nil {
-		return err
-	}
-	if blobFrozen > 0 {
-		return missFreezerEnvErr
-	}
-	return nil
+	return missFreezerEnvErr
 }
 
 func isCancun(env *ethdb.FreezerEnv, num *big.Int, time uint64) bool {

core/rawdb/chain_iterator.go

@@ -35,16 +35,16 @@ import (
 // injects into the database the block hash->number mappings.
 func InitDatabaseFromFreezer(db ethdb.Database) {
 	// If we can't access the freezer or it's empty, abort
-	frozen, err := db.ItemAmountInAncient()
+	frozen, err := db.BlockStore().ItemAmountInAncient()
 	if err != nil || frozen == 0 {
 		return
 	}
 	var (
-		batch  = db.NewBatch()
+		batch  = db.BlockStore().NewBatch()
 		start  = time.Now()
 		logged = start.Add(-7 * time.Second) // Unindex during import is fast, don't double log
 		hash   common.Hash
-		offset = db.AncientOffSet()
+		offset = db.BlockStore().AncientOffSet()
 	)
 	for i := uint64(0) + offset; i < frozen+offset; i++ {
 		// We read 100K hashes at a time, for a total of 3.2M
@@ -52,7 +52,7 @@ func InitDatabaseFromFreezer(db ethdb.Database) {
 		if i+count > frozen+offset {
 			count = frozen + offset - i
 		}
-		data, err := db.AncientRange(ChainFreezerHashTable, i, count, 32*count)
+		data, err := db.BlockStore().AncientRange(ChainFreezerHashTable, i, count, 32*count)
 		if err != nil {
 			log.Crit("Failed to init database from freezer", "err", err)
 		}
@@ -81,7 +81,7 @@ func InitDatabaseFromFreezer(db ethdb.Database) {
 	batch.Reset()
 
 	WriteHeadHeaderHash(db.BlockStore(), hash)
-	WriteHeadFastBlockHash(db, hash)
+	WriteHeadFastBlockHash(db.BlockStore(), hash)
 	log.Info("Initialized database from freezer", "blocks", frozen, "elapsed", common.PrettyDuration(time.Since(start)))
 }
 
@@ -100,7 +100,7 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool
 		number uint64
 		rlp    rlp.RawValue
 	}
-	if offset := db.AncientOffSet(); offset > from {
+	if offset := db.BlockStore().AncientOffSet(); offset > from {
 		from = offset
 	}
 	if to <= from {
@@ -122,7 +122,7 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool
 		}
 		defer close(rlpCh)
 		for n != end {
-			data := ReadCanonicalBodyRLP(db.BlockStore(), n)
+			data := ReadCanonicalBodyRLP(db, n)
 			// Feed the block to the aggregator, or abort on interrupt
 			select {
 			case rlpCh <- &numberRlp{n, data}:
@@ -187,7 +187,7 @@ func iterateTransactions(db ethdb.Database, from uint64, to uint64, reverse bool
 // signal received.
 func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool, report bool) {
 	// short circuit for invalid range
-	if offset := db.AncientOffSet(); offset > from {
+	if offset := db.BlockStore().AncientOffSet(); offset > from {
 		from = offset
 	}
 	if from >= to {
@@ -286,7 +286,7 @@ func indexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, inte
 // signal received.
 func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool, report bool) {
 	// short circuit for invalid range
-	if offset := db.AncientOffSet(); offset > from {
+	if offset := db.BlockStore().AncientOffSet(); offset > from {
 		from = offset
 	}
 	if from >= to {

core/rawdb/database.go

@@ -61,8 +61,10 @@ func (frdb *freezerdb) BlockStoreReader() ethdb.Reader {
 }
 
 func (frdb *freezerdb) BlockStoreWriter() ethdb.Writer {
-	// TODO implement me
-	panic("implement me")
+	if frdb.blockStore == nil {
+		return frdb
+	}
+	return frdb.blockStore
 }
 
 // AncientDatadir returns the path of root ancient directory.
@@ -116,6 +118,13 @@ func (frdb *freezerdb) StateStore() ethdb.Database {
 	return frdb.stateStore
 }
 
+func (frdb *freezerdb) GetStateStore() ethdb.Database {
+	if frdb.stateStore != nil {
+		return frdb.stateStore
+	}
+	return frdb
+}
+
 func (frdb *freezerdb) SetStateStore(state ethdb.Database) {
 	if frdb.stateStore != nil {
 		frdb.stateStore.Close()
@@ -254,6 +263,13 @@ func (db *nofreezedb) SetStateStore(state ethdb.Database) {
 	db.stateStore = state
 }
 
+func (db *nofreezedb) GetStateStore() ethdb.Database {
+	if db.stateStore != nil {
+		return db.stateStore
+	}
+	return db
+}
+
 func (db *nofreezedb) StateStoreReader() ethdb.Reader {
 	if db.stateStore != nil {
 		return db.stateStore
@@ -403,6 +419,7 @@ func (db *emptyfreezedb) Sync() error {
 func (db *emptyfreezedb) DiffStore() ethdb.KeyValueStore        { return db }
 func (db *emptyfreezedb) SetDiffStore(diff ethdb.KeyValueStore) {}
 func (db *emptyfreezedb) StateStore() ethdb.Database            { return db }
+func (db *emptyfreezedb) GetStateStore() ethdb.Database         { return db }
 func (db *emptyfreezedb) SetStateStore(state ethdb.Database)    {}
 func (db *emptyfreezedb) StateStoreReader() ethdb.Reader        { return db }
 func (db *emptyfreezedb) BlockStore() ethdb.Database            { return db }
@@ -518,8 +535,17 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st
 
 	// Create the idle freezer instance
 	frdb, err := newChainFreezer(resolveChainFreezerDir(ancient), namespace, readonly, offset, multiDatabase)
+
+	// We are creating the freezerdb here because the validation logic for db and freezer below requires certain interfaces
+	// that need a database type. Therefore, we are pre-creating it for subsequent use.
+	freezerDb := &freezerdb{
+		ancientRoot:    ancient,
+		KeyValueStore:  db,
+		AncientStore:   frdb,
+		AncientFreezer: frdb,
+	}
 	if err != nil {
-		printChainMetadata(db)
+		printChainMetadata(freezerDb)
 		return nil, err
 	}
 
@@ -555,10 +581,10 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st
 			// the freezer and the key-value store.
 			frgenesis, err := frdb.Ancient(ChainFreezerHashTable, 0)
 			if err != nil {
-				printChainMetadata(db)
+				printChainMetadata(freezerDb)
 				return nil, fmt.Errorf("failed to retrieve genesis from ancient %v", err)
 			} else if !bytes.Equal(kvgenesis, frgenesis) {
-				printChainMetadata(db)
+				printChainMetadata(freezerDb)
 				return nil, fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis)
 			}
 			// Key-value store and freezer belong to the same network. Ensure that they
@@ -566,7 +592,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st
 			if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 {
 				// Subsequent header after the freezer limit is missing from the database.
 				// Reject startup if the database has a more recent head.
-				if head := *ReadHeaderNumber(db, ReadHeadHeaderHash(db)); head > frozen-1 {
+				if head := *ReadHeaderNumber(freezerDb, ReadHeadHeaderHash(freezerDb)); head > frozen-1 {
 					// Find the smallest block stored in the key-value store
 					// in range of [frozen, head]
 					var number uint64
@@ -576,7 +602,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st
 						}
 					}
 					// We are about to exit on error. Print database metadata before exiting
-					printChainMetadata(db)
+					printChainMetadata(freezerDb)
 					return nil, fmt.Errorf("gap in the chain between ancients [0 - #%d] and leveldb [#%d - #%d] ",
 						frozen-1, number, head)
 				}
@@ -591,11 +617,11 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st
 			// store, otherwise we'll end up missing data. We check block #1 to decide
 			// if we froze anything previously or not, but do take care of databases with
 			// only the genesis block.
-			if ReadHeadHeaderHash(db) != common.BytesToHash(kvgenesis) {
+			if ReadHeadHeaderHash(freezerDb) != common.BytesToHash(kvgenesis) {
 				// Key-value store contains more data than the genesis block, make sure we
 				// didn't freeze anything yet.
 				if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 {
-					printChainMetadata(db)
+					printChainMetadata(freezerDb)
 					return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path")
 				}
 				// Block #1 is still in the database, we're allowed to init a new freezer
@@ -617,12 +643,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st
 			frdb.wg.Done()
 		}()
 	}
-	return &freezerdb{
-		ancientRoot:    ancient,
-		KeyValueStore:  db,
-		AncientStore:   frdb,
-		AncientFreezer: frdb,
-	}, nil
+	return freezerDb, nil
 }
 
 // NewMemoryDatabase creates an ephemeral in-memory key-value database without a
@@ -873,7 +894,7 @@ func DataTypeByKey(key []byte) DataType {
 				return StateDataType
 			}
 		}
-		for _, meta := range [][]byte{headHeaderKey, headFinalizedBlockKey} {
+		for _, meta := range [][]byte{headHeaderKey, headFinalizedBlockKey, headBlockKey, headFastBlockKey} {
 			if bytes.Equal(key, meta) {
 				return BlockDataType
 			}
@@ -1088,7 +1109,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 				hashNumPairings.Add(size)
 			default:
 				var accounted bool
-				for _, meta := range [][]byte{headHeaderKey, headFinalizedBlockKey} {
+				for _, meta := range [][]byte{headHeaderKey, headFinalizedBlockKey, headBlockKey, headFastBlockKey} {
 					if bytes.Equal(key, meta) {
 						metadata.Add(size)
 						accounted = true
@@ -1238,7 +1259,7 @@ func DeleteTrieState(db ethdb.Database) error {
 }
 
 // printChainMetadata prints out chain metadata to stderr.
-func printChainMetadata(db ethdb.KeyValueStore) {
+func printChainMetadata(db ethdb.Reader) {
 	fmt.Fprintf(os.Stderr, "Chain metadata\n")
 	for _, v := range ReadChainMetadata(db) {
 		fmt.Fprintf(os.Stderr, "  %s\n", strings.Join(v, ": "))
@@ -1249,7 +1270,7 @@ func printChainMetadata(db ethdb.KeyValueStore) {
 // ReadChainMetadata returns a set of key/value pairs that contains information
 // about the database chain status. This can be used for diagnostic purposes
 // when investigating the state of the node.
-func ReadChainMetadata(db ethdb.KeyValueStore) [][]string {
+func ReadChainMetadata(db ethdb.Reader) [][]string {
 	pp := func(val *uint64) string {
 		if val == nil {
 			return "<nil>"
@@ -1271,26 +1292,3 @@ func ReadChainMetadata(db ethdb.KeyValueStore) [][]string {
 	}
 	return data
 }
-
-func ReadChainMetadataFromMultiDatabase(db ethdb.Database) [][]string {
-	pp := func(val *uint64) string {
-		if val == nil {
-			return "<nil>"
-		}
-		return fmt.Sprintf("%d (%#x)", *val, *val)
-	}
-	data := [][]string{
-		{"databaseVersion", pp(ReadDatabaseVersion(db))},
-		{"headBlockHash", fmt.Sprintf("%v", ReadHeadBlockHash(db.BlockStore()))},
-		{"headFastBlockHash", fmt.Sprintf("%v", ReadHeadFastBlockHash(db))},
-		{"headHeaderHash", fmt.Sprintf("%v", ReadHeadHeaderHash(db.BlockStore()))},
-		{"lastPivotNumber", pp(ReadLastPivotNumber(db))},
-		{"len(snapshotSyncStatus)", fmt.Sprintf("%d bytes", len(ReadSnapshotSyncStatus(db)))},
-		{"snapshotDisabled", fmt.Sprintf("%v", ReadSnapshotDisabled(db))},
-		{"snapshotJournal", fmt.Sprintf("%d bytes", len(ReadSnapshotJournal(db)))},
-		{"snapshotRecoveryNumber", pp(ReadSnapshotRecoveryNumber(db))},
-		{"snapshotRoot", fmt.Sprintf("%v", ReadSnapshotRoot(db))},
-		{"txIndexTail", pp(ReadTxIndexTail(db))},
-	}
-	return data
-}

core/rawdb/freezer.go

@@ -541,41 +541,6 @@ func gcKvStore(db ethdb.KeyValueStore, ancients []common.Hash, first uint64, fro
 	}
 	batch.Reset()
 
-	// Step into the future and delete and dangling side chains
-	if frozen > 0 {
-		tip := frozen
-		nfdb := &nofreezedb{KeyValueStore: db}
-		for len(dangling) > 0 {
-			drop := make(map[common.Hash]struct{})
-			for _, hash := range dangling {
-				log.Debug("Dangling parent from freezer", "number", tip-1, "hash", hash)
-				drop[hash] = struct{}{}
-			}
-			children := ReadAllHashes(db, tip)
-			for i := 0; i < len(children); i++ {
-				// Dig up the child and ensure it's dangling
-				child := ReadHeader(nfdb, children[i], tip)
-				if child == nil {
-					log.Error("Missing dangling header", "number", tip, "hash", children[i])
-					continue
-				}
-				if _, ok := drop[child.ParentHash]; !ok {
-					children = append(children[:i], children[i+1:]...)
-					i--
-					continue
-				}
-				// Delete all block data associated with the child
-				log.Debug("Deleting dangling block", "number", tip, "hash", children[i], "parent", child.ParentHash)
-				DeleteBlock(batch, children[i], tip)
-			}
-			dangling = children
-			tip++
-		}
-		if err := batch.Write(); err != nil {
-			log.Crit("Failed to delete dangling side blocks", "err", err)
-		}
-	}
-
 	// Log something friendly for the user
 	context := []interface{}{
 		"blocks", frozen - first, "elapsed", common.PrettyDuration(time.Since(start)), "number", frozen - 1,

core/rawdb/freezer_table.go

@@ -127,6 +127,11 @@ func newFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerT
 	return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy, readonly)
 }
 
+// newAdditionTable opens the given path as a addition table.
+func newAdditionTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) {
+	return openAdditionTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy, readonly)
+}
+
 // newTable opens a freezer table, creating the data and index files if they are
 // non-existent. Both files are truncated to the shortest common length to ensure
 // they don't go out of sync.

core/rawdb/prunedfreezer.go

@@ -8,6 +8,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	"golang.org/x/exp/slices"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
@@ -66,28 +68,49 @@ func newPrunedFreezer(datadir string, db ethdb.KeyValueStore, offset uint64) (*p
 
 // repair init frozen , compatible disk-ancientdb and pruner-block-tool.
 func (f *prunedfreezer) repair(datadir string) error {
-	offset := atomic.LoadUint64(&f.frozen)
 	// compatible freezer
-	min := uint64(math.MaxUint64)
+	minItems := uint64(math.MaxUint64)
 	for name, disableSnappy := range chainFreezerNoSnappy {
-		table, err := newFreezerTable(datadir, name, disableSnappy, false)
+		var (
+			table *freezerTable
+			err   error
+		)
+		if slices.Contains(additionTables, name) {
+			table, err = newAdditionTable(datadir, name, disableSnappy, false)
+		} else {
+			table, err = newFreezerTable(datadir, name, disableSnappy, false)
+		}
 		if err != nil {
 			return err
 		}
+		// addition tables only align head
+		if slices.Contains(additionTables, name) {
+			if EmptyTable(table) {
+				continue
+			}
+		}
 		items := table.items.Load()
-		if min > items {
-			min = items
+		if minItems > items {
+			minItems = items
 		}
 		table.Close()
 	}
-	log.Info("Read ancientdb item counts", "items", min)
-	offset += min
 
-	if frozen := ReadFrozenOfAncientFreezer(f.db); frozen > offset {
-		offset = frozen
+	// If minItems is non-zero, it indicates that the chain freezer was previously enabled, and we should use minItems as the current frozen value.
+	// If minItems is zero, it indicates that the pruneAncient was previously enabled, and we should continue using frozen
+	//	(retrieved from CurrentAncientFreezer) as the current frozen value.
+	offset := minItems
+	if offset == 0 {
+		// no item in ancientDB, init `offset` to the `f.frozen`
+		offset = atomic.LoadUint64(&f.frozen)
 	}
+	log.Info("Read ancientdb item counts", "items", minItems, "offset", offset)
 
-	atomic.StoreUint64(&f.frozen, offset)
+	// FrozenOfAncientFreezer is the progress of the last prune-freezer freeze.
+	frozenInDB := ReadFrozenOfAncientFreezer(f.db)
+	maxOffset := max(offset, frozenInDB)
+
+	atomic.StoreUint64(&f.frozen, maxOffset)
 	if err := f.Sync(); err != nil {
 		return nil
 	}
@@ -299,9 +322,8 @@ func (f *prunedfreezer) freeze() {
 				log.Error("Append ancient err", "number", f.frozen, "hash", hash, "err", err)
 				break
 			}
-			if hash != (common.Hash{}) {
-				ancients = append(ancients, hash)
-			}
+			// may include common.Hash{}, will be delete in gcKvStore
+			ancients = append(ancients, hash)
 		}
 		// Batch of blocks have been frozen, flush them before wiping from leveldb
 		if err := f.Sync(); err != nil {

core/rawdb/table.go

@@ -251,6 +251,10 @@ func (t *table) SetStateStore(state ethdb.Database) {
 	panic("not implement")
 }
 
+func (t *table) GetStateStore() ethdb.Database {
+	return nil
+}
+
 func (t *table) StateStoreReader() ethdb.Reader {
 	return nil
 }

core/state/caching_versa_db.go

@@ -0,0 +1,521 @@
+package state
+
+import (
+	"errors"
+	"fmt"
+	"sync/atomic"
+
+	versa "github.com/bnb-chain/versioned-state-database"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/lru"
+	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/trienode"
+	"github.com/ethereum/go-ethereum/triedb"
+)
+
+const InvalidSateObjectVersion int64 = math.MinInt64
+
+type cachingVersaDB struct {
+	version       int64
+	triedb        *triedb.Database
+	versionDB     versa.Database
+	codeDB        ethdb.KeyValueStore
+	codeSizeCache *lru.Cache[common.Hash, int]
+	codeCache     *lru.SizeConstrainedCache[common.Hash, []byte]
+
+	accTree  *VersaTree
+	state    versa.StateHandler
+	root     common.Hash
+	mode     versa.StateMode
+	hasState atomic.Bool
+
+	//debug *DebugVersionState
+}
+
+// NewVersaDatabase should be call by NewDatabaseWithNodeDB
+// TODO:: NewDatabaseWithNodeDB should add mode param.
+func NewVersaDatabase(db ethdb.Database, triedb *triedb.Database, mode versa.StateMode) Database {
+	return &cachingVersaDB{
+		triedb:        triedb,
+		versionDB:     triedb.VersaDB(),
+		codeDB:        db,
+		codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize),
+		codeCache:     lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize),
+		mode:          mode,
+		state:         versa.ErrStateHandler,
+	}
+}
+
+func (cv *cachingVersaDB) Copy() Database {
+	cp := &cachingVersaDB{}
+	cp.codeCache = cv.codeCache
+	cp.codeSizeCache = cv.codeSizeCache
+	cp.triedb = cv.triedb
+	cp.versionDB = cv.versionDB
+	cp.codeDB = cv.codeDB
+	cp.mode = versa.S_RW // it is important
+
+	// TODO:: maybe add lock for cv.root
+	if cv.hasState.Load() {
+		_, err := cp.OpenTrie(cv.root)
+		if err != nil {
+			//if cv.debug != nil {
+			//	cv.debug.OnError(fmt.Errorf("failed to open trie in copy caching versa db, error: %s", err.Error()))
+			//}
+			return cp
+		}
+	}
+	return cp
+}
+
+// CopyTrie is used with Copy()
+func (cv *cachingVersaDB) CopyTrie(tr Trie) Trie {
+	vtr, ok := tr.(*VersaTree)
+	if !ok {
+		panic("caching versa db copy non versa tree")
+	}
+	if vtr.accountTree {
+		if cv.accTree != nil {
+			if cv.accTree.root.Cmp(vtr.root) != 0 {
+				panic("copy acc trie mismatch")
+			}
+			return cv.accTree
+		}
+		tree, err := cv.OpenTrie(vtr.root)
+		if err != nil {
+			//if cv.debug != nil {
+			//	cv.debug.OnError(fmt.Errorf("failed to open trie in copy versa trie, error: %s", err.Error()))
+			//}
+			return nil
+		}
+		return tree
+	} else {
+		tree, err := cv.OpenStorageTrie(vtr.stateRoot, vtr.address, vtr.root, nil)
+		if err != nil {
+			//if cv.debug != nil {
+			//	cv.debug.OnError(fmt.Errorf("failed to open storage trie in copy versa trie, error: %s", err.Error()))
+			//}
+			return nil
+		}
+		return tree
+	}
+	return nil
+}
+
+func (cv *cachingVersaDB) HasState(root common.Hash) bool {
+	return cv.versionDB.HasState(root)
+}
+
+func (cv *cachingVersaDB) OpenTrie(root common.Hash) (Trie, error) {
+	if cv.hasState.Load() {
+		//TODO:: will change to log.Error after stabilization
+		panic("account tree has open")
+	}
+
+	// TODO:: if root tree, versa db should ignore check version, temp use -1
+	state, err := cv.versionDB.OpenState(cv.version, root, cv.mode)
+	if err != nil {
+		//if cv.debug != nil {
+		//	cv.debug.OnError(fmt.Errorf("failed to open state, root:%s, error: %s", root.String(), err.Error()))
+		//}
+		return nil, err
+	}
+	//if cv.debug != nil {
+	//	cv.debug.OnOpenState(state)
+	//}
+
+	handler, err := cv.versionDB.OpenTree(state, cv.version, common.Hash{}, root)
+	if err != nil {
+		//if cv.debug != nil {
+		//	cv.debug.OnError(fmt.Errorf("failed to open account trie, root:%s, error: %s", root.String(), err.Error()))
+		//}
+		return nil, err
+	}
+
+	tree := &VersaTree{
+		db:          cv.versionDB,
+		handler:     handler,
+		accountTree: true,
+		root:        root,
+		mode:        cv.mode,
+		//debug:       cv.debug,
+	}
+
+	cv.state = state
+	cv.hasState.Store(true) // if set, can't change
+	cv.accTree = tree
+	cv.root = root
+
+	//if cv.debug != nil {
+	//	cv.debug.OnOpenTree(handler, common.Hash{}, common.Address{})
+	//}
+
+	return tree, nil
+}
+
+func (cv *cachingVersaDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, _ Trie) (Trie, error) {
+	version, _, err := cv.accTree.getAccountWithVersion(address)
+	if err != nil {
+		return nil, err
+	}
+	return cv.openStorageTreeWithVersion(version, stateRoot, address, root)
+}
+
+func (cv *cachingVersaDB) openStorageTreeWithVersion(version int64, stateRoot common.Hash, address common.Address, root common.Hash) (Trie, error) {
+	if !cv.hasState.Load() {
+		//TODO:: will change to log.Error after stabilization
+		panic("open account tree, before open storage tree")
+	}
+	if cv.root.Cmp(stateRoot) != 0 {
+		panic(fmt.Sprintf("account root mismatch, on open storage tree, actual: %s, expect: %s", root.String(), cv.root.String()))
+	}
+
+	owner := crypto.Keccak256Hash(address.Bytes())
+	handler, err := cv.versionDB.OpenTree(cv.state, version, owner, root)
+	if err != nil {
+		//if cv.debug != nil {
+		//	cv.debug.OnError(fmt.Errorf("failed to open storage trie, version: %d,stateRoot:%s, address:%s, root: %s, error: %s",
+		//		version, stateRoot.String(), address.String(), root.String(), err.Error()))
+		//}
+		return nil, err
+	}
+
+	//if cv.debug != nil {
+	//	cv.debug.OnOpenTree(handler, owner, address)
+	//}
+	tree := &VersaTree{
+		db:        cv.versionDB,
+		handler:   handler,
+		version:   version,
+		root:      stateRoot,
+		stateRoot: root,
+		address:   address,
+		mode:      cv.mode,
+		//debug:     cv.debug,
+	}
+	return tree, nil
+}
+
+// Flush unique to versa
+func (cv *cachingVersaDB) Flush() error {
+	err := cv.versionDB.Flush(cv.state)
+	//if err != nil && cv.debug != nil {
+	//	cv.debug.OnError(fmt.Errorf("failed to flush state, version: %d, root:%s, mode:%d, error: %s",
+	//		cv.accTree.version, cv.accTree.root.String(), cv.accTree.mode, err.Error()))
+	//}
+	return err
+}
+
+func (cv *cachingVersaDB) SetVersion(version int64) {
+	cv.version = version - 1
+	//cv.debug = NewDebugVersionState(cv.codeDB, cv.versionDB)
+	//cv.debug.Version = version
+}
+
+func (cv *cachingVersaDB) GetVersion() int64 {
+	return cv.version
+}
+
+// Release unique to versa
+func (cv *cachingVersaDB) Release() error {
+	//log.Info("close state", "state info", cv.versionDB.ParseStateHandler(cv.state))
+	if cv.state != versa.ErrStateHandler {
+		//if cv.debug != nil {
+		//	cv.debug.OnCloseState(cv.state)
+		//}
+		if err := cv.versionDB.CloseState(cv.state); err != nil {
+			//if cv.debug != nil {
+			//	cv.debug.OnError(fmt.Errorf("failed to close state in release, version: %d, root:%s, mode:%d, error: %s",
+			//		cv.accTree.version, cv.accTree.root.String(), cv.accTree.mode, err.Error()))
+			//}
+			return err
+		}
+		cv.hasState.Store(false)
+		cv.accTree = nil
+		cv.state = versa.ErrStateHandler
+		cv.root = common.Hash{}
+		//cv.debug = nil
+	}
+	return nil
+}
+
+func (cv *cachingVersaDB) Reset() {
+	if cv.state != versa.ErrStateHandler {
+		_ = cv.versionDB.CloseState(cv.state)
+		panic("close state in reset")
+	}
+	cv.hasState.Store(false)
+	cv.accTree = nil
+	cv.state = versa.ErrStateHandler
+	cv.root = common.Hash{}
+}
+
+func (cv *cachingVersaDB) HasTreeExpired(tr Trie) bool {
+	vtr, ok := tr.(*VersaTree)
+	if !ok {
+		panic("trie type mismatch")
+	}
+	return cv.versionDB.HasTreeExpired(vtr.handler)
+}
+
+func (cv *cachingVersaDB) Scheme() string {
+	return cv.triedb.Scheme()
+}
+
+func (cv *cachingVersaDB) ContractCode(addr common.Address, codeHash common.Hash) ([]byte, error) {
+	//if cv.debug != nil {
+	//	cv.debug.OnGetCode(addr, codeHash)
+	//}
+	code, _ := cv.codeCache.Get(codeHash)
+	if len(code) > 0 {
+		return code, nil
+	}
+	code = rawdb.ReadCode(cv.codeDB, codeHash)
+	if len(code) > 0 {
+		cv.codeCache.Add(codeHash, code)
+		cv.codeSizeCache.Add(codeHash, len(code))
+		return code, nil
+	}
+	return nil, errors.New("not found")
+}
+
+func (cv *cachingVersaDB) ContractCodeSize(addr common.Address, codeHash common.Hash) (int, error) {
+	if cached, ok := cv.codeSizeCache.Get(codeHash); ok {
+		return cached, nil
+	}
+	code, err := cv.ContractCode(addr, codeHash)
+	return len(code), err
+}
+
+func (cv *cachingVersaDB) ContractCodeWithPrefix(address common.Address, codeHash common.Hash) ([]byte, error) {
+	code, _ := cv.codeCache.Get(codeHash)
+	if len(code) > 0 {
+		return code, nil
+	}
+	code = rawdb.ReadCodeWithPrefix(cv.codeDB, codeHash)
+	if len(code) > 0 {
+		cv.codeCache.Add(codeHash, code)
+		cv.codeSizeCache.Add(codeHash, len(code))
+		return code, nil
+	}
+	return nil, errors.New("not found")
+}
+
+func (cv *cachingVersaDB) DiskDB() ethdb.KeyValueStore {
+	return cv.codeDB
+}
+
+func (cv *cachingVersaDB) TrieDB() *triedb.Database {
+	return cv.triedb
+}
+
+func (cv *cachingVersaDB) NoTries() bool {
+	// TODO:: not support fastnode
+	return false
+}
+
+type VersaTree struct {
+	db          versa.Database
+	handler     versa.TreeHandler
+	version     int64
+	accountTree bool
+	//debug       *DebugVersionState
+
+	// TODO:: debugging, used for logging
+	stateRoot common.Hash
+	root      common.Hash
+	address   common.Address
+	mode      versa.StateMode
+}
+
+func (vt *VersaTree) GetKey(key []byte) []byte {
+	_, val, err := vt.db.Get(vt.handler, key)
+	if err != nil {
+		log.Warn("failed to get key from version db")
+	}
+	return val
+}
+
+func (vt *VersaTree) GetAccount(address common.Address) (*types.StateAccount, error) {
+	_, res, err := vt.getAccountWithVersion(address)
+	return res, err
+}
+
+func (vt *VersaTree) getAccountWithVersion(address common.Address) (int64, *types.StateAccount, error) {
+	vt.CheckAccountTree()
+	ver, res, err := vt.db.Get(vt.handler, address.Bytes())
+	if res == nil || err != nil {
+		//if err != nil && vt.debug != nil {
+		//	vt.debug.OnError(fmt.Errorf("failed to get account, root: %s, address: %s, error: %s",
+		//		vt.root.String(), address.String(), err.Error()))
+		//}
+		return ver, nil, err
+	}
+	ret := new(types.StateAccount)
+	err = rlp.DecodeBytes(res, ret)
+	if err != nil {
+		//if vt.debug != nil {
+		//	vt.debug.OnError(fmt.Errorf("failed to rlp decode account, root: %s, address: %s, error: %s",
+		//		vt.root.String(), address.String(), err.Error()))
+		//}
+	}
+	//if vt.debug != nil {
+	//	vt.debug.OnGetAccount(address, ret)
+	//}
+	return ver, ret, err
+}
+
+func (vt *VersaTree) GetStorage(address common.Address, key []byte) ([]byte, error) {
+	if vt.address.Cmp(address) != 0 {
+		panic(fmt.Sprintf("address mismatch in get storage, expect: %s, actul: %s", vt.address.String(), address.String()))
+	}
+	vt.CheckStorageTree()
+	_, enc, err := vt.db.Get(vt.handler, key)
+	if err != nil || len(enc) == 0 {
+		//if err != nil && vt.debug != nil {
+		//	vt.debug.OnError(fmt.Errorf("failed to get storage, root: %s, stateRoot: %s, address:%s, key: %s, error: %s",
+		//		vt.root.String(), vt.stateRoot.String(), address.String(), common.Bytes2Hex(key), err.Error()))
+		//}
+		return nil, err
+	}
+	_, content, _, err := rlp.Split(enc)
+	if err != nil {
+		//if vt.debug != nil {
+		//	vt.debug.OnError(fmt.Errorf("failed to rlp decode storage, root: %s, stateRoot: %s, address: %s, key: %s,error: %s",
+		//		vt.root.String(), vt.stateRoot.String(), address.String(), common.Bytes2Hex(key), err.Error()))
+		//}
+	}
+	//if vt.debug != nil {
+	//	vt.debug.OnGetStorage(vt.handler, address, key, content)
+	//}
+	return content, err
+}
+
+func (vt *VersaTree) UpdateAccount(address common.Address, account *types.StateAccount) error {
+	vt.CheckAccountTree()
+	data, err := rlp.EncodeToBytes(account)
+	if err != nil {
+		//if vt.debug != nil {
+		//	vt.debug.OnError(fmt.Errorf("failed to update account, root: %s, address: %s, account: %s, error: %s",
+		//		vt.root.String(), address.String(), account.String(), err.Error()))
+		//}
+		return err
+	}
+	//if vt.debug != nil {
+	//	vt.debug.OnUpdateAccount(address, account)
+	//}
+	return vt.db.Put(vt.handler, address.Bytes(), data)
+}
+
+func (vt *VersaTree) UpdateStorage(address common.Address, key, value []byte) error {
+	if vt.address.Cmp(address) != 0 {
+		panic(fmt.Sprintf("address mismatch in get storage, expect: %s, actul: %s", vt.address.String(), address.String()))
+	}
+	vt.CheckStorageTree()
+	v, _ := rlp.EncodeToBytes(value)
+	err := vt.db.Put(vt.handler, key, v)
+	if err != nil {
+		//if vt.debug != nil {
+		//	vt.debug.OnError(fmt.Errorf("failed to update storage, root: %s, stateRoot: %s, address: %s, key: %s, val: %s, error: %s",
+		//		vt.root.String(), vt.stateRoot.String(), address.String(), common.Bytes2Hex(key), common.Bytes2Hex(value), err.Error()))
+		//}
+	}
+	//if vt.debug != nil {
+	//	vt.debug.OnUpdateStorage(vt.handler, address, key, value)
+	//}
+	return err
+}
+
+func (vt *VersaTree) DeleteAccount(address common.Address) error {
+	vt.CheckAccountTree()
+	err := vt.db.Delete(vt.handler, address.Bytes())
+	if err != nil {
+		//if vt.debug != nil {
+		//	vt.debug.OnError(fmt.Errorf("failed to delete account, root: %s, address: %s, error: %s",
+		//		vt.root.String(), address.String(), err.Error()))
+		//}
+	}
+	//if vt.debug != nil {
+	//	vt.debug.OnDeleteAccount(address)
+	//}
+	return err
+}
+
+func (vt *VersaTree) DeleteStorage(address common.Address, key []byte) error {
+	vt.CheckStorageTree()
+	err := vt.db.Delete(vt.handler, key)
+	if err != nil {
+		//if vt.debug != nil {
+		//	vt.debug.OnError(fmt.Errorf("failed to delete storage, root: %s, stateRoot: %s, address: %s, key: %s, error: %s",
+		//		vt.root.String(), vt.stateRoot.String(), address.String(), common.Bytes2Hex(key), err.Error()))
+		//}
+	}
+	//if vt.debug != nil {
+	//	vt.debug.OnDeleteStorage(vt.handler, address, key)
+	//}
+	return err
+}
+
+func (vt *VersaTree) UpdateContractCode(address common.Address, codeHash common.Hash, code []byte) error {
+	return nil
+}
+
+func (vt *VersaTree) Hash() common.Hash {
+	hash, err := vt.db.CalcRootHash(vt.handler)
+	if err != nil {
+		//if vt.debug != nil {
+		//	vt.debug.OnError(fmt.Errorf("failed to calc root, root: %s, stateRoot%s, error:%s",
+		//		vt.root.String(), vt.stateRoot.String(), err.Error()))
+		//}
+	}
+	//if vt.debug != nil {
+	//	vt.debug.OnCalcHash(vt.address, hash)
+	//}
+	return hash
+}
+
+func (vt *VersaTree) Commit(_ bool) (common.Hash, *trienode.NodeSet, error) {
+	hash, err := vt.db.Commit(vt.handler)
+	if err != nil {
+		//if vt.debug != nil {
+		//	vt.debug.OnError(fmt.Errorf("failed to commit versa tree, root: %s, stateRoot: %s, error: %s",
+		//		vt.root.String(), vt.stateRoot.String(), err.Error()))
+		//}
+	}
+	//if vt.debug != nil {
+	//	vt.debug.OnCalcHash(vt.address, hash)
+	//	vt.debug.OnCommitTree(vt.address, vt.handler)
+	//}
+	return hash, nil, err
+}
+
+func (vt *VersaTree) NodeIterator(startKey []byte) (trie.NodeIterator, error) {
+	panic("versa tree not support iterate")
+	return nil, nil
+}
+
+func (vt *VersaTree) Prove(key []byte, proofDb ethdb.KeyValueWriter) error {
+	panic("versa tree not support prove")
+	return nil
+}
+
+// TODO:: debug code, will be deleted after stabilization
+func (vt *VersaTree) CheckAccountTree() {
+	if !vt.accountTree {
+		panic("sub tree can't operate account")
+	}
+}
+
+// TODO:: debug code, will be deleted after stabilization
+func (vt *VersaTree) CheckStorageTree() {
+	if vt.accountTree {
+		panic("root tree can't operate storage")
+	}
+}

core/state/database.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 
+	versa "github.com/bnb-chain/versioned-state-database"
 	"github.com/crate-crypto/go-ipa/banderwagon"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/lru"
@@ -70,8 +71,33 @@ type Database interface {
 	// TrieDB returns the underlying trie database for managing trie nodes.
 	TrieDB() *triedb.Database
 
+	// Scheme returns triedb scheme, used to distinguish version triedb.
+	Scheme() string
+
+	// Flush used for version caching versa db to commit block state data.
+	Flush() error
+
+	// Release used for caching versa db to release resource.
+	Release() error
+
+	// Reset used for caching versa db to clean up meta data.
+	Reset()
+
+	// Copy used for caching versa db to copy db, main to transfer triedb with rw mode.
+	Copy() Database
+
+	// HasState returns the state data whether in the triedb.
+	HasState(root common.Hash) bool
+
+	// HasTreeExpired used for caching versa db, whether the state where the opened tree resides has been closed
+	HasTreeExpired(tr Trie) bool
+
 	// NoTries returns whether the database has tries storage.
 	NoTries() bool
+
+	SetVersion(version int64)
+
+	GetVersion() int64
 }
 
 // Trie is a Ethereum Merkle Patricia trie.
@@ -148,28 +174,43 @@ type Trie interface {
 // concurrent use, but does not retain any recent trie nodes in memory. To keep some
 // historical state in memory, use the NewDatabaseWithConfig constructor.
 func NewDatabase(db ethdb.Database) Database {
-	return NewDatabaseWithConfig(db, nil)
+	return NewDatabaseWithConfig(db, nil, false)
 }
 
 // NewDatabaseWithConfig creates a backing store for state. The returned database
 // is safe for concurrent use and retains a lot of collapsed RLP trie nodes in a
 // large memory cache.
-func NewDatabaseWithConfig(db ethdb.Database, config *triedb.Config) Database {
+func NewDatabaseWithConfig(db ethdb.Database, config *triedb.Config, needCommit bool) Database {
 	noTries := config != nil && config.NoTries
 
+	triedb := triedb.NewDatabase(db, config)
+	if triedb.Scheme() == rawdb.VersionScheme {
+		if needCommit {
+			return NewVersaDatabase(db, triedb, versa.S_COMMIT)
+		}
+		return NewVersaDatabase(db, triedb, versa.S_RW)
+	}
+
 	return &cachingDB{
 		disk:          db,
 		codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize),
 		codeCache:     lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize),
-		triedb:        triedb.NewDatabase(db, config),
+		triedb:        triedb,
 		noTries:       noTries,
 	}
 }
 
 // NewDatabaseWithNodeDB creates a state database with an already initialized node database.
-func NewDatabaseWithNodeDB(db ethdb.Database, triedb *triedb.Database) Database {
+func NewDatabaseWithNodeDB(db ethdb.Database, triedb *triedb.Database, needCommit bool) Database {
 	noTries := triedb != nil && triedb.Config() != nil && triedb.Config().NoTries
 
+	if triedb.Scheme() == rawdb.VersionScheme {
+		if needCommit {
+			return NewVersaDatabase(db, triedb, versa.S_COMMIT)
+		}
+		return NewVersaDatabase(db, triedb, versa.S_RW)
+	}
+
 	return &cachingDB{
 		disk:          db,
 		codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize),
@@ -185,6 +226,8 @@ type cachingDB struct {
 	codeCache     *lru.SizeConstrainedCache[common.Hash, []byte]
 	triedb        *triedb.Database
 	noTries       bool
+
+	//debug *DebugHashState
 }
 
 // OpenTrie opens the main account trie at a specific root hash.
@@ -197,8 +240,22 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
 	}
 	tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.triedb)
 	if err != nil {
+		//if db.debug != nil {
+		//	db.debug.OnError(fmt.Errorf("failed to open tree, root: %s, error: %s", root.String(), err.Error()))
+		//}
 		return nil, err
 	}
+	//ht := &HashTrie{
+	//	trie:    tr,
+	//	root:    root,
+	//	address: common.Address{},
+	//	owner:   common.Hash{},
+	//	debug:   db.debug,
+	//}
+	//if db.debug != nil {
+	//	db.debug.OnOpenTree(root, common.Hash{}, common.Address{})
+	//}
+	//return ht, nil
 	return tr, nil
 }
 
@@ -214,10 +271,27 @@ func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Addre
 	if db.triedb.IsVerkle() {
 		return self, nil
 	}
-	tr, err := trie.NewStateTrie(trie.StorageTrieID(stateRoot, crypto.Keccak256Hash(address.Bytes()), root), db.triedb)
+	owner := crypto.Keccak256Hash(address.Bytes())
+	tr, err := trie.NewStateTrie(trie.StorageTrieID(stateRoot, owner, root), db.triedb)
 	if err != nil {
+		//if db.debug != nil {
+		//	db.debug.OnError(fmt.Errorf("failed to storage open tree, stateRoot: %s, address: %s, root: %s, error: %s",
+		//		stateRoot.String(), address.String(), root.String(), err.Error()))
+		//}
 		return nil, err
 	}
+	//ht := &HashTrie{
+	//	trie:     tr,
+	//	root:     stateRoot,
+	//	statRoot: root,
+	//	address:  address,
+	//	owner:    owner,
+	//	debug:    db.debug,
+	//}
+	//if db.debug != nil {
+	//	db.debug.OnOpenTree(root, owner, address)
+	//}
+	//return ht, nil
 	return tr, nil
 }
 
@@ -235,6 +309,8 @@ func (db *cachingDB) CopyTrie(t Trie) Trie {
 		return t.Copy()
 	case *trie.EmptyTrie:
 		return t.Copy()
+	//case *HashTrie:
+	//	return db.CopyTrie(t.trie)
 	default:
 		panic(fmt.Errorf("unknown trie type %T", t))
 	}
@@ -242,6 +318,9 @@ func (db *cachingDB) CopyTrie(t Trie) Trie {
 
 // ContractCode retrieves a particular contract's code.
 func (db *cachingDB) ContractCode(address common.Address, codeHash common.Hash) ([]byte, error) {
+	//if db.debug != nil {
+	//	db.debug.OnGetCode(address, codeHash)
+	//}
 	code, _ := db.codeCache.Get(codeHash)
 	if len(code) > 0 {
 		return code, nil
@@ -290,3 +369,179 @@ func (db *cachingDB) DiskDB() ethdb.KeyValueStore {
 func (db *cachingDB) TrieDB() *triedb.Database {
 	return db.triedb
 }
+
+func (db *cachingDB) Reset() {
+	return
+}
+
+func (db *cachingDB) Scheme() string {
+	return db.triedb.Scheme()
+}
+
+func (db *cachingDB) Flush() error {
+	return nil
+}
+
+func (db *cachingDB) Release() error {
+	//db.debug.flush()
+	//db.debug = nil
+	return nil
+}
+
+func (db *cachingDB) SetVersion(version int64) {
+	//db.debug = NewDebugHashState(db.disk)
+	//db.debug.Version = version
+}
+
+func (db *cachingDB) GetVersion() int64 {
+	//return db.debug.Version
+	return 0
+}
+
+func (db *cachingDB) Copy() Database {
+	return db
+}
+
+func (db *cachingDB) HasState(root common.Hash) bool {
+	_, err := db.OpenTrie(root)
+	return err == nil
+}
+
+func (db *cachingDB) HasTreeExpired(_ Trie) bool {
+	return false
+}
+
+//type HashTrie struct {
+//	trie     Trie
+//	root     common.Hash
+//	statRoot common.Hash
+//	address  common.Address
+//	owner    common.Hash
+//
+//	debug *DebugHashState
+//}
+//
+//func (ht *HashTrie) GetKey(key []byte) []byte {
+//	return ht.trie.GetKey(key)
+//}
+//
+//func (ht *HashTrie) GetAccount(address common.Address) (*types.StateAccount, error) {
+//	acc, err := ht.trie.GetAccount(address)
+//	if err != nil {
+//		if ht.debug != nil {
+//			ht.debug.OnError(fmt.Errorf("failed to get account, address: %s, error: %s", address.String(), err.Error()))
+//		}
+//		return nil, err
+//	}
+//	if ht.debug != nil {
+//		ht.debug.OnGetAccount(address, acc)
+//	}
+//	return acc, nil
+//}
+//
+//func (ht *HashTrie) GetStorage(addr common.Address, key []byte) ([]byte, error) {
+//	val, err := ht.trie.GetStorage(addr, key)
+//	if err != nil {
+//		if ht.debug != nil {
+//			ht.debug.OnError(fmt.Errorf("failed to get storage, address: %s, error: %s", addr.String(), err.Error()))
+//		}
+//		return val, err
+//	}
+//	if ht.debug != nil {
+//		ht.debug.OnGetStorage(addr, key, val)
+//	}
+//	return val, err
+//}
+//
+//func (ht *HashTrie) UpdateAccount(address common.Address, account *types.StateAccount) error {
+//	err := ht.trie.UpdateAccount(address, account)
+//	if err != nil {
+//		if ht.debug != nil {
+//			ht.debug.OnError(fmt.Errorf("failed to update account, address: %s, account: %s, error: %s",
+//				address.String(), account.String(), err.Error()))
+//		}
+//		return err
+//	}
+//	if ht.debug != nil {
+//		ht.debug.OnUpdateAccount(address, account)
+//	}
+//	return nil
+//}
+//
+//func (ht *HashTrie) UpdateStorage(addr common.Address, key, value []byte) error {
+//	err := ht.trie.UpdateStorage(addr, key, value)
+//	if err != nil {
+//		if ht.debug != nil {
+//			ht.debug.OnError(fmt.Errorf("failed to update storage, address: %s, key: %s, val: %s, error: %s",
+//				addr.String(), common.Bytes2Hex(key), common.Bytes2Hex(value), err.Error()))
+//		}
+//		return err
+//	}
+//	if ht.debug != nil {
+//		ht.debug.OnUpdateStorage(addr, key, value)
+//	}
+//	return nil
+//}
+//
+//func (ht *HashTrie) DeleteAccount(address common.Address) error {
+//	err := ht.trie.DeleteAccount(address)
+//	if err != nil {
+//		if ht.debug != nil {
+//			ht.debug.OnError(fmt.Errorf("failed to delete account, address: %s, error: %s", address.String(), err.Error()))
+//		}
+//		return err
+//	}
+//	if ht.debug != nil {
+//		ht.debug.OnDeleteAccount(address)
+//	}
+//	return nil
+//}
+//
+//func (ht *HashTrie) DeleteStorage(addr common.Address, key []byte) error {
+//	err := ht.trie.DeleteStorage(addr, key)
+//	if err != nil {
+//		if ht.debug != nil {
+//			ht.debug.OnError(fmt.Errorf("failed to update storage, address: %s, key: %s, error: %s",
+//				addr.String(), common.Bytes2Hex(key), err.Error()))
+//		}
+//		return err
+//	}
+//	if ht.debug != nil {
+//		ht.debug.OnDeleteStorage(addr, key)
+//	}
+//	return nil
+//}
+//
+//func (ht *HashTrie) UpdateContractCode(address common.Address, codeHash common.Hash, code []byte) error {
+//	return ht.trie.UpdateContractCode(address, codeHash, code)
+//}
+//
+//func (ht *HashTrie) Hash() common.Hash {
+//	root := ht.trie.Hash()
+//	if ht.debug != nil {
+//		ht.debug.OnCalcHash(ht.address, root)
+//	}
+//	return root
+//}
+//
+//func (ht *HashTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) {
+//	hash, set, err := ht.trie.Commit(collectLeaf)
+//	if err != nil {
+//		ht.debug.OnError(fmt.Errorf("failed to commit tree, address: %s, error: %s",
+//			ht.address.String(), err.Error()))
+//		return hash, set, err
+//	}
+//	if ht.debug != nil {
+//		ht.debug.OnCalcHash(ht.address, hash)
+//		ht.debug.OnCommitTree(ht.address, hash)
+//	}
+//	return hash, set, nil
+//}
+//
+//func (ht *HashTrie) NodeIterator(startKey []byte) (trie.NodeIterator, error) {
+//	return ht.trie.NodeIterator(startKey)
+//}
+//
+//func (ht *HashTrie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error {
+//	return ht.trie.Prove(key, proofDb)
+//}

core/state/dump.go

@@ -161,7 +161,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []
 			address = &addr
 			account.Address = address
 		}
-		obj := newObject(s, addr, &data)
+		obj := newObject(s, addr, &data, InvalidSateObjectVersion)
 		if !conf.SkipCode {
 			account.Code = obj.Code()
 		}

core/state/pruner/pruner.go

@@ -232,6 +232,7 @@ func pruneAll(maindb ethdb.Database, g *core.Genesis) error {
 		}
 		log.Info("Database compaction finished", "elapsed", common.PrettyDuration(time.Since(cstart)))
 	}
+	// pruner should be not used to version db
 	statedb, _ := state.New(common.Hash{}, state.NewDatabase(maindb), nil)
 	for addr, account := range g.Alloc {
 		statedb.AddBalance(addr, uint256.MustFromBig(account.Balance))

core/state/state_object.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/metrics"
@@ -63,6 +64,7 @@ func (s Storage) Copy() Storage {
 // - Finally, call commit to return the changes of storage trie and update account data.
 type stateObject struct {
 	db       *StateDB
+	version  int64
 	address  common.Address      // address of ethereum account
 	addrHash common.Hash         // hash of ethereum address of the account
 	origin   *types.StateAccount // Account original data without any change applied, nil means it was not existent
@@ -99,7 +101,7 @@ func (s *stateObject) empty() bool {
 }
 
 // newObject creates a state object.
-func newObject(db *StateDB, address common.Address, acct *types.StateAccount) *stateObject {
+func newObject(db *StateDB, address common.Address, acct *types.StateAccount, version int64) *stateObject {
 	var (
 		origin  = acct
 		created = acct == nil // true if the account was not existent
@@ -112,9 +114,9 @@ func newObject(db *StateDB, address common.Address, acct *types.StateAccount) *s
 	if db != nil && db.storagePool != nil {
 		storageMap = db.GetStorage(address)
 	}
-
 	return &stateObject{
 		db:                  db,
+		version:             version,
 		address:             address,
 		addrHash:            crypto.Keccak256Hash(address[:]),
 		origin:              origin,
@@ -158,8 +160,18 @@ func (s *stateObject) getTrie() (Trie, error) {
 		//	s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root)
 		// }
 		// if s.trie == nil {
-		tr, err := s.db.db.OpenStorageTrie(s.db.originalRoot, s.address, s.data.Root, s.db.trie)
+		var (
+			tr  Trie
+			err error
+		)
+		if s.version == InvalidSateObjectVersion {
+			tr, err = s.db.db.OpenStorageTrie(s.db.originalRoot, s.address, s.data.Root, s.db.trie)
+		} else {
+			tr, err = s.db.db.(*cachingVersaDB).openStorageTreeWithVersion(s.version, s.db.originalRoot, s.address, s.data.Root)
+		}
+
 		if err != nil {
+			panic(fmt.Sprintf("open storage storage failed, root version: %d, storage version: %d, addrss: %s, storage root: %s, error: %s", s.db.db.GetVersion(), s.version, s.address.String(), s.data.Root.String(), err.Error()))
 			return nil, err
 		}
 		s.trie = tr
@@ -229,6 +241,7 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash {
 		value common.Hash
 	)
 	if s.db.snap != nil {
+		panic("snap is not nil")
 		start := time.Now()
 		enc, err = s.db.snap.Storage(s.addrHash, crypto.Keccak256Hash(key.Bytes()))
 		if metrics.EnabledExpensive {
@@ -302,6 +315,30 @@ func (s *stateObject) finalise(prefetch bool) {
 	}
 }
 
+func (s *stateObject) IsContractAccount() bool {
+	return s.data.Root.Cmp(types.EmptyRootHash) != 0 ||
+		bytes.Compare(s.data.CodeHash, types.EmptyCodeHash.Bytes()) != 0
+}
+
+func (s *stateObject) IsAccountChanged() bool {
+	if s.origin == nil {
+		return true
+	}
+	if s.data.Nonce != s.origin.Nonce {
+		return true
+	}
+	if s.data.Balance.Cmp(s.origin.Balance) != 0 {
+		return true
+	}
+	if s.data.Root.Cmp(s.origin.Root) != 0 {
+		return true
+	}
+	if bytes.Compare(s.data.CodeHash, s.origin.CodeHash) != 0 {
+		return true
+	}
+	return false
+}
+
 // updateTrie is responsible for persisting cached storage changes into the
 // object's storage trie. In case the storage trie is not yet loaded, this
 // function will load the trie automatically. If any issues arise during the
@@ -312,10 +349,28 @@ func (s *stateObject) updateTrie() (Trie, error) {
 	// Make sure all dirty slots are finalized into the pending storage area
 	s.finalise(false)
 
-	// Short circuit if nothing changed, don't bother with hashing anything
-	if len(s.pendingStorage) == 0 {
-		return s.trie, nil
+	// fix 33740 blocks issue, add 1002 contract balance, but not update 1002
+	// storage tree, the case lead to 1002 account version mismatch with 1002
+	// storage tree version, occurs 53409 block open 1002 storage tree error.
+	if s.db.db.Scheme() == rawdb.VersionScheme {
+		if len(s.pendingStorage) == 0 {
+			// transferring balance to a contract or upgrading the code, but
+			// without updating the storage key, a commit is still required to
+			// increment the version number of the storage tree.
+			if !s.IsContractAccount() {
+				return s.trie, nil
+			}
+			//if !s.IsAccountChanged() {
+			//	return s.trie, nil
+			//}
+		}
+	} else {
+		// Short circuit if nothing changed, don't bother with hashing anything
+		if len(s.pendingStorage) == 0 {
+			return s.trie, nil
+		}
 	}
+
 	// Track the amount of time wasted on updating the storage trie
 	if metrics.EnabledExpensive {
 		defer func(start time.Time) {
@@ -511,6 +566,7 @@ func (s *stateObject) setBalance(amount *uint256.Int) {
 }
 
 func (s *stateObject) deepCopy(db *StateDB) *stateObject {
+	//TODO:: debug code, deleted in the future
 	obj := &stateObject{
 		db:       db,
 		address:  s.address,

core/state/statedb.go

@@ -166,12 +166,18 @@ func NewWithSharedPool(root common.Hash, db Database, snaps *snapshot.Tree) (*St
 	if err != nil {
 		return nil, err
 	}
-	statedb.storagePool = NewStoragePool()
+	//statedb.storagePool = NewStoragePool()
 	return statedb, nil
 }
 
 // New creates a new state from a given trie.
 func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) {
+	if db.Scheme() == rawdb.VersionScheme && snaps != nil {
+		panic("statedb snapshot must be nil in version db.")
+	}
+	// clean up previous traces
+	db.Reset()
+
 	sdb := &StateDB{
 		db:                   db,
 		originalRoot:         root,
@@ -196,6 +202,7 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error)
 		sdb.snap = sdb.snaps.Snapshot(root)
 	}
 
+	// It should only one to open account tree
 	tr, err := db.OpenTrie(root)
 	// return error when 1. failed to open trie and 2. the snap is nil or the snap is not nil and done verification
 	if err != nil && (sdb.snap == nil || sdb.snap.Verified()) {
@@ -222,6 +229,9 @@ func (s *StateDB) TransferPrefetcher(prev *StateDB) {
 	prev.prefetcherLock.Lock()
 	fetcher = prev.prefetcher
 	prev.prefetcher = nil
+	if fetcher != nil {
+		panic("TransferPrefetcher is not nil")
+	}
 	prev.prefetcherLock.Unlock()
 
 	s.prefetcherLock.Lock()
@@ -243,6 +253,8 @@ func (s *StateDB) StartPrefetcher(namespace string) {
 		s.prefetcher = nil
 	}
 	if s.snap != nil {
+		// TODO:: debug code , will be deleted in the future
+		panic("snapshot is not nill, will start prefetch")
 		parent := s.snap.Parent()
 		if parent != nil {
 			s.prefetcher = newTriePrefetcher(s.db, s.originalRoot, parent.Root(), namespace)
@@ -305,6 +317,7 @@ func (s *StateDB) EnablePipeCommit() {
 	if s.snap != nil && s.snaps.Layers() > 1 {
 		// after big merge, disable pipeCommit for now,
 		// because `s.db.TrieDB().Update` should be called after `s.trie.Commit(true)`
+		panic("snapshot is not nil")
 		s.pipeCommit = false
 	}
 }
@@ -323,6 +336,8 @@ func (s *StateDB) MarkFullProcessed() {
 func (s *StateDB) setError(err error) {
 	if s.dbErr == nil {
 		s.dbErr = err
+	} else {
+		s.dbErr = fmt.Errorf(s.dbErr.Error()+", ", err.Error())
 	}
 }
 
@@ -338,6 +353,8 @@ func (s *StateDB) Error() error {
 // Not thread safe
 func (s *StateDB) Trie() (Trie, error) {
 	if s.trie == nil {
+		// TODO:: debug code, will be deleted in the future.
+		panic("state get trie is nil")
 		err := s.WaitPipeVerification()
 		if err != nil {
 			return nil, err
@@ -719,6 +736,7 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject {
 	// If no live objects are available, attempt to use snapshots
 	var data *types.StateAccount
 	if s.snap != nil {
+		panic("snapshot is not nil")
 		start := time.Now()
 		acc, err := s.snap.Account(crypto.HashData(s.hasher, addr.Bytes()))
 		if metrics.EnabledExpensive {
@@ -743,9 +761,12 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject {
 		}
 	}
 
+	version := InvalidSateObjectVersion
 	// If snapshot unavailable or reading from it failed, load from the database
 	if data == nil {
 		if s.trie == nil {
+			// TODO:: debug code, will be deleted in the future.
+			panic("getDeletedStateObject get trie is nil")
 			tr, err := s.db.OpenTrie(s.originalRoot)
 			if err != nil {
 				s.setError(errors.New("failed to open trie tree"))
@@ -755,7 +776,12 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject {
 		}
 		start := time.Now()
 		var err error
-		data, err = s.trie.GetAccount(addr)
+		if vtr, ok := s.trie.(*VersaTree); ok {
+			version, data, err = vtr.getAccountWithVersion(addr)
+		} else {
+			data, err = s.trie.GetAccount(addr)
+		}
+
 		if metrics.EnabledExpensive {
 			s.AccountReads += time.Since(start)
 		}
@@ -768,7 +794,7 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject {
 		}
 	}
 	// Insert into the live set
-	obj := newObject(s, addr, data)
+	obj := newObject(s, addr, data, version)
 	s.setStateObject(obj)
 	return obj
 }
@@ -790,7 +816,7 @@ func (s *StateDB) getOrNewStateObject(addr common.Address) *stateObject {
 // the given address, it is overwritten and returned as the second return value.
 func (s *StateDB) createObject(addr common.Address) (newobj, prev *stateObject) {
 	prev = s.getDeletedStateObject(addr) // Note, prev might have been deleted, we need that!
-	newobj = newObject(s, addr, nil)
+	newobj = newObject(s, addr, nil, InvalidSateObjectVersion)
 	if prev == nil {
 		s.journal.append(createObjectChange{account: &addr})
 	} else {
@@ -859,10 +885,12 @@ func (s *StateDB) CopyDoPrefetch() *StateDB {
 // If doPrefetch is true, it tries to reuse the prefetcher, the copied StateDB will do active trie prefetch.
 // otherwise, just do inactive copy trie prefetcher.
 func (s *StateDB) copyInternal(doPrefetch bool) *StateDB {
+	db := s.db.Copy()
+	tr := db.CopyTrie(s.trie)
 	// Copy all the basic fields, initialize the memory ones
 	state := &StateDB{
-		db:   s.db,
-		trie: s.db.CopyTrie(s.trie),
+		db:   db,
+		trie: tr,
 		// noTrie:s.noTrie,
 		// expectedRoot:         s.expectedRoot,
 		// stateRoot:            s.stateRoot,
@@ -933,8 +961,8 @@ func (s *StateDB) copyInternal(doPrefetch bool) *StateDB {
 	// along with their original values.
 	state.accounts = copySet(s.accounts)
 	state.storages = copy2DSet(s.storages)
-	state.accountsOrigin = copySet(state.accountsOrigin)
-	state.storagesOrigin = copy2DSet(state.storagesOrigin)
+	state.accountsOrigin = copySet(s.accountsOrigin)
+	state.storagesOrigin = copy2DSet(s.storagesOrigin)
 
 	// Deep copy the logs occurred in the scope of block
 	for hash, logs := range s.logs {
@@ -1003,6 +1031,7 @@ func (s *StateDB) GetRefund() uint64 {
 func (s *StateDB) WaitPipeVerification() error {
 	// Need to wait for the parent trie to commit
 	if s.snap != nil {
+		panic("snapshot is not nil")
 		if valid := s.snap.WaitAndGetVerifyRes(); !valid {
 			return errors.New("verification on parent snap failed")
 		}
@@ -1108,6 +1137,7 @@ func (s *StateDB) PopulateSnapAccountAndStorage() {
 	for addr := range s.stateObjectsPending {
 		if obj := s.stateObjects[addr]; !obj.deleted {
 			if s.snap != nil {
+				panic("snapshot is not nil")
 				s.populateSnapStorage(obj)
 				s.accounts[obj.addrHash] = types.SlimAccountRLP(obj.data)
 			}
@@ -1210,6 +1240,8 @@ func (s *StateDB) StateIntermediateRoot() common.Hash {
 		}
 	}
 	if s.trie == nil {
+		// TODO:: debug code, will be deleted in the future.
+		panic("StateIntermediateRoot get trie is nil")
 		tr, err := s.db.OpenTrie(s.originalRoot)
 		if err != nil {
 			panic(fmt.Sprintf("failed to open trie tree %s", s.originalRoot))
@@ -1369,6 +1401,7 @@ func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root
 	// generated, or it's internally corrupted. Fallback to the slow
 	// one just in case.
 	if s.snap != nil {
+		panic("snapshot is not nil")
 		aborted, size, slots, nodes, err = s.fastDeleteStorage(addrHash, root)
 	}
 	if s.snap == nil || err != nil {
@@ -1422,7 +1455,8 @@ func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.A
 	// considerable time and storage deletion isn't supported in hash mode, thus
 	// preemptively avoiding unnecessary expenses.
 	incomplete := make(map[common.Address]struct{})
-	if s.db.TrieDB().Scheme() == rawdb.HashScheme {
+	// Only pbss need handler incomplete destruction storage trie
+	if s.db.TrieDB().Scheme() != rawdb.PathScheme {
 		return incomplete, nil
 	}
 	for addr, prev := range s.stateObjectsDestruct {
@@ -1498,6 +1532,7 @@ func (s *StateDB) Commit(block uint64, failPostCommitFunc func(), postCommitFunc
 	)
 
 	if s.snap != nil {
+		panic("snapshot is not nil")
 		diffLayer = &types.DiffLayer{}
 	}
 	if s.pipeCommit {
@@ -1613,18 +1648,33 @@ func (s *StateDB) Commit(block uint64, failPostCommitFunc func(), postCommitFunc
 					origin = types.EmptyRootHash
 				}
 
-				if root != origin {
+				if s.db.Scheme() == rawdb.VersionScheme {
+					// flush and release will occur regardless of whether the root changes
 					start := time.Now()
-					set := triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete)
-					if err := s.db.TrieDB().Update(root, origin, block, nodes, set); err != nil {
+					if err := s.db.Flush(); err != nil {
+						return err
+					}
+					if err := s.db.Release(); err != nil {
 						return err
 					}
 					s.originalRoot = root
 					if metrics.EnabledExpensive {
 						s.TrieDBCommits += time.Since(start)
 					}
-					if s.onCommit != nil {
-						s.onCommit(set)
+				} else {
+					if root != origin {
+						start := time.Now()
+						set := triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete)
+						if err := s.db.TrieDB().Update(root, origin, block, nodes, set); err != nil {
+							return err
+						}
+						s.originalRoot = root
+						if metrics.EnabledExpensive {
+							s.TrieDBCommits += time.Since(start)
+						}
+						if s.onCommit != nil {
+							s.onCommit(set)
+						}
 					}
 				}
 			}
@@ -1663,8 +1713,21 @@ func (s *StateDB) Commit(block uint64, failPostCommitFunc func(), postCommitFunc
 					// Write any contract code associated with the state object
 					if obj.code != nil && obj.dirtyCode {
 						rawdb.WriteCode(codeWriter, common.BytesToHash(obj.CodeHash()), obj.code)
+						//switch d := s.db.(type) {
+						//case *cachingVersaDB:
+						//	if d.debug != nil {
+						//		d.debug.OnUpdateCode(obj.address, common.BytesToHash(obj.CodeHash()))
+						//	}
+						//case *cachingDB:
+						//	if d.debug != nil {
+						//		d.debug.OnUpdateCode(obj.address, common.BytesToHash(obj.CodeHash()))
+						//	}
+						//default:
+						//	panic("caching db type error")
+						//}
 						obj.dirtyCode = false
 						if s.snap != nil {
+							panic("snapshot is not nil")
 							diffLayer.Codes = append(diffLayer.Codes, types.DiffCode{
 								Hash: common.BytesToHash(obj.CodeHash()),
 								Code: obj.code,
@@ -1690,6 +1753,7 @@ func (s *StateDB) Commit(block uint64, failPostCommitFunc func(), postCommitFunc
 		func() error {
 			// If snapshotting is enabled, update the snapshot tree with this new version
 			if s.snap != nil {
+				panic("snapshot is not nil")
 				if metrics.EnabledExpensive {
 					defer func(start time.Time) { s.SnapshotCommits += time.Since(start) }(time.Now())
 				}

core/state/statedb_debug_diff.go

@@ -0,0 +1,207 @@
+package state
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+var (
+	VersionState = "version"
+	HashState    = "hash"
+)
+
+type DebugStateDiff struct {
+	DiffUpdateAccount map[string][]*VersaAccountInfo
+	DiffDeleteAccount map[string][]common.Address
+
+	DiffUpdateStorage map[string][]*VersaStorageInfo
+	DiffDeleteStorage map[string][]*VersaStorageInfo
+
+	DiffCalcHash map[string]map[common.Address]common.Hash
+	OwnerMap     map[common.Address]common.Hash
+	DiffErrs     map[string][]string
+}
+
+func (df *DebugStateDiff) diffUpdateAccount(vs []*VersaAccountInfo, hs []*VersaAccountInfo) {
+	count := len(vs)
+	if count > len(hs) {
+		count = len(hs)
+	}
+	idx := 0
+	for ; idx < count; idx++ {
+		if vs[idx].Address.Cmp(hs[idx].Address) != 0 {
+			break
+		}
+		if vs[idx].Account.Nonce != hs[idx].Account.Nonce {
+			break
+		}
+		if vs[idx].Account.Balance.Cmp(hs[idx].Account.Balance) != 0 {
+			break
+		}
+		if vs[idx].Account.Root.Cmp(hs[idx].Account.Root) != 0 {
+			break
+		}
+		if common.BytesToHash(vs[idx].Account.CodeHash).Cmp(common.BytesToHash(hs[idx].Account.CodeHash)) != 0 {
+			break
+		}
+	}
+	if idx < len(vs) {
+		df.DiffUpdateAccount[VersionState] = vs[idx:]
+	}
+	if idx < len(hs) {
+		df.DiffUpdateAccount[HashState] = hs[idx:]
+	}
+	return
+}
+
+func (df *DebugStateDiff) diffDeleteAccount(vs []common.Address, hs []common.Address) {
+	count := len(vs)
+	if count > len(hs) {
+		count = len(hs)
+	}
+	idx := 0
+	for ; idx < count; idx++ {
+		if vs[idx].Cmp(hs[idx]) != 0 {
+			break
+		}
+	}
+	if idx < len(vs) {
+		df.DiffDeleteAccount[VersionState] = vs[idx:]
+	}
+	if idx < len(hs) {
+		df.DiffDeleteAccount[HashState] = hs[idx:]
+	}
+	return
+}
+
+func (df *DebugStateDiff) diffUpdateStorage(vs []*VersaStorageInfo, hs []*VersaStorageInfo) {
+	count := len(vs)
+	if count > len(hs) {
+		count = len(hs)
+	}
+	idx := 0
+	for ; idx < count; idx++ {
+		if vs[idx].Address.Cmp(hs[idx].Address) != 0 {
+			break
+		}
+		if vs[idx].Key != hs[idx].Key {
+			break
+		}
+		if vs[idx].Val != hs[idx].Val {
+			break
+		}
+	}
+	if idx < len(vs) {
+		df.DiffUpdateStorage[VersionState] = vs[idx:]
+	}
+	if idx < len(hs) {
+		df.DiffUpdateStorage[HashState] = hs[idx:]
+	}
+	return
+}
+
+func (df *DebugStateDiff) diffDeleteStorage(vs []*VersaStorageInfo, hs []*VersaStorageInfo) {
+	count := len(vs)
+	if count > len(hs) {
+		count = len(hs)
+	}
+	idx := 0
+	for ; idx < count; idx++ {
+		if vs[idx].Address.Cmp(hs[idx].Address) != 0 {
+			break
+		}
+		if vs[idx].Key != hs[idx].Key {
+			break
+		}
+	}
+	if idx < len(vs) {
+		df.DiffDeleteStorage[VersionState] = vs[idx:]
+	}
+	if idx < len(hs) {
+		df.DiffDeleteStorage[HashState] = hs[idx:]
+	}
+	return
+}
+
+func (df *DebugStateDiff) diffCalcHash(vs map[common.Address]common.Hash, hs map[common.Address]common.Hash) {
+	record := make(map[common.Address]struct{})
+	for address, vch := range vs {
+		record[address] = struct{}{}
+		hch, ok := hs[address]
+		if !ok {
+			df.DiffCalcHash[VersionState][address] = vch
+		}
+		if vch.Cmp(hch) != 0 {
+			df.DiffCalcHash[VersionState][address] = vch
+			df.DiffCalcHash[HashState][address] = hch
+		}
+	}
+
+	for address := range record {
+		delete(vs, address)
+		delete(hs, address)
+	}
+
+	for address, hash := range vs {
+		df.DiffCalcHash[VersionState][address] = hash
+	}
+
+	for address, hash := range hs {
+		df.DiffCalcHash[HashState][address] = hash
+	}
+}
+
+func GenerateDebugStateDiff(vs *DebugVersionState, hs *DebugHashState) string {
+	diff := &DebugStateDiff{
+		DiffUpdateAccount: make(map[string][]*VersaAccountInfo),
+		DiffDeleteAccount: make(map[string][]common.Address),
+
+		DiffUpdateStorage: make(map[string][]*VersaStorageInfo),
+		DiffDeleteStorage: make(map[string][]*VersaStorageInfo),
+
+		DiffCalcHash: make(map[string]map[common.Address]common.Hash),
+		OwnerMap:     make(map[common.Address]common.Hash),
+		DiffErrs:     make(map[string][]string),
+	}
+	diff.DiffUpdateAccount[VersionState] = make([]*VersaAccountInfo, 0)
+	diff.DiffUpdateAccount[HashState] = make([]*VersaAccountInfo, 0)
+	diff.DiffDeleteAccount[VersionState] = make([]common.Address, 0)
+	diff.DiffDeleteAccount[HashState] = make([]common.Address, 0)
+
+	diff.DiffUpdateStorage[VersionState] = make([]*VersaStorageInfo, 0)
+	diff.DiffUpdateStorage[HashState] = make([]*VersaStorageInfo, 0)
+	diff.DiffDeleteStorage[VersionState] = make([]*VersaStorageInfo, 0)
+	diff.DiffDeleteStorage[HashState] = make([]*VersaStorageInfo, 0)
+
+	diff.DiffCalcHash[VersionState] = make(map[common.Address]common.Hash)
+	diff.DiffCalcHash[HashState] = make(map[common.Address]common.Hash)
+
+	diff.DiffErrs[VersionState] = make([]string, 0)
+	diff.DiffErrs[HashState] = make([]string, 0)
+
+	diff.diffUpdateAccount(vs.UpdateAccounts, hs.UpdateAccounts)
+	diff.diffDeleteAccount(vs.DeleteAccounts, hs.DeleteAccounts)
+	diff.diffUpdateStorage(vs.UpdateStorage, hs.UpdateStorage)
+	diff.diffDeleteStorage(vs.DeleteStorage, hs.DeleteStorage)
+	diff.diffCalcHash(vs.CalcHash, hs.CalcHash)
+
+	for address, _ := range diff.DiffCalcHash[HashState] {
+		diff.OwnerMap[address] = hs.StorageAddr2Owner[address]
+	}
+	for address, _ := range diff.DiffCalcHash[VersionState] {
+		diff.OwnerMap[address] = vs.StorageAddr2Owner[address]
+	}
+
+	if len(vs.Errs) != 0 || len(hs.Errs) != 0 {
+		diff.DiffErrs[VersionState] = vs.Errs
+		diff.DiffErrs[HashState] = hs.Errs
+	}
+
+	data, err := json.Marshal(diff)
+	if err != nil {
+		panic(fmt.Sprintf("failed to json encode debug info, err: %s", err.Error()))
+	}
+	return string(data)
+}

core/state/statedb_debug_hash.go

@@ -0,0 +1,216 @@
+package state
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strconv"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+type DebugHashState struct {
+	disk ethdb.KeyValueStore
+	lock sync.Mutex
+
+	Version     int64
+	AccessTrees map[common.Address][]common.Hash
+	CommitTrees map[common.Address][]common.Hash
+	CalcHash    map[common.Address]common.Hash
+
+	GetAccounts    []*VersaAccountInfo
+	UpdateAccounts []*VersaAccountInfo
+	DeleteAccounts []common.Address
+
+	GetStorage        []*VersaStorageInfo
+	UpdateStorage     []*VersaStorageInfo
+	DeleteStorage     []*VersaStorageInfo
+	StorageAddr2Owner map[common.Address]common.Hash
+
+	GetCode    map[common.Address][]common.Hash
+	UpdateCode map[common.Address][]common.Hash
+
+	Errs []string
+}
+
+func NewDebugHashState(disk ethdb.KeyValueStore) *DebugHashState {
+	return &DebugHashState{
+		disk:              disk,
+		AccessTrees:       make(map[common.Address][]common.Hash),
+		CommitTrees:       make(map[common.Address][]common.Hash),
+		CalcHash:          make(map[common.Address]common.Hash),
+		GetAccounts:       make([]*VersaAccountInfo, 0),
+		UpdateAccounts:    make([]*VersaAccountInfo, 0),
+		DeleteAccounts:    make([]common.Address, 0),
+		GetStorage:        make([]*VersaStorageInfo, 0),
+		UpdateStorage:     make([]*VersaStorageInfo, 0),
+		DeleteStorage:     make([]*VersaStorageInfo, 0),
+		StorageAddr2Owner: make(map[common.Address]common.Hash),
+		GetCode:           make(map[common.Address][]common.Hash),
+		UpdateCode:        make(map[common.Address][]common.Hash),
+		Errs:              make([]string, 0),
+	}
+}
+
+func (hs *DebugHashState) OnOpenTree(root common.Hash, owner common.Hash, address common.Address) {
+	hs.lock.Lock()
+	defer hs.lock.Unlock()
+
+	if _, ok := hs.AccessTrees[address]; !ok {
+		hs.AccessTrees[address] = make([]common.Hash, 0)
+	}
+	hs.AccessTrees[address] = append(hs.AccessTrees[address], root)
+	hs.StorageAddr2Owner[address] = owner
+}
+
+func (hs *DebugHashState) OnGetAccount(addr common.Address, acc *types.StateAccount) {
+	hs.lock.Lock()
+	defer hs.lock.Unlock()
+	hs.GetAccounts = append(hs.GetAccounts, &VersaAccountInfo{
+		Address: addr,
+		Account: acc,
+	})
+}
+
+func (hs *DebugHashState) OnUpdateAccount(addr common.Address, acc *types.StateAccount) {
+	hs.lock.Lock()
+	defer hs.lock.Unlock()
+	hs.UpdateAccounts = append(hs.UpdateAccounts, &VersaAccountInfo{
+		Address: addr,
+		Account: acc,
+	})
+}
+
+func (hs *DebugHashState) OnDeleteAccount(address common.Address) {
+	hs.lock.Lock()
+	defer hs.lock.Unlock()
+	hs.DeleteAccounts = append(hs.DeleteAccounts, address)
+}
+
+func (hs *DebugHashState) OnGetStorage(address common.Address, key []byte, val []byte) {
+	hs.lock.Lock()
+	defer hs.lock.Unlock()
+
+	hs.GetStorage = append(hs.GetStorage, &VersaStorageInfo{
+		Address: address,
+		Key:     common.Bytes2Hex(key),
+		Val:     common.Bytes2Hex(val),
+	})
+}
+
+func (hs *DebugHashState) OnUpdateStorage(address common.Address, key []byte, val []byte) {
+	hs.lock.Lock()
+	defer hs.lock.Unlock()
+
+	hs.UpdateStorage = append(hs.UpdateStorage, &VersaStorageInfo{
+		Address: address,
+		Key:     common.Bytes2Hex(key),
+		Val:     common.Bytes2Hex(val),
+	})
+}
+
+func (hs *DebugHashState) OnDeleteStorage(address common.Address, key []byte) {
+	hs.lock.Lock()
+	defer hs.lock.Unlock()
+
+	hs.DeleteStorage = append(hs.DeleteStorage, &VersaStorageInfo{
+		Address: address,
+		Key:     common.Bytes2Hex(key),
+	})
+}
+
+func (hs *DebugHashState) OnGetCode(addr common.Address, codeHash common.Hash) {
+	hs.lock.Lock()
+	defer hs.lock.Unlock()
+	if _, ok := hs.GetCode[addr]; !ok {
+		hs.GetCode[addr] = make([]common.Hash, 0)
+	}
+	hs.GetCode[addr] = append(hs.GetCode[addr], codeHash)
+}
+
+func (hs *DebugHashState) OnUpdateCode(addr common.Address, codeHash common.Hash) {
+	hs.lock.Lock()
+	defer hs.lock.Unlock()
+	if _, ok := hs.UpdateCode[addr]; !ok {
+		hs.UpdateCode[addr] = make([]common.Hash, 0)
+	}
+	hs.UpdateCode[addr] = append(hs.UpdateCode[addr], codeHash)
+}
+
+func (hs *DebugHashState) OnCalcHash(addr common.Address, root common.Hash) {
+	hs.lock.Lock()
+	defer hs.lock.Unlock()
+	hs.CalcHash[addr] = root
+}
+
+func (hs *DebugHashState) OnCommitTree(addr common.Address, root common.Hash) {
+	hs.lock.Lock()
+	defer hs.lock.Unlock()
+	if _, ok := hs.CommitTrees[addr]; !ok {
+		hs.CommitTrees[addr] = make([]common.Hash, 0)
+	}
+	hs.CommitTrees[addr] = append(hs.CommitTrees[addr], root)
+}
+
+func (hs *DebugHashState) OnError(err error) {
+	hs.lock.Lock()
+	defer hs.lock.Unlock()
+	hs.Errs = append(hs.Errs, err.Error())
+}
+
+func (hs *DebugHashState) flush() {
+	hs.lock.Lock()
+	defer hs.lock.Unlock()
+
+	hs.sortItems()
+	data, err := json.Marshal(hs)
+	if err != nil {
+		panic(fmt.Sprintf("failed to json encode debug info, err: %s", err.Error()))
+	}
+
+	err = hs.disk.Put(DebugHashStateKey(hs.Version), data)
+	if err != nil {
+		panic(fmt.Sprintf("failed to put debug version state into disk, err: %s", err.Error()))
+	}
+}
+
+func (hs *DebugHashState) sortItems() {
+	sort.Slice(hs.GetAccounts, func(i, j int) bool {
+		return hs.GetAccounts[i].Address.Cmp(hs.GetAccounts[j].Address) < 0
+	})
+	sort.Slice(hs.UpdateAccounts, func(i, j int) bool {
+		return hs.UpdateAccounts[i].Address.Cmp(hs.UpdateAccounts[j].Address) < 0
+	})
+	sort.Slice(hs.DeleteAccounts, func(i, j int) bool {
+		return hs.DeleteAccounts[i].Cmp(hs.DeleteAccounts[j]) < 0
+	})
+
+	sort.Slice(hs.GetStorage, func(i, j int) bool {
+		if hs.GetStorage[i].Address.Cmp(hs.GetStorage[j].Address) == 0 {
+			return hs.GetStorage[i].Key < hs.GetStorage[j].Key
+		}
+		return hs.GetStorage[i].Address.Cmp(hs.GetStorage[j].Address) < 0
+	})
+
+	sort.Slice(hs.UpdateStorage, func(i, j int) bool {
+		if hs.UpdateStorage[i].Address.Cmp(hs.UpdateStorage[j].Address) == 0 {
+			return hs.UpdateStorage[i].Key < hs.UpdateStorage[j].Key
+		}
+		return hs.UpdateStorage[i].Address.Cmp(hs.UpdateStorage[j].Address) < 0
+	})
+
+	sort.Slice(hs.DeleteStorage, func(i, j int) bool {
+		if hs.DeleteStorage[i].Address.Cmp(hs.DeleteStorage[j].Address) == 0 {
+			return hs.DeleteStorage[i].Key < hs.DeleteStorage[j].Key
+		}
+		return hs.DeleteStorage[i].Address.Cmp(hs.DeleteStorage[j].Address) < 0
+	})
+}
+
+func DebugHashStateKey(version int64) []byte {
+	key := "debug_hash_prefix" + strconv.FormatInt(version, 10)
+	return []byte(key)
+}

core/state/statedb_debug_version.go

@@ -0,0 +1,290 @@
+package state
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strconv"
+	"sync"
+
+	versa "github.com/bnb-chain/versioned-state-database"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+var (
+	DiffVersionCount = 0
+	DiskVersionCount = 0
+)
+
+type VersaAccountInfo struct {
+	Address common.Address
+	Account *types.StateAccount
+}
+
+type VersaStorageInfo struct {
+	Handler versa.TreeHandler
+	Address common.Address
+	Key     string
+	Val     string
+}
+
+type DebugVersionState struct {
+	disk      ethdb.KeyValueStore
+	versionDB versa.Database
+	lock      sync.Mutex
+
+	Version     int64
+	PreState    *versa.StateInfo
+	PostState   *versa.StateInfo
+	AccessTrees map[common.Address][]*versa.TreeInfo
+	CommitTrees map[common.Address][]*versa.TreeInfo
+	CalcHash    map[common.Address]common.Hash
+
+	GetAccounts    []*VersaAccountInfo
+	UpdateAccounts []*VersaAccountInfo
+	DeleteAccounts []common.Address
+
+	GetStorage        []*VersaStorageInfo
+	UpdateStorage     []*VersaStorageInfo
+	DeleteStorage     []*VersaStorageInfo
+	StorageAddr2Owner map[common.Address]common.Hash
+
+	GetCode    map[common.Address][]common.Hash
+	UpdateCode map[common.Address][]common.Hash
+
+	Errs []string
+}
+
+func NewDebugVersionState(disk ethdb.KeyValueStore, versionDB versa.Database) *DebugVersionState {
+	return &DebugVersionState{
+		disk:              disk,
+		versionDB:         versionDB,
+		AccessTrees:       make(map[common.Address][]*versa.TreeInfo, 0),
+		CommitTrees:       make(map[common.Address][]*versa.TreeInfo, 0),
+		CalcHash:          make(map[common.Address]common.Hash),
+		GetAccounts:       make([]*VersaAccountInfo, 0),
+		UpdateAccounts:    make([]*VersaAccountInfo, 0),
+		DeleteAccounts:    make([]common.Address, 0),
+		GetStorage:        make([]*VersaStorageInfo, 0),
+		UpdateStorage:     make([]*VersaStorageInfo, 0),
+		DeleteStorage:     make([]*VersaStorageInfo, 0),
+		StorageAddr2Owner: make(map[common.Address]common.Hash),
+		GetCode:           make(map[common.Address][]common.Hash, 0),
+		UpdateCode:        make(map[common.Address][]common.Hash, 0),
+		Errs:              make([]string, 0),
+	}
+}
+func (ds *DebugVersionState) SetVersion(version int64) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+	ds.Version = version
+}
+
+func (ds *DebugVersionState) OnOpenState(handler versa.StateHandler) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+
+	stateInfo, err := ds.versionDB.GetStateInfo(handler)
+	if err != nil {
+		panic(fmt.Sprintf("failed to get state info on open state, err: %s", err.Error()))
+	}
+	ds.PreState = stateInfo
+}
+
+func (ds *DebugVersionState) OnOpenTree(handler versa.TreeHandler, owner common.Hash, address common.Address) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+
+	treeInfo, err := ds.versionDB.GetTreeInfo(handler)
+	if err != nil {
+		panic(fmt.Sprintf("failed to get tree info on open tree, err: %s", err.Error()))
+	}
+	if _, ok := ds.AccessTrees[address]; !ok {
+		ds.AccessTrees[address] = make([]*versa.TreeInfo, 0)
+	}
+	ds.AccessTrees[address] = append(ds.AccessTrees[address], treeInfo)
+	ds.StorageAddr2Owner[address] = owner
+}
+
+func (ds *DebugVersionState) OnGetAccount(addr common.Address, acc *types.StateAccount) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+	ds.GetAccounts = append(ds.GetAccounts, &VersaAccountInfo{
+		Address: addr,
+		Account: acc,
+	})
+}
+
+func (ds *DebugVersionState) OnUpdateAccount(addr common.Address, acc *types.StateAccount) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+	ds.UpdateAccounts = append(ds.UpdateAccounts, &VersaAccountInfo{
+		Address: addr,
+		Account: acc,
+	})
+}
+
+func (ds *DebugVersionState) OnDeleteAccount(address common.Address) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+	ds.DeleteAccounts = append(ds.DeleteAccounts, address)
+}
+
+func (ds *DebugVersionState) OnGetStorage(handler versa.TreeHandler, address common.Address, key []byte, val []byte) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+
+	ds.GetStorage = append(ds.GetStorage, &VersaStorageInfo{
+		Handler: handler,
+		Address: address,
+		Key:     common.Bytes2Hex(key),
+		Val:     common.Bytes2Hex(val),
+	})
+}
+
+func (ds *DebugVersionState) OnUpdateStorage(handler versa.TreeHandler, address common.Address, key []byte, val []byte) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+
+	ds.UpdateStorage = append(ds.UpdateStorage, &VersaStorageInfo{
+		Handler: handler,
+		Address: address,
+		Key:     common.Bytes2Hex(key),
+		Val:     common.Bytes2Hex(val),
+	})
+}
+
+func (ds *DebugVersionState) OnDeleteStorage(handler versa.TreeHandler, address common.Address, key []byte) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+
+	ds.DeleteStorage = append(ds.DeleteStorage, &VersaStorageInfo{
+		Handler: handler,
+		Address: address,
+		Key:     common.Bytes2Hex(key),
+	})
+}
+
+func (ds *DebugVersionState) OnGetCode(addr common.Address, codeHash common.Hash) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+	if _, ok := ds.GetCode[addr]; !ok {
+		ds.GetCode[addr] = make([]common.Hash, 0)
+	}
+	ds.GetCode[addr] = append(ds.GetCode[addr], codeHash)
+}
+
+func (ds *DebugVersionState) OnUpdateCode(addr common.Address, codeHash common.Hash) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+	if _, ok := ds.UpdateCode[addr]; !ok {
+		ds.UpdateCode[addr] = make([]common.Hash, 0)
+	}
+	ds.UpdateCode[addr] = append(ds.UpdateCode[addr], codeHash)
+}
+
+func (ds *DebugVersionState) OnCalcHash(addr common.Address, root common.Hash) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+	ds.CalcHash[addr] = root
+}
+
+func (ds *DebugVersionState) OnCommitTree(addr common.Address, handler versa.TreeHandler) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+	treeInfo, err := ds.versionDB.GetTreeInfo(handler)
+	if err != nil {
+		panic(fmt.Sprintf("failed to get tree info on commit tree, err: %s", err.Error()))
+	}
+	if _, ok := ds.CommitTrees[addr]; !ok {
+		ds.CommitTrees[addr] = make([]*versa.TreeInfo, 0)
+	}
+	ds.CommitTrees[addr] = append(ds.CommitTrees[addr], treeInfo)
+}
+
+func (ds *DebugVersionState) OnError(err error) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+	ds.Errs = append(ds.Errs, err.Error())
+}
+
+func (ds *DebugVersionState) OnCloseState(handler versa.StateHandler) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+
+	stateInfo, err := ds.versionDB.GetStateInfo(handler)
+	if err != nil {
+		panic(fmt.Sprintf("failed to get state info on close state, err: %s", err.Error()))
+	}
+	ds.PostState = stateInfo
+
+	oldDiskVersionCount := DiskVersionCount
+	if ds.PreState.Root.Cmp(ds.PostState.Root) != 0 {
+		DiffVersionCount++
+		if ds.PostState.IsDiskVersion {
+			DiskVersionCount++
+		}
+	}
+
+	if ds.Version%1000 == 0 || oldDiskVersionCount != DiskVersionCount {
+		log.Info("version state info", "current block", ds.Version, "diff version count", DiffVersionCount, "disk version count", DiskVersionCount)
+	}
+
+	ds.sortItems()
+
+	data, err := json.Marshal(ds)
+	if err != nil {
+		panic(fmt.Sprintf("failed to json encode debug info, err: %s", err.Error()))
+	}
+
+	err = ds.disk.Put(DebugVersionStateKey(ds.Version), data)
+	if err != nil {
+		panic(fmt.Sprintf("failed to put debug version state into disk, err: %s", err.Error()))
+	}
+
+	if len(ds.Errs) != 0 {
+		log.Info("version state occurs error", "debug info", string(data))
+		log.Crit("exit....")
+	}
+}
+
+func (ds *DebugVersionState) sortItems() {
+	sort.Slice(ds.GetAccounts, func(i, j int) bool {
+		return ds.GetAccounts[i].Address.Cmp(ds.GetAccounts[j].Address) < 0
+	})
+	sort.Slice(ds.UpdateAccounts, func(i, j int) bool {
+		return ds.UpdateAccounts[i].Address.Cmp(ds.UpdateAccounts[j].Address) < 0
+	})
+	sort.Slice(ds.DeleteAccounts, func(i, j int) bool {
+		return ds.DeleteAccounts[i].Cmp(ds.DeleteAccounts[j]) < 0
+	})
+
+	sort.Slice(ds.GetStorage, func(i, j int) bool {
+		if ds.GetStorage[i].Address.Cmp(ds.GetStorage[j].Address) == 0 {
+			return ds.GetStorage[i].Key < ds.GetStorage[j].Key
+		}
+		return ds.GetStorage[i].Address.Cmp(ds.GetStorage[j].Address) < 0
+	})
+
+	sort.Slice(ds.UpdateStorage, func(i, j int) bool {
+		if ds.UpdateStorage[i].Address.Cmp(ds.UpdateStorage[j].Address) == 0 {
+			return ds.UpdateStorage[i].Key < ds.UpdateStorage[j].Key
+		}
+		return ds.UpdateStorage[i].Address.Cmp(ds.UpdateStorage[j].Address) < 0
+	})
+
+	sort.Slice(ds.DeleteStorage, func(i, j int) bool {
+		if ds.DeleteStorage[i].Address.Cmp(ds.DeleteStorage[j].Address) == 0 {
+			return ds.DeleteStorage[i].Key < ds.DeleteStorage[j].Key
+		}
+		return ds.DeleteStorage[i].Address.Cmp(ds.DeleteStorage[j].Address) < 0
+	})
+}
+
+func DebugVersionStateKey(version int64) []byte {
+	key := "debug_version_prefix" + strconv.FormatInt(version, 10)
+	return []byte(key)
+}

core/state/sync.go

@@ -25,7 +25,7 @@ import (
 )
 
 // NewStateSync creates a new state trie download scheduler.
-func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(keys [][]byte, leaf []byte) error, scheme string) *trie.Sync {
+func NewStateSync(root common.Hash, database ethdb.Database, onLeaf func(keys [][]byte, leaf []byte) error, scheme string) *trie.Sync {
 	// Register the storage slot callback if the external callback is specified.
 	var onSlot func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error
 	if onLeaf != nil {

core/state/sync_test.go

@@ -268,7 +268,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool, s
 			}
 		}
 		batch := dstDb.NewBatch()
-		if err := sched.Commit(batch); err != nil {
+		if err := sched.Commit(batch, nil); err != nil {
 			t.Fatalf("failed to commit data: %v", err)
 		}
 		batch.Write()
@@ -369,7 +369,7 @@ func testIterativeDelayedStateSync(t *testing.T, scheme string) {
 			nodeProcessed = len(nodeResults)
 		}
 		batch := dstDb.NewBatch()
-		if err := sched.Commit(batch); err != nil {
+		if err := sched.Commit(batch, nil); err != nil {
 			t.Fatalf("failed to commit data: %v", err)
 		}
 		batch.Write()
@@ -469,7 +469,7 @@ func testIterativeRandomStateSync(t *testing.T, count int, scheme string) {
 			}
 		}
 		batch := dstDb.NewBatch()
-		if err := sched.Commit(batch); err != nil {
+		if err := sched.Commit(batch, nil); err != nil {
 			t.Fatalf("failed to commit data: %v", err)
 		}
 		batch.Write()
@@ -575,7 +575,7 @@ func testIterativeRandomDelayedStateSync(t *testing.T, scheme string) {
 			}
 		}
 		batch := dstDb.NewBatch()
-		if err := sched.Commit(batch); err != nil {
+		if err := sched.Commit(batch, nil); err != nil {
 			t.Fatalf("failed to commit data: %v", err)
 		}
 		batch.Write()
@@ -688,7 +688,7 @@ func testIncompleteStateSync(t *testing.T, scheme string) {
 			}
 		}
 		batch := dstDb.NewBatch()
-		if err := sched.Commit(batch); err != nil {
+		if err := sched.Commit(batch, nil); err != nil {
 			t.Fatalf("failed to commit data: %v", err)
 		}
 		batch.Write()

core/state/trie_prefetcher.go

@@ -81,6 +81,7 @@ type triePrefetcher struct {
 
 // newTriePrefetcher
 func newTriePrefetcher(db Database, root, rootParent common.Hash, namespace string) *triePrefetcher {
+	panic("prefetcher not support")
 	prefix := triePrefetchMetricsPrefix + namespace
 	p := &triePrefetcher{
 		db:         db,

core/state_prefetcher.go

@@ -49,6 +49,7 @@ func NewStatePrefetcher(config *params.ChainConfig, bc *BlockChain, engine conse
 // the transaction messages using the statedb, but any changes are discarded. The
 // only goal is to pre-cache transaction signatures and state trie nodes.
 func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, cfg *vm.Config, interruptCh <-chan struct{}) {
+	panic("prefetcher not support")
 	var (
 		header = block.Header()
 		signer = types.MakeSigner(p.config, header.Number, header.Time)
@@ -101,6 +102,7 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c
 // the transaction messages using the statedb, but any changes are discarded. The
 // only goal is to pre-cache transaction signatures and snapshot clean state. Only used for mining stage
 func (p *statePrefetcher) PrefetchMining(txs TransactionsByPriceAndNonce, header *types.Header, gasLimit uint64, statedb *state.StateDB, cfg vm.Config, interruptCh <-chan struct{}, txCurr **types.Transaction) {
+	panic("prefetcher not support")
 	var signer = types.MakeSigner(p.config, header.Number, header.Time)
 
 	txCh := make(chan *types.Transaction, 2*prefetchThread)

core/systemcontracts/haber_fix/types.go

@@ -0,0 +1,19 @@
+package haber_fix
+
+import _ "embed"
+
+// contract codes for Chapel upgrade
+var (
+	//go:embed chapel/ValidatorContract
+	ChapelValidatorContract string
+	//go:embed chapel/SlashContract
+	ChapelSlashContract string
+)
+
+// contract codes for Mainnet upgrade
+var (
+	//go:embed mainnet/ValidatorContract
+	MainnetValidatorContract string
+	//go:embed mainnet/SlashContract
+	MainnetSlashContract string
+)

core/systemcontracts/upgrade.go

@@ -4,17 +4,16 @@ import (
 	"encoding/hex"
 	"fmt"
 	"math/big"
+	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/params"
-
 	"github.com/ethereum/go-ethereum/core/systemcontracts/bruno"
 	"github.com/ethereum/go-ethereum/core/systemcontracts/euler"
 	"github.com/ethereum/go-ethereum/core/systemcontracts/feynman"
 	feynmanFix "github.com/ethereum/go-ethereum/core/systemcontracts/feynman_fix"
 	"github.com/ethereum/go-ethereum/core/systemcontracts/gibbs"
+	haberFix "github.com/ethereum/go-ethereum/core/systemcontracts/haber_fix"
 	"github.com/ethereum/go-ethereum/core/systemcontracts/kepler"
 	"github.com/ethereum/go-ethereum/core/systemcontracts/luban"
 	"github.com/ethereum/go-ethereum/core/systemcontracts/mirror"
@@ -23,6 +22,8 @@ import (
 	"github.com/ethereum/go-ethereum/core/systemcontracts/planck"
 	"github.com/ethereum/go-ethereum/core/systemcontracts/plato"
 	"github.com/ethereum/go-ethereum/core/systemcontracts/ramanujan"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/params"
 )
 
 type UpgradeConfig struct {
@@ -75,6 +76,8 @@ var (
 	feynmanUpgrade = make(map[string]*Upgrade)
 
 	feynmanFixUpgrade = make(map[string]*Upgrade)
+
+	haberFixUpgrade = make(map[string]*Upgrade)
 )
 
 func init() {
@@ -701,6 +704,38 @@ func init() {
 			},
 		},
 	}
+
+	haberFixUpgrade[mainNet] = &Upgrade{
+		UpgradeName: "haberFix",
+		Configs: []*UpgradeConfig{
+			{
+				ContractAddr: common.HexToAddress(ValidatorContract),
+				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/b743ce3f1f1e94c349b175cd6593bc263463b33b",
+				Code:         haberFix.MainnetValidatorContract,
+			},
+			{
+				ContractAddr: common.HexToAddress(SlashContract),
+				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/b743ce3f1f1e94c349b175cd6593bc263463b33b",
+				Code:         haberFix.MainnetSlashContract,
+			},
+		},
+	}
+
+	haberFixUpgrade[chapelNet] = &Upgrade{
+		UpgradeName: "haberFix",
+		Configs: []*UpgradeConfig{
+			{
+				ContractAddr: common.HexToAddress(ValidatorContract),
+				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/b743ce3f1f1e94c349b175cd6593bc263463b33b",
+				Code:         haberFix.ChapelValidatorContract,
+			},
+			{
+				ContractAddr: common.HexToAddress(SlashContract),
+				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/b743ce3f1f1e94c349b175cd6593bc263463b33b",
+				Code:         haberFix.ChapelSlashContract,
+			},
+		},
+	}
 }
 
 func UpgradeBuildInSystemContract(config *params.ChainConfig, blockNumber *big.Int, lastBlockTime uint64, blockTime uint64, statedb *state.StateDB) {
@@ -777,6 +812,10 @@ func UpgradeBuildInSystemContract(config *params.ChainConfig, blockNumber *big.I
 		applySystemContractUpgrade(feynmanFixUpgrade[network], blockNumber, statedb, logger)
 	}
 
+	if config.IsOnHaberFix(blockNumber, lastBlockTime, blockTime) {
+		applySystemContractUpgrade(haberFixUpgrade[network], blockNumber, statedb, logger)
+	}
+
 	/*
 		apply other upgrades
 	*/
@@ -799,7 +838,7 @@ func applySystemContractUpgrade(upgrade *Upgrade, blockNumber *big.Int, statedb
 			}
 		}
 
-		newContractCode, err := hex.DecodeString(cfg.Code)
+		newContractCode, err := hex.DecodeString(strings.TrimSpace(cfg.Code))
 		if err != nil {
 			panic(fmt.Errorf("failed to decode new contract code: %s", err.Error()))
 		}

core/types/bid.go

@@ -193,5 +193,7 @@ type MevParams struct {
 	ValidatorCommission   uint64 // 100 means 1%
 	BidSimulationLeftOver time.Duration
 	GasCeil               uint64
+	GasPrice              *big.Int // Minimum avg gas price for bid block
 	BuilderFeeCeil        *big.Int
+	Version               string
 }

core/types/state_account.go

@@ -18,6 +18,7 @@ package types
 
 import (
 	"bytes"
+	"fmt"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -58,6 +59,11 @@ func (acct *StateAccount) Copy() *StateAccount {
 	}
 }
 
+func (acct *StateAccount) String() string {
+	return fmt.Sprintf("nonce: %d, balance: %d, root: %s, codeHash: %s",
+		acct.Nonce, acct.Balance, acct.Root.String(), common.Bytes2Hex(acct.CodeHash))
+}
+
 // SlimAccount is a modified version of an Account, where the root is replaced
 // with a byte slice. This format can be used to represent full-consensus format
 // or slim format which replaces the empty root and code hash as nil byte slice.

core/vote/vote_manager.go

@@ -4,9 +4,11 @@ import (
 	"bytes"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus"
+	"github.com/ethereum/go-ethereum/consensus/parlia"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/eth/downloader"
@@ -139,6 +141,15 @@ func (voteManager *VoteManager) loop() {
 			}
 
 			curHead := cHead.Block.Header()
+			if p, ok := voteManager.engine.(*parlia.Parlia); ok {
+				nextBlockMinedTime := time.Unix(int64((curHead.Time + p.Period())), 0)
+				timeForBroadcast := 50 * time.Millisecond // enough to broadcast a vote
+				if time.Now().Add(timeForBroadcast).After(nextBlockMinedTime) {
+					log.Warn("too late to vote", "Head.Time(Second)", curHead.Time, "Now(Millisecond)", time.Now().UnixMilli())
+					continue
+				}
+			}
+
 			// Check if cur validator is within the validatorSet at curHead
 			if !voteManager.engine.IsActiveValidatorAt(voteManager.chain, curHead,
 				func(bLSPublicKey *types.BLSPublicKey) bool {

eth/api_backend.go

@@ -484,6 +484,10 @@ func (b *EthAPIBackend) RemoveBuilder(builder common.Address) error {
 	return b.Miner().RemoveBuilder(builder)
 }
 
+func (b *EthAPIBackend) HasBuilder(builder common.Address) bool {
+	return b.Miner().HasBuilder(builder)
+}
+
 func (b *EthAPIBackend) SendBid(ctx context.Context, bid *types.BidArgs) (common.Hash, error) {
 	return b.Miner().SendBid(ctx, bid)
 }

eth/backend.go

@@ -123,6 +123,18 @@ type Ethereum struct {
 // New creates a new Ethereum object (including the
 // initialisation of the common Ethereum object)
 func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
+	onlyFullSync := false
+	if config.StateScheme == rawdb.VersionScheme {
+		config.SnapshotCache = 0
+		onlyFullSync = true
+		config.SyncMode = downloader.FullSync
+	}
+
+	// TODO:: debug code
+	config.SnapshotCache = 0
+	onlyFullSync = true
+	config.SyncMode = downloader.FullSync
+
 	// Ensure configuration values are compatible and sane
 	if config.SyncMode == downloader.LightSync {
 		return nil, errors.New("can't run eth.Ethereum in light sync mode, light mode has been deprecated")
@@ -161,12 +173,18 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	// Optimize memory distribution by reallocating surplus allowance from the
 	// dirty cache to the clean cache.
 	if config.StateScheme == rawdb.PathScheme && config.TrieDirtyCache > pathdb.MaxDirtyBufferSize/1024/1024 {
-		log.Info("Capped dirty cache size", "provided", common.StorageSize(config.TrieDirtyCache)*1024*1024, "adjusted", common.StorageSize(pathdb.MaxDirtyBufferSize))
-		log.Info("Clean cache size", "provided", common.StorageSize(config.TrieCleanCache)*1024*1024)
+		log.Info("Capped dirty cache size", "provided", common.StorageSize(config.TrieDirtyCache)*1024*1024,
+			"adjusted", common.StorageSize(pathdb.MaxDirtyBufferSize))
+		log.Info("Clean cache size", "provided", common.StorageSize(config.TrieCleanCache)*1024*1024,
+			"adjusted", common.StorageSize(config.TrieCleanCache+config.TrieDirtyCache-pathdb.MaxDirtyBufferSize/1024/1024)*1024*1024)
+		config.TrieCleanCache += config.TrieDirtyCache - pathdb.MaxDirtyBufferSize/1024/1024
 		config.TrieDirtyCache = pathdb.MaxDirtyBufferSize / 1024 / 1024
 	}
-	log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024)
-
+	log.Info("Allocated memory caches",
+		"state_scheme", config.StateScheme,
+		"trie_clean_cache", common.StorageSize(config.TrieCleanCache)*1024*1024,
+		"trie_dirty_cache", common.StorageSize(config.TrieDirtyCache)*1024*1024,
+		"snapshot_cache", common.StorageSize(config.SnapshotCache)*1024*1024)
 	// Try to recover offline state pruning only in hash-based.
 	if config.StateScheme == rawdb.HashScheme {
 		if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, config.TriesInMemory); err != nil {
@@ -179,14 +197,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	}
 	// Override the chain config with provided settings.
 	var overrides core.ChainOverrides
-	if config.OverrideCancun != nil {
-		chainConfig.CancunTime = config.OverrideCancun
-		overrides.OverrideCancun = config.OverrideCancun
-	}
-	if config.OverrideHaber != nil {
-		chainConfig.HaberTime = config.OverrideHaber
-		overrides.OverrideHaber = config.OverrideHaber
-	}
 	if config.OverrideBohr != nil {
 		chainConfig.BohrTime = config.OverrideBohr
 		overrides.OverrideBohr = config.OverrideBohr
@@ -330,6 +340,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 		DirectBroadcast:        config.DirectBroadcast,
 		DisablePeerTxBroadcast: config.DisablePeerTxBroadcast,
 		PeerSet:                peers,
+		OnlyFullSync:           onlyFullSync,
 	}); err != nil {
 		return nil, err
 	}

eth/downloader/downloader.go

@@ -558,8 +558,8 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td, ttd *
 		} else {
 			d.ancientLimit = 0
 		}
-		frozen, _ := d.stateDB.Ancients() // Ignore the error here since light client can also hit here.
-		itemAmountInAncient, _ := d.stateDB.ItemAmountInAncient()
+		frozen, _ := d.stateDB.BlockStore().Ancients() // Ignore the error here since light client can also hit here.
+		itemAmountInAncient, _ := d.stateDB.BlockStore().ItemAmountInAncient()
 		// If a part of blockchain data has already been written into active store,
 		// disable the ancient style insertion explicitly.
 		if origin >= frozen && itemAmountInAncient != 0 {
@@ -1671,9 +1671,9 @@ func (d *Downloader) reportSnapSyncProgress(force bool) {
 	}
 	// Don't report anything until we have a meaningful progress
 	var (
-		headerBytes, _  = d.stateDB.AncientSize(rawdb.ChainFreezerHeaderTable)
-		bodyBytes, _    = d.stateDB.AncientSize(rawdb.ChainFreezerBodiesTable)
-		receiptBytes, _ = d.stateDB.AncientSize(rawdb.ChainFreezerReceiptTable)
+		headerBytes, _  = d.stateDB.BlockStore().AncientSize(rawdb.ChainFreezerHeaderTable)
+		bodyBytes, _    = d.stateDB.BlockStore().AncientSize(rawdb.ChainFreezerBodiesTable)
+		receiptBytes, _ = d.stateDB.BlockStore().AncientSize(rawdb.ChainFreezerReceiptTable)
 	)
 	syncedBytes := common.StorageSize(headerBytes + bodyBytes + receiptBytes)
 	if syncedBytes == 0 {

eth/ethconfig/config.go

@@ -188,12 +188,6 @@ type Config struct {
 	// send-transaction variants. The unit is ether.
 	RPCTxFeeCap float64
 
-	// OverrideCancun (TODO: remove after the fork)
-	OverrideCancun *uint64 `toml:",omitempty"`
-
-	// OverrideHaber (TODO: remove after the fork)
-	OverrideHaber *uint64 `toml:",omitempty"`
-
 	// OverrideBohr (TODO: remove after the fork)
 	OverrideBohr *uint64 `toml:",omitempty"`
 

eth/ethconfig/gen_config.go

@@ -70,8 +70,6 @@ func (c Config) MarshalTOML() (interface{}, error) {
 		RPCGasCap               uint64
 		RPCEVMTimeout           time.Duration
 		RPCTxFeeCap             float64
-		OverrideCancun          *uint64 `toml:",omitempty"`
-		OverrideHaber           *uint64 `toml:",omitempty"`
 		OverrideBohr            *uint64 `toml:",omitempty"`
 		OverrideVerkle          *uint64 `toml:",omitempty"`
 		BlobExtraReserve        uint64
@@ -130,8 +128,6 @@ func (c Config) MarshalTOML() (interface{}, error) {
 	enc.RPCGasCap = c.RPCGasCap
 	enc.RPCEVMTimeout = c.RPCEVMTimeout
 	enc.RPCTxFeeCap = c.RPCTxFeeCap
-	enc.OverrideCancun = c.OverrideCancun
-	enc.OverrideHaber = c.OverrideHaber
 	enc.OverrideBohr = c.OverrideBohr
 	enc.OverrideVerkle = c.OverrideVerkle
 	enc.BlobExtraReserve = c.BlobExtraReserve
@@ -194,8 +190,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 		RPCGasCap               *uint64
 		RPCEVMTimeout           *time.Duration
 		RPCTxFeeCap             *float64
-		OverrideCancun          *uint64 `toml:",omitempty"`
-		OverrideHaber           *uint64 `toml:",omitempty"`
 		OverrideBohr            *uint64 `toml:",omitempty"`
 		OverrideVerkle          *uint64 `toml:",omitempty"`
 		BlobExtraReserve        *uint64
@@ -363,12 +357,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 	if dec.RPCTxFeeCap != nil {
 		c.RPCTxFeeCap = *dec.RPCTxFeeCap
 	}
-	if dec.OverrideCancun != nil {
-		c.OverrideCancun = dec.OverrideCancun
-	}
-	if dec.OverrideHaber != nil {
-		c.OverrideHaber = dec.OverrideHaber
-	}
 	if dec.OverrideBohr != nil {
 		c.OverrideBohr = dec.OverrideBohr
 	}

eth/fetcher/block_fetcher.go

@@ -731,9 +731,6 @@ func (f *BlockFetcher) loop() {
 						matched = true
 						if f.getBlock(hash) == nil {
 							block := types.NewBlockWithHeader(announce.header).WithBody(task.transactions[i], task.uncles[i])
-							if block.Header().EmptyWithdrawalsHash() {
-								block = block.WithWithdrawals(make([]*types.Withdrawal, 0))
-							}
 							block = block.WithSidecars(task.sidecars[i])
 							block.ReceivedAt = task.time
 							blocks = append(blocks, block)
@@ -919,6 +916,10 @@ func (f *BlockFetcher) importBlocks(op *blockOrHeaderInject) {
 			return
 		}
 
+		if block.Header().EmptyWithdrawalsHash() {
+			block = block.WithWithdrawals(make([]*types.Withdrawal, 0))
+		}
+
 		defer func() { f.done <- hash }()
 		// Quickly validate the header and propagate the block if it passes
 		switch err := f.verifyHeader(block.Header()); err {

eth/handler.go

@@ -124,6 +124,7 @@ type handlerConfig struct {
 	DirectBroadcast        bool
 	DisablePeerTxBroadcast bool
 	PeerSet                *peerSet
+	OnlyFullSync           bool
 }
 
 type handler struct {
@@ -202,35 +203,37 @@ func newHandler(config *handlerConfig) (*handler, error) {
 		handlerStartCh:         make(chan struct{}),
 		stopCh:                 make(chan struct{}),
 	}
-	if config.Sync == downloader.FullSync {
-		// The database seems empty as the current block is the genesis. Yet the snap
-		// block is ahead, so snap sync was enabled for this node at a certain point.
-		// The scenarios where this can happen is
-		// * if the user manually (or via a bad block) rolled back a snap sync node
-		//   below the sync point.
-		// * the last snap sync is not finished while user specifies a full sync this
-		//   time. But we don't have any recent state for full sync.
-		// In these cases however it's safe to reenable snap sync.
-		fullBlock, snapBlock := h.chain.CurrentBlock(), h.chain.CurrentSnapBlock()
-		if fullBlock.Number.Uint64() == 0 && snapBlock.Number.Uint64() > 0 {
-			if rawdb.ReadAncientType(h.database) == rawdb.PruneFreezerType {
-				log.Crit("Fast Sync not finish, can't enable pruneancient mode")
+	if !config.OnlyFullSync {
+		if config.Sync == downloader.FullSync {
+			// The database seems empty as the current block is the genesis. Yet the snap
+			// block is ahead, so snap sync was enabled for this node at a certain point.
+			// The scenarios where this can happen is
+			// * if the user manually (or via a bad block) rolled back a snap sync node
+			//   below the sync point.
+			// * the last snap sync is not finished while user specifies a full sync this
+			//   time. But we don't have any recent state for full sync.
+			// In these cases however it's safe to reenable snap sync.
+			fullBlock, snapBlock := h.chain.CurrentBlock(), h.chain.CurrentSnapBlock()
+			if fullBlock.Number.Uint64() == 0 && snapBlock.Number.Uint64() > 0 {
+				if rawdb.ReadAncientType(h.database) == rawdb.PruneFreezerType {
+					log.Crit("Fast Sync not finish, can't enable pruneancient mode")
+				}
+				h.snapSync.Store(true)
+				log.Warn("Switch sync mode from full sync to snap sync", "reason", "snap sync incomplete")
+			} else if !h.chain.NoTries() && !h.chain.HasState(fullBlock.Root) {
+				h.snapSync.Store(true)
+				log.Warn("Switch sync mode from full sync to snap sync", "reason", "head state missing")
 			}
-			h.snapSync.Store(true)
-			log.Warn("Switch sync mode from full sync to snap sync", "reason", "snap sync incomplete")
-		} else if !h.chain.NoTries() && !h.chain.HasState(fullBlock.Root) {
-			h.snapSync.Store(true)
-			log.Warn("Switch sync mode from full sync to snap sync", "reason", "head state missing")
-		}
-	} else {
-		head := h.chain.CurrentBlock()
-		if head.Number.Uint64() > 0 && h.chain.HasState(head.Root) {
-			// Print warning log if database is not empty to run snap sync.
-			log.Warn("Switch sync mode from snap sync to full sync", "reason", "snap sync complete")
 		} else {
-			// If snap sync was requested and our database is empty, grant it
-			h.snapSync.Store(true)
-			log.Info("Enabled snap sync", "head", head.Number, "hash", head.Hash())
+			head := h.chain.CurrentBlock()
+			if head.Number.Uint64() > 0 && h.chain.HasState(head.Root) {
+				// Print warning log if database is not empty to run snap sync.
+				log.Warn("Switch sync mode from snap sync to full sync", "reason", "snap sync complete")
+			} else {
+				// If snap sync was requested and our database is empty, grant it
+				h.snapSync.Store(true)
+				log.Info("Enabled snap sync", "head", head.Number, "hash", head.Hash())
+			}
 		}
 	}
 	// If snap sync is requested but snapshots are disabled, fail loudly

eth/handler_eth_test.go

@@ -633,6 +633,9 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) {
 		go source.handler.runEthPeer(sourcePeer, func(peer *eth.Peer) error {
 			return eth.Handle((*ethHandler)(source.handler), peer)
 		})
+		// Wait a bit for the above handlers to start
+		time.Sleep(100 * time.Millisecond)
+
 		if err := sinkPeer.Handshake(1, td, genesis.Hash(), genesis.Hash(), forkid.NewIDWithChain(source.chain), forkid.NewFilter(source.chain), nil); err != nil {
 			t.Fatalf("failed to run protocol handshake")
 		}

eth/protocols/snap/sync.go

@@ -409,8 +409,8 @@ type SyncPeer interface {
 //   - The peer delivers a stale response after a previous timeout
 //   - The peer delivers a refusal to serve the requested state
 type Syncer struct {
-	db     ethdb.KeyValueStore // Database to store the trie nodes into (and dedup)
-	scheme string              // Node scheme used in node database
+	db     ethdb.Database // Database to store the trie nodes into (and dedup)
+	scheme string         // Node scheme used in node database
 
 	root    common.Hash    // Current state trie root being synced
 	tasks   []*accountTask // Current account task set being synced
@@ -478,7 +478,7 @@ type Syncer struct {
 
 // NewSyncer creates a new snapshot syncer to download the Ethereum state over the
 // snap protocol.
-func NewSyncer(db ethdb.KeyValueStore, scheme string) *Syncer {
+func NewSyncer(db ethdb.Database, scheme string) *Syncer {
 	return &Syncer{
 		db:     db,
 		scheme: scheme,
@@ -719,11 +719,11 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error {
 
 // cleanPath is used to remove the dangling nodes in the stackTrie.
 func (s *Syncer) cleanPath(batch ethdb.Batch, owner common.Hash, path []byte) {
-	if owner == (common.Hash{}) && rawdb.ExistsAccountTrieNode(s.db, path) {
+	if owner == (common.Hash{}) && rawdb.ExistsAccountTrieNode(s.db.StateStoreReader(), path) {
 		rawdb.DeleteAccountTrieNode(batch, path)
 		deletionGauge.Inc(1)
 	}
-	if owner != (common.Hash{}) && rawdb.ExistsStorageTrieNode(s.db, owner, path) {
+	if owner != (common.Hash{}) && rawdb.ExistsStorageTrieNode(s.db.StateStoreReader(), owner, path) {
 		rawdb.DeleteStorageTrieNode(batch, owner, path)
 		deletionGauge.Inc(1)
 	}
@@ -735,6 +735,7 @@ func (s *Syncer) cleanPath(batch ethdb.Batch, owner common.Hash, path []byte) {
 func (s *Syncer) loadSyncStatus() {
 	var progress SyncProgress
 
+	stateDiskDB := s.db.GetStateStore()
 	if status := rawdb.ReadSnapshotSyncStatus(s.db); status != nil {
 		if err := json.Unmarshal(status, &progress); err != nil {
 			log.Error("Failed to decode snap sync status", "err", err)
@@ -747,7 +748,7 @@ func (s *Syncer) loadSyncStatus() {
 				task := task // closure for task.genBatch in the stacktrie writer callback
 
 				task.genBatch = ethdb.HookedBatch{
-					Batch: s.db.NewBatch(),
+					Batch: stateDiskDB.NewBatch(),
 					OnPut: func(key []byte, value []byte) {
 						s.accountBytes += common.StorageSize(len(key) + len(value))
 					},
@@ -773,7 +774,7 @@ func (s *Syncer) loadSyncStatus() {
 						subtask := subtask // closure for subtask.genBatch in the stacktrie writer callback
 
 						subtask.genBatch = ethdb.HookedBatch{
-							Batch: s.db.NewBatch(),
+							Batch: stateDiskDB.NewBatch(),
 							OnPut: func(key []byte, value []byte) {
 								s.storageBytes += common.StorageSize(len(key) + len(value))
 							},
@@ -841,7 +842,7 @@ func (s *Syncer) loadSyncStatus() {
 			last = common.MaxHash
 		}
 		batch := ethdb.HookedBatch{
-			Batch: s.db.NewBatch(),
+			Batch: stateDiskDB.NewBatch(),
 			OnPut: func(key []byte, value []byte) {
 				s.accountBytes += common.StorageSize(len(key) + len(value))
 			},
@@ -1894,7 +1895,7 @@ func (s *Syncer) processAccountResponse(res *accountResponse) {
 		}
 		// Check if the account is a contract with an unknown storage trie
 		if account.Root != types.EmptyRootHash {
-			if !rawdb.HasTrieNode(s.db, res.hashes[i], nil, account.Root, s.scheme) {
+			if !rawdb.HasTrieNode(s.db.StateStoreReader(), res.hashes[i], nil, account.Root, s.scheme) {
 				// If there was a previous large state retrieval in progress,
 				// don't restart it from scratch. This happens if a sync cycle
 				// is interrupted and resumed later. However, *do* update the
@@ -1986,12 +1987,25 @@ func (s *Syncer) processStorageResponse(res *storageResponse) {
 	if res.subTask != nil {
 		res.subTask.req = nil
 	}
+
+	var usingMultDatabase bool
 	batch := ethdb.HookedBatch{
-		Batch: s.db.NewBatch(),
+		Batch: s.db.GetStateStore().NewBatch(),
 		OnPut: func(key []byte, value []byte) {
 			s.storageBytes += common.StorageSize(len(key) + len(value))
 		},
 	}
+	var snapBatch ethdb.HookedBatch
+	if s.db.StateStore() != nil {
+		usingMultDatabase = true
+		snapBatch = ethdb.HookedBatch{
+			Batch: s.db.NewBatch(),
+			OnPut: func(key []byte, value []byte) {
+				s.storageBytes += common.StorageSize(len(key) + len(value))
+			},
+		}
+	}
+
 	var (
 		slots           int
 		oldStorageBytes = s.storageBytes
@@ -2061,7 +2075,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) {
 					}
 					// Our first task is the one that was just filled by this response.
 					batch := ethdb.HookedBatch{
-						Batch: s.db.NewBatch(),
+						Batch: s.db.GetStateStore().NewBatch(),
 						OnPut: func(key []byte, value []byte) {
 							s.storageBytes += common.StorageSize(len(key) + len(value))
 						},
@@ -2088,7 +2102,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) {
 					})
 					for r.Next() {
 						batch := ethdb.HookedBatch{
-							Batch: s.db.NewBatch(),
+							Batch: s.db.GetStateStore().NewBatch(),
 							OnPut: func(key []byte, value []byte) {
 								s.storageBytes += common.StorageSize(len(key) + len(value))
 							},
@@ -2184,8 +2198,11 @@ func (s *Syncer) processStorageResponse(res *storageResponse) {
 		// outdated during the sync, but it can be fixed later during the
 		// snapshot generation.
 		for j := 0; j < len(res.hashes[i]); j++ {
-			rawdb.WriteStorageSnapshot(batch, account, res.hashes[i][j], res.slots[i][j])
-
+			if usingMultDatabase {
+				rawdb.WriteStorageSnapshot(snapBatch, account, res.hashes[i][j], res.slots[i][j])
+			} else {
+				rawdb.WriteStorageSnapshot(batch, account, res.hashes[i][j], res.slots[i][j])
+			}
 			// If we're storing large contracts, generate the trie nodes
 			// on the fly to not trash the gluing points
 			if i == len(res.hashes)-1 && res.subTask != nil {
@@ -2205,7 +2222,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) {
 			// If the chunk's root is an overflown but full delivery,
 			// clear the heal request.
 			accountHash := res.accounts[len(res.accounts)-1]
-			if root == res.subTask.root && rawdb.HasStorageTrieNode(s.db, accountHash, nil, root) {
+			if root == res.subTask.root && rawdb.HasStorageTrieNode(s.db.StateStoreReader(), accountHash, nil, root) {
 				for i, account := range res.mainTask.res.hashes {
 					if account == accountHash {
 						res.mainTask.needHeal[i] = false
@@ -2225,6 +2242,11 @@ func (s *Syncer) processStorageResponse(res *storageResponse) {
 	if err := batch.Write(); err != nil {
 		log.Crit("Failed to persist storage slots", "err", err)
 	}
+	if usingMultDatabase {
+		if err := snapBatch.Write(); err != nil {
+			log.Crit("Failed to persist storage slots", "err", err)
+		}
+	}
 	s.storageSynced += uint64(slots)
 
 	log.Debug("Persisted set of storage slots", "accounts", len(res.hashes), "slots", slots, "bytes", s.storageBytes-oldStorageBytes)
@@ -2323,12 +2345,25 @@ func (s *Syncer) commitHealer(force bool) {
 		return
 	}
 	batch := s.db.NewBatch()
-	if err := s.healer.scheduler.Commit(batch); err != nil {
+	var stateBatch ethdb.Batch
+	var err error
+	if s.db.StateStore() != nil {
+		stateBatch = s.db.StateStore().NewBatch()
+		err = s.healer.scheduler.Commit(batch, stateBatch)
+	} else {
+		err = s.healer.scheduler.Commit(batch, nil)
+	}
+	if err != nil {
 		log.Error("Failed to commit healing data", "err", err)
 	}
 	if err := batch.Write(); err != nil {
 		log.Crit("Failed to persist healing data", "err", err)
 	}
+	if s.db.StateStore() != nil {
+		if err := stateBatch.Write(); err != nil {
+			log.Crit("Failed to persist healing data", "err", err)
+		}
+	}
 	log.Debug("Persisted set of healing data", "type", "trienodes", "bytes", common.StorageSize(batch.ValueSize()))
 }
 

eth/state_accessor.go

@@ -71,7 +71,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u
 			// the internal junks created by tracing will be persisted into the disk.
 			// TODO(rjl493456442), clean cache is disabled to prevent memory leak,
 			// please re-enable it for better performance.
-			database = state.NewDatabaseWithConfig(eth.chainDb, triedb.HashDefaults)
+			database = state.NewDatabaseWithConfig(eth.chainDb, triedb.HashDefaults, false)
 			if statedb, err = state.New(block.Root(), database, nil); err == nil {
 				log.Info("Found disk backend for state trie", "root", block.Root(), "number", block.Number())
 				return statedb, noopReleaser, nil
@@ -92,7 +92,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u
 		// TODO(rjl493456442), clean cache is disabled to prevent memory leak,
 		// please re-enable it for better performance.
 		tdb = triedb.NewDatabase(eth.chainDb, triedb.HashDefaults)
-		database = state.NewDatabaseWithNodeDB(eth.chainDb, tdb)
+		database = state.NewDatabaseWithNodeDB(eth.chainDb, tdb, false)
 
 		// If we didn't check the live database, do check state over ephemeral database,
 		// otherwise we would rewind past a persisted block (specific corner case is
@@ -221,6 +221,7 @@ func (eth *Ethereum) stateAtBlock(ctx context.Context, block *types.Block, reexe
 	if eth.blockchain.TrieDB().Scheme() == rawdb.HashScheme {
 		return eth.hashState(ctx, block, reexec, base, readOnly, preferDisk)
 	}
+	// path and version schema use the same interface
 	return eth.pathState(block)
 }
 

eth/sync.go

@@ -191,33 +191,40 @@ func peerToSyncOp(mode downloader.SyncMode, p *eth.Peer) *chainSyncOp {
 }
 
 func (cs *chainSyncer) modeAndLocalHead() (downloader.SyncMode, *big.Int) {
-	// If we're in snap sync mode, return that directly
-	if cs.handler.snapSync.Load() {
-		block := cs.handler.chain.CurrentSnapBlock()
-		td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64())
-		return downloader.SnapSync, td
-	}
-	// We are probably in full sync, but we might have rewound to before the
-	// snap sync pivot, check if we should re-enable snap sync.
 	head := cs.handler.chain.CurrentBlock()
-	if pivot := rawdb.ReadLastPivotNumber(cs.handler.database); pivot != nil {
-		if head.Number.Uint64() < *pivot {
-			if rawdb.ReadAncientType(cs.handler.database) == rawdb.PruneFreezerType {
-				log.Crit("Current rewound to before the fast sync pivot, can't enable pruneancient mode", "current block number", head.Number.Uint64(), "pivot", *pivot)
-			}
+	if cs.handler.chain.TrieDB().Scheme() != rawdb.VersionScheme {
+		// If we're in snap sync mode, return that directly
+		if cs.handler.snapSync.Load() {
 			block := cs.handler.chain.CurrentSnapBlock()
 			td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64())
 			return downloader.SnapSync, td
 		}
-	}
-	// We are in a full sync, but the associated head state is missing. To complete
-	// the head state, forcefully rerun the snap sync. Note it doesn't mean the
-	// persistent state is corrupted, just mismatch with the head block.
-	if !cs.handler.chain.NoTries() && !cs.handler.chain.HasState(head.Root) {
-		block := cs.handler.chain.CurrentSnapBlock()
-		td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64())
-		log.Info("Reenabled snap sync as chain is stateless")
-		return downloader.SnapSync, td
+		// We are probably in full sync, but we might have rewound to before the
+		// snap sync pivot, check if we should re-enable snap sync.
+		head = cs.handler.chain.CurrentBlock()
+		if pivot := rawdb.ReadLastPivotNumber(cs.handler.database); pivot != nil {
+			if head.Number.Uint64() < *pivot {
+				if rawdb.ReadAncientType(cs.handler.database) == rawdb.PruneFreezerType {
+					log.Crit("Current rewound to before the fast sync pivot, can't enable pruneancient mode", "current block number", head.Number.Uint64(), "pivot", *pivot)
+				}
+				block := cs.handler.chain.CurrentSnapBlock()
+				td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64())
+				return downloader.SnapSync, td
+			}
+		}
+		// We are in a full sync, but the associated head state is missing. To complete
+		// the head state, forcefully rerun the snap sync. Note it doesn't mean the
+		// persistent state is corrupted, just mismatch with the head block.
+		if !cs.handler.chain.NoTries() && !cs.handler.chain.HasState(head.Root) {
+			block := cs.handler.chain.CurrentSnapBlock()
+			td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64())
+			log.Info("Reenabled snap sync as chain is stateless")
+			return downloader.SnapSync, td
+		}
+	} else {
+		if !cs.handler.chain.HasState(head.Root) {
+			panic("version db not support snap sync")
+		}
 	}
 	// Nope, we're really full syncing
 	td := cs.handler.chain.GetTd(head.Hash(), head.Number.Uint64())

eth/sync_test.go

@@ -112,7 +112,14 @@ func testChainSyncWithBlobs(t *testing.T, mode downloader.SyncMode, preCancunBlk
 	cancunTime := (preCancunBlks + 1) * 10
 	config.CancunTime = &cancunTime
 
-	// Create a full handler and ensure snap sync ends up disabled
+	// Create an empty handler
+	empty := newTestParliaHandlerAfterCancun(t, &config, mode, 0, 0)
+	defer empty.close()
+	if downloader.SnapSync == mode && !empty.handler.snapSync.Load() {
+		t.Fatalf("snap sync disabled on pristine blockchain")
+	}
+
+	// Create a full handler
 	full := newTestParliaHandlerAfterCancun(t, &config, mode, preCancunBlks, postCancunBlks)
 	defer full.close()
 	if downloader.SnapSync == mode && full.handler.snapSync.Load() {
@@ -122,13 +129,6 @@ func testChainSyncWithBlobs(t *testing.T, mode downloader.SyncMode, preCancunBlk
 	// check blocks and blobs
 	checkChainWithBlobs(t, full.chain, preCancunBlks, postCancunBlks)
 
-	// Create an empty handler and ensure it's in snap sync mode
-	empty := newTestParliaHandlerAfterCancun(t, &config, mode, 0, 0)
-	defer empty.close()
-	if downloader.SnapSync == mode && !empty.handler.snapSync.Load() {
-		t.Fatalf("snap sync disabled on pristine blockchain")
-	}
-
 	// Sync up the two handlers via both `eth` and `snap`
 	ethVer := uint(eth.ETH68)
 	snapVer := uint(snap.SNAP1)
@@ -165,14 +165,17 @@ func testChainSyncWithBlobs(t *testing.T, mode downloader.SyncMode, preCancunBlk
 	go full.handler.runSnapExtension(fullPeerSnap, func(peer *snap.Peer) error {
 		return snap.Handle((*snapHandler)(full.handler), peer)
 	})
-	// Wait a bit for the above handlers to start
-	time.Sleep(250 * time.Millisecond)
 
-	// Check that snap sync was disabled
+	for empty.handler.peers.snapLen() < 1 {
+		// Wait a bit for the above handlers to start
+		time.Sleep(100 * time.Millisecond)
+	}
+
 	op := peerToSyncOp(mode, empty.handler.peers.peerWithHighestTD())
 	if err := empty.handler.doSync(op); err != nil {
 		t.Fatal("sync failed:", err)
 	}
+	// Check that snap sync was disabled
 	if !empty.handler.synced.Load() {
 		t.Fatalf("full sync not done after successful synchronisation")
 	}

ethclient/ethclient.go

@@ -133,7 +133,7 @@ func (ec *Client) BlockReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumb
 // BlobSidecars return the Sidecars of a given block number or hash.
 func (ec *Client) BlobSidecars(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]*types.BlobTxSidecar, error) {
 	var r []*types.BlobTxSidecar
-	err := ec.c.CallContext(ctx, &r, "eth_getBlobSidecars", blockNrOrHash.String(), true)
+	err := ec.c.CallContext(ctx, &r, "eth_getBlobSidecars", blockNrOrHash.String())
 	if err == nil && r == nil {
 		return nil, ethereum.NotFound
 	}
@@ -143,7 +143,7 @@ func (ec *Client) BlobSidecars(ctx context.Context, blockNrOrHash rpc.BlockNumbe
 // BlobSidecarByTxHash return a sidecar of a given blob transaction
 func (ec *Client) BlobSidecarByTxHash(ctx context.Context, hash common.Hash) (*types.BlobTxSidecar, error) {
 	var r *types.BlobTxSidecar
-	err := ec.c.CallContext(ctx, &r, "eth_getBlockSidecarByTxHash", hash, true)
+	err := ec.c.CallContext(ctx, &r, "eth_getBlobSidecarByTxHash", hash)
 	if err == nil && r == nil {
 		return nil, ethereum.NotFound
 	}
@@ -752,6 +752,13 @@ func (ec *Client) MevRunning(ctx context.Context) (bool, error) {
 	return result, err
 }
 
+// HasBuilder returns whether the builder is registered
+func (ec *Client) HasBuilder(ctx context.Context, address common.Address) (bool, error) {
+	var result bool
+	err := ec.c.CallContext(ctx, &result, "mev_hasBuilder", address)
+	return result, err
+}
+
 // SendBid sends a bid
 func (ec *Client) SendBid(ctx context.Context, args types.BidArgs) (common.Hash, error) {
 	var hash common.Hash

ethdb/database.go

@@ -180,12 +180,6 @@ type StateStoreReader interface {
 	StateStoreReader() Reader
 }
 
-type BlockStore interface {
-	BlockStore() Database
-	SetBlockStore(block Database)
-	HasSeparateBlockStore() bool
-}
-
 type BlockStoreReader interface {
 	BlockStoreReader() Reader
 }
@@ -194,6 +188,14 @@ type BlockStoreWriter interface {
 	BlockStoreWriter() Writer
 }
 
+// MultiDatabaseReader contains the methods required to read data from both key-value as well as
+// blockStore or stateStore.
+type MultiDatabaseReader interface {
+	KeyValueReader
+	StateStoreReader
+	BlockStoreReader
+}
+
 // Reader contains the methods required to read data from both key-value as well as
 // immutable ancient data.
 type Reader interface {
@@ -234,6 +236,13 @@ type DiffStore interface {
 type StateStore interface {
 	StateStore() Database
 	SetStateStore(state Database)
+	GetStateStore() Database
+}
+
+type BlockStore interface {
+	BlockStore() Database
+	SetBlockStore(block Database)
+	HasSeparateBlockStore() bool
 }
 
 // Database contains all the methods required by the high level database to not

ethdb/memorydb/memorydb.go

@@ -39,6 +39,9 @@ var (
 	// errSnapshotReleased is returned if callers want to retrieve data from a
 	// released snapshot.
 	errSnapshotReleased = errors.New("snapshot released")
+
+	// errNotSupported is returned if the database doesn't support the required operation.
+	errNotSupported = errors.New("this operation is not supported")
 )
 
 // Database is an ephemeral key-value store. Apart from basic data storage
@@ -47,6 +50,84 @@ var (
 type Database struct {
 	db   map[string][]byte
 	lock sync.RWMutex
+
+	stateStore ethdb.Database
+	blockStore ethdb.Database
+}
+
+func (db *Database) ModifyAncients(f func(ethdb.AncientWriteOp) error) (int64, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (db *Database) TruncateHead(n uint64) (uint64, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (db *Database) TruncateTail(n uint64) (uint64, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (db *Database) Sync() error {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (db *Database) TruncateTableTail(kind string, tail uint64) (uint64, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (db *Database) ResetTable(kind string, startAt uint64, onlyEmpty bool) error {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (db *Database) HasAncient(kind string, number uint64) (bool, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (db *Database) Ancient(kind string, number uint64) ([]byte, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (db *Database) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (db *Database) Ancients() (uint64, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (db *Database) Tail() (uint64, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (db *Database) AncientSize(kind string) (uint64, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (db *Database) ItemAmountInAncient() (uint64, error) {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (db *Database) AncientOffSet() uint64 {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (db *Database) ReadAncients(fn func(ethdb.AncientReaderOp) error) (err error) {
+	//TODO implement me
+	panic("implement me")
 }
 
 // New returns a wrapped map with all the required database interface methods
@@ -204,6 +285,37 @@ func (db *Database) Len() int {
 	return len(db.db)
 }
 
+func (db *Database) StateStoreReader() ethdb.Reader {
+	if db.stateStore == nil {
+		return db
+	}
+	return db.stateStore
+}
+
+func (db *Database) BlockStoreReader() ethdb.Reader {
+	if db.blockStore == nil {
+		return db
+	}
+	return db.blockStore
+}
+
+func (db *Database) BlockStoreWriter() ethdb.Writer {
+	if db.blockStore == nil {
+		return db
+	}
+	return db.blockStore
+}
+
+// convertLegacyFn takes a raw freezer entry in an older format and
+// returns it in the new format.
+type convertLegacyFn = func([]byte) ([]byte, error)
+
+// MigrateTable processes the entries in a given table in sequence
+// converting them to a new format if they're of an old format.
+func (db *Database) MigrateTable(kind string, convert convertLegacyFn) error {
+	return errNotSupported
+}
+
 // keyvalue is a key-value tuple tagged with a deletion field to allow creating
 // memory-database write batches.
 type keyvalue struct {

ethdb/remotedb/remotedb.go

@@ -122,6 +122,10 @@ func (db *Database) SetStateStore(state ethdb.Database) {
 	panic("not supported")
 }
 
+func (db *Database) GetStateStore() ethdb.Database {
+	panic("not supported")
+}
+
 func (db *Database) StateStoreReader() ethdb.Reader {
 	return db
 }

go.mod

@@ -2,8 +2,6 @@ module github.com/ethereum/go-ethereum
 
 go 1.21
 
-toolchain go1.21.5
-
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0
 	github.com/Microsoft/go-winio v0.6.1
@@ -14,11 +12,12 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/route53 v1.30.2
 	github.com/bnb-chain/fastssz v0.1.2
 	github.com/bnb-chain/ics23 v0.1.0
+	github.com/bnb-chain/versioned-state-database v0.0.0-00010101000000-000000000000
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2
 	github.com/cespare/cp v1.1.1
 	github.com/cloudflare/cloudflare-go v0.79.0
-	github.com/cometbft/cometbft v0.37.0
 	github.com/cockroachdb/pebble v1.1.0
+	github.com/cometbft/cometbft v0.37.0
 	github.com/consensys/gnark-crypto v0.12.1
 	github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233
 	github.com/crate-crypto/go-kzg-4844 v0.7.0
@@ -26,7 +25,7 @@ require (
 	github.com/deckarep/golang-set/v2 v2.5.0
 	github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127
 	github.com/ethereum/c-kzg-4844 v0.4.0
-	github.com/fatih/color v1.13.0
+	github.com/fatih/color v1.16.0
 	github.com/fatih/structs v1.1.0
 	github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e
 	github.com/fjl/memsize v0.0.2
@@ -39,14 +38,14 @@ require (
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb
 	github.com/google/gofuzz v1.2.0
 	github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5
-	github.com/google/uuid v1.4.0
+	github.com/google/uuid v1.5.0
 	github.com/gorilla/websocket v1.5.1
 	github.com/graph-gophers/graphql-go v1.3.0
 	github.com/hashicorp/go-bexpr v0.1.10
 	github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d
 	github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4
 	github.com/holiman/bloomfilter/v2 v2.0.3
-	github.com/holiman/uint256 v1.2.4
+	github.com/holiman/uint256 v1.3.0
 	github.com/huin/goupnp v1.3.0
 	github.com/influxdata/influxdb-client-go/v2 v2.4.0
 	github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c
@@ -69,7 +68,7 @@ require (
 	github.com/rs/cors v1.8.2
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/status-im/keycard-go v0.2.0
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	github.com/supranational/blst v0.3.11
 	github.com/syndtr/goleveldb v1.0.1
 	github.com/tendermint/go-amino v0.14.1
@@ -81,13 +80,13 @@ require (
 	github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4 v1.1.3
 	github.com/willf/bitset v1.1.3
 	go.uber.org/automaxprocs v1.5.2
-	golang.org/x/crypto v0.21.0
+	golang.org/x/crypto v0.25.0
 	golang.org/x/exp v0.0.0-20240213143201-ec583247a57a
-	golang.org/x/sync v0.6.0
-	golang.org/x/sys v0.18.0
-	golang.org/x/text v0.14.0
+	golang.org/x/sync v0.7.0
+	golang.org/x/sys v0.22.0
+	golang.org/x/text v0.16.0
 	golang.org/x/time v0.5.0
-	golang.org/x/tools v0.18.0
+	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -98,6 +97,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
 	github.com/BurntSushi/toml v1.3.2 // indirect
 	github.com/DataDog/zstd v1.5.5 // indirect
+	github.com/allegro/bigcache v1.2.1 // indirect
 	github.com/aristanetworks/goarista v0.0.0-20200805130819-fd197cf57d96 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect
@@ -112,7 +112,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.11.0 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/chzyer/readline v1.5.1 // indirect
 	github.com/cockroachdb/errors v1.11.1 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
@@ -160,7 +160,7 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.0.1 // indirect
 	github.com/gtank/merlin v0.1.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
 	github.com/herumi/bls-eth-go-binary v0.0.0-20210917013441-d37c07cfda4e // indirect
 	github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
@@ -233,12 +233,10 @@ require (
 	github.com/prometheus/common v0.47.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/prometheus/prom2json v1.3.0 // indirect
-	github.com/prysmaticlabs/eth2-types v0.0.0-20210303084904-c9735a06829d // indirect
 	github.com/prysmaticlabs/fastssz v0.0.0-20221107182844-78142813af44 // indirect
 	github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 // indirect
 	github.com/prysmaticlabs/gohashtree v0.0.4-beta // indirect
 	github.com/prysmaticlabs/prombbolt v0.0.0-20210126082820-9b7adba6db7c // indirect
-	github.com/prysmaticlabs/prysm v0.0.0-20220124113610-e26cde5e091b // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
 	github.com/quic-go/quic-go v0.42.0 // indirect
 	github.com/quic-go/webtransport-go v0.6.0 // indirect
@@ -249,10 +247,12 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sasha-s/go-deadlock v0.3.1 // indirect
 	github.com/schollz/progressbar/v3 v3.3.4 // indirect
-	github.com/sirupsen/logrus v1.9.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/afero v1.10.0 // indirect
+	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
 	github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e // indirect
+	github.com/tidwall/btree v1.7.0 // indirect
 	github.com/tidwall/gjson v1.10.2 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
@@ -266,21 +266,23 @@ require (
 	github.com/wealdtech/go-eth2-util v1.6.3 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
-	go.etcd.io/bbolt v1.3.7 // indirect
+	go.etcd.io/bbolt v1.3.9 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.uber.org/dig v1.17.1 // indirect
 	go.uber.org/fx v1.20.1 // indirect
 	go.uber.org/mock v0.4.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/mod v0.15.0 // indirect
-	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/mod v0.17.0 // indirect
+	golang.org/x/net v0.25.0 // indirect
 	golang.org/x/oauth2 v0.16.0 // indirect
-	golang.org/x/term v0.18.0 // indirect
+	golang.org/x/term v0.22.0 // indirect
 	google.golang.org/api v0.44.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
-	google.golang.org/grpc v1.56.3 // indirect
+	google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
+	google.golang.org/grpc v1.59.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
@@ -295,8 +297,8 @@ require (
 )
 
 replace (
-	github.com/btcsuite/btcd => github.com/btcsuite/btcd v0.23.0
-	github.com/cometbft/cometbft => github.com/bnb-chain/greenfield-tendermint v0.0.0-20230417032003-4cda1f296fb2
+	github.com/bnb-chain/versioned-state-database => ../versioned-state-database
+	github.com/cometbft/cometbft => github.com/bnb-chain/greenfield-cometbft v1.3.1
 	github.com/grpc-ecosystem/grpc-gateway/v2 => github.com/prysmaticlabs/grpc-gateway/v2 v2.3.1-0.20210702154020-550e1cd83ec1
 	github.com/syndtr/goleveldb v1.0.1 => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	github.com/tendermint/tendermint => github.com/bnb-chain/tendermint v0.31.16

internal/ethapi/api_mev.go

@@ -87,6 +87,10 @@ func (m *MevAPI) Params() *types.MevParams {
 	return m.b.MevParams()
 }
 
+func (m *MevAPI) HasBuilder(builder common.Address) bool {
+	return m.b.HasBuilder(builder)
+}
+
 // Running returns true if mev is running
 func (m *MevAPI) Running() bool {
 	return m.b.MevRunning()

internal/ethapi/api_test.go

@@ -650,7 +650,8 @@ func (b testBackend) ServiceFilter(ctx context.Context, session *bloombits.Match
 	panic("implement me")
 }
 
-func (b *testBackend) MevRunning() bool { return false }
+func (b *testBackend) MevRunning() bool                       { return false }
+func (b *testBackend) HasBuilder(builder common.Address) bool { return false }
 func (b *testBackend) MevParams() *types.MevParams {
 	return &types.MevParams{}
 }

internal/ethapi/backend.go

@@ -115,6 +115,8 @@ type Backend interface {
 	AddBuilder(builder common.Address, builderUrl string) error
 	// RemoveBuilder removes a builder from the bid simulator.
 	RemoveBuilder(builder common.Address) error
+	// HasBuilder returns true if the builder is in the builder list.
+	HasBuilder(builder common.Address) bool
 	// SendBid receives bid from the builders.
 	SendBid(ctx context.Context, bid *types.BidArgs) (common.Hash, error)
 	// BestBidGasFee returns the gas fee of the best bid for the given parent hash.

internal/ethapi/dbapi.go

@@ -33,11 +33,11 @@ func (api *DebugAPI) DbGet(key string) (hexutil.Bytes, error) {
 // DbAncient retrieves an ancient binary blob from the append-only immutable files.
 // It is a mapping to the `AncientReaderOp.Ancient` method
 func (api *DebugAPI) DbAncient(kind string, number uint64) (hexutil.Bytes, error) {
-	return api.b.ChainDb().Ancient(kind, number)
+	return api.b.ChainDb().BlockStore().Ancient(kind, number)
 }
 
 // DbAncients returns the ancient item numbers in the ancient store.
 // It is a mapping to the `AncientReaderOp.Ancients` method
 func (api *DebugAPI) DbAncients() (uint64, error) {
-	return api.b.ChainDb().Ancients()
+	return api.b.ChainDb().BlockStore().Ancients()
 }

internal/ethapi/transaction_args.go

@@ -204,7 +204,8 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro
 	// Sanity check the EIP-1559 fee parameters if present.
 	if args.GasPrice == nil && eip1559ParamsSet {
 		if args.MaxFeePerGas.ToInt().Sign() == 0 {
-			return errors.New("maxFeePerGas must be non-zero")
+			// return errors.New("maxFeePerGas must be non-zero")
+			log.Warn("EIP-1559 Tx with zero maxFeePerGas") // BSC accepts zero gas price.
 		}
 		if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 {
 			return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas)
@@ -217,7 +218,8 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) erro
 	if args.GasPrice != nil && !eip1559ParamsSet {
 		// Zero gas-price is not allowed after London fork
 		if args.GasPrice.ToInt().Sign() == 0 && isLondon {
-			return errors.New("gasPrice must be non-zero after london fork")
+			// return errors.New("gasPrice must be non-zero after london fork")
+			log.Warn("non EIP-1559 Tx with zero gasPrice") // BSC accepts zero gas price.
 		}
 		return nil // No need to set anything, user already set GasPrice
 	}

internal/ethapi/transaction_args_test.go

@@ -85,8 +85,8 @@ func TestSetFeeDefaults(t *testing.T) {
 			"legacy tx post-London with zero price",
 			"london",
 			&TransactionArgs{GasPrice: zero},
-			nil,
-			errors.New("gasPrice must be non-zero after london fork"),
+			&TransactionArgs{GasPrice: zero},
+			nil, // errors.New("gasPrice must be non-zero after london fork"),
 		},
 
 		// Access list txs
@@ -180,8 +180,8 @@ func TestSetFeeDefaults(t *testing.T) {
 			"dynamic fee tx post-London, explicit gas price",
 			"london",
 			&TransactionArgs{MaxFeePerGas: zero, MaxPriorityFeePerGas: zero},
-			nil,
-			errors.New("maxFeePerGas must be non-zero"),
+			&TransactionArgs{MaxFeePerGas: zero, MaxPriorityFeePerGas: zero},
+			nil, // errors.New("maxFeePerGas must be non-zero"),
 		},
 
 		// Misc
@@ -416,7 +416,8 @@ func (b *backendMock) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent)
 
 func (b *backendMock) Engine() consensus.Engine { return nil }
 
-func (b *backendMock) MevRunning() bool { return false }
+func (b *backendMock) MevRunning() bool                       { return false }
+func (b *backendMock) HasBuilder(builder common.Address) bool { return false }
 func (b *backendMock) MevParams() *types.MevParams {
 	return &types.MevParams{}
 }

miner/bid_simulator.go

@@ -7,6 +7,7 @@ import (
 	"math/big"
 	"net"
 	"net/http"
+	"strconv"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -68,6 +69,12 @@ type simBidReq struct {
 	interruptCh chan int32
 }
 
+// newBidPackage is the warp of a new bid and a feedback channel
+type newBidPackage struct {
+	bid      *types.Bid
+	feedback chan error
+}
+
 // bidSimulator is in charge of receiving bid from builders, reporting issue to builders.
 // And take care of bid simulation, rewards computing, best bid maintaining.
 type bidSimulator struct {
@@ -95,7 +102,7 @@ type bidSimulator struct {
 
 	// channels
 	simBidCh chan *simBidReq
-	newBidCh chan *types.Bid
+	newBidCh chan newBidPackage
 
 	pendingMu sync.RWMutex
 	pending   map[uint64]map[common.Address]map[common.Hash]struct{} // blockNumber -> builder -> bidHash -> struct{}
@@ -128,7 +135,7 @@ func newBidSimulator(
 		chainHeadCh:   make(chan core.ChainHeadEvent, chainHeadChanSize),
 		builders:      make(map[common.Address]*builderclient.Client),
 		simBidCh:      make(chan *simBidReq),
-		newBidCh:      make(chan *types.Bid, 100),
+		newBidCh:      make(chan newBidPackage, 100),
 		pending:       make(map[uint64]map[common.Address]map[common.Hash]struct{}),
 		bestBid:       make(map[common.Hash]*BidRuntime),
 		simulatingBid: make(map[common.Hash]*BidRuntime),
@@ -327,6 +334,10 @@ func (b *bidSimulator) newBidLoop() {
 		}
 	}
 
+	genDiscardedReply := func(betterBid *BidRuntime) error {
+		return fmt.Errorf("bid is discarded, current bestBid is [blockReward: %s, validatorReward: %s]", betterBid.expectedBlockReward, betterBid.expectedValidatorReward)
+	}
+
 	for {
 		select {
 		case newBid := <-b.newBidCh:
@@ -334,60 +345,47 @@ func (b *bidSimulator) newBidLoop() {
 				continue
 			}
 
-			// check the block reward and validator reward of the newBid
-			expectedBlockReward := newBid.GasFee
-			expectedValidatorReward := new(big.Int).Mul(expectedBlockReward, big.NewInt(int64(b.config.ValidatorCommission)))
-			expectedValidatorReward.Div(expectedValidatorReward, big.NewInt(10000))
-			expectedValidatorReward.Sub(expectedValidatorReward, newBid.BuilderFee)
-
-			if expectedValidatorReward.Cmp(big.NewInt(0)) < 0 {
-				// damage self profit, ignore
-				log.Debug("BidSimulator: invalid bid, validator reward is less than 0, ignore",
-					"builder", newBid.Builder, "bidHash", newBid.Hash().Hex())
-				continue
-			}
-
-			bidRuntime := &BidRuntime{
-				bid:                     newBid,
-				expectedBlockReward:     expectedBlockReward,
-				expectedValidatorReward: expectedValidatorReward,
-				packedBlockReward:       big.NewInt(0),
-				packedValidatorReward:   big.NewInt(0),
-				finished:                make(chan struct{}),
-			}
-
-			simulatingBid := b.GetSimulatingBid(newBid.ParentHash)
-			// simulatingBid is nil means there is no bid in simulation
-			if simulatingBid == nil {
-				// bestBid is nil means bid is the first bid
-				bestBid := b.GetBestBid(newBid.ParentHash)
-				if bestBid == nil {
-					commit(commitInterruptBetterBid, bidRuntime)
-					continue
+			bidRuntime, err := newBidRuntime(newBid.bid, b.config.ValidatorCommission)
+			if err != nil {
+				if newBid.feedback != nil {
+					newBid.feedback <- err
 				}
+				continue
+			}
 
-				// if bestBid is not nil, check if newBid is better than bestBid
-				if bidRuntime.expectedBlockReward.Cmp(bestBid.expectedBlockReward) >= 0 &&
-					bidRuntime.expectedValidatorReward.Cmp(bestBid.expectedValidatorReward) >= 0 {
-					// if both reward are better than last simulating newBid, commit for simulation
+			var replyErr error
+			// simulatingBid will be nil if there is no bid in simulation, compare with the bestBid instead
+			if simulatingBid := b.GetSimulatingBid(newBid.bid.ParentHash); simulatingBid != nil {
+				// simulatingBid always better than bestBid, so only compare with simulatingBid if a simulatingBid exists
+				if bidRuntime.isExpectedBetterThan(simulatingBid) {
 					commit(commitInterruptBetterBid, bidRuntime)
-					continue
+				} else {
+					replyErr = genDiscardedReply(simulatingBid)
+				}
+			} else {
+				// bestBid is nil means the bid is the first bid, otherwise the bid should compare with the bestBid
+				if bestBid := b.GetBestBid(newBid.bid.ParentHash); bestBid == nil ||
+					bidRuntime.isExpectedBetterThan(bestBid) {
+					commit(commitInterruptBetterBid, bidRuntime)
+				} else {
+					replyErr = genDiscardedReply(bestBid)
 				}
-
-				log.Debug("BidSimulator: lower reward, ignore",
-					"builder", bidRuntime.bid.Builder, "bidHash", newBid.Hash().Hex())
-				continue
 			}
 
-			// simulatingBid must be better than bestBid, if newBid is better than simulatingBid, commit for simulation
-			if bidRuntime.expectedBlockReward.Cmp(simulatingBid.expectedBlockReward) >= 0 &&
-				bidRuntime.expectedValidatorReward.Cmp(simulatingBid.expectedValidatorReward) >= 0 {
-				// if both reward are better than last simulating newBid, commit for simulation
-				commit(commitInterruptBetterBid, bidRuntime)
-				continue
+			if newBid.feedback != nil {
+				newBid.feedback <- replyErr
+
+				log.Info("[BID ARRIVED]",
+					"block", newBid.bid.BlockNumber,
+					"builder", newBid.bid.Builder,
+					"accepted", replyErr == nil,
+					"blockReward", weiToEtherStringF6(bidRuntime.expectedBlockReward),
+					"validatorReward", weiToEtherStringF6(bidRuntime.expectedValidatorReward),
+					"tx", len(newBid.bid.Txs),
+					"hash", newBid.bid.Hash().TerminalString(),
+				)
 			}
 
-			log.Debug("BidSimulator: lower reward, ignore", "builder", newBid.Builder, "bidHash", newBid.Hash().Hex())
 		case <-b.exitCh:
 			return
 		}
@@ -419,10 +417,6 @@ func (b *bidSimulator) clearLoop() {
 		b.bestBidMu.Unlock()
 
 		b.simBidMu.Lock()
-		if bid, ok := b.simulatingBid[parentHash]; ok {
-			bid.env.discard()
-		}
-		delete(b.simulatingBid, parentHash)
 		for k, v := range b.simulatingBid {
 			if v.bid.BlockNumber <= blockNumber-core.TriesInMemory {
 				v.env.discard()
@@ -446,10 +440,19 @@ func (b *bidSimulator) clearLoop() {
 func (b *bidSimulator) sendBid(_ context.Context, bid *types.Bid) error {
 	timer := time.NewTimer(1 * time.Second)
 	defer timer.Stop()
+
+	replyCh := make(chan error, 1)
+
 	select {
-	case b.newBidCh <- bid:
+	case b.newBidCh <- newBidPackage{bid: bid, feedback: replyCh}:
 		b.AddPending(bid.BlockNumber, bid.Builder, bid.Hash())
-		return nil
+	case <-timer.C:
+		return types.ErrMevBusy
+	}
+
+	select {
+	case reply := <-replyCh:
+		return reply
 	case <-timer.C:
 		return types.ErrMevBusy
 	}
@@ -495,6 +498,8 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) {
 	}
 
 	var (
+		startTS = time.Now()
+
 		blockNumber = bidRuntime.bid.BlockNumber
 		parentHash  = bidRuntime.bid.ParentHash
 		builder     = bidRuntime.bid.Builder
@@ -546,12 +551,12 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) {
 			}
 
 			select {
-			case b.newBidCh <- bidRuntime.bid:
+			case b.newBidCh <- newBidPackage{bid: bidRuntime.bid}:
 				log.Debug("BidSimulator: recommit", "builder", bidRuntime.bid.Builder, "bidHash", bidRuntime.bid.Hash().Hex())
 			default:
 			}
 		}
-	}(time.Now())
+	}(startTS)
 
 	// prepareWork will configure header with a suitable time according to consensus
 	// prepareWork will start trie prefetching
@@ -637,7 +642,7 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) {
 	if b.config.GreedyMergeTx {
 		delay := b.engine.Delay(b.chain, bidRuntime.env.header, &b.delayLeftOver)
 		if delay != nil && *delay > 0 {
-			bidTxsSet := mapset.NewSet[common.Hash]()
+			bidTxsSet := mapset.NewThreadUnsafeSetWithSize[common.Hash](len(bidRuntime.bid.Txs))
 			for _, tx := range bidRuntime.bid.Txs {
 				bidTxsSet.Add(tx.Hash())
 			}
@@ -662,13 +667,30 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) {
 	}
 
 	bestBid := b.GetBestBid(parentHash)
-
 	if bestBid == nil {
+		log.Info("[BID RESULT]", "win", "true[first]", "builder", bidRuntime.bid.Builder, "hash", bidRuntime.bid.Hash().TerminalString())
 		b.SetBestBid(bidRuntime.bid.ParentHash, bidRuntime)
 		success = true
 		return
 	}
 
+	if bidRuntime.bid.Hash() != bestBid.bid.Hash() {
+		log.Info("[BID RESULT]",
+			"win", bidRuntime.packedBlockReward.Cmp(bestBid.packedBlockReward) >= 0,
+
+			"bidHash", bidRuntime.bid.Hash().TerminalString(),
+			"bestHash", bestBid.bid.Hash().TerminalString(),
+
+			"bidGasFee", weiToEtherStringF6(bidRuntime.packedBlockReward),
+			"bestGasFee", weiToEtherStringF6(bestBid.packedBlockReward),
+
+			"bidBlockTx", bidRuntime.env.tcount,
+			"bestBlockTx", bestBid.env.tcount,
+
+			"simElapsed", time.Since(startTS),
+		)
+	}
+
 	// this is the simplest strategy: best for all the delegators.
 	if bidRuntime.packedBlockReward.Cmp(bestBid.packedBlockReward) >= 0 {
 		b.SetBestBid(bidRuntime.bid.ParentHash, bidRuntime)
@@ -682,7 +704,7 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) {
 	}
 
 	select {
-	case b.newBidCh <- bestBid.bid:
+	case b.newBidCh <- newBidPackage{bid: bestBid.bid}:
 		log.Debug("BidSimulator: recommit last bid", "builder", bidRuntime.bid.Builder, "bidHash", bidRuntime.bid.Hash().Hex())
 	default:
 	}
@@ -702,7 +724,7 @@ func (b *bidSimulator) reportIssue(bidRuntime *BidRuntime, err error) {
 		})
 
 		if err != nil {
-			log.Error("BidSimulator: failed to report issue", "builder", bidRuntime.bid.Builder, "err", err)
+			log.Warn("BidSimulator: failed to report issue", "builder", bidRuntime.bid.Builder, "err", err)
 		}
 	}
 }
@@ -722,11 +744,42 @@ type BidRuntime struct {
 	duration time.Duration
 }
 
+func newBidRuntime(newBid *types.Bid, validatorCommission uint64) (*BidRuntime, error) {
+	// check the block reward and validator reward of the newBid
+	expectedBlockReward := newBid.GasFee
+	expectedValidatorReward := new(big.Int).Mul(expectedBlockReward, big.NewInt(int64(validatorCommission)))
+	expectedValidatorReward.Div(expectedValidatorReward, big.NewInt(10000))
+	expectedValidatorReward.Sub(expectedValidatorReward, newBid.BuilderFee)
+
+	if expectedValidatorReward.Cmp(big.NewInt(0)) < 0 {
+		// damage self profit, ignore
+		log.Debug("BidSimulator: invalid bid, validator reward is less than 0, ignore",
+			"builder", newBid.Builder, "bidHash", newBid.Hash().Hex())
+		return nil, fmt.Errorf("validator reward is less than 0, value: %s, commissionConfig: %d", expectedValidatorReward, validatorCommission)
+	}
+
+	bidRuntime := &BidRuntime{
+		bid:                     newBid,
+		expectedBlockReward:     expectedBlockReward,
+		expectedValidatorReward: expectedValidatorReward,
+		packedBlockReward:       big.NewInt(0),
+		packedValidatorReward:   big.NewInt(0),
+		finished:                make(chan struct{}),
+	}
+
+	return bidRuntime, nil
+}
+
 func (r *BidRuntime) validReward() bool {
 	return r.packedBlockReward.Cmp(r.expectedBlockReward) >= 0 &&
 		r.packedValidatorReward.Cmp(r.expectedValidatorReward) >= 0
 }
 
+func (r *BidRuntime) isExpectedBetterThan(other *BidRuntime) bool {
+	return r.expectedBlockReward.Cmp(other.expectedBlockReward) >= 0 &&
+		r.expectedValidatorReward.Cmp(other.expectedValidatorReward) >= 0
+}
+
 // packReward calculates packedBlockReward and packedValidatorReward
 func (r *BidRuntime) packReward(validatorCommission uint64) {
 	r.packedBlockReward = r.env.state.GetBalance(consensus.SystemAddress).ToBig()
@@ -782,3 +835,8 @@ func (r *BidRuntime) commitTransaction(chain *core.BlockChain, chainConfig *para
 
 	return nil
 }
+
+func weiToEtherStringF6(wei *big.Int) string {
+	f, _ := new(big.Float).Quo(new(big.Float).SetInt(wei), big.NewFloat(params.Ether)).Float64()
+	return strconv.FormatFloat(f, 'f', 6, 64)
+}

miner/miner_mev.go

@@ -9,6 +9,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/params"
 )
 
 type BuilderConfig struct {
@@ -59,6 +60,11 @@ func (miner *Miner) RemoveBuilder(builderAddr common.Address) error {
 	return miner.bidSimulator.RemoveBuilder(builderAddr)
 }
 
+// HasBuilder returns true if the builder is in the builder list.
+func (miner *Miner) HasBuilder(builder common.Address) bool {
+	return miner.bidSimulator.ExistBuilder(builder)
+}
+
 func (miner *Miner) SendBid(ctx context.Context, bidArgs *types.BidArgs) (common.Hash, error) {
 	builder, err := bidArgs.EcrecoverSender()
 	if err != nil {
@@ -117,6 +123,8 @@ func (miner *Miner) MevParams() *types.MevParams {
 		ValidatorCommission:   miner.worker.config.Mev.ValidatorCommission,
 		BidSimulationLeftOver: miner.worker.config.Mev.BidSimulationLeftOver,
 		GasCeil:               miner.worker.config.GasCeil,
+		GasPrice:              miner.worker.config.GasPrice,
 		BuilderFeeCeil:        builderFeeCeil,
+		Version:               params.Version,
 	}
 }

miner/worker.go

@@ -1371,7 +1371,13 @@ LOOP:
 				bestWork = bestBid.env
 				from = bestBid.bid.Builder
 
-				log.Debug("BidSimulator: bid win", "block", bestWork.header.Number.Uint64(), "bid", bestBid.bid.Hash())
+				log.Info("[BUILDER BLOCK]",
+					"block", bestWork.header.Number.Uint64(),
+					"builder", from,
+					"blockReward", weiToEtherStringF6(bestBid.packedBlockReward),
+					"validatorReward", weiToEtherStringF6(bestBid.packedValidatorReward),
+					"bid", bestBid.bid.Hash().TerminalString(),
+				)
 			}
 		}
 	}

node/node.go

@@ -74,11 +74,11 @@ const (
 	initializingState = iota
 	runningState
 	closedState
-	blockDbCacheSize        = 256
-	blockDbHandlesMinSize   = 1000
-	blockDbHandlesMaxSize   = 2000
-	chainDbMemoryPercentage = 50
-	chainDbHandlesPercentage
+	blockDbCacheSize           = 256
+	blockDbHandlesMinSize      = 1000
+	blockDbHandlesMaxSize      = 2000
+	chainDbMemoryPercentage    = 50
+	chainDbHandlesPercentage   = 50
 	diffStoreHandlesPercentage = 20
 )
 
@@ -791,14 +791,15 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r
 
 func (n *Node) OpenAndMergeDatabase(name string, namespace string, readonly bool, config *ethconfig.Config) (ethdb.Database, error) {
 	var (
-		err                  error
-		stateDiskDb          ethdb.Database
-		blockDb              ethdb.Database
-		disableChainDbFreeze = false
-		blockDbHandlesSize   int
-		diffStoreHandles     int
-		chainDataHandles     = config.DatabaseHandles
-		chainDbCache         = config.DatabaseCache
+		err                          error
+		stateDiskDb                  ethdb.Database
+		blockDb                      ethdb.Database
+		disableChainDbFreeze         = false
+		blockDbHandlesSize           int
+		diffStoreHandles             int
+		chainDataHandles             = config.DatabaseHandles
+		chainDbCache                 = config.DatabaseCache
+		stateDbCache, stateDbHandles int
 	)
 
 	if config.PersistDiff {
@@ -818,10 +819,17 @@ func (n *Node) OpenAndMergeDatabase(name string, namespace string, readonly bool
 		} else {
 			blockDbHandlesSize = blockDbHandlesMinSize
 		}
-		stateDbCache := config.DatabaseCache - chainDbCache - blockDbCacheSize
-		stateDbHandles := config.DatabaseHandles - chainDataHandles - blockDbHandlesSize
+		stateDbCache = config.DatabaseCache - chainDbCache - blockDbCacheSize
+		stateDbHandles = config.DatabaseHandles - chainDataHandles - blockDbHandlesSize
 		disableChainDbFreeze = true
+	}
 
+	chainDB, err := n.OpenDatabaseWithFreezer(name, chainDbCache, chainDataHandles, config.DatabaseFreezer, namespace, readonly, disableChainDbFreeze, false, config.PruneAncientData)
+	if err != nil {
+		return nil, err
+	}
+
+	if isMultiDatabase {
 		// Allocate half of the  handles and chainDbCache to this separate state data database
 		stateDiskDb, err = n.OpenDatabaseWithFreezer(name+"/state", stateDbCache, stateDbHandles, "", "eth/db/statedata/", readonly, true, false, config.PruneAncientData)
 		if err != nil {
@@ -833,14 +841,6 @@ func (n *Node) OpenAndMergeDatabase(name string, namespace string, readonly bool
 			return nil, err
 		}
 		log.Warn("Multi-database is an experimental feature")
-	}
-
-	chainDB, err := n.OpenDatabaseWithFreezer(name, chainDbCache, chainDataHandles, config.DatabaseFreezer, namespace, readonly, disableChainDbFreeze, false, config.PruneAncientData)
-	if err != nil {
-		return nil, err
-	}
-
-	if isMultiDatabase {
 		chainDB.SetStateStore(stateDiskDb)
 		chainDB.SetBlockStore(blockDb)
 	}

params/config.go

@@ -153,6 +153,7 @@ var (
 		FeynmanFixTime:      newUint64(1713419340), // 2024-04-18 05:49:00 AM UTC
 		CancunTime:          newUint64(1718863500), // 2024-06-20 06:05:00 AM UTC
 		HaberTime:           newUint64(1718863500), // 2024-06-20 06:05:00 AM UTC
+		HaberFixTime:        nil,                   // TBD
 		BohrTime:            nil,
 
 		Parlia: &ParliaConfig{
@@ -193,6 +194,7 @@ var (
 		FeynmanFixTime:      newUint64(1711342800), // 2024-03-25 5:00:00 AM UTC
 		CancunTime:          newUint64(1713330442), // 2024-04-17 05:07:22 AM UTC
 		HaberTime:           newUint64(1716962820), // 2024-05-29 06:07:00 AM UTC
+		HaberFixTime:        newUint64(1719986788), // 2024-07-03 06:06:28 AM UTC
 		BohrTime:            nil,
 
 		Parlia: &ParliaConfig{
@@ -234,6 +236,7 @@ var (
 		FeynmanFixTime:      newUint64(0),
 		CancunTime:          newUint64(0),
 		HaberTime:           newUint64(0),
+		HaberFixTime:        newUint64(0),
 		BohrTime:            newUint64(0),
 
 		Parlia: &ParliaConfig{
@@ -512,6 +515,7 @@ type ChainConfig struct {
 	FeynmanFixTime *uint64 `json:"feynmanFixTime,omitempty"` // FeynmanFix switch time (nil = no fork, 0 = already activated)
 	CancunTime     *uint64 `json:"cancunTime,omitempty"`     // Cancun switch time (nil = no fork, 0 = already on cancun)
 	HaberTime      *uint64 `json:"haberTime,omitempty"`      // Haber switch time (nil = no fork, 0 = already on haber)
+	HaberFixTime   *uint64 `json:"haberFixTime,omitempty"`   // HaberFix switch time (nil = no fork, 0 = already on haberFix)
 	BohrTime       *uint64 `json:"bohrTime,omitempty"`       // Bohr switch time (nil = no fork, 0 = already on bohr)
 	PragueTime     *uint64 `json:"pragueTime,omitempty"`     // Prague switch time (nil = no fork, 0 = already on prague)
 	VerkleTime     *uint64 `json:"verkleTime,omitempty"`     // Verkle switch time (nil = no fork, 0 = already on verkle)
@@ -623,12 +627,17 @@ func (c *ChainConfig) String() string {
 		HaberTime = big.NewInt(0).SetUint64(*c.HaberTime)
 	}
 
+	var HaberFixTime *big.Int
+	if c.HaberFixTime != nil {
+		HaberFixTime = big.NewInt(0).SetUint64(*c.HaberFixTime)
+	}
+
 	var BohrTime *big.Int
 	if c.BohrTime != nil {
 		BohrTime = big.NewInt(0).SetUint64(*c.BohrTime)
 	}
 
-	return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, Ramanujan: %v, Niels: %v, MirrorSync: %v, Bruno: %v, Berlin: %v, YOLO v3: %v, CatalystBlock: %v, London: %v, ArrowGlacier: %v, MergeFork:%v, Euler: %v, Gibbs: %v, Nano: %v, Moran: %v, Planck: %v,Luban: %v, Plato: %v, Hertz: %v, Hertzfix: %v, ShanghaiTime: %v, KeplerTime: %v, FeynmanTime: %v, FeynmanFixTime: %v, CancunTime: %v, HaberTime: %v, BohrTime: %v, Engine: %v}",
+	return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, Ramanujan: %v, Niels: %v, MirrorSync: %v, Bruno: %v, Berlin: %v, YOLO v3: %v, CatalystBlock: %v, London: %v, ArrowGlacier: %v, MergeFork:%v, Euler: %v, Gibbs: %v, Nano: %v, Moran: %v, Planck: %v,Luban: %v, Plato: %v, Hertz: %v, Hertzfix: %v, ShanghaiTime: %v, KeplerTime: %v, FeynmanTime: %v, FeynmanFixTime: %v, CancunTime: %v, HaberTime: %v, HaberFixTime: %v, BohrTime: %v, Engine: %v}",
 		c.ChainID,
 		c.HomesteadBlock,
 		c.DAOForkBlock,
@@ -666,6 +675,7 @@ func (c *ChainConfig) String() string {
 		FeynmanFixTime,
 		CancunTime,
 		HaberTime,
+		HaberFixTime,
 		BohrTime,
 		engine,
 	)
@@ -939,6 +949,20 @@ func (c *ChainConfig) IsHaber(num *big.Int, time uint64) bool {
 	return c.IsLondon(num) && isTimestampForked(c.HaberTime, time)
 }
 
+// IsHaberFix returns whether time is either equal to the HaberFix fork time or greater.
+func (c *ChainConfig) IsHaberFix(num *big.Int, time uint64) bool {
+	return c.IsLondon(num) && isTimestampForked(c.HaberFixTime, time)
+}
+
+// IsOnHaberFix returns whether currentBlockTime is either equal to the HaberFix fork time or greater firstly.
+func (c *ChainConfig) IsOnHaberFix(currentBlockNumber *big.Int, lastBlockTime uint64, currentBlockTime uint64) bool {
+	lastBlockNumber := new(big.Int)
+	if currentBlockNumber.Cmp(big.NewInt(1)) >= 0 {
+		lastBlockNumber.Sub(currentBlockNumber, big.NewInt(1))
+	}
+	return !c.IsHaberFix(lastBlockNumber, lastBlockTime) && c.IsHaberFix(currentBlockNumber, currentBlockTime)
+}
+
 // IsBohr returns whether time is either equal to the Bohr fork time or greater.
 func (c *ChainConfig) IsBohr(num *big.Int, time uint64) bool {
 	return c.IsLondon(num) && isTimestampForked(c.BohrTime, time)
@@ -1017,6 +1041,7 @@ func (c *ChainConfig) CheckConfigForkOrder() error {
 		{name: "feynmanFixTime", timestamp: c.FeynmanFixTime},
 		{name: "cancunTime", timestamp: c.CancunTime},
 		{name: "haberTime", timestamp: c.HaberTime},
+		{name: "haberFixTime", timestamp: c.HaberFixTime},
 		{name: "bohrTime", timestamp: c.BohrTime},
 		{name: "pragueTime", timestamp: c.PragueTime, optional: true},
 		{name: "verkleTime", timestamp: c.VerkleTime, optional: true},
@@ -1165,6 +1190,15 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, headNumber *big.Int,
 	if isForkTimestampIncompatible(c.CancunTime, newcfg.CancunTime, headTimestamp) {
 		return newTimestampCompatError("Cancun fork timestamp", c.CancunTime, newcfg.CancunTime)
 	}
+	if isForkTimestampIncompatible(c.HaberTime, newcfg.HaberTime, headTimestamp) {
+		return newTimestampCompatError("Haber fork timestamp", c.HaberTime, newcfg.HaberTime)
+	}
+	if isForkTimestampIncompatible(c.HaberFixTime, newcfg.HaberFixTime, headTimestamp) {
+		return newTimestampCompatError("HaberFix fork timestamp", c.HaberFixTime, newcfg.HaberFixTime)
+	}
+	if isForkTimestampIncompatible(c.BohrTime, newcfg.BohrTime, headTimestamp) {
+		return newTimestampCompatError("Bohr fork timestamp", c.BohrTime, newcfg.BohrTime)
+	}
 	if isForkTimestampIncompatible(c.PragueTime, newcfg.PragueTime, headTimestamp) {
 		return newTimestampCompatError("Prague fork timestamp", c.PragueTime, newcfg.PragueTime)
 	}

params/version.go

@@ -23,7 +23,7 @@ import (
 const (
 	VersionMajor = 1  // Major version component of the current release
 	VersionMinor = 4  // Minor version component of the current release
-	VersionPatch = 9  // Patch version component of the current release
+	VersionPatch = 12 // Patch version component of the current release
 	VersionMeta  = "" // Version metadata to append to the version string
 )
 

signer/fourbyte/abi.go

@@ -98,7 +98,7 @@ func parseCallData(calldata []byte, unescapedAbidata string) (*decodedCallData,
 	if len(argdata)%32 != 0 {
 		return nil, fmt.Errorf("invalid call data; length should be a multiple of 32 bytes (was %d)", len(argdata))
 	}
-	// Validate the called method and upack the call data accordingly
+	// Validate the called method and unpack the call data accordingly
 	abispec, err := abi.JSON(strings.NewReader(unescapedAbidata))
 	if err != nil {
 		return nil, fmt.Errorf("invalid method signature (%q): %v", unescapedAbidata, err)

trie/sync.go

@@ -229,7 +229,7 @@ func (batch *syncMemBatch) delNode(owner common.Hash, path []byte) {
 // and reconstructs the trie step by step until all is done.
 type Sync struct {
 	scheme   string                       // Node scheme descriptor used in database.
-	database ethdb.KeyValueReader         // Persistent database to check for existing entries
+	database ethdb.Database               // Persistent database to check for existing entries
 	membatch *syncMemBatch                // Memory buffer to avoid frequent database writes
 	nodeReqs map[string]*nodeRequest      // Pending requests pertaining to a trie node path
 	codeReqs map[common.Hash]*codeRequest // Pending requests pertaining to a code hash
@@ -238,7 +238,7 @@ type Sync struct {
 }
 
 // NewSync creates a new trie data download scheduler.
-func NewSync(root common.Hash, database ethdb.KeyValueReader, callback LeafCallback, scheme string) *Sync {
+func NewSync(root common.Hash, database ethdb.Database, callback LeafCallback, scheme string) *Sync {
 	ts := &Sync{
 		scheme:   scheme,
 		database: database,
@@ -420,7 +420,7 @@ func (s *Sync) ProcessNode(result NodeSyncResult) error {
 // Commit flushes the data stored in the internal membatch out to persistent
 // storage, returning any occurred error. The whole data set will be flushed
 // in an atomic database batch.
-func (s *Sync) Commit(dbw ethdb.Batch) error {
+func (s *Sync) Commit(dbw ethdb.Batch, stateBatch ethdb.Batch) error {
 	// Flush the pending node writes into database batch.
 	var (
 		account int
@@ -430,9 +430,17 @@ func (s *Sync) Commit(dbw ethdb.Batch) error {
 		if op.isDelete() {
 			// node deletion is only supported in path mode.
 			if op.owner == (common.Hash{}) {
-				rawdb.DeleteAccountTrieNode(dbw, op.path)
+				if stateBatch != nil {
+					rawdb.DeleteAccountTrieNode(stateBatch, op.path)
+				} else {
+					rawdb.DeleteAccountTrieNode(dbw, op.path)
+				}
 			} else {
-				rawdb.DeleteStorageTrieNode(dbw, op.owner, op.path)
+				if stateBatch != nil {
+					rawdb.DeleteStorageTrieNode(stateBatch, op.owner, op.path)
+				} else {
+					rawdb.DeleteStorageTrieNode(dbw, op.owner, op.path)
+				}
 			}
 			deletionGauge.Inc(1)
 		} else {
@@ -441,7 +449,11 @@ func (s *Sync) Commit(dbw ethdb.Batch) error {
 			} else {
 				storage += 1
 			}
-			rawdb.WriteTrieNode(dbw, op.owner, op.path, op.hash, op.blob, s.scheme)
+			if stateBatch != nil {
+				rawdb.WriteTrieNode(stateBatch, op.owner, op.path, op.hash, op.blob, s.scheme)
+			} else {
+				rawdb.WriteTrieNode(dbw, op.owner, op.path, op.hash, op.blob, s.scheme)
+			}
 		}
 	}
 	accountNodeSyncedGauge.Inc(int64(account))
@@ -546,9 +558,9 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) {
 				// the performance impact negligible.
 				var exists bool
 				if owner == (common.Hash{}) {
-					exists = rawdb.ExistsAccountTrieNode(s.database, append(inner, key[:i]...))
+					exists = rawdb.ExistsAccountTrieNode(s.database.StateStoreReader(), append(inner, key[:i]...))
 				} else {
-					exists = rawdb.ExistsStorageTrieNode(s.database, owner, append(inner, key[:i]...))
+					exists = rawdb.ExistsStorageTrieNode(s.database.StateStoreReader(), owner, append(inner, key[:i]...))
 				}
 				if exists {
 					s.membatch.delNode(owner, append(inner, key[:i]...))
@@ -687,15 +699,15 @@ func (s *Sync) commitCodeRequest(req *codeRequest) error {
 func (s *Sync) hasNode(owner common.Hash, path []byte, hash common.Hash) (exists bool, inconsistent bool) {
 	// If node is running with hash scheme, check the presence with node hash.
 	if s.scheme == rawdb.HashScheme {
-		return rawdb.HasLegacyTrieNode(s.database, hash), false
+		return rawdb.HasLegacyTrieNode(s.database.StateStoreReader(), hash), false
 	}
 	// If node is running with path scheme, check the presence with node path.
 	var blob []byte
 	var dbHash common.Hash
 	if owner == (common.Hash{}) {
-		blob, dbHash = rawdb.ReadAccountTrieNode(s.database, path)
+		blob, dbHash = rawdb.ReadAccountTrieNode(s.database.StateStoreReader(), path)
 	} else {
-		blob, dbHash = rawdb.ReadStorageTrieNode(s.database, owner, path)
+		blob, dbHash = rawdb.ReadStorageTrieNode(s.database.StateStoreReader(), owner, path)
 	}
 	exists = hash == dbHash
 	inconsistent = !exists && len(blob) != 0

trie/sync_test.go

@@ -27,7 +27,6 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/ethdb/memorydb"
 	"github.com/ethereum/go-ethereum/trie/trienode"
 )
 
@@ -143,7 +142,7 @@ func TestEmptySync(t *testing.T) {
 	emptyD, _ := New(TrieID(types.EmptyRootHash), dbD)
 
 	for i, trie := range []*Trie{emptyA, emptyB, emptyC, emptyD} {
-		sync := NewSync(trie.Hash(), memorydb.New(), nil, []*testDb{dbA, dbB, dbC, dbD}[i].Scheme())
+		sync := NewSync(trie.Hash(), rawdb.NewMemoryDatabase(), nil, []*testDb{dbA, dbB, dbC, dbD}[i].Scheme())
 		if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 {
 			t.Errorf("test %d: content requested for empty trie: %v, %v, %v", i, paths, nodes, codes)
 		}
@@ -212,7 +211,7 @@ func testIterativeSync(t *testing.T, count int, bypath bool, scheme string) {
 			}
 		}
 		batch := diskdb.NewBatch()
-		if err := sched.Commit(batch); err != nil {
+		if err := sched.Commit(batch, nil); err != nil {
 			t.Fatalf("failed to commit data: %v", err)
 		}
 		batch.Write()
@@ -278,7 +277,7 @@ func testIterativeDelayedSync(t *testing.T, scheme string) {
 			}
 		}
 		batch := diskdb.NewBatch()
-		if err := sched.Commit(batch); err != nil {
+		if err := sched.Commit(batch, nil); err != nil {
 			t.Fatalf("failed to commit data: %v", err)
 		}
 		batch.Write()
@@ -348,7 +347,7 @@ func testIterativeRandomSync(t *testing.T, count int, scheme string) {
 			}
 		}
 		batch := diskdb.NewBatch()
-		if err := sched.Commit(batch); err != nil {
+		if err := sched.Commit(batch, nil); err != nil {
 			t.Fatalf("failed to commit data: %v", err)
 		}
 		batch.Write()
@@ -419,7 +418,7 @@ func testIterativeRandomDelayedSync(t *testing.T, scheme string) {
 			}
 		}
 		batch := diskdb.NewBatch()
-		if err := sched.Commit(batch); err != nil {
+		if err := sched.Commit(batch, nil); err != nil {
 			t.Fatalf("failed to commit data: %v", err)
 		}
 		batch.Write()
@@ -491,7 +490,7 @@ func testDuplicateAvoidanceSync(t *testing.T, scheme string) {
 			}
 		}
 		batch := diskdb.NewBatch()
-		if err := sched.Commit(batch); err != nil {
+		if err := sched.Commit(batch, nil); err != nil {
 			t.Fatalf("failed to commit data: %v", err)
 		}
 		batch.Write()
@@ -563,7 +562,7 @@ func testIncompleteSync(t *testing.T, scheme string) {
 			}
 		}
 		batch := diskdb.NewBatch()
-		if err := sched.Commit(batch); err != nil {
+		if err := sched.Commit(batch, nil); err != nil {
 			t.Fatalf("failed to commit data: %v", err)
 		}
 		batch.Write()
@@ -653,7 +652,7 @@ func testSyncOrdering(t *testing.T, scheme string) {
 			}
 		}
 		batch := diskdb.NewBatch()
-		if err := sched.Commit(batch); err != nil {
+		if err := sched.Commit(batch, nil); err != nil {
 			t.Fatalf("failed to commit data: %v", err)
 		}
 		batch.Write()
@@ -723,7 +722,7 @@ func syncWithHookWriter(t *testing.T, root common.Hash, db ethdb.Database, srcDb
 			}
 		}
 		batch := db.NewBatch()
-		if err := sched.Commit(batch); err != nil {
+		if err := sched.Commit(batch, nil); err != nil {
 			t.Fatalf("failed to commit data: %v", err)
 		}
 		if hookWriter != nil {

triedb/database.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"strings"
 
+	versa "github.com/bnb-chain/versioned-state-database"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -30,6 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/triedb/database"
 	"github.com/ethereum/go-ethereum/triedb/hashdb"
 	"github.com/ethereum/go-ethereum/triedb/pathdb"
+	"github.com/ethereum/go-ethereum/triedb/versiondb"
 )
 
 // Config defines all necessary options for database.
@@ -37,7 +39,9 @@ type Config struct {
 	Preimages bool // Flag whether the preimage of node key is recorded
 	Cache     int
 	NoTries   bool
-	IsVerkle  bool           // Flag whether the db is holding a verkle tree
+	IsVerkle  bool // Flag whether the db is holding a verkle tree
+	IsVersion bool
+	VersionDB *versiondb.Config
 	HashDB    *hashdb.Config // Configs for hash-based scheme
 	PathDB    *pathdb.Config // Configs for experimental path-based scheme
 }
@@ -92,6 +96,16 @@ type Database struct {
 // NewDatabase initializes the trie database with default settings, note
 // the legacy hash-based scheme is used by default.
 func NewDatabase(diskdb ethdb.Database, config *Config) *Database {
+	if config != nil && config.IsVersion {
+		// TODO:: Wait for debugging to stabilize, and then consider initialization compatibility with other databases
+		db := &Database{
+			config:  config,
+			diskdb:  diskdb,
+			backend: versiondb.New(config.VersionDB),
+		}
+		return db
+	}
+
 	// Sanitize the config and use the default one if it's not specified.
 	var triediskdb ethdb.Database
 	if diskdb != nil && diskdb.StateStore() != nil {
@@ -409,3 +423,12 @@ func (db *Database) GetAllRooHash() [][]string {
 func (db *Database) IsVerkle() bool {
 	return db.config.IsVerkle
 }
+
+// VersaDB returns versioned database instance, it is useless for hashdb and pathdb
+func (db *Database) VersaDB() versa.Database {
+	vdb, ok := db.backend.(*versiondb.VersionDB)
+	if !ok {
+		log.Crit("only version db support")
+	}
+	return vdb.VersaDB()
+}

triedb/hashdb/database.go

@@ -139,6 +139,7 @@ func New(diskdb ethdb.Database, config *Config, resolver ChildResolver) *Databas
 	if config.CleanCacheSize > 0 {
 		cleans = fastcache.New(config.CleanCacheSize)
 	}
+	log.Info("success to init hash mode triedb")
 	return &Database{
 		diskdb:   diskdb,
 		resolver: resolver,

triedb/pathdb/database.go

@@ -226,6 +226,7 @@ func New(diskdb ethdb.Database, config *Config) *Database {
 			log.Crit("Failed to disable database", "err", err) // impossible to happen
 		}
 	}
+	log.Info("success to init path mode triedb")
 	return db
 }
 

triedb/pathdb/difflayer.go

@@ -26,6 +26,106 @@ import (
 	"github.com/ethereum/go-ethereum/trie/triestate"
 )
 
+type RefTrieNode struct {
+	refCount uint32
+	node     *trienode.Node
+}
+
+type HashNodeCache struct {
+	lock  sync.RWMutex
+	cache map[common.Hash]*RefTrieNode
+}
+
+func (h *HashNodeCache) length() int {
+	if h == nil {
+		return 0
+	}
+	h.lock.RLock()
+	defer h.lock.RUnlock()
+	return len(h.cache)
+}
+
+func (h *HashNodeCache) set(hash common.Hash, node *trienode.Node) {
+	if h == nil {
+		return
+	}
+	h.lock.Lock()
+	defer h.lock.Unlock()
+	if n, ok := h.cache[hash]; ok {
+		n.refCount++
+	} else {
+		h.cache[hash] = &RefTrieNode{1, node}
+	}
+}
+
+func (h *HashNodeCache) Get(hash common.Hash) *trienode.Node {
+	if h == nil {
+		return nil
+	}
+	h.lock.RLock()
+	defer h.lock.RUnlock()
+	if n, ok := h.cache[hash]; ok {
+		return n.node
+	}
+	return nil
+}
+
+func (h *HashNodeCache) del(hash common.Hash) {
+	if h == nil {
+		return
+	}
+	h.lock.Lock()
+	defer h.lock.Unlock()
+	n, ok := h.cache[hash]
+	if !ok {
+		return
+	}
+	if n.refCount > 0 {
+		n.refCount--
+	}
+	if n.refCount == 0 {
+		delete(h.cache, hash)
+	}
+}
+
+func (h *HashNodeCache) Add(ly layer) {
+	if h == nil {
+		return
+	}
+	dl, ok := ly.(*diffLayer)
+	if !ok {
+		return
+	}
+	beforeAdd := h.length()
+	for _, subset := range dl.nodes {
+		for _, node := range subset {
+			h.set(node.Hash, node)
+		}
+	}
+	diffHashCacheLengthGauge.Update(int64(h.length()))
+	log.Debug("Add difflayer to hash map", "root", ly.rootHash(), "block_number", dl.block, "map_len", h.length(), "add_delta", h.length()-beforeAdd)
+}
+
+func (h *HashNodeCache) Remove(ly layer) {
+	if h == nil {
+		return
+	}
+	dl, ok := ly.(*diffLayer)
+	if !ok {
+		return
+	}
+	go func() {
+		beforeDel := h.length()
+		for _, subset := range dl.nodes {
+			for _, node := range subset {
+				h.del(node.Hash)
+			}
+		}
+		diffHashCacheLengthGauge.Update(int64(h.length()))
+		log.Debug("Remove difflayer from hash map", "root", ly.rootHash(), "block_number", dl.block, "map_len", h.length(), "del_delta", beforeDel-h.length())
+	}()
+}
+
 // diffLayer represents a collection of modifications made to the in-memory tries
 // along with associated state changes after running a block on top.
 //
@@ -39,7 +139,10 @@ type diffLayer struct {
 	nodes  map[common.Hash]map[string]*trienode.Node // Cached trie nodes indexed by owner and path
 	states *triestate.Set                            // Associated state change set for building history
 	memory uint64                                    // Approximate guess as to how much memory we use
+	cache  *HashNodeCache                            // trienode cache by hash key. cache is immutable, but cache's item can be add/del.
 
+	// mutables
+	origin *diskLayer   // The current difflayer corresponds to the underlying disklayer and is updated during cap.
 	parent layer        // Parent layer modified by this one, never nil, **can be changed**
 	lock   sync.RWMutex // Lock used to protect parent
 }
@@ -58,6 +161,20 @@ func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes
 		states: states,
 		parent: parent,
 	}
+
+	switch l := parent.(type) {
+	case *diskLayer:
+		dl.origin = l
+		dl.cache = &HashNodeCache{
+			cache: make(map[common.Hash]*RefTrieNode),
+		}
+	case *diffLayer:
+		dl.origin = l.originDiskLayer()
+		dl.cache = l.cache
+	default:
+		panic("unknown parent type")
+	}
+
 	for _, subset := range nodes {
 		for path, n := range subset {
 			dl.memory += uint64(n.Size() + len(path))
@@ -75,6 +192,12 @@ func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes
 	return dl
 }
 
+func (dl *diffLayer) originDiskLayer() *diskLayer {
+	dl.lock.RLock()
+	defer dl.lock.RUnlock()
+	return dl.origin
+}
+
 // rootHash implements the layer interface, returning the root hash of
 // corresponding state.
 func (dl *diffLayer) rootHash() common.Hash {
@@ -133,6 +256,32 @@ func (dl *diffLayer) node(owner common.Hash, path []byte, hash common.Hash, dept
 // Node implements the layer interface, retrieving the trie node blob with the
 // provided node information. No error will be returned if the node is not found.
 func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
+	if n := dl.cache.Get(hash); n != nil {
+		// The query from the hash map is fastpath,
+		// avoiding recursive query of 128 difflayers.
+		diffHashCacheHitMeter.Mark(1)
+		diffHashCacheReadMeter.Mark(int64(len(n.Blob)))
+		return n.Blob, nil
+	}
+	diffHashCacheMissMeter.Mark(1)
+
+	persistLayer := dl.originDiskLayer()
+	if persistLayer != nil {
+		blob, err := persistLayer.Node(owner, path, hash)
+		if err != nil {
+			// This is a bad case with a very low probability.
+			// r/w the difflayer cache and r/w the disklayer are not in the same lock,
+			// so in extreme cases, both reading the difflayer cache and reading the disklayer may fail, eg, disklayer is stale.
+			// In this case, fallback to the original 128-layer recursive difflayer query path.
+			diffHashCacheSlowPathMeter.Mark(1)
+			log.Debug("Retry difflayer due to query origin failed", "owner", owner, "path", path, "hash", hash.String(), "error", err)
+			return dl.node(owner, path, hash, 0)
+		} else { // This is the fastpath.
+			return blob, nil
+		}
+	}
+	diffHashCacheSlowPathMeter.Mark(1)
+	log.Debug("Retry difflayer due to origin is nil", "owner", owner, "path", path, "hash", hash.String())
 	return dl.node(owner, path, hash, 0)
 }
 

triedb/pathdb/disklayer.go

@@ -286,6 +286,9 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) {
 		}
 		log.Debug("Pruned state history", "items", pruned, "tailid", oldest)
 	}
+
+	// The bottom has been eaten by disklayer, releasing the hash cache of bottom difflayer.
+	bottom.cache.Remove(bottom)
 	return ndl, nil
 }
 

triedb/pathdb/layertree.go

@@ -51,9 +51,20 @@ func (tree *layerTree) reset(head layer) {
 	tree.lock.Lock()
 	defer tree.lock.Unlock()
 
+	for _, ly := range tree.layers {
+		if dl, ok := ly.(*diffLayer); ok {
+			// Clean up the hash cache of difflayers due to reset.
+			dl.cache.Remove(dl)
+		}
+	}
+
 	var layers = make(map[common.Hash]layer)
 	for head != nil {
 		layers[head.rootHash()] = head
+		if dl, ok := head.(*diffLayer); ok {
+			// Add the hash cache of difflayers due to reset.
+			dl.cache.Add(dl)
+		}
 		head = head.parentLayer()
 	}
 	tree.layers = layers
@@ -98,12 +109,19 @@ func (tree *layerTree) add(root common.Hash, parentRoot common.Hash, block uint6
 	if root == parentRoot {
 		return errors.New("layer cycle")
 	}
+	if tree.get(root) != nil {
+		log.Info("Skip add repeated difflayer", "root", root.String(), "block_id", block)
+		return nil
+	}
 	parent := tree.get(parentRoot)
 	if parent == nil {
 		return fmt.Errorf("triedb parent [%#x] layer missing", parentRoot)
 	}
 	l := parent.update(root, parent.stateID()+1, block, nodes.Flatten(), states)
 
+	// Before adding layertree, update the hash cache.
+	l.cache.Add(l)
+
 	tree.lock.Lock()
 	tree.layers[l.rootHash()] = l
 	tree.lock.Unlock()
@@ -132,8 +150,15 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 		if err != nil {
 			return err
 		}
+		for _, ly := range tree.layers {
+			if dl, ok := ly.(*diffLayer); ok {
+				dl.cache.Remove(dl)
+				log.Debug("Cleanup difflayer hash cache due to cap all", "diff_root", dl.root.String(), "diff_block_number", dl.block)
+			}
+		}
 		// Replace the entire layer tree with the flat base
 		tree.layers = map[common.Hash]layer{base.rootHash(): base}
+		log.Debug("Cap all difflayers to disklayer", "disk_root", base.rootHash().String())
 		return nil
 	}
 	// Dive until we run out of layers or reach the persistent database
@@ -146,6 +171,7 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 			return nil
 		}
 	}
+	var persisted *diskLayer
 	// We're out of layers, flatten anything below, stopping if it's the disk or if
 	// the memory limit is not yet exceeded.
 	switch parent := diff.parentLayer().(type) {
@@ -166,6 +192,7 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 		diff.parent = base
 
 		diff.lock.Unlock()
+		persisted = base.(*diskLayer)
 
 	default:
 		panic(fmt.Sprintf("unknown data layer in triedb: %T", parent))
@@ -180,6 +207,13 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 	}
 	var remove func(root common.Hash)
 	remove = func(root common.Hash) {
+		if df, exist := tree.layers[root]; exist {
+			if dl, ok := df.(*diffLayer); ok {
+				// Clean up the hash cache of the child difflayer corresponding to the stale parent, include the re-org case.
+				dl.cache.Remove(dl)
+				log.Debug("Cleanup difflayer hash cache due to reorg", "diff_root", dl.root.String(), "diff_block_number", dl.block)
+			}
+		}
 		delete(tree.layers, root)
 		for _, child := range children[root] {
 			remove(child)
@@ -189,8 +223,25 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 	for root, layer := range tree.layers {
 		if dl, ok := layer.(*diskLayer); ok && dl.isStale() {
 			remove(root)
+			log.Debug("Remove stale the disklayer", "disk_root", dl.root.String())
 		}
 	}
+
+	if persisted != nil {
+		var updateOriginFunc func(root common.Hash)
+		updateOriginFunc = func(root common.Hash) {
+			if diff, ok := tree.layers[root].(*diffLayer); ok {
+				diff.lock.Lock()
+				diff.origin = persisted
+				diff.lock.Unlock()
+			}
+			for _, child := range children[root] {
+				updateOriginFunc(child)
+			}
+		}
+		updateOriginFunc(persisted.root)
+	}
+
 	return nil
 }
 

triedb/pathdb/metrics.go

@@ -47,4 +47,10 @@ var (
 	historyBuildTimeMeter  = metrics.NewRegisteredTimer("pathdb/history/time", nil)
 	historyDataBytesMeter  = metrics.NewRegisteredMeter("pathdb/history/bytes/data", nil)
 	historyIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/index", nil)
+
+	diffHashCacheHitMeter      = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/hit", nil)
+	diffHashCacheReadMeter     = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/read", nil)
+	diffHashCacheMissMeter     = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/miss", nil)
+	diffHashCacheSlowPathMeter = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/slowpath", nil)
+	diffHashCacheLengthGauge   = metrics.NewRegisteredGauge("pathdb/difflayer/hashcache/size", nil)
 )

triedb/versiondb/versadb.go

@@ -0,0 +1,77 @@
+package versiondb
+
+import (
+	versa "github.com/bnb-chain/versioned-state-database"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/trie/trienode"
+	"github.com/ethereum/go-ethereum/trie/triestate"
+)
+
+type Config struct {
+	Path           string
+	FlushInterval  int64
+	MaxStatesInMem int
+}
+
+type VersionDB struct {
+	db versa.Database
+}
+
+func New(config *Config) *VersionDB {
+	var (
+		cfg  *versa.VersaDBConfig
+		path = "./node/version_db" // TODO:: debug code
+	)
+
+	if config != nil {
+		path = config.Path
+		cfg = &versa.VersaDBConfig{
+			FlushInterval:  config.FlushInterval,
+			MaxStatesInMem: config.MaxStatesInMem,
+		}
+	}
+	db, err := versa.NewVersaDB(path, cfg)
+	if err != nil {
+		log.Crit("failed to new version db", "error", err)
+	}
+	v := &VersionDB{
+		db: db,
+	}
+	log.Info("success to init version mode triedb")
+	return v
+}
+
+func (v *VersionDB) Scheme() string {
+	return rawdb.VersionScheme
+}
+
+func (v *VersionDB) Initialized(genesisRoot common.Hash) bool {
+	version, _ := v.db.LatestStoreDiskVersionInfo()
+	return version >= 0
+}
+
+func (v *VersionDB) Size() (common.StorageSize, common.StorageSize, common.StorageSize) {
+	// TODO:: waiting versa db supported
+	return 0, 0, 0
+}
+
+func (v *VersionDB) Update(root common.Hash, parent common.Hash, block uint64, nodes *trienode.MergedNodeSet, states *triestate.Set) error {
+	// TODO:: debug code, will change to return error.
+	panic("version db not supported")
+}
+
+func (v *VersionDB) Commit(root common.Hash, report bool) error {
+	// TODO:: debug code, will change to return error.
+	panic("version db not supported")
+}
+
+func (v *VersionDB) Close() error {
+	return v.db.Close()
+}
+
+func (v *VersionDB) VersaDB() versa.Database {
+	return v.db
+}