pool: address PR comments

merge PRs from master to develop branch

.nancy-ignore

@@ -0,0 +1 @@
+CVE-2024-34478 # "CWE-754: Improper Check for Unusual or Exceptional Conditions." This vulnerability is BTC only, BSC does not have the issue.

CHANGELOG.md

@@ -1,4 +1,29 @@
 # Changelog
+## v1.4.8
+### FEATURE
+* [\#2483](https://github.com/bnb-chain/bsc/pull/2483) core/vm: add secp256r1 into PrecompiledContractsHaber
+* [\#2400](https://github.com/bnb-chain/bsc/pull/2400) RIP-7212: Precompile for secp256r1 Curve Support
+
+### IMPROVEMENT
+NA
+
+### BUGFIX
+NA
+
+## v1.4.7
+### FEATURE
+* [\#2439](https://github.com/bnb-chain/bsc/pull/2439) config: setup Mainnet Tycho(Cancun) hardfork date
+
+### IMPROVEMENT
+* [\#2396](https://github.com/bnb-chain/bsc/pull/2396) metrics: add blockInsertMgaspsGauge to trace mgasps
+* [\#2411](https://github.com/bnb-chain/bsc/pull/2411) build(deps): bump golang.org/x/net from 0.19.0 to 0.23.0
+* [\#2435](https://github.com/bnb-chain/bsc/pull/2435) txpool: limit max gas when mining is enabled
+* [\#2438](https://github.com/bnb-chain/bsc/pull/2438) fix: performance issue when load journal
+* [\#2440](https://github.com/bnb-chain/bsc/pull/2440) nancy: add files .nancy-ignore
+
+### BUGFIX
+NA
+
 ## v1.4.6
 ### FEATURE
 * [\#2227](https://github.com/bnb-chain/bsc/pull/2227) core: separated databases for block data

README.md

@@ -36,7 +36,7 @@ To combine DPoS and PoA for consensus, BNB Smart Chain implement a novel consens
 2. Validators take turns to produce blocks in a PoA manner, similar to Ethereum's Clique consensus engine.
 3. Validator set are elected in and out based on a staking based governance on BNB Beacon Chain.
 4. The validator set change is relayed via a cross-chain communication mechanism.
-5. Parlia consensus engine will interact with a set of [system contracts](https://docs.bnbchain.org/docs/learn/system-contract) to achieve liveness slash, revenue distributing and validator set renewing func.
+5. Parlia consensus engine will interact with a set of [system contracts](https://docs.bnbchain.org/bnb-smart-chain/staking/overview/#system-contracts) to achieve liveness slash, revenue distributing and validator set renewing func.
 
  
 ### Light Client of BNB Beacon Chain
@@ -183,7 +183,7 @@ This tool is optional and if you leave it out you can always attach to an alread
 
 #### 7. More
 
-More details about [running a node](https://docs.bnbchain.org/docs/validator/fullnode) and [becoming a validator](https://docs.bnbchain.org/docs/validator/create-val)
+More details about [running a node](https://docs.bnbchain.org/bnb-smart-chain/developers/node_operators/full_node/) and [becoming a validator](https://docs.bnbchain.org/bnb-smart-chain/validator/create-val/)
 
 *Note: Although some internal protective measures prevent transactions from
 crossing over between the main network and test network, you should always

cmd/geth/chaincmd.go

@@ -63,6 +63,8 @@ var (
 		Flags: flags.Merge([]cli.Flag{
 			utils.CachePreimagesFlag,
 			utils.OverrideCancun,
+			utils.OverrideHaber,
+			utils.OverrideBohr,
 			utils.OverrideVerkle,
 		}, utils.DatabaseFlags),
 		Description: `
@@ -256,6 +258,14 @@ func initGenesis(ctx *cli.Context) error {
 		v := ctx.Uint64(utils.OverrideCancun.Name)
 		overrides.OverrideCancun = &v
 	}
+	if ctx.IsSet(utils.OverrideHaber.Name) {
+		v := ctx.Uint64(utils.OverrideHaber.Name)
+		overrides.OverrideHaber = &v
+	}
+	if ctx.IsSet(utils.OverrideBohr.Name) {
+		v := ctx.Uint64(utils.OverrideBohr.Name)
+		overrides.OverrideBohr = &v
+	}
 	if ctx.IsSet(utils.OverrideVerkle.Name) {
 		v := ctx.Uint64(utils.OverrideVerkle.Name)
 		overrides.OverrideVerkle = &v

cmd/geth/config.go

@@ -189,18 +189,18 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 		v := ctx.Uint64(utils.OverrideCancun.Name)
 		cfg.Eth.OverrideCancun = &v
 	}
+	if ctx.IsSet(utils.OverrideHaber.Name) {
+		v := ctx.Uint64(utils.OverrideHaber.Name)
+		cfg.Eth.OverrideHaber = &v
+	}
+	if ctx.IsSet(utils.OverrideBohr.Name) {
+		v := ctx.Uint64(utils.OverrideBohr.Name)
+		cfg.Eth.OverrideBohr = &v
+	}
 	if ctx.IsSet(utils.OverrideVerkle.Name) {
 		v := ctx.Uint64(utils.OverrideVerkle.Name)
 		cfg.Eth.OverrideVerkle = &v
 	}
-	if ctx.IsSet(utils.OverrideFeynman.Name) {
-		v := ctx.Uint64(utils.OverrideFeynman.Name)
-		cfg.Eth.OverrideFeynman = &v
-	}
-	if ctx.IsSet(utils.OverrideFeynmanFix.Name) {
-		v := ctx.Uint64(utils.OverrideFeynmanFix.Name)
-		cfg.Eth.OverrideFeynmanFix = &v
-	}
 	if ctx.IsSet(utils.OverrideFullImmutabilityThreshold.Name) {
 		params.FullImmutabilityThreshold = ctx.Uint64(utils.OverrideFullImmutabilityThreshold.Name)
 		downloader.FullMaxForkAncestry = ctx.Uint64(utils.OverrideFullImmutabilityThreshold.Name)
@@ -211,6 +211,9 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 	if ctx.IsSet(utils.OverrideDefaultExtraReserveForBlobRequests.Name) {
 		params.DefaultExtraReserveForBlobRequests = ctx.Uint64(utils.OverrideDefaultExtraReserveForBlobRequests.Name)
 	}
+	if ctx.IsSet(utils.OverrideBreatheBlockInterval.Name) {
+		params.BreatheBlockInterval = ctx.Uint64(utils.OverrideBreatheBlockInterval.Name)
+	}
 
 	backend, eth := utils.RegisterEthService(stack, &cfg.Eth)
 

cmd/geth/dbcmd.go

@@ -106,12 +106,12 @@ Remove blockchain and state databases`,
 	dbInspectTrieCmd = &cli.Command{
 		Action:    inspectTrie,
 		Name:      "inspect-trie",
-		ArgsUsage: "<blocknum> <jobnum>",
+		ArgsUsage: "<blocknum> <jobnum> <topn>",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
 			utils.SyncModeFlag,
 		},
-		Usage:       "Inspect the MPT tree of the account and contract.",
+		Usage:       "Inspect the MPT tree of the account and contract. 'blocknum' can be latest/snapshot/number. 'topn' means output the top N storage tries info ranked by the total number of TrieNodes",
 		Description: `This commands iterates the entrie WorldState.`,
 	}
 	dbCheckStateContentCmd = &cli.Command{
@@ -386,6 +386,7 @@ func inspectTrie(ctx *cli.Context) error {
 		blockNumber  uint64
 		trieRootHash common.Hash
 		jobnum       uint64
+		topN         uint64
 	)
 
 	stack, _ := makeConfigNode(ctx)
@@ -411,12 +412,25 @@ func inspectTrie(ctx *cli.Context) error {
 
 		if ctx.NArg() == 1 {
 			jobnum = 1000
+			topN = 10
+		} else if ctx.NArg() == 2 {
+			var err error
+			jobnum, err = strconv.ParseUint(ctx.Args().Get(1), 10, 64)
+			if err != nil {
+				return fmt.Errorf("failed to Parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err)
+			}
+			topN = 10
 		} else {
 			var err error
 			jobnum, err = strconv.ParseUint(ctx.Args().Get(1), 10, 64)
 			if err != nil {
 				return fmt.Errorf("failed to Parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err)
 			}
+
+			topN, err = strconv.ParseUint(ctx.Args().Get(2), 10, 64)
+			if err != nil {
+				return fmt.Errorf("failed to Parse topn, Args[1]: %v, err: %v", ctx.Args().Get(1), err)
+			}
 		}
 
 		if blockNumber != math.MaxUint64 {
@@ -437,6 +451,7 @@ func inspectTrie(ctx *cli.Context) error {
 		if dbScheme == rawdb.PathScheme {
 			config = &triedb.Config{
 				PathDB: utils.PathDBConfigAddJournalFilePath(stack, pathdb.ReadOnly),
+				Cache:  0,
 			}
 		} else if dbScheme == rawdb.HashScheme {
 			config = triedb.HashDefaults
@@ -448,7 +463,7 @@ func inspectTrie(ctx *cli.Context) error {
 			fmt.Printf("fail to new trie tree, err: %v, rootHash: %v\n", err, trieRootHash.String())
 			return err
 		}
-		theInspect, err := trie.NewInspector(theTrie, triedb, trieRootHash, blockNumber, jobnum)
+		theInspect, err := trie.NewInspector(theTrie, triedb, trieRootHash, blockNumber, jobnum, int(topN))
 		if err != nil {
 			return err
 		}

cmd/geth/main.go

@@ -25,6 +25,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ethereum/go-ethereum/params"
+
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -71,12 +73,13 @@ var (
 		utils.SmartCardDaemonPathFlag,
 		utils.RialtoHash,
 		utils.OverrideCancun,
+		utils.OverrideHaber,
+		utils.OverrideBohr,
 		utils.OverrideVerkle,
-		utils.OverrideFeynman,
-		utils.OverrideFeynmanFix,
 		utils.OverrideFullImmutabilityThreshold,
 		utils.OverrideMinBlocksForBlobRequests,
 		utils.OverrideDefaultExtraReserveForBlobRequests,
+		utils.OverrideBreatheBlockInterval,
 		utils.EnablePersonal,
 		utils.TxPoolLocalsFlag,
 		utils.TxPoolNoLocalsFlag,
@@ -456,6 +459,10 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon
 		// Set the gas price to the limits from the CLI and start mining
 		gasprice := flags.GlobalBig(ctx, utils.MinerGasPriceFlag.Name)
 		ethBackend.TxPool().SetGasTip(gasprice)
+		gasCeil := ethBackend.Miner().GasCeil()
+		if gasCeil > params.SystemTxsGas {
+			ethBackend.TxPool().SetMaxGas(gasCeil - params.SystemTxsGas)
+		}
 		if err := ethBackend.StartMining(); err != nil {
 			utils.Fatalf("Failed to start mining: %v", err)
 		}

cmd/geth/pruneblock_test.go

@@ -75,7 +75,7 @@ func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, ancient
 	if err != nil {
 		return nil, err
 	}
-	frdb, err := rawdb.NewDatabaseWithFreezer(kvdb, ancient, namespace, readonly, disableFreeze, isLastOffset, pruneAncientData)
+	frdb, err := rawdb.NewDatabaseWithFreezer(kvdb, ancient, namespace, readonly, disableFreeze, isLastOffset, pruneAncientData, false)
 	if err != nil {
 		kvdb.Close()
 		return nil, err
@@ -178,11 +178,10 @@ func BlockchainCreator(t *testing.T, chaindbPath, AncientPath string, blockRemai
 
 	// Force run a freeze cycle
 	type freezer interface {
-		Freeze() error
+		Freeze(threshold uint64) error
 		Ancients() (uint64, error)
 	}
-	blockchain.SetFinalized(blocks[len(blocks)-1].Header())
+	db.(freezer).Freeze(10)
-	db.(freezer).Freeze()
 
 	frozen, err := db.Ancients()
 	//make sure there're frozen items

cmd/geth/snapshot.go

@@ -43,9 +43,11 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/ethereum/go-ethereum/triedb"
+	"github.com/ethereum/go-ethereum/triedb/pathdb"
 	cli "github.com/urfave/cli/v2"
 )
 
@@ -245,7 +247,16 @@ func accessDb(ctx *cli.Context, stack *node.Node) (ethdb.Database, error) {
 		NoBuild:    true,
 		AsyncBuild: false,
 	}
-	snaptree, err := snapshot.New(snapconfig, chaindb, triedb.NewDatabase(chaindb, nil), headBlock.Root(), TriesInMemory, false)
+	dbScheme := rawdb.ReadStateScheme(chaindb)
+	var config *triedb.Config
+	if dbScheme == rawdb.PathScheme {
+		config = &triedb.Config{
+			PathDB: utils.PathDBConfigAddJournalFilePath(stack, pathdb.ReadOnly),
+		}
+	} else if dbScheme == rawdb.HashScheme {
+		config = triedb.HashDefaults
+	}
+	snaptree, err := snapshot.New(snapconfig, chaindb, triedb.NewDatabase(chaindb, config), headBlock.Root(), TriesInMemory, false)
 	if err != nil {
 		log.Error("snaptree error", "err", err)
 		return nil, err // The relevant snapshot(s) might not exist
@@ -333,6 +344,9 @@ func pruneBlock(ctx *cli.Context) error {
 	stack, config = makeConfigNode(ctx)
 	defer stack.Close()
 	blockAmountReserved = ctx.Uint64(utils.BlockAmountReserved.Name)
+	if blockAmountReserved < params.FullImmutabilityThreshold {
+		return fmt.Errorf("block-amount-reserved must be greater than or equal to %d", params.FullImmutabilityThreshold)
+	}
 	chaindb, err = accessDb(ctx, stack)
 	if err != nil {
 		return err
@@ -409,7 +423,7 @@ func pruneBlock(ctx *cli.Context) error {
 	}
 
 	if _, err := os.Stat(newAncientPath); err == nil {
-		// No file lock found for old ancientDB but new ancientDB exsisted, indicating the geth was interrupted
+		// No file lock found for old ancientDB but new ancientDB existed, indicating the geth was interrupted
 		// after old ancientDB removal, this happened after backup successfully, so just rename the new ancientDB
 		if err := blockpruner.AncientDbReplacer(); err != nil {
 			log.Error("Failed to rename new ancient directory")

cmd/jsutils/check_blobtx.js

@@ -0,0 +1,51 @@
+import { ethers } from "ethers";
+import program from "commander";
+
+// depends on ethjs v6.11.0+ for 4844, https://github.com/ethers-io/ethers.js/releases/tag/v6.11.0
+// BSC testnet enabled 4844 on block: 39539137
+// Usage:
+//   nvm use 20
+//   node check_blobtx.js --rpc https://data-seed-prebsc-1-s1.binance.org:8545 --startNum 39539137
+//   node check_blobtx.js --rpc https://data-seed-prebsc-1-s1.binance.org:8545 --startNum 39539137 --endNum 40345994
+program.option("--rpc <Rpc>", "Rpc Server URL");
+program.option("--startNum <Num>", "start block", 0);
+program.option("--endNum <Num>", "end block", 0);
+program.parse(process.argv);
+
+const provider = new ethers.JsonRpcProvider(program.rpc);
+const main = async () => {
+    var startBlock = parseInt(program.startNum)
+    var endBlock = parseInt(program.endNum)
+    if (isNaN(endBlock) || isNaN(startBlock) || startBlock == 0) {
+        console.error("invalid input, --startNum", program.startNum, "--end", program.endNum)
+        return
+    }
+    // if --endNum is not specified, set it to the latest block number.
+    if (endBlock == 0) {
+        endBlock = await provider.getBlockNumber();
+    }
+    if (startBlock > endBlock) {
+        console.error("invalid input, startBlock:",startBlock, " endBlock:", endBlock);
+        return
+    }
+
+    for (let i = startBlock; i <= endBlock; i++) {
+        let blockData = await provider.getBlock(i);
+        console.log("startBlock:",startBlock, "endBlock:", endBlock, "curBlock", i, "blobGasUsed", blockData.blobGasUsed);
+        if (blockData.blobGasUsed == 0) {
+            continue
+        }
+        for  (let txIndex = 0; txIndex<= blockData.transactions.length - 1; txIndex++) {
+            let txHash = blockData.transactions[txIndex]
+            let txData =  await provider.getTransaction(txHash);
+            if (txData.type == 3) {
+                console.log("BlobTx in block:",i, " txIndex:", txIndex, " txHash:", txHash);
+            }
+        }
+    }
+};
+main().then(() => process.exit(0))
+    .catch((error) => {
+        console.error(error);
+        process.exit(1);
+    });

cmd/jsutils/faucet_request.js

@@ -0,0 +1,49 @@
+import { ethers } from "ethers";
+import program from "commander";
+
+// Usage:
+//   node faucet_request.js --rpc localhost:8545 --startNum 39539137
+//   node faucet_request.js --rpc localhost:8545 --startNum 39539137 --endNum 40345994
+
+//   node faucet_request.js --rpc https://data-seed-prebsc-1-s1.bnbchain.org:8545 --startNum 39539137 --endNum 40345994
+program.option("--rpc <Rpc>", "Rpc Server URL");
+program.option("--startNum <Num>", "start block", 0);
+program.option("--endNum <Num>", "end block", 0);
+program.parse(process.argv);
+
+const provider = new ethers.JsonRpcProvider(program.rpc);
+const main = async () => {
+    var startBlock = parseInt(program.startNum)
+    var endBlock = parseInt(program.endNum)
+    if (isNaN(endBlock) || isNaN(startBlock) || startBlock == 0) {
+        console.error("invalid input, --startNum", program.startNum, "--end", program.endNum)
+        return
+    }
+    // if --endNum is not specified, set it to the latest block number.
+    if (endBlock == 0) {
+        endBlock = await provider.getBlockNumber();
+    }
+    if (startBlock > endBlock) {
+        console.error("invalid input, startBlock:",startBlock, " endBlock:", endBlock);
+        return
+    }
+
+    let startBalance = await provider.getBalance("0xaa25Aa7a19f9c426E07dee59b12f944f4d9f1DD3", startBlock)
+    let endBalance = await provider.getBalance("0xaa25Aa7a19f9c426E07dee59b12f944f4d9f1DD3", endBlock)
+    const faucetAmount = BigInt(0.3 * 10**18); // Convert 0.3 ether to wei as a BigInt
+    const numFaucetRequest = (startBalance - endBalance) / faucetAmount;
+
+    // Convert BigInt to ether
+    const startBalanceEth = Number(startBalance) / 10**18;
+    const endBalanceEth = Number(endBalance) / 10**18;
+
+    console.log(`Start Balance: ${startBalanceEth} ETH`);
+    console.log(`End Balance: ${endBalanceEth} ETH`);
+
+    console.log("successful faucet request: ",numFaucetRequest);
+};
+main().then(() => process.exit(0))
+    .catch((error) => {
+        console.error(error);
+        process.exit(1);
+    });

cmd/utils/flags.go

@@ -310,20 +310,21 @@ var (
 		Usage:    "Manually specify the Cancun fork timestamp, overriding the bundled setting",
 		Category: flags.EthCategory,
 	}
+	OverrideHaber = &cli.Uint64Flag{
+		Name:     "override.haber",
+		Usage:    "Manually specify the Haber fork timestamp, overriding the bundled setting",
+		Category: flags.EthCategory,
+	}
+	OverrideBohr = &cli.Uint64Flag{
+		Name:     "override.bohr",
+		Usage:    "Manually specify the Bohr fork timestamp, overriding the bundled setting",
+		Category: flags.EthCategory,
+	}
 	OverrideVerkle = &cli.Uint64Flag{
 		Name:     "override.verkle",
 		Usage:    "Manually specify the Verkle fork timestamp, overriding the bundled setting",
 		Category: flags.EthCategory,
 	}
-	OverrideFeynman = &cli.Uint64Flag{
-		Name:     "override.feynman",
-		Usage:    "Manually specify the Feynman fork timestamp, overriding the bundled setting",
-		Category: flags.EthCategory,
-	}
-	OverrideFeynmanFix = &cli.Uint64Flag{
-		Name:  "override.feynmanfix",
-		Usage: "Manually specify the FeynmanFix fork timestamp, overriding the bundled setting",
-	}
 	OverrideFullImmutabilityThreshold = &cli.Uint64Flag{
 		Name:     "override.immutabilitythreshold",
 		Usage:    "It is the number of blocks after which a chain segment is considered immutable, only for testing purpose",
@@ -342,6 +343,12 @@ var (
 		Value:    params.DefaultExtraReserveForBlobRequests,
 		Category: flags.EthCategory,
 	}
+	OverrideBreatheBlockInterval = &cli.Uint64Flag{
+		Name:     "override.breatheblockinterval",
+		Usage:    "It changes the interval between breathe blocks, only for testing purpose",
+		Value:    params.BreatheBlockInterval,
+		Category: flags.EthCategory,
+	}
 	SyncModeFlag = &flags.TextMarshalerFlag{
 		Name:     "syncmode",
 		Usage:    `Blockchain sync mode ("snap" or "full")`,
@@ -1080,6 +1087,7 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server.
 		Name:     "block-amount-reserved",
 		Usage:    "Sets the expected remained amount of blocks for offline block prune",
 		Category: flags.BlockHistoryCategory,
+		Value:    params.FullImmutabilityThreshold,
 	}
 
 	CheckSnapshotWithMPT = &cli.BoolFlag{

cmd/utils/history_test.go

@@ -163,7 +163,7 @@ func TestHistoryImportAndExport(t *testing.T) {
 
 	// Now import Era.
 	freezer := t.TempDir()
-	db2, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), freezer, "", false, false, false, false)
+	db2, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), freezer, "", false, false, false, false, false)
 	if err != nil {
 		panic(err)
 	}

consensus/parlia/feynmanfork.go

@@ -15,14 +15,13 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
 )
 
-const SecondsPerDay uint64 = 86400
-
 // the params should be two blocks' time(timestamp)
 func sameDayInUTC(first, second uint64) bool {
-	return first/SecondsPerDay == second/SecondsPerDay
+	return first/params.BreatheBlockInterval == second/params.BreatheBlockInterval
 }
 
 func isBreatheBlock(lastBlockTime, blockTime uint64) bool {

core/block_validator.go

@@ -66,6 +66,31 @@ func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engin
 	return validator
 }
 
+// ValidateListsInBody validates that UncleHash, WithdrawalsHash, and WithdrawalsHash correspond to the lists in the block body, respectively.
+func ValidateListsInBody(block *types.Block) error {
+	header := block.Header()
+	if hash := types.CalcUncleHash(block.Uncles()); hash != header.UncleHash {
+		return fmt.Errorf("uncle root hash mismatch (header value %x, calculated %x)", header.UncleHash, hash)
+	}
+	if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash {
+		return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash)
+	}
+	// Withdrawals are present after the Shanghai fork.
+	if header.WithdrawalsHash != nil {
+		// Withdrawals list must be present in body after Shanghai.
+		if block.Withdrawals() == nil {
+			return errors.New("missing withdrawals in block body")
+		}
+		if hash := types.DeriveSha(block.Withdrawals(), trie.NewStackTrie(nil)); hash != *header.WithdrawalsHash {
+			return fmt.Errorf("withdrawals root hash mismatch (header value %x, calculated %x)", *header.WithdrawalsHash, hash)
+		}
+	} else if block.Withdrawals() != nil { // Withdrawals turn into empty from nil when BlockBody has Sidecars
+		// Withdrawals are not allowed prior to shanghai fork
+		return errors.New("withdrawals present in block body")
+	}
+	return nil
+}
+
 // ValidateBody validates the given block's uncles and verifies the block
 // header's transaction and uncle roots. The headers are assumed to be already
 // validated at this point.
@@ -83,31 +108,12 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	if err := v.engine.VerifyUncles(v.bc, block); err != nil {
 		return err
 	}
-	if hash := types.CalcUncleHash(block.Uncles()); hash != header.UncleHash {
-		return fmt.Errorf("uncle root hash mismatch (header value %x, calculated %x)", header.UncleHash, hash)
-	}
 
 	validateFuns := []func() error{
 		func() error {
-			if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash {
+			return ValidateListsInBody(block)
-				return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash)
-			}
-			return nil
 		},
 		func() error {
-			// Withdrawals are present after the Shanghai fork.
-			if header.WithdrawalsHash != nil {
-				// Withdrawals list must be present in body after Shanghai.
-				if block.Withdrawals() == nil {
-					return errors.New("missing withdrawals in block body")
-				}
-				if hash := types.DeriveSha(block.Withdrawals(), trie.NewStackTrie(nil)); hash != *header.WithdrawalsHash {
-					return fmt.Errorf("withdrawals root hash mismatch (header value %x, calculated %x)", *header.WithdrawalsHash, hash)
-				}
-			} else if block.Withdrawals() != nil { // Withdrawals turn into empty from nil when BlockBody has Sidecars
-				// Withdrawals are not allowed prior to shanghai fork
-				return errors.New("withdrawals present in block body")
-			}
 			// Blob transactions may be present after the Cancun fork.
 			var blobs int
 			for i, tx := range block.Transactions() {

core/blockchain.go

@@ -71,6 +71,8 @@ var (
 	justifiedBlockGauge = metrics.NewRegisteredGauge("chain/head/justified", nil)
 	finalizedBlockGauge = metrics.NewRegisteredGauge("chain/head/finalized", nil)
 
+	blockInsertMgaspsGauge = metrics.NewRegisteredGauge("chain/insert/mgasps", nil)
+
 	chainInfoGauge = metrics.NewRegisteredGaugeInfo("chain/info", nil)
 
 	accountReadTimer   = metrics.NewRegisteredTimer("chain/account/reads", nil)

core/blockchain_insert.go

@@ -58,11 +58,13 @@ func (st *insertStats) report(chain []*types.Block, index int, snapDiffItems, sn
 		end := chain[index]
 
 		// Assemble the log context and send it to the logger
+		mgasps := float64(st.usedGas) * 1000 / float64(elapsed)
 		context := []interface{}{
 			"number", end.Number(), "hash", end.Hash(), "miner", end.Coinbase(),
 			"blocks", st.processed, "txs", txs, "blobs", blobs, "mgas", float64(st.usedGas) / 1000000,
-			"elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
+			"elapsed", common.PrettyDuration(elapsed), "mgasps", mgasps,
 		}
+		blockInsertMgaspsGauge.Update(int64(mgasps))
 		if timestamp := time.Unix(int64(end.Time()), 0); time.Since(timestamp) > time.Minute {
 			context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
 		}

core/blockchain_reader.go

@@ -511,3 +511,12 @@ func (bc *BlockChain) SubscribeBlockProcessingEvent(ch chan<- bool) event.Subscr
 func (bc *BlockChain) SubscribeFinalizedHeaderEvent(ch chan<- FinalizedHeaderEvent) event.Subscription {
 	return bc.scope.Track(bc.finalizedHeaderFeed.Subscribe(ch))
 }
+
+// AncientTail retrieves the tail the ancients blocks
+func (bc *BlockChain) AncientTail() (uint64, error) {
+	tail, err := bc.db.Tail()
+	if err != nil {
+		return 0, err
+	}
+	return tail, nil
+}

core/blockchain_repair_test.go

@@ -1832,14 +1832,10 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
 	}
 	// Force run a freeze cycle
 	type freezer interface {
-		Freeze() error
+		Freeze(threshold uint64) error
 		Ancients() (uint64, error)
 	}
-	if tt.freezeThreshold < uint64(tt.canonicalBlocks) {
+	db.(freezer).Freeze(tt.freezeThreshold)
-		final := uint64(tt.canonicalBlocks) - tt.freezeThreshold
-		chain.SetFinalized(canonblocks[int(final)-1].Header())
-	}
-	db.(freezer).Freeze()
 
 	// Set the simulated pivot block
 	if tt.pivotBlock != nil {

core/blockchain_sethead_test.go

@@ -2045,14 +2045,10 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme
 
 	// Force run a freeze cycle
 	type freezer interface {
-		Freeze() error
+		Freeze(threshold uint64) error
 		Ancients() (uint64, error)
 	}
-	if tt.freezeThreshold < uint64(tt.canonicalBlocks) {
+	db.(freezer).Freeze(tt.freezeThreshold)
-		final := uint64(tt.canonicalBlocks) - tt.freezeThreshold
-		chain.SetFinalized(canonblocks[int(final)-1].Header())
-	}
-	db.(freezer).Freeze()
 
 	// Set the simulated pivot block
 	if tt.pivotBlock != nil {

core/blockchain_test.go

@@ -974,7 +974,7 @@ func testFastVsFullChains(t *testing.T, scheme string) {
 		t.Fatalf("failed to insert receipt %d: %v", n, err)
 	}
 	// Freezer style fast import the chain.
-	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false)
+	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false)
 	if err != nil {
 		t.Fatalf("failed to create temp freezer db: %v", err)
 	}
@@ -1069,7 +1069,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {
 
 	// makeDb creates a db instance for testing.
 	makeDb := func() ethdb.Database {
-		db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false)
+		db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false)
 		if err != nil {
 			t.Fatalf("failed to create temp freezer db: %v", err)
 		}
@@ -1957,7 +1957,7 @@ func testLargeReorgTrieGC(t *testing.T, scheme string) {
 	competitor, _ := GenerateChain(genesis.Config, shared[len(shared)-1], engine, genDb, 2*TriesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) })
 
 	// Import the shared chain and the original canonical one
-	db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false)
+	db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false)
 	defer db.Close()
 
 	chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil)
@@ -2026,7 +2026,7 @@ func testBlockchainRecovery(t *testing.T, scheme string) {
 	_, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), int(height), nil)
 
 	// Import the chain as a ancient-first node and ensure all pointers are updated
-	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false)
+	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false)
 	if err != nil {
 		t.Fatalf("failed to create temp freezer db: %v", err)
 	}
@@ -2097,7 +2097,7 @@ func testInsertReceiptChainRollback(t *testing.T, scheme string) {
 	}
 
 	// Set up a BlockChain that uses the ancient store.
-	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false)
+	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false)
 	if err != nil {
 		t.Fatalf("failed to create temp freezer db: %v", err)
 	}
@@ -2167,7 +2167,7 @@ func testLowDiffLongChain(t *testing.T, scheme string) {
 	})
 
 	// Import the canonical chain
-	diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false)
+	diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false)
 	defer diskdb.Close()
 
 	chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil, nil)
@@ -2384,7 +2384,7 @@ func testInsertKnownChainData(t *testing.T, typ string, scheme string) {
 		b.OffsetTime(-9) // A higher difficulty
 	})
 	// Import the shared chain and the original canonical one
-	chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false)
+	chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false)
 	if err != nil {
 		t.Fatalf("failed to create temp freezer db: %v", err)
 	}
@@ -2555,7 +2555,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i
 		}
 	})
 	// Import the shared chain and the original canonical one
-	chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false)
+	chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false)
 	if err != nil {
 		t.Fatalf("failed to create temp freezer db: %v", err)
 	}
@@ -3858,7 +3858,7 @@ func testSetCanonical(t *testing.T, scheme string) {
 		}
 		gen.AddTx(tx)
 	})
-	diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false)
+	diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false)
 	defer diskdb.Close()
 
 	chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil)
@@ -4483,7 +4483,7 @@ func (c *mockParlia) CalcDifficulty(chain consensus.ChainHeaderReader, time uint
 func TestParliaBlobFeeReward(t *testing.T) {
 	// Have N headers in the freezer
 	frdir := t.TempDir()
-	db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false, false)
+	db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false, false, false)
 	if err != nil {
 		t.Fatalf("failed to create database with ancient backend")
 	}

core/genesis.go

@@ -216,10 +216,10 @@ func (e *GenesisMismatchError) Error() string {
 // ChainOverrides contains the changes to chain config
 // Typically, these modifications involve hardforks that are not enabled on the BSC mainnet, intended for testing purposes.
 type ChainOverrides struct {
-	OverrideCancun     *uint64
+	OverrideCancun *uint64
-	OverrideVerkle     *uint64
+	OverrideHaber  *uint64
-	OverrideFeynman    *uint64
+	OverrideBohr   *uint64
-	OverrideFeynmanFix *uint64
+	OverrideVerkle *uint64
 }
 
 // SetupGenesisBlock writes or updates the genesis block in db.
@@ -248,15 +248,15 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g
 			if overrides != nil && overrides.OverrideCancun != nil {
 				config.CancunTime = overrides.OverrideCancun
 			}
+			if overrides != nil && overrides.OverrideHaber != nil {
+				config.HaberTime = overrides.OverrideHaber
+			}
+			if overrides != nil && overrides.OverrideBohr != nil {
+				config.BohrTime = overrides.OverrideBohr
+			}
 			if overrides != nil && overrides.OverrideVerkle != nil {
 				config.VerkleTime = overrides.OverrideVerkle
 			}
-			if overrides != nil && overrides.OverrideFeynman != nil {
-				config.FeynmanTime = overrides.OverrideFeynman
-			}
-			if overrides != nil && overrides.OverrideFeynmanFix != nil {
-				config.FeynmanFixTime = overrides.OverrideFeynmanFix
-			}
 		}
 	}
 	// Just commit the new block if there is no stored genesis block.

core/rawdb/accessors_chain.go

@@ -316,8 +316,8 @@ func ReadHeaderRange(db ethdb.Reader, number uint64, count uint64) []rlp.RawValu
 	if count == 0 {
 		return rlpHeaders
 	}
-	// read remaining from ancients
+	// read remaining from ancients, cap at 2M
-	data, err := db.AncientRange(ChainFreezerHeaderTable, i+1-count, count, 0)
+	data, err := db.AncientRange(ChainFreezerHeaderTable, i+1-count, count, 2*1024*1024)
 	if err != nil {
 		log.Error("Failed to read headers from freezer", "err", err)
 		return rlpHeaders

core/rawdb/accessors_chain_test.go

@@ -518,7 +518,7 @@ func checkBlobSidecarsRLP(have, want types.BlobSidecars) error {
 func TestAncientStorage(t *testing.T) {
 	// Freezer style fast import the chain.
 	frdir := t.TempDir()
-	db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, false, false, false)
+	db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, false, false, false, false)
 	if err != nil {
 		t.Fatalf("failed to create database with ancient backend")
 	}
@@ -657,7 +657,7 @@ func TestHashesInRange(t *testing.T) {
 func BenchmarkWriteAncientBlocks(b *testing.B) {
 	// Open freezer database.
 	frdir := b.TempDir()
-	db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, false, false, false)
+	db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, false, false, false, false)
 	if err != nil {
 		b.Fatalf("failed to create database with ancient backend")
 	}
@@ -1001,7 +1001,7 @@ func TestHeadersRLPStorage(t *testing.T) {
 	// Have N headers in the freezer
 	frdir := t.TempDir()
 
-	db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, false, false, false)
+	db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, false, false, false, false)
 	if err != nil {
 		t.Fatalf("failed to create database with ancient backend")
 	}

core/rawdb/chain_freezer.go

@@ -24,12 +24,12 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/ethereum/go-ethereum/rlp"
-
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
 const (
@@ -51,25 +51,31 @@ var (
 // The background thread will keep moving ancient chain segments from key-value
 // database to flat files for saving space on live database.
 type chainFreezer struct {
+	threshold atomic.Uint64 // Number of recent blocks not to freeze (params.FullImmutabilityThreshold apart from tests)
+
 	*Freezer
 	quit    chan struct{}
 	wg      sync.WaitGroup
 	trigger chan chan struct{} // Manual blocking freeze trigger, test determinism
 
 	freezeEnv atomic.Value
+
+	multiDatabase bool
 }
 
 // newChainFreezer initializes the freezer for ancient chain data.
-func newChainFreezer(datadir string, namespace string, readonly bool, offset uint64) (*chainFreezer, error) {
+func newChainFreezer(datadir string, namespace string, readonly bool, offset uint64, multiDatabase bool) (*chainFreezer, error) {
 	freezer, err := NewChainFreezer(datadir, namespace, readonly, offset)
 	if err != nil {
 		return nil, err
 	}
-	return &chainFreezer{
+	cf := chainFreezer{
 		Freezer: freezer,
 		quit:    make(chan struct{}),
 		trigger: make(chan chan struct{}),
-	}, nil
+	}
+	cf.threshold.Store(params.FullImmutabilityThreshold)
+	return &cf, nil
 }
 
 // Close closes the chain freezer instance and terminates the background thread.
@@ -185,29 +191,101 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
 			continue
 		}
 
-		threshold, err := f.freezeThreshold(nfdb)
+		var (
-		if err != nil {
+			frozen    uint64
-			backoff = true
+			threshold uint64
-			log.Debug("Current full block not old enough to freeze", "err", err)
+			first     uint64 // the first block to freeze
-			continue
+			last      uint64 // the last block to freeze
-		}
-		frozen := f.frozen.Load()
 
-		// Short circuit if the blocks below threshold are already frozen.
+			hash   common.Hash
-		if frozen != 0 && frozen-1 >= threshold {
+			number *uint64
-			backoff = true
+			head   *types.Header
-			log.Debug("Ancient blocks frozen already", "threshold", threshold, "frozen", frozen)
+		)
-			continue
+
+		// use finalized block as the chain freeze indicator was used for multiDatabase feature, if multiDatabase is false, keep 9W blocks in db
+		if f.multiDatabase {
+			threshold, err = f.freezeThreshold(nfdb)
+			if err != nil {
+				backoff = true
+				log.Debug("Current full block not old enough to freeze", "err", err)
+				continue
+			}
+			frozen = f.frozen.Load()
+
+			// Short circuit if the blocks below threshold are already frozen.
+			if frozen != 0 && frozen-1 >= threshold {
+				backoff = true
+				log.Debug("Ancient blocks frozen already", "threshold", threshold, "frozen", frozen)
+				continue
+			}
+
+			hash = ReadHeadBlockHash(nfdb)
+			if hash == (common.Hash{}) {
+				log.Debug("Current full block hash unavailable") // new chain, empty database
+				backoff = true
+				continue
+			}
+			number = ReadHeaderNumber(nfdb, hash)
+			if number == nil {
+				log.Error("Current full block number unavailable", "hash", hash)
+				backoff = true
+				continue
+			}
+			head = ReadHeader(nfdb, hash, *number)
+			if head == nil {
+				log.Error("Current full block unavailable", "number", *number, "hash", hash)
+				backoff = true
+				continue
+			}
+
+			first = frozen
+			last = threshold
+			if last-first+1 > freezerBatchLimit {
+				last = freezerBatchLimit + first - 1
+			}
+		} else {
+			// Retrieve the freezing threshold.
+			hash = ReadHeadBlockHash(nfdb)
+			if hash == (common.Hash{}) {
+				log.Debug("Current full block hash unavailable") // new chain, empty database
+				backoff = true
+				continue
+			}
+			number = ReadHeaderNumber(nfdb, hash)
+			threshold = f.threshold.Load()
+			frozen = f.frozen.Load()
+			switch {
+			case number == nil:
+				log.Error("Current full block number unavailable", "hash", hash)
+				backoff = true
+				continue
+
+			case *number < threshold:
+				log.Debug("Current full block not old enough to freeze", "number", *number, "hash", hash, "delay", threshold)
+				backoff = true
+				continue
+
+			case *number-threshold <= frozen:
+				log.Debug("Ancient blocks frozen already", "number", *number, "hash", hash, "frozen", frozen)
+				backoff = true
+				continue
+			}
+			head = ReadHeader(nfdb, hash, *number)
+			if head == nil {
+				log.Error("Current full block unavailable", "number", *number, "hash", hash)
+				backoff = true
+				continue
+			}
+			first, _ = f.Ancients()
+			last = *number - threshold
+			if last-first > freezerBatchLimit {
+				last = first + freezerBatchLimit
+			}
 		}
 		// Seems we have data ready to be frozen, process in usable batches
 		var (
 			start = time.Now()
-			first = frozen    // the first block to freeze
-			last  = threshold // the last block to freeze
 		)
-		if last-first+1 > freezerBatchLimit {
-			last = freezerBatchLimit + first - 1
-		}
 
 		ancients, err := f.freezeRangeWithBlobs(nfdb, first, last)
 		if err != nil {
@@ -295,24 +373,6 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
 		log.Debug("Deep froze chain segment", context...)
 
 		env, _ := f.freezeEnv.Load().(*ethdb.FreezerEnv)
-		hash := ReadHeadBlockHash(nfdb)
-		if hash == (common.Hash{}) {
-			log.Debug("Current full block hash unavailable") // new chain, empty database
-			backoff = true
-			continue
-		}
-		number := ReadHeaderNumber(nfdb, hash)
-		if number == nil {
-			log.Error("Current full block number unavailable", "hash", hash)
-			backoff = true
-			continue
-		}
-		head := ReadHeader(nfdb, hash, *number)
-		if head == nil {
-			log.Error("Current full block unavailable", "number", *number, "hash", hash)
-			backoff = true
-			continue
-		}
 		// try prune blob data after cancun fork
 		if isCancun(env, head.Number, head.Time) {
 			f.tryPruneBlobAncientTable(env, *number)

core/rawdb/database.go

@@ -138,13 +138,22 @@ func (frdb *freezerdb) SetBlockStore(block ethdb.Database) {
 	frdb.blockStore = block
 }
 
+func (frdb *freezerdb) HasSeparateBlockStore() bool {
+	return frdb.blockStore != nil
+}
+
 // Freeze is a helper method used for external testing to trigger and block until
 // a freeze cycle completes, without having to sleep for a minute to trigger the
 // automatic background run.
-func (frdb *freezerdb) Freeze() error {
+func (frdb *freezerdb) Freeze(threshold uint64) error {
 	if frdb.AncientStore.(*chainFreezer).readonly {
 		return errReadOnly
 	}
+	// Set the freezer threshold to a temporary value
+	defer func(old uint64) {
+		frdb.AncientStore.(*chainFreezer).threshold.Store(old)
+	}(frdb.AncientStore.(*chainFreezer).threshold.Load())
+	frdb.AncientStore.(*chainFreezer).threshold.Store(threshold)
 	// Trigger a freeze cycle and block until it's done
 	trigger := make(chan struct{}, 1)
 	frdb.AncientStore.(*chainFreezer).trigger <- trigger
@@ -263,6 +272,10 @@ func (db *nofreezedb) SetBlockStore(block ethdb.Database) {
 	db.blockStore = block
 }
 
+func (db *nofreezedb) HasSeparateBlockStore() bool {
+	return db.blockStore != nil
+}
+
 func (db *nofreezedb) BlockStoreReader() ethdb.Reader {
 	if db.blockStore != nil {
 		return db.blockStore
@@ -358,7 +371,7 @@ func resolveChainFreezerDir(ancient string) string {
 // value data store with a freezer moving immutable chain segments into cold
 // storage. The passed ancient indicates the path of root ancient directory
 // where the chain freezer can be opened.
-func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace string, readonly, disableFreeze, isLastOffset, pruneAncientData bool) (ethdb.Database, error) {
+func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace string, readonly, disableFreeze, isLastOffset, pruneAncientData, multiDatabase bool) (ethdb.Database, error) {
 	var offset uint64
 	// The offset of ancientDB should be handled differently in different scenarios.
 	if isLastOffset {
@@ -394,7 +407,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st
 	}
 
 	// Create the idle freezer instance
-	frdb, err := newChainFreezer(resolveChainFreezerDir(ancient), namespace, readonly, offset)
+	frdb, err := newChainFreezer(resolveChainFreezerDir(ancient), namespace, readonly, offset, multiDatabase)
 	if err != nil {
 		printChainMetadata(db)
 		return nil, err
@@ -575,6 +588,8 @@ type OpenOptions struct {
 	// Ephemeral means that filesystem sync operations should be avoided: data integrity in the face of
 	// a crash is not important. This option should typically be used in tests.
 	Ephemeral bool
+
+	MultiDataBase bool
 }
 
 // openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble.
@@ -625,7 +640,7 @@ func Open(o OpenOptions) (ethdb.Database, error) {
 	if len(o.AncientsDirectory) == 0 {
 		return kvdb, nil
 	}
-	frdb, err := NewDatabaseWithFreezer(kvdb, o.AncientsDirectory, o.Namespace, o.ReadOnly, o.DisableFreeze, o.IsLastOffset, o.PruneAncientData)
+	frdb, err := NewDatabaseWithFreezer(kvdb, o.AncientsDirectory, o.Namespace, o.ReadOnly, o.DisableFreeze, o.IsLastOffset, o.PruneAncientData, o.MultiDataBase)
 	if err != nil {
 		kvdb.Close()
 		return nil, err
@@ -769,7 +784,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		trieIter = db.StateStore().NewIterator(keyPrefix, nil)
 		defer trieIter.Release()
 	}
-	if db.BlockStore() != db {
+	if db.HasSeparateBlockStore() {
 		blockIter = db.BlockStore().NewIterator(keyPrefix, nil)
 		defer blockIter.Release()
 	}

core/rawdb/freezer.go

@@ -239,7 +239,7 @@ func (f *Freezer) Ancient(kind string, number uint64) ([]byte, error) {
 //   - if maxBytes is not specified, 'count' items will be returned if they are present.
 func (f *Freezer) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) {
 	if table := f.tables[kind]; table != nil {
-		return table.RetrieveItems(start, count, maxBytes)
+		return table.RetrieveItems(start-f.offset, count, maxBytes)
 	}
 	return nil, errUnknownTable
 }
@@ -252,7 +252,7 @@ func (f *Freezer) Ancients() (uint64, error) {
 func (f *Freezer) TableAncients(kind string) (uint64, error) {
 	f.writeLock.RLock()
 	defer f.writeLock.RUnlock()
-	return f.tables[kind].items.Load(), nil
+	return f.tables[kind].items.Load() + f.offset, nil
 }
 
 // ItemAmountInAncient returns the actual length of current ancientDB.

core/rawdb/table.go

@@ -43,6 +43,10 @@ func (t *table) SetBlockStore(block ethdb.Database) {
 	panic("not implement")
 }
 
+func (t *table) HasSeparateBlockStore() bool {
+	panic("not implement")
+}
+
 // NewTable returns a database object that prefixes all keys with a given string.
 func NewTable(db ethdb.Database, prefix string) ethdb.Database {
 	return &table{

core/state/pruner/pruner.go

@@ -382,7 +382,7 @@ func (p *BlockPruner) backUpOldDb(name string, cache, handles int, namespace str
 	log.Info("chainDB opened successfully")
 
 	// Get the number of items in old ancient db.
-	itemsOfAncient, err := chainDb.ItemAmountInAncient()
+	itemsOfAncient, err := chainDb.BlockStore().ItemAmountInAncient()
 	log.Info("the number of items in ancientDB is ", "itemsOfAncient", itemsOfAncient)
 
 	// If we can't access the freezer or it's empty, abort.
@@ -402,7 +402,7 @@ func (p *BlockPruner) backUpOldDb(name string, cache, handles int, namespace str
 
 	var oldOffSet uint64
 	if interrupt {
-		// The interrupt scecario within this function is specific for old and new ancientDB exsisted concurrently,
+		// The interrupt scecario within this function is specific for old and new ancientDB existed concurrently,
 		// should use last version of offset for oldAncientDB, because current offset is
 		// actually of the new ancientDB_Backup, but what we want is the offset of ancientDB being backup.
 		oldOffSet = rawdb.ReadOffSetOfLastAncientFreezer(chainDb)

core/systemcontracts/feynman/types.go

@@ -57,31 +57,3 @@ var (
 	//go:embed chapel/TokenRecoverPortalContract
 	ChapelTokenRecoverPortalContract string
 )
-
-// contract codes for Rialto upgrade
-var (
-	//go:embed rialto/ValidatorContract
-	RialtoValidatorContract string
-	//go:embed rialto/SlashContract
-	RialtoSlashContract string
-	//go:embed rialto/TokenHubContract
-	RialtoTokenHubContract string
-	//go:embed rialto/GovHubContract
-	RialtoGovHubContract string
-	//go:embed rialto/CrossChainContract
-	RialtoCrossChainContract string
-	//go:embed rialto/StakingContract
-	RialtoStakingContract string
-	//go:embed rialto/StakeHubContract
-	RialtoStakeHubContract string
-	//go:embed rialto/StakeCreditContract
-	RialtoStakeCreditContract string
-	//go:embed rialto/GovernorContract
-	RialtoGovernorContract string
-	//go:embed rialto/GovTokenContract
-	RialtoGovTokenContract string
-	//go:embed rialto/TimelockContract
-	RialtoTimelockContract string
-	//go:embed rialto/TokenRecoverPortalContract
-	RialtoTokenRecoverPortalContract string
-)

core/systemcontracts/upgrade.go

@@ -680,72 +680,6 @@ func init() {
 		},
 	}
 
-	feynmanUpgrade[rialtoNet] = &Upgrade{
-		UpgradeName: "feynman",
-		Configs: []*UpgradeConfig{
-			{
-				ContractAddr: common.HexToAddress(ValidatorContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoValidatorContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(SlashContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoSlashContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(TokenHubContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoTokenHubContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(GovHubContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoGovHubContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(CrossChainContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoCrossChainContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(StakingContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoStakingContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(StakeHubContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoStakeHubContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(StakeCreditContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.MainnetStakeCreditContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(GovernorContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoGovernorContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(GovTokenContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.MainnetGovTokenContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(TimelockContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoTimelockContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(TokenRecoverPortalContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoTokenRecoverPortalContract,
-			},
-		},
-	}
-
 	// This upgrade is to fix an error on testnet only. So the upgrade config of mainnet is empty.
 	feynmanFixUpgrade[mainNet] = &Upgrade{
 		UpgradeName: "feynmanFix",
@@ -767,11 +701,6 @@ func init() {
 			},
 		},
 	}
-
-	feynmanFixUpgrade[rialtoNet] = &Upgrade{
-		UpgradeName: "feynmanFix",
-		Configs:     []*UpgradeConfig{},
-	}
 }
 
 func UpgradeBuildInSystemContract(config *params.ChainConfig, blockNumber *big.Int, lastBlockTime uint64, blockTime uint64, statedb *state.StateDB) {

core/systemcontracts/upgrade_test.go

@@ -26,7 +26,7 @@ func TestAllCodesHash(t *testing.T) {
 
 	allCodes := make([]byte, 0, 10_000_000)
 	for _, hardfork := range upgradesList {
-		for _, network := range []string{mainNet, chapelNet, rialtoNet} {
+		for _, network := range []string{mainNet, chapelNet} {
 			allCodes = append(allCodes, []byte(network)...)
 			if hardfork[network] != nil {
 				for _, addressConfig := range hardfork[network].Configs {
@@ -37,6 +37,5 @@ func TestAllCodesHash(t *testing.T) {
 		}
 	}
 	allCodeHash := sha256.Sum256(allCodes)
-
+	require.Equal(t, allCodeHash[:], common.Hex2Bytes("833cc0fc87c46ad8a223e44ccfdc16a51a7e7383525136441bd0c730f06023df"))
-	require.Equal(t, allCodeHash[:], common.Hex2Bytes("3d68c07faa6b9385e981a45bd539f15d4cbb712426c604b9cab22591af446fc8"))
 }

core/txindexer_test.go

@@ -212,7 +212,7 @@ func TestTxIndexer(t *testing.T) {
 	}
 	for _, c := range cases {
 		frdir := t.TempDir()
-		db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false, false)
+		db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false, false, false)
 		rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...), big.NewInt(0))
 
 		// Index the initial blocks from ancient store

core/txpool/blobpool/blobpool.go

@@ -27,6 +27,7 @@ import (
 	"path/filepath"
 	"sort"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -305,6 +306,7 @@ type BlobPool struct {
 	head   *types.Header  // Current head of the chain
 	state  *state.StateDB // Current state at the head of the chain
 	gasTip *uint256.Int   // Currently accepted minimum gas tip
+	maxGas atomic.Uint64  // Currently accepted max gas, it will be modified by MinerAPI
 
 	lookup map[common.Hash]uint64           // Lookup table mapping hashes to tx billy entries
 	index  map[common.Address][]*blobTxMeta // Blob transactions grouped by accounts, sorted by nonce
@@ -1098,6 +1100,7 @@ func (p *BlobPool) validateTx(tx *types.Transaction) error {
 		Accept:  1 << types.BlobTxType,
 		MaxSize: txMaxSize,
 		MinTip:  p.gasTip.ToBig(),
+		MaxGas:  p.GetMaxGas(),
 	}
 	if err := txpool.ValidateTransaction(tx, p.head, p.signer, baseOpts); err != nil {
 		return err
@@ -1671,3 +1674,11 @@ func (p *BlobPool) Status(hash common.Hash) txpool.TxStatus {
 	}
 	return txpool.TxStatusUnknown
 }
+
+func (p *BlobPool) SetMaxGas(maxGas uint64) {
+	p.maxGas.Store(maxGas)
+}
+
+func (p *BlobPool) GetMaxGas() uint64 {
+	return p.maxGas.Load()
+}

core/txpool/legacypool/legacypool.go

@@ -55,6 +55,8 @@ const (
 
 	// txReannoMaxNum is the maximum number of transactions a reannounce action can include.
 	txReannoMaxNum = 1024
+
+	maxBufferSize = 1000 // maximum size of tx buffer
 )
 
 var (
@@ -219,6 +221,7 @@ type LegacyPool struct {
 	scope        event.SubscriptionScope
 	signer       types.Signer
 	mu           sync.RWMutex
+	maxGas       atomic.Uint64 // Currently accepted max gas, it will be modified by MinerAPI
 
 	currentHead   atomic.Pointer[types.Header] // Current head of the blockchain
 	currentState  *state.StateDB               // Current state in the blockchain head
@@ -243,6 +246,10 @@ type LegacyPool struct {
 	initDoneCh      chan struct{}  // is closed once the pool is initialized (for tests)
 
 	changesSinceReorg int // A counter for how many drops we've performed in-between reorg.
+
+	// A buffer to store transactions that would otherwise be discarded
+	buffer     []*types.Transaction
+	bufferLock sync.Mutex
 }
 
 type txpoolResetRequest struct {
@@ -354,11 +361,13 @@ func (pool *LegacyPool) loop() {
 		evict      = time.NewTicker(evictionInterval)
 		reannounce = time.NewTicker(reannounceInterval)
 		journal    = time.NewTicker(pool.config.Rejournal)
+		readd      = time.NewTicker(time.Minute) // ticker to re-add buffered transactions periodically
 	)
 	defer report.Stop()
 	defer evict.Stop()
 	defer reannounce.Stop()
 	defer journal.Stop()
+	defer readd.Stop() // Stop the ticker when the loop exits
 
 	// Notify tests that the init phase is done
 	close(pool.initDoneCh)
@@ -435,6 +444,9 @@ func (pool *LegacyPool) loop() {
 				}
 				pool.mu.Unlock()
 			}
+			// Handle re-adding buffered transactions
+		case <-readd.C:
+			pool.readdBufferedTransactions()
 		}
 	}
 }
@@ -670,6 +682,7 @@ func (pool *LegacyPool) validateTxBasics(tx *types.Transaction, local bool) erro
 			1<<types.DynamicFeeTxType,
 		MaxSize: txMaxSize,
 		MinTip:  pool.gasTip.Load().ToBig(),
+		MaxGas:  pool.GetMaxGas(),
 	}
 	if local {
 		opts.MinTip = new(big.Int)
@@ -779,12 +792,21 @@ func (pool *LegacyPool) add(tx *types.Transaction, local bool) (replaced bool, e
 			}
 		}()
 	}
-	// If the transaction pool is full, discard underpriced transactions
+	// If the transaction pool is full, buffer underpriced transactions
 	if uint64(pool.all.Slots()+numSlots(tx)) > pool.config.GlobalSlots+pool.config.GlobalQueue {
-		// If the new transaction is underpriced, don't accept it
+		// If the new transaction is underpriced, buffer it
 		if !isLocal && pool.priced.Underpriced(tx) {
-			log.Trace("Discarding underpriced transaction", "hash", hash, "gasTipCap", tx.GasTipCap(), "gasFeeCap", tx.GasFeeCap())
+			log.Trace("Buffering underpriced transaction", "hash", hash, "gasTipCap", tx.GasTipCap(), "gasFeeCap", tx.GasFeeCap())
 			underpricedTxMeter.Mark(1)
+
+			pool.bufferLock.Lock()
+			if len(pool.buffer) < maxBufferSize {
+				pool.buffer = append(pool.buffer, tx)
+			} else {
+				log.Warn("Buffer is full, discarding transaction", "hash", hash)
+			}
+			pool.bufferLock.Unlock()
+
 			return false, txpool.ErrUnderpriced
 		}
 
@@ -802,6 +824,16 @@ func (pool *LegacyPool) add(tx *types.Transaction, local bool) (replaced bool, e
 		// Otherwise if we can't make enough room for new one, abort the operation.
 		drop, success := pool.priced.Discard(pool.all.Slots()-int(pool.config.GlobalSlots+pool.config.GlobalQueue)+numSlots(tx), isLocal)
 
+		// Add dropped transactions to the buffer
+		pool.bufferLock.Lock()
+		availableSpace := maxBufferSize - len(pool.buffer)
+		// Determine how many elements to take from drop
+		if availableSpace > len(drop) {
+			availableSpace = len(drop)
+		}
+		pool.buffer = append(pool.buffer, drop[:availableSpace]...)
+		pool.bufferLock.Unlock()
+
 		// Special case, we still can't make the room for the new remote one.
 		if !isLocal && !success {
 			log.Trace("Discarding overflown transaction", "hash", hash)
@@ -1769,6 +1801,59 @@ func (pool *LegacyPool) demoteUnexecutables() {
 	}
 }
 
+func (pool *LegacyPool) GetMaxGas() uint64 {
+	return pool.maxGas.Load()
+}
+
+func (pool *LegacyPool) SetMaxGas(maxGas uint64) {
+	pool.maxGas.Store(maxGas)
+}
+
+func (pool *LegacyPool) readdBufferedTransactions() {
+	pool.mu.Lock()
+	defer pool.mu.Unlock()
+
+	// Check if there is space in the pool
+	if uint64(pool.all.Slots()) >= pool.config.GlobalSlots+pool.config.GlobalQueue {
+		return // No space available, skip re-adding
+	}
+
+	var readded []*types.Transaction
+
+	pool.bufferLock.Lock()
+	for _, tx := range pool.buffer {
+		// Check if adding this transaction will exceed the pool capacity
+		if uint64(pool.all.Slots()+numSlots(tx)) > pool.config.GlobalSlots+pool.config.GlobalQueue {
+			break // Stop if adding the transaction will exceed the pool capacity
+		}
+
+		if _, err := pool.add(tx, false); err == nil {
+			readded = append(readded, tx)
+		}
+	}
+	pool.bufferLock.Unlock()
+
+	// Remove successfully re-added transactions from the buffer
+	if len(readded) > 0 {
+		remaining := pool.buffer[:0]
+		for _, tx := range pool.buffer {
+			if !containsTransaction(readded, tx) {
+				remaining = append(remaining, tx)
+			}
+		}
+		pool.buffer = remaining
+	}
+}
+
+func containsTransaction(txs []*types.Transaction, tx *types.Transaction) bool {
+	for _, t := range txs {
+		if t.Hash() == tx.Hash() {
+			return true
+		}
+	}
+	return false
+}
+
 // addressByHeartbeat is an account address tagged with its last activity timestamp.
 type addressByHeartbeat struct {
 	address   common.Address

core/txpool/subpool.go

@@ -166,4 +166,7 @@ type SubPool interface {
 	// Status returns the known status (unknown/pending/queued) of a transaction
 	// identified by their hashes.
 	Status(hash common.Hash) TxStatus
+
+	// SetMaxGas limit max acceptable tx gas when mine is enabled
+	SetMaxGas(maxGas uint64)
 }

core/txpool/txpool.go

@@ -284,6 +284,12 @@ func (p *TxPool) SetGasTip(tip *big.Int) {
 	}
 }
 
+func (p *TxPool) SetMaxGas(gas uint64) {
+	for _, subpool := range p.subpools {
+		subpool.SetMaxGas(gas)
+	}
+}
+
 // Has returns an indicator whether the pool has a transaction cached with the
 // given hash.
 func (p *TxPool) Has(hash common.Hash) bool {

core/txpool/validation.go

@@ -45,6 +45,7 @@ type ValidationOptions struct {
 	Accept  uint8    // Bitmap of transaction types that should be accepted for the calling pool
 	MaxSize uint64   // Maximum size of a transaction that the caller can meaningfully handle
 	MinTip  *big.Int // Minimum gas tip needed to allow a transaction into the caller pool
+	MaxGas  uint64   // Max acceptable transaction gas in the txpool
 }
 
 // ValidateTransaction is a helper method to check whether a transaction is valid
@@ -86,6 +87,12 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
 	if head.GasLimit < tx.Gas() {
 		return ErrGasLimit
 	}
+
+	// Ensure the transaction doesn't exceed the current miner max acceptable limit gas
+	if opts.MaxGas > 0 && opts.MaxGas < tx.Gas() {
+		return ErrGasLimit
+	}
+
 	// Sanity check for extremely large numbers (supported by RLP or RPC)
 	if tx.GasFeeCap().BitLen() > 256 {
 		return core.ErrFeeCapVeryHigh

core/types/bid.go

@@ -6,6 +6,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	mapset "github.com/deckarep/golang-set/v2"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -40,6 +42,12 @@ func (b *BidArgs) ToBid(builder common.Address, signer Signer) (*Bid, error) {
 		return nil, err
 	}
 
+	if len(b.RawBid.UnRevertible) > len(txs) {
+		return nil, fmt.Errorf("expect NonRevertible no more than %d", len(txs))
+	}
+	unRevertibleHashes := mapset.NewThreadUnsafeSetWithSize[common.Hash](len(b.RawBid.UnRevertible))
+	unRevertibleHashes.Append(b.RawBid.UnRevertible...)
+
 	if len(b.PayBidTx) != 0 {
 		var payBidTx = new(Transaction)
 		err = payBidTx.UnmarshalBinary(b.PayBidTx)
@@ -51,14 +59,15 @@ func (b *BidArgs) ToBid(builder common.Address, signer Signer) (*Bid, error) {
 	}
 
 	bid := &Bid{
-		Builder:     builder,
+		Builder:      builder,
-		BlockNumber: b.RawBid.BlockNumber,
+		BlockNumber:  b.RawBid.BlockNumber,
-		ParentHash:  b.RawBid.ParentHash,
+		ParentHash:   b.RawBid.ParentHash,
-		Txs:         txs,
+		Txs:          txs,
-		GasUsed:     b.RawBid.GasUsed + b.PayBidTxGasUsed,
+		UnRevertible: unRevertibleHashes,
-		GasFee:      b.RawBid.GasFee,
+		GasUsed:      b.RawBid.GasUsed + b.PayBidTxGasUsed,
-		BuilderFee:  b.RawBid.BuilderFee,
+		GasFee:       b.RawBid.GasFee,
-		rawBid:      *b.RawBid,
+		BuilderFee:   b.RawBid.BuilderFee,
+		rawBid:       *b.RawBid,
 	}
 
 	if bid.BuilderFee == nil {
@@ -70,12 +79,13 @@ func (b *BidArgs) ToBid(builder common.Address, signer Signer) (*Bid, error) {
 
 // RawBid represents a raw bid from builder directly.
 type RawBid struct {
-	BlockNumber uint64          `json:"blockNumber"`
+	BlockNumber  uint64          `json:"blockNumber"`
-	ParentHash  common.Hash     `json:"parentHash"`
+	ParentHash   common.Hash     `json:"parentHash"`
-	Txs         []hexutil.Bytes `json:"txs"`
+	Txs          []hexutil.Bytes `json:"txs"`
-	GasUsed     uint64          `json:"gasUsed"`
+	UnRevertible []common.Hash   `json:"unRevertible"`
-	GasFee      *big.Int        `json:"gasFee"`
+	GasUsed      uint64          `json:"gasUsed"`
-	BuilderFee  *big.Int        `json:"builderFee"`
+	GasFee       *big.Int        `json:"gasFee"`
+	BuilderFee   *big.Int        `json:"builderFee"`
 
 	hash atomic.Value
 }
@@ -154,13 +164,14 @@ func (b *RawBid) Hash() common.Hash {
 
 // Bid represents a bid.
 type Bid struct {
-	Builder     common.Address
+	Builder      common.Address
-	BlockNumber uint64
+	BlockNumber  uint64
-	ParentHash  common.Hash
+	ParentHash   common.Hash
-	Txs         Transactions
+	Txs          Transactions
-	GasUsed     uint64
+	UnRevertible mapset.Set[common.Hash]
-	GasFee      *big.Int
+	GasUsed      uint64
-	BuilderFee  *big.Int
+	GasFee       *big.Int
+	BuilderFee   *big.Int
 
 	rawBid RawBid
 }

core/vm/contracts.go

@@ -36,6 +36,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto/bn256"
 	"github.com/ethereum/go-ethereum/crypto/kzg4844"
 	"github.com/ethereum/go-ethereum/crypto/secp256k1"
+	"github.com/ethereum/go-ethereum/crypto/secp256r1"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -247,6 +248,36 @@ var PrecompiledContractsCancun = map[common.Address]PrecompiledContract{
 	common.BytesToAddress([]byte{105}): &secp256k1SignatureRecover{},
 }
 
+// PrecompiledContractsHaber contains the default set of pre-compiled Ethereum
+// contracts used in the Haber release.
+var PrecompiledContractsHaber = map[common.Address]PrecompiledContract{
+	common.BytesToAddress([]byte{1}):    &ecrecover{},
+	common.BytesToAddress([]byte{2}):    &sha256hash{},
+	common.BytesToAddress([]byte{3}):    &ripemd160hash{},
+	common.BytesToAddress([]byte{4}):    &dataCopy{},
+	common.BytesToAddress([]byte{5}):    &bigModExp{eip2565: true},
+	common.BytesToAddress([]byte{6}):    &bn256AddIstanbul{},
+	common.BytesToAddress([]byte{7}):    &bn256ScalarMulIstanbul{},
+	common.BytesToAddress([]byte{8}):    &bn256PairingIstanbul{},
+	common.BytesToAddress([]byte{9}):    &blake2F{},
+	common.BytesToAddress([]byte{0x0a}): &kzgPointEvaluation{},
+
+	common.BytesToAddress([]byte{100}): &tmHeaderValidate{},
+	common.BytesToAddress([]byte{101}): &iavlMerkleProofValidatePlato{},
+	common.BytesToAddress([]byte{102}): &blsSignatureVerify{},
+	common.BytesToAddress([]byte{103}): &cometBFTLightBlockValidateHertz{},
+	common.BytesToAddress([]byte{104}): &verifyDoubleSignEvidence{},
+	common.BytesToAddress([]byte{105}): &secp256k1SignatureRecover{},
+
+	common.BytesToAddress([]byte{0x01, 0x00}): &p256Verify{},
+}
+
+// PrecompiledContractsP256Verify contains the precompiled Ethereum
+// contract specified in EIP-7212. This is exported for testing purposes.
+var PrecompiledContractsP256Verify = map[common.Address]PrecompiledContract{
+	common.BytesToAddress([]byte{0x01, 0x00}): &p256Verify{},
+}
+
 // PrecompiledContractsBLS contains the set of pre-compiled Ethereum
 // contracts specified in EIP-2537. These are exported for testing purposes.
 var PrecompiledContractsBLS = map[common.Address]PrecompiledContract{
@@ -262,6 +293,7 @@ var PrecompiledContractsBLS = map[common.Address]PrecompiledContract{
 }
 
 var (
+	PrecompiledAddressesHaber     []common.Address
 	PrecompiledAddressesCancun    []common.Address
 	PrecompiledAddressesFeynman   []common.Address
 	PrecompiledAddressesHertz     []common.Address
@@ -313,11 +345,16 @@ func init() {
 	for k := range PrecompiledContractsCancun {
 		PrecompiledAddressesCancun = append(PrecompiledAddressesCancun, k)
 	}
+	for k := range PrecompiledContractsHaber {
+		PrecompiledAddressesHaber = append(PrecompiledAddressesHaber, k)
+	}
 }
 
 // ActivePrecompiles returns the precompiles enabled with the current configuration.
 func ActivePrecompiles(rules params.Rules) []common.Address {
 	switch {
+	case rules.IsHaber:
+		return PrecompiledAddressesHaber
 	case rules.IsCancun:
 		return PrecompiledAddressesCancun
 	case rules.IsFeynman:
@@ -1389,6 +1426,40 @@ func kZGToVersionedHash(kzg kzg4844.Commitment) common.Hash {
 	return h
 }
 
+// P256VERIFY (secp256r1 signature verification)
+// implemented as a native contract
+type p256Verify struct{}
+
+// RequiredGas returns the gas required to execute the precompiled contract
+func (c *p256Verify) RequiredGas(input []byte) uint64 {
+	return params.P256VerifyGas
+}
+
+// Run executes the precompiled contract with given 160 bytes of param, returning the output and the used gas
+func (c *p256Verify) Run(input []byte) ([]byte, error) {
+	// Required input length is 160 bytes
+	const p256VerifyInputLength = 160
+	// Check the input length
+	if len(input) != p256VerifyInputLength {
+		// Input length is invalid
+		return nil, nil
+	}
+
+	// Extract the hash, r, s, x, y from the input
+	hash := input[0:32]
+	r, s := new(big.Int).SetBytes(input[32:64]), new(big.Int).SetBytes(input[64:96])
+	x, y := new(big.Int).SetBytes(input[96:128]), new(big.Int).SetBytes(input[128:160])
+
+	// Verify the secp256r1 signature
+	if secp256r1.Verify(hash, r, s, x, y) {
+		// Signature is valid
+		return common.LeftPadBytes(common.Big1.Bytes(), 32), nil
+	} else {
+		// Signature is invalid
+		return nil, nil
+	}
+}
+
 // verifyDoubleSignEvidence implements bsc header verification precompile.
 type verifyDoubleSignEvidence struct{}
 

core/vm/contracts_test.go

@@ -46,17 +46,19 @@ type precompiledFailureTest struct {
 // allPrecompiles does not map to the actual set of precompiles, as it also contains
 // repriced versions of precompiles at certain slots
 var allPrecompiles = map[common.Address]PrecompiledContract{
-	common.BytesToAddress([]byte{1}):          &ecrecover{},
+	common.BytesToAddress([]byte{1}):    &ecrecover{},
-	common.BytesToAddress([]byte{2}):          &sha256hash{},
+	common.BytesToAddress([]byte{2}):    &sha256hash{},
-	common.BytesToAddress([]byte{3}):          &ripemd160hash{},
+	common.BytesToAddress([]byte{3}):    &ripemd160hash{},
-	common.BytesToAddress([]byte{4}):          &dataCopy{},
+	common.BytesToAddress([]byte{4}):    &dataCopy{},
-	common.BytesToAddress([]byte{5}):          &bigModExp{eip2565: false},
+	common.BytesToAddress([]byte{5}):    &bigModExp{eip2565: false},
-	common.BytesToAddress([]byte{0xf5}):       &bigModExp{eip2565: true},
+	common.BytesToAddress([]byte{0xf5}): &bigModExp{eip2565: true},
-	common.BytesToAddress([]byte{6}):          &bn256AddIstanbul{},
+	common.BytesToAddress([]byte{6}):    &bn256AddIstanbul{},
-	common.BytesToAddress([]byte{7}):          &bn256ScalarMulIstanbul{},
+	common.BytesToAddress([]byte{7}):    &bn256ScalarMulIstanbul{},
-	common.BytesToAddress([]byte{8}):          &bn256PairingIstanbul{},
+	common.BytesToAddress([]byte{8}):    &bn256PairingIstanbul{},
-	common.BytesToAddress([]byte{9}):          &blake2F{},
+	common.BytesToAddress([]byte{9}):    &blake2F{},
-	common.BytesToAddress([]byte{0x0a}):       &kzgPointEvaluation{},
+	common.BytesToAddress([]byte{0x0a}): &kzgPointEvaluation{},
+
+	common.BytesToAddress([]byte{0x01, 0x00}): &p256Verify{},
 	common.BytesToAddress([]byte{0x0f, 0x0a}): &bls12381G1Add{},
 	common.BytesToAddress([]byte{0x0f, 0x0b}): &bls12381G1Mul{},
 	common.BytesToAddress([]byte{0x0f, 0x0c}): &bls12381G1MultiExp{},
@@ -407,6 +409,18 @@ func BenchmarkPrecompiledBLS12381G2MultiExpWorstCase(b *testing.B) {
 	benchmarkPrecompiled("0f", testcase, b)
 }
 
+// Benchmarks the sample inputs from the P256VERIFY precompile.
+func BenchmarkPrecompiledP256Verify(bench *testing.B) {
+	t := precompiledTest{
+		Input:    "4cee90eb86eaa050036147a12d49004b6b9c72bd725d39d4785011fe190f0b4da73bd4903f0ce3b639bbbf6e8e80d16931ff4bcf5993d58468e8fb19086e8cac36dbcd03009df8c59286b162af3bd7fcc0450c9aa81be5d10d312af6c66b1d604aebd3099c618202fcfe16ae7770b0c49ab5eadf74b754204a3bb6060e44eff37618b065f9832de4ca6ca971a7a1adc826d0f7c00181a5fb2ddf79ae00b4e10e",
+		Expected: "0000000000000000000000000000000000000000000000000000000000000001",
+		Name:     "p256Verify",
+	}
+	benchmarkPrecompiled("100", t, bench)
+}
+
+func TestPrecompiledP256Verify(t *testing.T) { testJson("p256Verify", "100", t) }
+
 func TestDoubleSignSlash(t *testing.T) {
 	tc := precompiledTest{
 		Input:    "f906278202cab9030ff9030ca01062d3d5015b9242bc193a9b0769f3d3780ecb55f97f40a752ae26d0b68cd0d8a0fae1a05fcb14bfd9b8a9f2b65007a9b6c2000de0627a73be644dd993d32342c494976ea74026e726554db657fa54763abd0c3a0aa9a0f385cc58ed297ff0d66eb5580b02853d3478ba418b1819ac659ee05df49b9794a0bf88464af369ed6b8cf02db00f0b9556ffa8d49cd491b00952a7f83431446638a00a6d0870e586a76278fbfdcedf76ef6679af18fc1f9137cfad495f434974ea81b901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001820cdf830f4240830f4240846555fa64b90111d983010301846765746888676f312e32302e378664617277696e00007abd731ef8ae07b86091cb8836d58f5444b883422a18825d899035d3e6ea39ad1a50069bf0b86da8b5573dde1cb4a0a34f19ce94e0ef78ff7518c80265b8a3ca56e3c60167523590d4e8dcc324900559465fc0fa403774096614e135de280949b58a45cc96f2ba9e17f848820d41a08429d0d8b33ee72a84f750fefea846cbca54e487129c7961c680bb72309ca888820d42a08c9db14d938b19f9e2261bbeca2679945462be2b58103dfff73665d0d150fb8a804ae755e0fe64b59753f4db6308a1f679747bce186aa2c62b95fa6eeff3fbd08f3b0667e45428a54ade15bad19f49641c499b431b36f65803ea71b379e6b61de501a0232c9ba2d41b40d36ed794c306747bcbc49bf61a0f37409c18bfe2b5bef26a2d880000000000000000b9030ff9030ca01062d3d5015b9242bc193a9b0769f3d3780ecb55f97f40a752ae26d0b68cd0d8a0b2789a5357827ed838335283e15c4dcc42b9bebcbf2919a18613246787e2f96094976ea74026e726554db657fa54763abd0c3a0aa9a071ce4c09ee275206013f0063761bc19c93c13990582f918cc57333634c94ce89a00e095703e5c9b149f253fe89697230029e32484a410b4b1f2c61442d73c3095aa0d317ae19ede7c8a2d3ac9ef98735b049bcb7278d12f48c42b924538b60a25e12b901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001820cdf830f4240830f4240846555fa64b90111d983010301846765746888676f312e32302e378664617277696e00007abd731ef8ae07b86091cb8836d58f5444b883422a18825d899035d3e6ea39ad1a50069bf0b86da8b5573dde1cb4a0a34f19ce94e0ef78ff7518c80265b8a3ca56e3c60167523590d4e8dcc324900559465fc0fa403774096614e135de280949b58a45cc96f2ba9e17f848820d41a08429d0d8b33ee72a84f750fefea846cbca54e487129c7961c680bb72309ca888820d42a08c9db14d938b19f9e2261bbeca2679945462be2b58103dfff73665d0d150fb8a80c0b17bfe88534296ff064cb7156548f6deba2d6310d5044ed6485f087dc6ef232e051c28e1909c2b50a3b4f29345d66681c319bef653e52e5d746480d5a3983b00a0b56228685be711834d0f154292d07826dea42a0fad3e4f56c31470b7fbfbea26880000000000000000",

core/vm/evm.go

@@ -48,6 +48,8 @@ type (
 func (evm *EVM) precompile(addr common.Address) (PrecompiledContract, bool) {
 	var precompiles map[common.Address]PrecompiledContract
 	switch {
+	case evm.chainRules.IsHaber:
+		precompiles = PrecompiledContractsHaber
 	case evm.chainRules.IsCancun:
 		precompiles = PrecompiledContractsCancun
 	case evm.chainRules.IsFeynman:

crypto/secp256r1/publickey.go

@@ -0,0 +1,21 @@
+package secp256r1
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"math/big"
+)
+
+// Generates appropriate public key format from given coordinates
+func newPublicKey(x, y *big.Int) *ecdsa.PublicKey {
+	// Check if the given coordinates are valid
+	if x == nil || y == nil || !elliptic.P256().IsOnCurve(x, y) {
+		return nil
+	}
+
+	return &ecdsa.PublicKey{
+		Curve: elliptic.P256(),
+		X:     x,
+		Y:     y,
+	}
+}

crypto/secp256r1/verifier.go

@@ -0,0 +1,22 @@
+package secp256r1
+
+import (
+	"crypto/ecdsa"
+	"math/big"
+)
+
+// Verify verifies the given signature (r, s) for the given hash and public key (x, y).
+// It returns true if the signature is valid, false otherwise.
+func Verify(hash []byte, r, s, x, y *big.Int) bool {
+	// Create the public key format
+	publicKey := newPublicKey(x, y)
+
+	// Check if they are invalid public key coordinates
+	if publicKey == nil {
+		return false
+	}
+
+	// Verify the signature with the public key,
+	// then return true if it's valid, false otherwise
+	return ecdsa.Verify(publicKey, hash, r, s)
+}

eth/api_admin.go

@@ -24,7 +24,6 @@ import (
 	"os"
 	"strings"
 
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -142,31 +141,3 @@ func (api *AdminAPI) ImportChain(file string) (bool, error) {
 	}
 	return true, nil
 }
-
-// MevRunning returns true if the validator accept bids from builder
-func (api *AdminAPI) MevRunning() bool {
-	return api.eth.APIBackend.MevRunning()
-}
-
-// StartMev starts mev. It notifies the miner to start to receive bids.
-func (api *AdminAPI) StartMev() {
-	api.eth.APIBackend.StartMev()
-}
-
-// StopMev stops mev. It notifies the miner to stop receiving bids from this moment,
-// but the bids before this moment would still been taken into consideration by mev.
-func (api *AdminAPI) StopMev() {
-	api.eth.APIBackend.StopMev()
-}
-
-// AddBuilder adds a builder to the bid simulator.
-// url is the endpoint of the builder, for example, "https://mev-builder.amazonaws.com",
-// if validator is equipped with sentry, ignore the url.
-func (api *AdminAPI) AddBuilder(builder common.Address, url string) error {
-	return api.eth.APIBackend.AddBuilder(builder, url)
-}
-
-// RemoveBuilder removes a builder from the bid simulator.
-func (api *AdminAPI) RemoveBuilder(builder common.Address) error {
-	return api.eth.APIBackend.RemoveBuilder(builder)
-}

eth/api_miner.go

@@ -20,6 +20,8 @@ import (
 	"math/big"
 	"time"
 
+	"github.com/ethereum/go-ethereum/params"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 )
@@ -71,6 +73,9 @@ func (api *MinerAPI) SetGasPrice(gasPrice hexutil.Big) bool {
 // SetGasLimit sets the gaslimit to target towards during mining.
 func (api *MinerAPI) SetGasLimit(gasLimit hexutil.Uint64) bool {
 	api.e.Miner().SetGasCeil(uint64(gasLimit))
+	if api.e.Miner().Mining() && uint64(gasLimit) > params.SystemTxsGas {
+		api.e.TxPool().SetMaxGas(uint64(gasLimit) - params.SystemTxsGas)
+	}
 	return true
 }
 
@@ -84,3 +89,31 @@ func (api *MinerAPI) SetEtherbase(etherbase common.Address) bool {
 func (api *MinerAPI) SetRecommitInterval(interval int) {
 	api.e.Miner().SetRecommitInterval(time.Duration(interval) * time.Millisecond)
 }
+
+// MevRunning returns true if the validator accept bids from builder
+func (api *MinerAPI) MevRunning() bool {
+	return api.e.APIBackend.MevRunning()
+}
+
+// StartMev starts mev. It notifies the miner to start to receive bids.
+func (api *MinerAPI) StartMev() {
+	api.e.APIBackend.StartMev()
+}
+
+// StopMev stops mev. It notifies the miner to stop receiving bids from this moment,
+// but the bids before this moment would still been taken into consideration by mev.
+func (api *MinerAPI) StopMev() {
+	api.e.APIBackend.StopMev()
+}
+
+// AddBuilder adds a builder to the bid simulator.
+// url is the endpoint of the builder, for example, "https://mev-builder.amazonaws.com",
+// if validator is equipped with sentry, ignore the url.
+func (api *MinerAPI) AddBuilder(builder common.Address, url string) error {
+	return api.e.APIBackend.AddBuilder(builder, url)
+}
+
+// RemoveBuilder removes a builder from the bid simulator.
+func (api *MinerAPI) RemoveBuilder(builder common.Address) error {
+	return api.e.APIBackend.RemoveBuilder(builder)
+}

eth/backend.go

@@ -183,18 +183,18 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 		chainConfig.CancunTime = config.OverrideCancun
 		overrides.OverrideCancun = config.OverrideCancun
 	}
+	if config.OverrideHaber != nil {
+		chainConfig.HaberTime = config.OverrideHaber
+		overrides.OverrideHaber = config.OverrideHaber
+	}
+	if config.OverrideBohr != nil {
+		chainConfig.BohrTime = config.OverrideBohr
+		overrides.OverrideBohr = config.OverrideBohr
+	}
 	if config.OverrideVerkle != nil {
 		chainConfig.VerkleTime = config.OverrideVerkle
 		overrides.OverrideVerkle = config.OverrideVerkle
 	}
-	if config.OverrideFeynman != nil {
-		chainConfig.FeynmanTime = config.OverrideFeynman
-		overrides.OverrideFeynman = config.OverrideFeynman
-	}
-	if config.OverrideFeynmanFix != nil {
-		chainConfig.FeynmanFixTime = config.OverrideFeynmanFix
-		overrides.OverrideFeynmanFix = config.OverrideFeynmanFix
-	}
 
 	// startup ancient freeze
 	if err = chainDb.SetupFreezerEnv(&ethdb.FreezerEnv{

eth/downloader/downloader.go

@@ -209,6 +209,9 @@ type BlockChain interface {
 
 	// UpdateChasingHead update remote best chain head, used by DA check now.
 	UpdateChasingHead(head *types.Header)
+
+	// AncientTail retrieves the tail the ancients blocks
+	AncientTail() (uint64, error)
 }
 
 type DownloadOption func(downloader *Downloader) *Downloader
@@ -797,6 +800,11 @@ func (d *Downloader) findAncestor(p *peerConnection, localHeight uint64, remoteH
 		// We're above the max reorg threshold, find the earliest fork point
 		floor = int64(localHeight - maxForkAncestry)
 	}
+	// if we have pruned too much history, reset the floor
+	if tail, err := d.blockchain.AncientTail(); err == nil && tail > uint64(floor) {
+		floor = int64(tail)
+	}
+
 	// If we're doing a light sync, ensure the floor doesn't go below the CHT, as
 	// all headers before that point will be missing.
 	if mode == LightSync {

eth/downloader/downloader_test.go

@@ -60,7 +60,7 @@ func newTester(t *testing.T) *downloadTester {
 // newTester creates a new downloader test mocker.
 func newTesterWithNotification(t *testing.T, success func()) *downloadTester {
 	freezer := t.TempDir()
-	db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), freezer, "", false, false, false, false)
+	db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), freezer, "", false, false, false, false, false)
 	if err != nil {
 		panic(err)
 	}

eth/ethconfig/config.go

@@ -191,15 +191,15 @@ type Config struct {
 	// OverrideCancun (TODO: remove after the fork)
 	OverrideCancun *uint64 `toml:",omitempty"`
 
+	// OverrideHaber (TODO: remove after the fork)
+	OverrideHaber *uint64 `toml:",omitempty"`
+
+	// OverrideBohr (TODO: remove after the fork)
+	OverrideBohr *uint64 `toml:",omitempty"`
+
 	// OverrideVerkle (TODO: remove after the fork)
 	OverrideVerkle *uint64 `toml:",omitempty"`
 
-	// OverrideFeynman (TODO: remove after the fork)
-	OverrideFeynman *uint64 `toml:",omitempty"`
-
-	// OverrideFeynmanFix (TODO: remove after the fork)
-	OverrideFeynmanFix *uint64 `toml:",omitempty"`
-
 	// blob setting
 	BlobExtraReserve uint64
 }

eth/ethconfig/gen_config.go

@@ -32,11 +32,12 @@ func (c Config) MarshalTOML() (interface{}, error) {
 		EnableTrustProtocol     bool
 		PipeCommit              bool
 		RangeLimit              bool
-		TxLookupLimit           uint64                 `toml:",omitempty"`
+		TxLookupLimit           uint64 `toml:",omitempty"`
-		TransactionHistory      uint64                 `toml:",omitempty"`
+		TransactionHistory      uint64 `toml:",omitempty"`
-		StateHistory            uint64                 `toml:",omitempty"`
+		StateHistory            uint64 `toml:",omitempty"`
-		StateScheme             string                 `toml:",omitempty"`
+		StateScheme             string `toml:",omitempty"`
-		PathSyncFlush           bool                   `toml:",omitempty"`
+		PathSyncFlush           bool   `toml:",omitempty"`
+		JournalFileEnabled      bool
 		RequiredBlocks          map[uint64]common.Hash `toml:"-"`
 		LightServ               int                    `toml:",omitempty"`
 		LightIngress            int                    `toml:",omitempty"`
@@ -70,9 +71,10 @@ func (c Config) MarshalTOML() (interface{}, error) {
 		RPCEVMTimeout           time.Duration
 		RPCTxFeeCap             float64
 		OverrideCancun          *uint64 `toml:",omitempty"`
+		OverrideHaber           *uint64 `toml:",omitempty"`
+		OverrideBohr            *uint64 `toml:",omitempty"`
 		OverrideVerkle          *uint64 `toml:",omitempty"`
-		OverrideFeynman         *uint64 `toml:",omitempty"`
+		BlobExtraReserve        uint64
-		OverrideFeynmanFix      *uint64 `toml:",omitempty"`
 	}
 	var enc Config
 	enc.Genesis = c.Genesis
@@ -95,6 +97,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
 	enc.StateHistory = c.StateHistory
 	enc.StateScheme = c.StateScheme
 	enc.PathSyncFlush = c.PathSyncFlush
+	enc.JournalFileEnabled = c.JournalFileEnabled
 	enc.RequiredBlocks = c.RequiredBlocks
 	enc.LightServ = c.LightServ
 	enc.LightIngress = c.LightIngress
@@ -128,9 +131,10 @@ func (c Config) MarshalTOML() (interface{}, error) {
 	enc.RPCEVMTimeout = c.RPCEVMTimeout
 	enc.RPCTxFeeCap = c.RPCTxFeeCap
 	enc.OverrideCancun = c.OverrideCancun
+	enc.OverrideHaber = c.OverrideHaber
+	enc.OverrideBohr = c.OverrideBohr
 	enc.OverrideVerkle = c.OverrideVerkle
-	enc.OverrideFeynman = c.OverrideFeynman
+	enc.BlobExtraReserve = c.BlobExtraReserve
-	enc.OverrideFeynmanFix = c.OverrideFeynmanFix
 	return &enc, nil
 }
 
@@ -152,11 +156,12 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 		EnableTrustProtocol     *bool
 		PipeCommit              *bool
 		RangeLimit              *bool
-		TxLookupLimit           *uint64                `toml:",omitempty"`
+		TxLookupLimit           *uint64 `toml:",omitempty"`
-		TransactionHistory      *uint64                `toml:",omitempty"`
+		TransactionHistory      *uint64 `toml:",omitempty"`
-		StateHistory            *uint64                `toml:",omitempty"`
+		StateHistory            *uint64 `toml:",omitempty"`
-		StateScheme             *string                `toml:",omitempty"`
+		StateScheme             *string `toml:",omitempty"`
-		PathSyncFlush           *bool                  `toml:",omitempty"`
+		PathSyncFlush           *bool   `toml:",omitempty"`
+		JournalFileEnabled      *bool
 		RequiredBlocks          map[uint64]common.Hash `toml:"-"`
 		LightServ               *int                   `toml:",omitempty"`
 		LightIngress            *int                   `toml:",omitempty"`
@@ -190,9 +195,10 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 		RPCEVMTimeout           *time.Duration
 		RPCTxFeeCap             *float64
 		OverrideCancun          *uint64 `toml:",omitempty"`
+		OverrideHaber           *uint64 `toml:",omitempty"`
+		OverrideBohr            *uint64 `toml:",omitempty"`
 		OverrideVerkle          *uint64 `toml:",omitempty"`
-		OverrideFeynman         *uint64 `toml:",omitempty"`
+		BlobExtraReserve        *uint64
-		OverrideFeynmanFix      *uint64 `toml:",omitempty"`
 	}
 	var dec Config
 	if err := unmarshal(&dec); err != nil {
@@ -258,6 +264,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 	if dec.PathSyncFlush != nil {
 		c.PathSyncFlush = *dec.PathSyncFlush
 	}
+	if dec.JournalFileEnabled != nil {
+		c.JournalFileEnabled = *dec.JournalFileEnabled
+	}
 	if dec.RequiredBlocks != nil {
 		c.RequiredBlocks = dec.RequiredBlocks
 	}
@@ -357,14 +366,17 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 	if dec.OverrideCancun != nil {
 		c.OverrideCancun = dec.OverrideCancun
 	}
+	if dec.OverrideHaber != nil {
+		c.OverrideHaber = dec.OverrideHaber
+	}
+	if dec.OverrideBohr != nil {
+		c.OverrideBohr = dec.OverrideBohr
+	}
 	if dec.OverrideVerkle != nil {
 		c.OverrideVerkle = dec.OverrideVerkle
 	}
-	if dec.OverrideFeynman != nil {
+	if dec.BlobExtraReserve != nil {
-		c.OverrideFeynman = dec.OverrideFeynman
+		c.BlobExtraReserve = *dec.BlobExtraReserve
-	}
-	if dec.OverrideFeynmanFix != nil {
-		c.OverrideFeynmanFix = dec.OverrideFeynmanFix
 	}
 	return nil
 }

eth/fetcher/block_fetcher.go

@@ -22,6 +22,7 @@ import (
 	"math/rand"
 	"time"
 
+	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/prque"
 	"github.com/ethereum/go-ethereum/consensus"
@@ -66,6 +67,10 @@ var (
 	headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/out", nil)
 	bodyFilterInMeter    = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/in", nil)
 	bodyFilterOutMeter   = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/out", nil)
+
+	blockInsertFailRecords      = mapset.NewSet[common.Hash]()
+	blockInsertFailRecordslimit = 1000
+	blockInsertFailGauge        = metrics.NewRegisteredGauge("chain/insert/failed", nil)
 )
 
 var errTerminated = errors.New("terminated")
@@ -934,6 +939,10 @@ func (f *BlockFetcher) importBlocks(op *blockOrHeaderInject) {
 		}
 		// Run the actual import and log any issues
 		if _, err := f.insertChain(types.Blocks{block}); err != nil {
+			if blockInsertFailRecords.Cardinality() < blockInsertFailRecordslimit {
+				blockInsertFailRecords.Add(block.Hash())
+				blockInsertFailGauge.Update(int64(blockInsertFailRecords.Cardinality()))
+			}
 			log.Debug("Propagated block import failed", "peer", peer, "number", block.Number(), "hash", hash, "err", err)
 			return
 		}

eth/filters/api.go

@@ -44,6 +44,9 @@ var (
 // The maximum number of topic criteria allowed, vm.LOG4 - vm.LOG0
 const maxTopics = 4
 
+// The maximum number of allowed topics within a topic criteria
+const maxSubTopics = 1000
+
 // filter is a helper struct that holds meta information over the filter type
 // and associated subscription in the event system.
 type filter struct {
@@ -671,6 +674,9 @@ func (args *FilterCriteria) UnmarshalJSON(data []byte) error {
 			return errors.New("invalid addresses in query")
 		}
 	}
+	if len(raw.Topics) > maxTopics {
+		return errExceedMaxTopics
+	}
 
 	// topics is an array consisting of strings and/or arrays of strings.
 	// JSON null values are converted to common.Hash{} and ignored by the filter manager.
@@ -691,6 +697,9 @@ func (args *FilterCriteria) UnmarshalJSON(data []byte) error {
 
 			case []interface{}:
 				// or case e.g. [null, "topic0", "topic1"]
+				if len(topic) > maxSubTopics {
+					return errExceedMaxTopics
+				}
 				for _, rawTopic := range topic {
 					if rawTopic == nil {
 						// null component, match all

eth/handler.go

@@ -318,7 +318,30 @@ func newHandler(config *handlerConfig) (*handler, error) {
 		}
 		return h.chain.InsertChain(blocks)
 	}
-	h.blockFetcher = fetcher.NewBlockFetcher(false, nil, h.chain.GetBlockByHash, validator, h.BroadcastBlock,
+
+	broadcastBlockWithCheck := func(block *types.Block, propagate bool) {
+		if propagate {
+			checkErrs := make(chan error, 2)
+
+			go func() {
+				checkErrs <- core.ValidateListsInBody(block)
+			}()
+			go func() {
+				checkErrs <- core.IsDataAvailable(h.chain, block)
+			}()
+
+			for i := 0; i < cap(checkErrs); i++ {
+				err := <-checkErrs
+				if err != nil {
+					log.Error("Propagating invalid block", "number", block.Number(), "hash", block.Hash(), "err", err)
+					return
+				}
+			}
+		}
+		h.BroadcastBlock(block, propagate)
+	}
+
+	h.blockFetcher = fetcher.NewBlockFetcher(false, nil, h.chain.GetBlockByHash, validator, broadcastBlockWithCheck,
 		heighter, finalizeHeighter, nil, inserter, h.removePeer)
 
 	fetchTx := func(peer string, hashes []common.Hash) error {

eth/handler_test.go

@@ -258,7 +258,7 @@ func (c *mockParlia) CalcDifficulty(chain consensus.ChainHeaderReader, time uint
 func newTestParliaHandlerAfterCancun(t *testing.T, config *params.ChainConfig, mode downloader.SyncMode, preCancunBlks, postCancunBlks uint64) *testHandler {
 	// Have N headers in the freezer
 	frdir := t.TempDir()
-	db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false, false)
+	db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false, false, false, false, false)
 	if err != nil {
 		t.Fatalf("failed to create database with ancient backend")
 	}

ethclient/ethclient.go

@@ -133,7 +133,7 @@ func (ec *Client) BlockReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumb
 // BlobSidecars return the Sidecars of a given block number or hash.
 func (ec *Client) BlobSidecars(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]*types.BlobTxSidecar, error) {
 	var r []*types.BlobTxSidecar
-	err := ec.c.CallContext(ctx, &r, "eth_getBlobSidecars", blockNrOrHash.String())
+	err := ec.c.CallContext(ctx, &r, "eth_getBlobSidecars", blockNrOrHash.String(), true)
 	if err == nil && r == nil {
 		return nil, ethereum.NotFound
 	}
@@ -143,7 +143,7 @@ func (ec *Client) BlobSidecars(ctx context.Context, blockNrOrHash rpc.BlockNumbe
 // BlobSidecarByTxHash return a sidecar of a given blob transaction
 func (ec *Client) BlobSidecarByTxHash(ctx context.Context, hash common.Hash) (*types.BlobTxSidecar, error) {
 	var r *types.BlobTxSidecar
-	err := ec.c.CallContext(ctx, &r, "eth_getBlockSidecarByTxHash", hash)
+	err := ec.c.CallContext(ctx, &r, "eth_getBlockSidecarByTxHash", hash, true)
 	if err == nil && r == nil {
 		return nil, ethereum.NotFound
 	}

ethdb/database.go

@@ -183,6 +183,7 @@ type StateStoreReader interface {
 type BlockStore interface {
 	BlockStore() Database
 	SetBlockStore(block Database)
+	HasSeparateBlockStore() bool
 }
 
 type BlockStoreReader interface {

ethdb/pebble/pebble.go

@@ -48,6 +48,9 @@ const (
 
 	// numLevels is the level number of pebble sst files
 	numLevels = 7
+	// degradationWarnInterval specifies how often warning should be printed if the
+	// leveldb database cannot keep up with requested writes.
+	degradationWarnInterval = time.Minute
 )
 
 // Database is a persistent key-value store based on the pebble storage engine.
@@ -79,14 +82,16 @@ type Database struct {
 
 	log log.Logger // Contextual logger tracking the database path
 
-	activeComp          int           // Current number of active compactions
+	activeComp    int           // Current number of active compactions
-	compStartTime       time.Time     // The start time of the earliest currently-active compaction
+	compStartTime time.Time     // The start time of the earliest currently-active compaction
-	compTime            atomic.Int64  // Total time spent in compaction in ns
+	compTime      atomic.Int64  // Total time spent in compaction in ns
-	level0Comp          atomic.Uint32 // Total number of level-zero compactions
+	level0Comp    atomic.Uint32 // Total number of level-zero compactions
-	nonLevel0Comp       atomic.Uint32 // Total number of non level-zero compactions
+	nonLevel0Comp atomic.Uint32 // Total number of non level-zero compactions
-	writeDelayStartTime time.Time     // The start time of the latest write stall
+
-	writeDelayCount     atomic.Int64  // Total number of write stall counts
+	writeStalled        atomic.Bool  // Flag whether the write is stalled
-	writeDelayTime      atomic.Int64  // Total time spent in write stalls
+	writeDelayStartTime time.Time    // The start time of the latest write stall
+	writeDelayCount     atomic.Int64 // Total number of write stall counts
+	writeDelayTime      atomic.Int64 // Total time spent in write stalls
 
 	writeOptions *pebble.WriteOptions
 }
@@ -115,10 +120,13 @@ func (d *Database) onCompactionEnd(info pebble.CompactionInfo) {
 
 func (d *Database) onWriteStallBegin(b pebble.WriteStallBeginInfo) {
 	d.writeDelayStartTime = time.Now()
+	d.writeDelayCount.Add(1)
+	d.writeStalled.Store(true)
 }
 
 func (d *Database) onWriteStallEnd() {
 	d.writeDelayTime.Add(int64(time.Since(d.writeDelayStartTime)))
+	d.writeStalled.Store(false)
 }
 
 // panicLogger is just a noop logger to disable Pebble's internal logger.
@@ -461,13 +469,15 @@ func (d *Database) meter(refresh time.Duration, namespace string) {
 
 	// Create storage and warning log tracer for write delay.
 	var (
-		compTimes        [2]int64
+		compTimes  [2]int64
-		writeDelayTimes  [2]int64
+		compWrites [2]int64
-		writeDelayCounts [2]int64
+		compReads  [2]int64
-		compWrites       [2]int64
-		compReads        [2]int64
 
 		nWrites [2]int64
+
+		writeDelayTimes      [2]int64
+		writeDelayCounts     [2]int64
+		lastWriteStallReport time.Time
 	)
 
 	// Iterate ad infinitum and collect the stats
@@ -507,6 +517,13 @@ func (d *Database) meter(refresh time.Duration, namespace string) {
 		if d.writeDelayMeter != nil {
 			d.writeDelayMeter.Mark(writeDelayTimes[i%2] - writeDelayTimes[(i-1)%2])
 		}
+		// Print a warning log if writing has been stalled for a while. The log will
+		// be printed per minute to avoid overwhelming users.
+		if d.writeStalled.Load() && writeDelayCounts[i%2] == writeDelayCounts[(i-1)%2] &&
+			time.Now().After(lastWriteStallReport.Add(degradationWarnInterval)) {
+			d.log.Warn("Database compacting, degraded performance")
+			lastWriteStallReport = time.Now()
+		}
 		if d.compTimeMeter != nil {
 			d.compTimeMeter.Mark(compTimes[i%2] - compTimes[(i-1)%2])
 		}
@@ -600,8 +617,8 @@ func (b *batch) Reset() {
 func (b *batch) Replay(w ethdb.KeyValueWriter) error {
 	reader := b.b.Reader()
 	for {
-		kind, k, v, ok := reader.Next()
+		kind, k, v, ok, err := reader.Next()
-		if !ok {
+		if !ok || err != nil {
 			break
 		}
 		// The (k,v) slices might be overwritten if the batch is reset/reused,

ethdb/remotedb/remotedb.go

@@ -44,6 +44,10 @@ func (db *Database) BlockStore() ethdb.Database {
 	return db
 }
 
+func (db *Database) HasSeparateBlockStore() bool {
+	return false
+}
+
 func (db *Database) SetBlockStore(block ethdb.Database) {
 	panic("not supported")
 }

go.mod

@@ -17,8 +17,8 @@ require (
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2
 	github.com/cespare/cp v1.1.1
 	github.com/cloudflare/cloudflare-go v0.79.0
-	github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593
 	github.com/cometbft/cometbft v0.37.0
+	github.com/cockroachdb/pebble v1.1.0
 	github.com/consensys/gnark-crypto v0.12.1
 	github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233
 	github.com/crate-crypto/go-kzg-4844 v0.7.0
@@ -81,10 +81,10 @@ require (
 	github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4 v1.1.3
 	github.com/willf/bitset v1.1.3
 	go.uber.org/automaxprocs v1.5.2
-	golang.org/x/crypto v0.19.0
+	golang.org/x/crypto v0.21.0
 	golang.org/x/exp v0.0.0-20240213143201-ec583247a57a
 	golang.org/x/sync v0.6.0
-	golang.org/x/sys v0.17.0
+	golang.org/x/sys v0.18.0
 	golang.org/x/text v0.14.0
 	golang.org/x/time v0.5.0
 	golang.org/x/tools v0.18.0
@@ -274,9 +274,9 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/mod v0.15.0 // indirect
-	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/oauth2 v0.16.0 // indirect
-	golang.org/x/term v0.17.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
 	google.golang.org/api v0.44.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect

go.sum

@@ -275,8 +275,8 @@ github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZ
 github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvDmeyi+l/FS/VtA00IWvjeFvjen1m1l1A=
+github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4=
-github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo=
+github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E=
 github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
@@ -1743,8 +1743,8 @@ golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1857,8 +1857,8 @@ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -2007,13 +2007,13 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

internal/ethapi/api.go

@@ -1010,7 +1010,11 @@ func (s *BlockChainAPI) GetBlockReceipts(ctx context.Context, blockNrOrHash rpc.
 	return result, nil
 }
 
-func (s *BlockChainAPI) GetBlobSidecars(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]map[string]interface{}, error) {
+func (s *BlockChainAPI) GetBlobSidecars(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash, fullBlob *bool) ([]map[string]interface{}, error) {
+	showBlob := true
+	if fullBlob != nil {
+		showBlob = *fullBlob
+	}
 	header, err := s.b.HeaderByNumberOrHash(ctx, blockNrOrHash)
 	if header == nil || err != nil {
 		// When the block doesn't exist, the RPC method should return JSON null
@@ -1023,12 +1027,16 @@ func (s *BlockChainAPI) GetBlobSidecars(ctx context.Context, blockNrOrHash rpc.B
 	}
 	result := make([]map[string]interface{}, len(blobSidecars))
 	for i, sidecar := range blobSidecars {
-		result[i] = marshalBlobSidecar(sidecar)
+		result[i] = marshalBlobSidecar(sidecar, showBlob)
 	}
 	return result, nil
 }
 
-func (s *BlockChainAPI) GetBlobSidecarByTxHash(ctx context.Context, hash common.Hash) (map[string]interface{}, error) {
+func (s *BlockChainAPI) GetBlobSidecarByTxHash(ctx context.Context, hash common.Hash, fullBlob *bool) (map[string]interface{}, error) {
+	showBlob := true
+	if fullBlob != nil {
+		showBlob = *fullBlob
+	}
 	txTarget, blockHash, _, Index := rawdb.ReadTransaction(s.b.ChainDb(), hash)
 	if txTarget == nil {
 		return nil, nil
@@ -1045,7 +1053,7 @@ func (s *BlockChainAPI) GetBlobSidecarByTxHash(ctx context.Context, hash common.
 	}
 	for _, sidecar := range blobSidecars {
 		if sidecar.TxIndex == Index {
-			return marshalBlobSidecar(sidecar), nil
+			return marshalBlobSidecar(sidecar, showBlob), nil
 		}
 	}
 
@@ -2142,13 +2150,31 @@ func marshalReceipt(receipt *types.Receipt, blockHash common.Hash, blockNumber u
 	return fields
 }
 
-func marshalBlobSidecar(sidecar *types.BlobSidecar) map[string]interface{} {
+func marshalBlobSidecar(sidecar *types.BlobSidecar, fullBlob bool) map[string]interface{} {
 	fields := map[string]interface{}{
 		"blockHash":   sidecar.BlockHash,
 		"blockNumber": hexutil.EncodeUint64(sidecar.BlockNumber.Uint64()),
 		"txHash":      sidecar.TxHash,
 		"txIndex":     hexutil.EncodeUint64(sidecar.TxIndex),
-		"blobSidecar": sidecar.BlobTxSidecar,
+	}
+	fields["blobSidecar"] = marshalBlob(sidecar.BlobTxSidecar, fullBlob)
+	return fields
+}
+
+func marshalBlob(blobTxSidecar types.BlobTxSidecar, fullBlob bool) map[string]interface{} {
+	fields := map[string]interface{}{
+		"blobs":       blobTxSidecar.Blobs,
+		"commitments": blobTxSidecar.Commitments,
+		"proofs":      blobTxSidecar.Proofs,
+	}
+	if !fullBlob {
+		var blobs []common.Hash
+		for _, blob := range blobTxSidecar.Blobs {
+			var value common.Hash
+			copy(value[:], blob[:32])
+			blobs = append(blobs, value)
+		}
+		fields["blobs"] = blobs
 	}
 	return fields
 }

internal/ethapi/api_test.go

@@ -2139,48 +2139,63 @@ func TestRPCGetBlobSidecars(t *testing.T) {
 	}
 
 	var testSuite = []struct {
-		test rpc.BlockNumberOrHash
+		test     rpc.BlockNumberOrHash
-		file string
+		fullBlob bool
+		file     string
 	}{
 		// 1. block without any txs(number)
 		{
-			test: rpc.BlockNumberOrHashWithNumber(0),
+			test:     rpc.BlockNumberOrHashWithNumber(0),
-			file: "number-1",
+			fullBlob: true,
+			file:     "number-1",
 		},
 		// 2. earliest tag
 		{
-			test: rpc.BlockNumberOrHashWithNumber(rpc.EarliestBlockNumber),
+			test:     rpc.BlockNumberOrHashWithNumber(rpc.EarliestBlockNumber),
-			file: "tag-earliest",
+			fullBlob: true,
+			file:     "tag-earliest",
 		},
 		// 3. latest tag
 		{
-			test: rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber),
+			test:     rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber),
-			file: "tag-latest",
+			fullBlob: true,
+			file:     "tag-latest",
 		},
 		// 4. block is empty
 		{
-			test: rpc.BlockNumberOrHashWithHash(common.Hash{}, false),
+			test:     rpc.BlockNumberOrHashWithHash(common.Hash{}, false),
-			file: "hash-empty",
+			fullBlob: true,
+			file:     "hash-empty",
 		},
 		// 5. block is not found
 		{
-			test: rpc.BlockNumberOrHashWithHash(common.HexToHash("deadbeef"), false),
+			test:     rpc.BlockNumberOrHashWithHash(common.HexToHash("deadbeef"), false),
-			file: "hash-notfound",
+			fullBlob: true,
+			file:     "hash-notfound",
 		},
 		// 6. block is not found
 		{
-			test: rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(genBlocks + 1)),
+			test:     rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(genBlocks + 1)),
-			file: "block-notfound",
+			fullBlob: true,
+			file:     "block-notfound",
 		},
 		// 7. block with blob tx
 		{
-			test: rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(6)),
+			test:     rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(6)),
-			file: "block-with-blob-tx",
+			fullBlob: true,
+			file:     "block-with-blob-tx",
 		},
 		// 8. block with sidecar
 		{
-			test: rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(7)),
+			test:     rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(7)),
-			file: "block-with-blobSidecars",
+			fullBlob: true,
+			file:     "block-with-blobSidecars",
+		},
+		// 9. block with sidecar but show little
+		{
+			test:     rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(7)),
+			fullBlob: false,
+			file:     "block-with-blobSidecars-show-little",
 		},
 	}
 
@@ -2189,7 +2204,7 @@ func TestRPCGetBlobSidecars(t *testing.T) {
 			result interface{}
 			err    error
 		)
-		result, err = api.GetBlobSidecars(context.Background(), tt.test)
+		result, err = api.GetBlobSidecars(context.Background(), tt.test, &tt.fullBlob)
 		if err != nil {
 			t.Errorf("test %d: want no error, have %v", i, err)
 			continue
@@ -2205,33 +2220,45 @@ func TestGetBlobSidecarByTxHash(t *testing.T) {
 		api              = NewBlockChainAPI(backend)
 	)
 	var testSuite = []struct {
-		test common.Hash
+		test     common.Hash
-		file string
+		fullBlob bool
+		file     string
 	}{
 		// 0. txHash is empty
 		{
-			test: common.Hash{},
+			test:     common.Hash{},
-			file: "hash-empty",
+			fullBlob: true,
+			file:     "hash-empty",
 		},
 		// 1. txHash is not found
 		{
-			test: common.HexToHash("deadbeef"),
+			test:     common.HexToHash("deadbeef"),
-			file: "hash-notfound",
+			fullBlob: true,
+			file:     "hash-notfound",
 		},
 		// 2. txHash is not blob tx
 		{
-			test: common.HexToHash("deadbeef"),
+			test:     common.HexToHash("deadbeef"),
-			file: "not-blob-tx",
+			fullBlob: true,
+			file:     "not-blob-tx",
 		},
 		// 3. block with blob tx without sidecar
 		{
-			test: txHashs[5],
+			test:     txHashs[5],
-			file: "block-with-blob-tx",
+			fullBlob: true,
+			file:     "block-with-blob-tx",
 		},
 		// 4. block with sidecar
 		{
-			test: txHashs[6],
+			test:     txHashs[6],
-			file: "block-with-blobSidecars",
+			fullBlob: true,
+			file:     "block-with-blobSidecars",
+		},
+		// 5. block show part blobs
+		{
+			test:     txHashs[6],
+			fullBlob: false,
+			file:     "block-with-blobSidecars-show-little",
 		},
 	}
 
@@ -2240,7 +2267,7 @@ func TestGetBlobSidecarByTxHash(t *testing.T) {
 			result interface{}
 			err    error
 		)
-		result, err = api.GetBlobSidecarByTxHash(context.Background(), tt.test)
+		result, err = api.GetBlobSidecarByTxHash(context.Background(), tt.test, &tt.fullBlob)
 		if err != nil {
 			t.Errorf("test %d: want no error, have %v", i, err)
 			continue

internal/ethapi/testdata/eth_getBlobSidecarByTxHash-block-with-blobSidecars-show-little.json

@@ -0,0 +1,17 @@
+{
+  "blobSidecar": {
+    "blobs": [
+      "0x0000000000000000000000000000000000000000000000000000000000000000"
+    ],
+    "commitments": [
+      "0xc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
+    ],
+    "proofs": [
+      "0xc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
+    ]
+  },
+  "blockHash": "0x6fe5e7eea22c44f700c95da066d6e0740894b6c1e993825cc63f634ad4f74250",
+  "blockNumber": "0x7",
+  "txHash": "0xc520427e696154779f6b21ab03a0735769e1c029035a484f5876f60383a0a7ce",
+  "txIndex": "0x0"
+}

internal/ethapi/testdata/eth_getBlobSidecars-block-with-blobSidecars-show-little.json

@@ -0,0 +1,19 @@
+[
+  {
+    "blobSidecar": {
+      "blobs": [
+        "0x0000000000000000000000000000000000000000000000000000000000000000"
+      ],
+      "commitments": [
+        "0xc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
+      ],
+      "proofs": [
+        "0xc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
+      ]
+    },
+    "blockHash": "0x6fe5e7eea22c44f700c95da066d6e0740894b6c1e993825cc63f634ad4f74250",
+    "blockNumber": "0x7",
+    "txHash": "0xc520427e696154779f6b21ab03a0735769e1c029035a484f5876f60383a0a7ce",
+    "txIndex": "0x0"
+  }
+]

internal/web3ext/web3ext.go

@@ -617,12 +617,12 @@ web3._extend({
 		new web3._extend.Method({
 			name: 'getBlobSidecars',
 			call: 'eth_getBlobSidecars',
-			params: 1,
+			params: 2,
 		}),
 		new web3._extend.Method({
 			name: 'getBlobSidecarByTxHash',
 			call: 'eth_getBlobSidecarByTxHash',
-			params: 1,
+			params: 2,
 		}),
 	],
 	properties: [
@@ -659,6 +659,30 @@ web3._extend({
 			name: 'stop',
 			call: 'miner_stop'
 		}),
+		new web3._extend.Method({
+			name: 'mevRunning',
+			call: 'miner_mevRunning'
+		}),
+		new web3._extend.Method({
+			name: 'startMev',
+			call: 'miner_startMev'
+		}),
+		new web3._extend.Method({
+			name: 'stopMev',
+			call: 'miner_stopMev'
+		}),
+		new web3._extend.Method({
+			name: 'addBuilder',
+			call: 'miner_addBuilder',
+			params: 2,
+			inputFormatter: [web3._extend.formatters.inputAddressFormatter, null]
+		}),
+		new web3._extend.Method({
+			name: 'removeBuilder',
+			call: 'miner_removeBuilder',
+			params: 1,
+			inputFormatter: [web3._extend.formatters.inputAddressFormatter]
+		}),
 		new web3._extend.Method({
 			name: 'setEtherbase',
 			call: 'miner_setEtherbase',

miner/bid_simulator.go

@@ -78,6 +78,7 @@ type simBidReq struct {
 type bidSimulator struct {
 	config        *MevConfig
 	delayLeftOver time.Duration
+	minGasPrice   *big.Int
 	chain         *core.BlockChain
 	chainConfig   *params.ChainConfig
 	engine        consensus.Engine
@@ -114,6 +115,7 @@ type bidSimulator struct {
 func newBidSimulator(
 	config *MevConfig,
 	delayLeftOver time.Duration,
+	minGasPrice *big.Int,
 	chain *core.BlockChain,
 	chainConfig *params.ChainConfig,
 	engine consensus.Engine,
@@ -122,6 +124,7 @@ func newBidSimulator(
 	b := &bidSimulator{
 		config:        config,
 		delayLeftOver: delayLeftOver,
+		minGasPrice:   minGasPrice,
 		chain:         chain,
 		chainConfig:   chainConfig,
 		engine:        engine,
@@ -592,6 +595,7 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) {
 		return
 	}
 
+	// commit transactions in bid
 	for _, tx := range bidRuntime.bid.Txs {
 		select {
 		case <-interruptCh:
@@ -609,7 +613,7 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) {
 			break
 		}
 
-		err = bidRuntime.commitTransaction(b.chain, b.chainConfig, tx)
+		err = bidRuntime.commitTransaction(b.chain, b.chainConfig, tx, bidRuntime.bid.UnRevertible.Contains(tx.Hash()))
 		if err != nil {
 			log.Error("BidSimulator: failed to commit tx", "bidHash", bidRuntime.bid.Hash(), "tx", tx.Hash(), "err", err)
 			err = fmt.Errorf("invalid tx in bid, %v", err)
@@ -617,15 +621,32 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) {
 		}
 	}
 
-	bidRuntime.packReward(b.config.ValidatorCommission)
+	// check if bid reward is valid
-
+	{
-	// return if bid is invalid, reportIssue issue to mev-sentry/builder if simulation is fully done
+		bidRuntime.packReward(b.config.ValidatorCommission)
-	if !bidRuntime.validReward() {
+		if !bidRuntime.validReward() {
-		err = errors.New("reward does not achieve the expectation")
+			err = errors.New("reward does not achieve the expectation")
-		return
+			return
+		}
 	}
 
-	// fill transactions from mempool
+	// check if bid gas price is lower than min gas price
+	{
+		bidGasUsed := uint64(0)
+		bidGasFee := bidRuntime.env.state.GetBalance(consensus.SystemAddress)
+
+		for _, receipt := range bidRuntime.env.receipts {
+			bidGasUsed += receipt.GasUsed
+		}
+
+		bidGasPrice := new(big.Int).Div(bidGasFee.ToBig(), new(big.Int).SetUint64(bidGasUsed))
+		if bidGasPrice.Cmp(b.minGasPrice) < 0 {
+			err = errors.New("bid gas price is lower than min gas price")
+			return
+		}
+	}
+
+	// if enable greedy merge, fill bid env with transactions from mempool
 	if b.config.GreedyMergeTx {
 		delay := b.engine.Delay(b.chain, bidRuntime.env.header, &b.delayLeftOver)
 		if delay != nil && *delay > 0 {
@@ -645,8 +666,9 @@ func (b *bidSimulator) simBid(interruptCh chan int32, bidRuntime *BidRuntime) {
 		}
 	}
 
+	// commit payBidTx at the end of the block
 	bidRuntime.env.gasPool.AddGas(params.PayBidTxGasLimit)
-	err = bidRuntime.commitTransaction(b.chain, b.chainConfig, payBidTx)
+	err = bidRuntime.commitTransaction(b.chain, b.chainConfig, payBidTx, true)
 	if err != nil {
 		log.Error("BidSimulator: failed to commit tx", "builder", bidRuntime.bid.Builder,
 			"bidHash", bidRuntime.bid.Hash(), "tx", payBidTx.Hash(), "err", err)
@@ -727,12 +749,10 @@ func (r *BidRuntime) packReward(validatorCommission uint64) {
 	r.packedValidatorReward.Sub(r.packedValidatorReward, r.bid.BuilderFee)
 }
 
-func (r *BidRuntime) commitTransaction(chain *core.BlockChain, chainConfig *params.ChainConfig, tx *types.Transaction) error {
+func (r *BidRuntime) commitTransaction(chain *core.BlockChain, chainConfig *params.ChainConfig, tx *types.Transaction, unRevertible bool) error {
 	var (
-		env  = r.env
+		env = r.env
-		snap = env.state.Snapshot()
+		sc  *types.BlobSidecar
-		gp   = env.gasPool.Gas()
-		sc   *types.BlobSidecar
 	)
 
 	// Start executing the transaction
@@ -755,9 +775,9 @@ func (r *BidRuntime) commitTransaction(chain *core.BlockChain, chainConfig *para
 	receipt, err := core.ApplyTransaction(chainConfig, chain, &env.coinbase, env.gasPool, env.state, env.header, tx,
 		&env.header.GasUsed, *chain.GetVMConfig(), core.NewReceiptBloomGenerator())
 	if err != nil {
-		env.state.RevertToSnapshot(snap)
-		env.gasPool.SetGas(gp)
 		return err
+	} else if unRevertible && receipt.Status == types.ReceiptStatusFailed {
+		return errors.New("no revertible transaction failed")
 	}
 
 	if tx.Type() == types.BlobTxType {

miner/miner.go

@@ -102,7 +102,7 @@ func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *even
 		worker:  newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, false),
 	}
 
-	miner.bidSimulator = newBidSimulator(&config.Mev, config.DelayLeftOver, eth.BlockChain(), chainConfig, engine, miner.worker)
+	miner.bidSimulator = newBidSimulator(&config.Mev, config.DelayLeftOver, config.GasPrice, eth.BlockChain(), chainConfig, engine, miner.worker)
 	miner.worker.setBestBidFetcher(miner.bidSimulator)
 
 	miner.wg.Add(1)
@@ -295,3 +295,7 @@ func (miner *Miner) SubscribePendingLogs(ch chan<- []*types.Log) event.Subscript
 func (miner *Miner) BuildPayload(args *BuildPayloadArgs) (*Payload, error) {
 	return miner.worker.buildPayload(args)
 }
+
+func (miner *Miner) GasCeil() uint64 {
+	return miner.worker.getGasCeil()
+}

miner/worker.go

@@ -329,6 +329,12 @@ func (w *worker) setGasCeil(ceil uint64) {
 	w.config.GasCeil = ceil
 }
 
+func (w *worker) getGasCeil() uint64 {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return w.config.GasCeil
+}
+
 // setExtra sets the content used to initialize the block extra field.
 func (w *worker) setExtra(extra []byte) {
 	w.mu.Lock()

node/node.go

@@ -773,12 +773,13 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r
 		db = rawdb.NewMemoryDatabase()
 	} else {
 		db, err = rawdb.Open(rawdb.OpenOptions{
-			Type:      n.config.DBEngine,
+			Type:          n.config.DBEngine,
-			Directory: n.ResolvePath(name),
+			Directory:     n.ResolvePath(name),
-			Namespace: namespace,
+			Namespace:     namespace,
-			Cache:     cache,
+			Cache:         cache,
-			Handles:   handles,
+			Handles:       handles,
-			ReadOnly:  readonly,
+			ReadOnly:      readonly,
+			MultiDataBase: n.CheckIfMultiDataBase(),
 		})
 	}
 

params/config.go

@@ -147,13 +147,13 @@ var (
 		LondonBlock:         big.NewInt(31302048),
 		HertzBlock:          big.NewInt(31302048),
 		HertzfixBlock:       big.NewInt(34140700),
-		// UnixTime: 1705996800 is January 23, 2024 8:00:00 AM UTC
+		ShanghaiTime:        newUint64(1705996800), // 2024-01-23 08:00:00 AM UTC
-		ShanghaiTime:   newUint64(1705996800),
+		KeplerTime:          newUint64(1705996800), // 2024-01-23 08:00:00 AM UTC
-		KeplerTime:     newUint64(1705996800),
+		FeynmanTime:         newUint64(1713419340), // 2024-04-18 05:49:00 AM UTC
-		FeynmanTime:    newUint64(1713419340),
+		FeynmanFixTime:      newUint64(1713419340), // 2024-04-18 05:49:00 AM UTC
-		FeynmanFixTime: newUint64(1713419340),
+		CancunTime:          newUint64(1718863500), // 2024-06-20 06:05:00 AM UTC
-		// TODO(GalaIO): enable cancun fork time later
+		HaberTime:           newUint64(1718863500), // 2024-06-20 06:05:00 AM UTC
-		//CancunTime: newUint64(),
+		BohrTime:            nil,
 
 		Parlia: &ParliaConfig{
 			Period: 3,
@@ -192,6 +192,8 @@ var (
 		FeynmanTime:         newUint64(1710136800), // 2024-03-11 6:00:00 AM UTC
 		FeynmanFixTime:      newUint64(1711342800), // 2024-03-25 5:00:00 AM UTC
 		CancunTime:          newUint64(1713330442), // 2024-04-17 05:07:22 AM UTC
+		HaberTime:           newUint64(1716962820), // 2024-05-29 06:07:00 AM UTC
+		BohrTime:            nil,
 
 		Parlia: &ParliaConfig{
 			Period: 3,
@@ -231,6 +233,8 @@ var (
 		FeynmanTime:         newUint64(0),
 		FeynmanFixTime:      newUint64(0),
 		CancunTime:          newUint64(0),
+		HaberTime:           newUint64(0),
+		BohrTime:            newUint64(0),
 
 		Parlia: &ParliaConfig{
 			Period: 3,
@@ -269,6 +273,7 @@ var (
 		FeynmanTime:         newUint64(0),
 		FeynmanFixTime:      newUint64(0),
 		CancunTime:          newUint64(0),
+
 		Parlia: &ParliaConfig{
 			Period: 3,
 			Epoch:  200,
@@ -506,6 +511,8 @@ type ChainConfig struct {
 	FeynmanTime    *uint64 `json:"feynmanTime,omitempty"`    // Feynman switch time (nil = no fork, 0 = already activated)
 	FeynmanFixTime *uint64 `json:"feynmanFixTime,omitempty"` // FeynmanFix switch time (nil = no fork, 0 = already activated)
 	CancunTime     *uint64 `json:"cancunTime,omitempty"`     // Cancun switch time (nil = no fork, 0 = already on cancun)
+	HaberTime      *uint64 `json:"haberTime,omitempty"`      // Haber switch time (nil = no fork, 0 = already on haber)
+	BohrTime       *uint64 `json:"bohrTime,omitempty"`       // Bohr switch time (nil = no fork, 0 = already on bohr)
 	PragueTime     *uint64 `json:"pragueTime,omitempty"`     // Prague switch time (nil = no fork, 0 = already on prague)
 	VerkleTime     *uint64 `json:"verkleTime,omitempty"`     // Verkle switch time (nil = no fork, 0 = already on verkle)
 
@@ -611,7 +618,17 @@ func (c *ChainConfig) String() string {
 		CancunTime = big.NewInt(0).SetUint64(*c.CancunTime)
 	}
 
-	return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, Ramanujan: %v, Niels: %v, MirrorSync: %v, Bruno: %v, Berlin: %v, YOLO v3: %v, CatalystBlock: %v, London: %v, ArrowGlacier: %v, MergeFork:%v, Euler: %v, Gibbs: %v, Nano: %v, Moran: %v, Planck: %v,Luban: %v, Plato: %v, Hertz: %v, Hertzfix: %v, ShanghaiTime: %v, KeplerTime: %v, FeynmanTime: %v, FeynmanFixTime: %v, CancunTime: %v, Engine: %v}",
+	var HaberTime *big.Int
+	if c.HaberTime != nil {
+		HaberTime = big.NewInt(0).SetUint64(*c.HaberTime)
+	}
+
+	var BohrTime *big.Int
+	if c.BohrTime != nil {
+		BohrTime = big.NewInt(0).SetUint64(*c.BohrTime)
+	}
+
+	return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, Ramanujan: %v, Niels: %v, MirrorSync: %v, Bruno: %v, Berlin: %v, YOLO v3: %v, CatalystBlock: %v, London: %v, ArrowGlacier: %v, MergeFork:%v, Euler: %v, Gibbs: %v, Nano: %v, Moran: %v, Planck: %v,Luban: %v, Plato: %v, Hertz: %v, Hertzfix: %v, ShanghaiTime: %v, KeplerTime: %v, FeynmanTime: %v, FeynmanFixTime: %v, CancunTime: %v, HaberTime: %v, BohrTime: %v, Engine: %v}",
 		c.ChainID,
 		c.HomesteadBlock,
 		c.DAOForkBlock,
@@ -648,6 +665,8 @@ func (c *ChainConfig) String() string {
 		FeynmanTime,
 		FeynmanFixTime,
 		CancunTime,
+		HaberTime,
+		BohrTime,
 		engine,
 	)
 }
@@ -915,6 +934,25 @@ func (c *ChainConfig) IsCancun(num *big.Int, time uint64) bool {
 	return c.IsLondon(num) && isTimestampForked(c.CancunTime, time)
 }
 
+// IsHaber returns whether time is either equal to the Haber fork time or greater.
+func (c *ChainConfig) IsHaber(num *big.Int, time uint64) bool {
+	return c.IsLondon(num) && isTimestampForked(c.HaberTime, time)
+}
+
+// IsBohr returns whether time is either equal to the Bohr fork time or greater.
+func (c *ChainConfig) IsBohr(num *big.Int, time uint64) bool {
+	return c.IsLondon(num) && isTimestampForked(c.BohrTime, time)
+}
+
+// IsOnBohr returns whether currentBlockTime is either equal to the Bohr fork time or greater firstly.
+func (c *ChainConfig) IsOnBohr(currentBlockNumber *big.Int, lastBlockTime uint64, currentBlockTime uint64) bool {
+	lastBlockNumber := new(big.Int)
+	if currentBlockNumber.Cmp(big.NewInt(1)) >= 0 {
+		lastBlockNumber.Sub(currentBlockNumber, big.NewInt(1))
+	}
+	return !c.IsBohr(lastBlockNumber, lastBlockTime) && c.IsBohr(currentBlockNumber, currentBlockTime)
+}
+
 // IsPrague returns whether num is either equal to the Prague fork time or greater.
 func (c *ChainConfig) IsPrague(num *big.Int, time uint64) bool {
 	return c.IsLondon(num) && isTimestampForked(c.PragueTime, time)
@@ -978,6 +1016,8 @@ func (c *ChainConfig) CheckConfigForkOrder() error {
 		{name: "feynmanTime", timestamp: c.FeynmanTime},
 		{name: "feynmanFixTime", timestamp: c.FeynmanFixTime},
 		{name: "cancunTime", timestamp: c.CancunTime},
+		{name: "haberTime", timestamp: c.HaberTime},
+		{name: "bohrTime", timestamp: c.BohrTime},
 		{name: "pragueTime", timestamp: c.PragueTime, optional: true},
 		{name: "verkleTime", timestamp: c.VerkleTime, optional: true},
 	} {
@@ -1307,8 +1347,8 @@ type Rules struct {
 	IsPlato                                                 bool
 	IsHertz                                                 bool
 	IsHertzfix                                              bool
-	IsShanghai, IsKepler, IsFeynman, IsCancun, IsPrague     bool
+	IsShanghai, IsKepler, IsFeynman, IsCancun, IsHaber      bool
-	IsVerkle                                                bool
+	IsBohr, IsPrague, IsVerkle                              bool
 }
 
 // Rules ensures c's ChainID is not nil.
@@ -1343,6 +1383,8 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool, timestamp uint64) Rules
 		IsKepler:         c.IsKepler(num, timestamp),
 		IsFeynman:        c.IsFeynman(num, timestamp),
 		IsCancun:         c.IsCancun(num, timestamp),
+		IsHaber:          c.IsHaber(num, timestamp),
+		IsBohr:           c.IsBohr(num, timestamp),
 		IsPrague:         c.IsPrague(num, timestamp),
 		IsVerkle:         c.IsVerkle(num, timestamp),
 	}

params/protocol_params.go

@@ -170,6 +170,8 @@ const (
 	Bls12381MapG1Gas          uint64 = 5500   // Gas price for BLS12-381 mapping field element to G1 operation
 	Bls12381MapG2Gas          uint64 = 110000 // Gas price for BLS12-381 mapping field element to G2 operation
 
+	P256VerifyGas uint64 = 3450 // secp256r1 elliptic curve signature verifier gas price
+
 	// The Refund Quotient is the cap on how much of the used gas can be refunded. Before EIP-3529,
 	// up to half the consumed gas could be refunded. Redefined as 1/5th in EIP-3529
 	RefundQuotient        uint64 = 2
@@ -189,6 +191,7 @@ const (
 var (
 	MinBlocksForBlobRequests           uint64 = 524288              // it keeps blob data available for ~18.2 days in local, ref: https://github.com/bnb-chain/BEPs/blob/master/BEPs/BEP-336.md#51-parameters.
 	DefaultExtraReserveForBlobRequests uint64 = 1 * (24 * 3600) / 3 // it adds more time for expired blobs for some request cases, like expiry blob when remote peer is syncing, default 1 day.
+	BreatheBlockInterval               uint64 = 86400               // Controls the interval for updateValidatorSetV2
 )
 
 // Gas discount table for BLS12-381 G1 and G2 multi exponentiation operations

params/version.go

@@ -23,7 +23,7 @@ import (
 const (
 	VersionMajor = 1  // Major version component of the current release
 	VersionMinor = 4  // Minor version component of the current release
-	VersionPatch = 6  // Patch version component of the current release
+	VersionPatch = 8  // Patch version component of the current release
 	VersionMeta  = "" // Version metadata to append to the version string
 )
 

trie/hbss2pbss.go

@@ -181,7 +181,7 @@ func (h2p *Hbss2Pbss) ConcurrentTraversal(theTrie *Trie, theNode node, path []by
 	}
 }
 
-// copy from trie/Commiter (*committer).commit
+// copy from trie/Committer (*committer).commit
 func (h2p *Hbss2Pbss) commitChildren(path []byte, n *fullNode) [17]node {
 	var children [17]node
 	for i := 0; i < 16; i++ {

trie/inspect_trie.go

@@ -4,17 +4,15 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"math/big"
-	"os"
 	"runtime"
-	"sort"
+	"strings"
-	"strconv"
 	"sync"
 	"sync/atomic"
 
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/mclock"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -26,63 +24,113 @@ import (
 	"golang.org/x/sync/semaphore"
 )
 
-type Account struct {
-	Nonce    uint64
-	Balance  *big.Int
-	Root     common.Hash // merkle root of the storage trie
-	CodeHash []byte
-}
-
 type Database interface {
 	database.Database
 	Scheme() string
 	Cap(limit common.StorageSize) error
 	DiskDB() ethdb.Database
 }
+
+const TopN = 3
+
 type Inspector struct {
 	trie           *Trie // traverse trie
 	db             Database
 	stateRootHash  common.Hash
-	blocknum       uint64
+	blockNum       uint64
 	root           node // root of triedb
-	totalNum       uint64
-	wg             sync.WaitGroup
-	statLock       sync.RWMutex
-	result         map[string]*TrieTreeStat
 	sem            *semaphore.Weighted
 	eoaAccountNums uint64
+
+	wg sync.WaitGroup
+
+	results stat
+	topN    int
+
+	totalAccountNum atomic.Uint64
+	totalStorageNum atomic.Uint64
+	lastTime        mclock.AbsTime
 }
 
-type TrieTreeStat struct {
+type stat struct {
-	is_account_trie    bool
+	lock             sync.RWMutex
-	theNodeStatByLevel [15]NodeStat
+	account          *trieStat
-	totalNodeStat      NodeStat
+	storageTopN      []*trieStat
+	storageTopNTotal []uint64
+	storageTotal     nodeStat
+	storageTrieNum   uint64
 }
 
-type NodeStat struct {
+type trieStat struct {
-	ShortNodeCnt uint64
+	owner           common.Hash
-	FullNodeCnt  uint64
+	totalNodeStat   nodeStat
-	ValueNodeCnt uint64
+	nodeStatByLevel [16]nodeStat
 }
 
-func (trieStat *TrieTreeStat) AtomicAdd(theNode node, height uint32) {
+type nodeStat struct {
+	ShortNodeCnt atomic.Uint64
+	FullNodeCnt  atomic.Uint64
+	ValueNodeCnt atomic.Uint64
+}
+
+func (ns *nodeStat) IsEmpty() bool {
+	if ns.FullNodeCnt.Load() == 0 && ns.ShortNodeCnt.Load() == 0 && ns.ValueNodeCnt.Load() == 0 {
+		return true
+	}
+	return false
+}
+
+func (s *stat) add(ts *trieStat, topN int) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	if ts.owner == (common.Hash{}) {
+		s.account = ts
+		return
+	}
+
+	total := ts.totalNodeStat.ValueNodeCnt.Load() + ts.totalNodeStat.FullNodeCnt.Load() + ts.totalNodeStat.ShortNodeCnt.Load()
+	if len(s.storageTopNTotal) == 0 || total > s.storageTopNTotal[len(s.storageTopNTotal)-1] {
+		var (
+			i int
+			t uint64
+		)
+		for i, t = range s.storageTopNTotal {
+			if total < t {
+				continue
+			}
+			break
+		}
+		s.storageTopNTotal = append(s.storageTopNTotal[:i], append([]uint64{total}, s.storageTopNTotal[i:]...)...)
+		s.storageTopN = append(s.storageTopN[:i], append([]*trieStat{ts}, s.storageTopN[i:]...)...)
+		if len(s.storageTopN) > topN {
+			s.storageTopNTotal = s.storageTopNTotal[:topN]
+			s.storageTopN = s.storageTopN[:topN]
+		}
+	}
+
+	s.storageTotal.ShortNodeCnt.Add(ts.totalNodeStat.ShortNodeCnt.Load())
+	s.storageTotal.ValueNodeCnt.Add(ts.totalNodeStat.ValueNodeCnt.Load())
+	s.storageTotal.FullNodeCnt.Add(ts.totalNodeStat.FullNodeCnt.Load())
+	s.storageTrieNum++
+}
+
+func (trieStat *trieStat) add(theNode node, height int) {
 	switch (theNode).(type) {
 	case *shortNode:
-		atomic.AddUint64(&trieStat.totalNodeStat.ShortNodeCnt, 1)
+		trieStat.totalNodeStat.ShortNodeCnt.Add(1)
-		atomic.AddUint64(&(trieStat.theNodeStatByLevel[height].ShortNodeCnt), 1)
+		trieStat.nodeStatByLevel[height].ShortNodeCnt.Add(1)
 	case *fullNode:
-		atomic.AddUint64(&trieStat.totalNodeStat.FullNodeCnt, 1)
+		trieStat.totalNodeStat.FullNodeCnt.Add(1)
-		atomic.AddUint64(&trieStat.theNodeStatByLevel[height].FullNodeCnt, 1)
+		trieStat.nodeStatByLevel[height].FullNodeCnt.Add(1)
 	case valueNode:
-		atomic.AddUint64(&trieStat.totalNodeStat.ValueNodeCnt, 1)
+		trieStat.totalNodeStat.ValueNodeCnt.Add(1)
-		atomic.AddUint64(&((trieStat.theNodeStatByLevel[height]).ValueNodeCnt), 1)
+		trieStat.nodeStatByLevel[height].ValueNodeCnt.Add(1)
-	default:
-		panic(errors.New("Invalid node type to statistics"))
 	}
 }
 
-func (trieStat *TrieTreeStat) Display(ownerAddress string, treeType string) {
+func (trieStat *trieStat) Display(ownerAddress string, treeType string) string {
-	table := tablewriter.NewWriter(os.Stdout)
+	sw := new(strings.Builder)
+	table := tablewriter.NewWriter(sw)
 	table.SetHeader([]string{"-", "Level", "ShortNodeCnt", "FullNodeCnt", "ValueNodeCnt"})
 	if ownerAddress == "" {
 		table.SetCaption(true, fmt.Sprintf("%v", treeType))
@@ -90,38 +138,27 @@ func (trieStat *TrieTreeStat) Display(ownerAddress string, treeType string) {
 		table.SetCaption(true, fmt.Sprintf("%v-%v", treeType, ownerAddress))
 	}
 	table.SetAlignment(1)
-	for i := 0; i < len(trieStat.theNodeStatByLevel); i++ {
+
-		nodeStat := trieStat.theNodeStatByLevel[i]
+	for i := range trieStat.nodeStatByLevel {
-		if nodeStat.FullNodeCnt == 0 && nodeStat.ShortNodeCnt == 0 && nodeStat.ValueNodeCnt == 0 {
+		if trieStat.nodeStatByLevel[i].IsEmpty() {
-			break
+			continue
 		}
 		table.AppendBulk([][]string{
-			{"-", strconv.Itoa(i), nodeStat.ShortNodeCount(), nodeStat.FullNodeCount(), nodeStat.ValueNodeCount()},
+			{"-", fmt.Sprintf("%d", i),
+				fmt.Sprintf("%d", trieStat.nodeStatByLevel[i].ShortNodeCnt.Load()),
+				fmt.Sprintf("%d", trieStat.nodeStatByLevel[i].FullNodeCnt.Load()),
+				fmt.Sprintf("%d", trieStat.nodeStatByLevel[i].ValueNodeCnt.Load())},
 		})
 	}
 	table.AppendBulk([][]string{
-		{"Total", "-", trieStat.totalNodeStat.ShortNodeCount(), trieStat.totalNodeStat.FullNodeCount(), trieStat.totalNodeStat.ValueNodeCount()},
+		{"Total", "-", fmt.Sprintf("%d", trieStat.totalNodeStat.ShortNodeCnt.Load()), fmt.Sprintf("%d", trieStat.totalNodeStat.FullNodeCnt.Load()), fmt.Sprintf("%d", trieStat.totalNodeStat.ValueNodeCnt.Load())},
 	})
 	table.Render()
-}
+	return sw.String()
-
-func Uint64ToString(cnt uint64) string {
-	return fmt.Sprintf("%v", cnt)
-}
-
-func (nodeStat *NodeStat) ShortNodeCount() string {
-	return Uint64ToString(nodeStat.ShortNodeCnt)
-}
-
-func (nodeStat *NodeStat) FullNodeCount() string {
-	return Uint64ToString(nodeStat.FullNodeCnt)
-}
-func (nodeStat *NodeStat) ValueNodeCount() string {
-	return Uint64ToString(nodeStat.ValueNodeCnt)
 }
 
 // NewInspector return a inspector obj
-func NewInspector(tr *Trie, db Database, stateRootHash common.Hash, blocknum uint64, jobnum uint64) (*Inspector, error) {
+func NewInspector(tr *Trie, db Database, stateRootHash common.Hash, blockNum uint64, jobNum uint64, topN int) (*Inspector, error) {
 	if tr == nil {
 		return nil, errors.New("trie is nil")
 	}
@@ -131,15 +168,20 @@ func NewInspector(tr *Trie, db Database, stateRootHash common.Hash, blocknum uin
 	}
 
 	ins := &Inspector{
-		trie:           tr,
+		trie:            tr,
-		db:             db,
+		db:              db,
-		stateRootHash:  stateRootHash,
+		stateRootHash:   stateRootHash,
-		blocknum:       blocknum,
+		blockNum:        blockNum,
-		root:           tr.root,
+		root:            tr.root,
-		result:         make(map[string]*TrieTreeStat),
+		results:         stat{},
-		totalNum:       (uint64)(0),
+		topN:            topN,
-		wg:             sync.WaitGroup{},
+		totalAccountNum: atomic.Uint64{},
-		sem:            semaphore.NewWeighted(int64(jobnum)),
+		totalStorageNum: atomic.Uint64{},
+		lastTime:        mclock.Now(),
+		sem:             semaphore.NewWeighted(int64(jobNum)),
+
+		wg: sync.WaitGroup{},
+
 		eoaAccountNums: 0,
 	}
 
@@ -147,155 +189,123 @@ func NewInspector(tr *Trie, db Database, stateRootHash common.Hash, blocknum uin
 }
 
 // Run statistics, external call
-func (inspect *Inspector) Run() {
+func (s *Inspector) Run() {
-	accountTrieStat := &TrieTreeStat{
+	ticker := time.NewTicker(30 * time.Second)
-		is_account_trie: true,
+	go func() {
-	}
+		defer ticker.Stop()
-	if inspect.db.Scheme() == rawdb.HashScheme {
+		for range ticker.C {
-		ticker := time.NewTicker(30 * time.Second)
+			if s.db.Scheme() == rawdb.HashScheme {
-		go func() {
+				s.db.Cap(DEFAULT_TRIEDBCACHE_SIZE)
-			defer ticker.Stop()
-			for range ticker.C {
-				inspect.db.Cap(DEFAULT_TRIEDBCACHE_SIZE)
 			}
-		}()
+			runtime.GC()
-	}
+		}
+	}()
 
-	if _, ok := inspect.result[""]; !ok {
+	log.Info("Find Account Trie Tree", "rootHash: ", s.trie.Hash().String(), "BlockNum: ", s.blockNum)
-		inspect.result[""] = accountTrieStat
-	}
-	log.Info("Find Account Trie Tree", "rootHash: ", inspect.trie.Hash().String(), "BlockNum: ", inspect.blocknum)
 
-	inspect.ConcurrentTraversal(inspect.trie, accountTrieStat, inspect.root, 0, []byte{})
+	ts := &trieStat{
-	inspect.wg.Wait()
+		owner: common.Hash{},
+	}
+	s.traversal(s.trie, ts, s.root, 0, []byte{})
+	s.results.add(ts, s.topN)
+	s.wg.Wait()
 }
 
-func (inspect *Inspector) SubConcurrentTraversal(theTrie *Trie, theTrieTreeStat *TrieTreeStat, theNode node, height uint32, path []byte) {
+func (s *Inspector) traversal(trie *Trie, ts *trieStat, n node, height int, path []byte) {
-	inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, theNode, height, path)
-	inspect.wg.Done()
-}
-
-func (inspect *Inspector) ConcurrentTraversal(theTrie *Trie, theTrieTreeStat *TrieTreeStat, theNode node, height uint32, path []byte) {
-	// print process progress
-	total_num := atomic.AddUint64(&inspect.totalNum, 1)
-	if total_num%100000 == 0 {
-		fmt.Printf("Complete progress: %v, go routines Num: %v\n", total_num, runtime.NumGoroutine())
-	}
-
 	// nil node
-	if theNode == nil {
+	if n == nil {
 		return
 	}
 
-	switch current := (theNode).(type) {
+	ts.add(n, height)
+
+	switch current := (n).(type) {
 	case *shortNode:
-		inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, current.Val, height, append(path, current.Key...))
+		s.traversal(trie, ts, current.Val, height, append(path, current.Key...))
 	case *fullNode:
 		for idx, child := range current.Children {
 			if child == nil {
 				continue
 			}
-			childPath := append(path, byte(idx))
+			p := common.CopyBytes(append(path, byte(idx)))
-			if inspect.sem.TryAcquire(1) {
+			s.traversal(trie, ts, child, height+1, p)
-				inspect.wg.Add(1)
-				dst := make([]byte, len(childPath))
-				copy(dst, childPath)
-				go inspect.SubConcurrentTraversal(theTrie, theTrieTreeStat, child, height+1, dst)
-			} else {
-				inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, child, height+1, childPath)
-			}
 		}
 	case hashNode:
-		n, err := theTrie.resloveWithoutTrack(current, path)
+		tn, err := trie.resloveWithoutTrack(current, path)
 		if err != nil {
-			fmt.Printf("Resolve HashNode error: %v, TrieRoot: %v, Height: %v, Path: %v\n", err, theTrie.Hash().String(), height+1, path)
+			fmt.Printf("Resolve HashNode error: %v, TrieRoot: %v, Height: %v, Path: %v\n", err, trie.Hash().String(), height+1, path)
 			return
 		}
-		inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, n, height, path)
+		s.PrintProgress(trie)
-		return
+		s.traversal(trie, ts, tn, height, path)
 	case valueNode:
 		if !hasTerm(path) {
 			break
 		}
-		var account Account
+		var account types.StateAccount
 		if err := rlp.Decode(bytes.NewReader(current), &account); err != nil {
 			break
 		}
 		if common.BytesToHash(account.CodeHash) == types.EmptyCodeHash {
-			inspect.eoaAccountNums++
+			s.eoaAccountNums++
 		}
 		if account.Root == (common.Hash{}) || account.Root == types.EmptyRootHash {
 			break
 		}
 		ownerAddress := common.BytesToHash(hexToCompact(path))
-		contractTrie, err := New(StorageTrieID(inspect.stateRootHash, ownerAddress, account.Root), inspect.db)
+		contractTrie, err := New(StorageTrieID(s.stateRootHash, ownerAddress, account.Root), s.db)
 		if err != nil {
-			fmt.Printf("New contract trie node: %v, error: %v, Height: %v, Path: %v\n", theNode, err, height, path)
+			panic(err)
-			break
 		}
 		contractTrie.tracer.reset()
-		trieStat := &TrieTreeStat{
-			is_account_trie: false,
-		}
 
-		inspect.statLock.Lock()
+		if s.sem.TryAcquire(1) {
-		if _, ok := inspect.result[ownerAddress.String()]; !ok {
+			s.wg.Add(1)
-			inspect.result[ownerAddress.String()] = trieStat
+			go func() {
+				t := &trieStat{
+					owner: ownerAddress,
+				}
+				s.traversal(contractTrie, t, contractTrie.root, 0, []byte{})
+				s.results.add(t, s.topN)
+				s.sem.Release(1)
+				s.wg.Done()
+			}()
+		} else {
+			t := &trieStat{
+				owner: ownerAddress,
+			}
+			s.traversal(contractTrie, t, contractTrie.root, 0, []byte{})
+			s.results.add(t, s.topN)
 		}
-		inspect.statLock.Unlock()
-
-		// log.Info("Find Contract Trie Tree, rootHash: ", contractTrie.Hash().String(), "")
-		inspect.wg.Add(1)
-		go inspect.SubConcurrentTraversal(contractTrie, trieStat, contractTrie.root, 0, []byte{})
 	default:
-		panic(errors.New("Invalid node type to traverse."))
+		panic(errors.New("invalid node type to traverse"))
 	}
-	theTrieTreeStat.AtomicAdd(theNode, height)
 }
 
-func (inspect *Inspector) DisplayResult() {
+func (s *Inspector) PrintProgress(t *Trie) {
+	var (
+		elapsed = mclock.Now().Sub(s.lastTime)
+	)
+	if t.owner == (common.Hash{}) {
+		s.totalAccountNum.Add(1)
+	} else {
+		s.totalStorageNum.Add(1)
+	}
+	if elapsed > 4*time.Second {
+		log.Info("traversal progress", "TotalAccountNum", s.totalAccountNum.Load(), "TotalStorageNum", s.totalStorageNum.Load(), "Goroutine", runtime.NumGoroutine())
+		s.lastTime = mclock.Now()
+	}
+}
+
+func (s *Inspector) DisplayResult() {
 	// display root hash
-	if _, ok := inspect.result[""]; !ok {
+	fmt.Println(s.results.account.Display("", "AccountTrie"))
-		log.Info("Display result error", "missing account trie")
+	fmt.Println("EOA accounts num: ", s.eoaAccountNums)
-		return
-	}
-	inspect.result[""].Display("", "AccountTrie")
 
-	type SortedTrie struct {
-		totalNum     uint64
-		ownerAddress string
-	}
 	// display contract trie
-	var sortedTriesByNums []SortedTrie
+	for _, st := range s.results.storageTopN {
-	var totalContactsNodeStat NodeStat
+		fmt.Println(st.Display(st.owner.String(), "StorageTrie"))
-	var contractTrieCnt uint64 = 0
-
-	for ownerAddress, stat := range inspect.result {
-		if ownerAddress == "" {
-			continue
-		}
-		contractTrieCnt++
-		totalContactsNodeStat.ShortNodeCnt += stat.totalNodeStat.ShortNodeCnt
-		totalContactsNodeStat.FullNodeCnt += stat.totalNodeStat.FullNodeCnt
-		totalContactsNodeStat.ValueNodeCnt += stat.totalNodeStat.ValueNodeCnt
-		totalNodeCnt := stat.totalNodeStat.ShortNodeCnt + stat.totalNodeStat.ValueNodeCnt + stat.totalNodeStat.FullNodeCnt
-		sortedTriesByNums = append(sortedTriesByNums, SortedTrie{totalNum: totalNodeCnt, ownerAddress: ownerAddress})
-	}
-	sort.Slice(sortedTriesByNums, func(i, j int) bool {
-		return sortedTriesByNums[i].totalNum > sortedTriesByNums[j].totalNum
-	})
-	fmt.Println("EOA accounts num: ", inspect.eoaAccountNums)
-	// only display top 5
-	for i, t := range sortedTriesByNums {
-		if i > 5 {
-			break
-		}
-		if stat, ok := inspect.result[t.ownerAddress]; !ok {
-			log.Error("Storage trie stat not found", "ownerAddress", t.ownerAddress)
-		} else {
-			stat.Display(t.ownerAddress, "ContractTrie")
-		}
 	}
 	fmt.Printf("Contract Trie, total trie num: %v, ShortNodeCnt: %v, FullNodeCnt: %v, ValueNodeCnt: %v\n",
-		contractTrieCnt, totalContactsNodeStat.ShortNodeCnt, totalContactsNodeStat.FullNodeCnt, totalContactsNodeStat.ValueNodeCnt)
+		s.results.storageTrieNum, s.results.storageTotal.ShortNodeCnt.Load(), s.results.storageTotal.FullNodeCnt.Load(), s.results.storageTotal.ValueNodeCnt.Load())
 }

triedb/pathdb/database_test.go

@@ -98,7 +98,7 @@ type tester struct {
 
 func newTester(t *testing.T, historyLimit uint64) *tester {
 	var (
-		disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false)
+		disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, false, false, false, false)
 		db      = New(disk, &Config{
 			StateHistory:   historyLimit,
 			CleanCacheSize: 256 * 1024,

triedb/pathdb/journal.go

@@ -152,12 +152,13 @@ func (kr *JournalKVReader) Close() {
 }
 
 func newJournalWriter(file string, db ethdb.Database, journalType JournalType) JournalWriter {
-	log.Info("New journal writer", "path", file, "journalType", journalType)
 	if journalType == JournalKVType {
+		log.Info("New journal writer for journal kv")
 		return &JournalKVWriter{
 			diskdb: db,
 		}
 	} else {
+		log.Info("New journal writer for journal file", "path", file)
 		fd, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
 		if err != nil {
 			return nil
@@ -169,8 +170,8 @@ func newJournalWriter(file string, db ethdb.Database, journalType JournalType) J
 }
 
 func newJournalReader(file string, db ethdb.Database, journalType JournalType) (JournalReader, error) {
-	log.Info("New journal reader", "path", file, "journalType", journalType)
 	if journalType == JournalKVType {
+		log.Info("New journal reader for journal kv")
 		journal := rawdb.ReadTrieJournal(db)
 		if len(journal) == 0 {
 			return nil, errMissJournal
@@ -179,6 +180,7 @@ func newJournalReader(file string, db ethdb.Database, journalType JournalType) (
 			journalBuf: bytes.NewBuffer(journal),
 		}, nil
 	} else {
+		log.Info("New journal reader for journal file", "path", file)
 		fd, err := os.Open(file)
 		if errors.Is(err, fs.ErrNotExist) {
 			return nil, errMissJournal
@@ -195,7 +197,8 @@ func newJournalReader(file string, db ethdb.Database, journalType JournalType) (
 // loadJournal tries to parse the layer journal from the disk.
 func (db *Database) loadJournal(diskRoot common.Hash) (layer, error) {
 	start := time.Now()
-	reader, err := newJournalReader(db.config.JournalFilePath, db.diskdb, db.DetermineJournalTypeForReader())
+	journalTypeForReader := db.DetermineJournalTypeForReader()
+	reader, err := newJournalReader(db.config.JournalFilePath, db.diskdb, journalTypeForReader)
 
 	if err != nil {
 		return nil, err
@@ -226,12 +229,12 @@ func (db *Database) loadJournal(diskRoot common.Hash) (layer, error) {
 		return nil, fmt.Errorf("%w want %x got %x", errUnmatchedJournal, root, diskRoot)
 	}
 	// Load the disk layer from the journal
-	base, err := db.loadDiskLayer(r)
+	base, err := db.loadDiskLayer(r, journalTypeForReader)
 	if err != nil {
 		return nil, err
 	}
 	// Load all the diff layers from the journal
-	head, err := db.loadDiffLayer(base, r)
+	head, err := db.loadDiffLayer(base, r, journalTypeForReader)
 	if err != nil {
 		return nil, err
 	}
@@ -262,14 +265,14 @@ func (db *Database) loadLayers() layer {
 
 // loadDiskLayer reads the binary blob from the layer journal, reconstructing
 // a new disk layer on it.
-func (db *Database) loadDiskLayer(r *rlp.Stream) (layer, error) {
+func (db *Database) loadDiskLayer(r *rlp.Stream, journalTypeForReader JournalType) (layer, error) {
 	// Resolve disk layer root
 	var (
 		root               common.Hash
 		journalBuf         *rlp.Stream
 		journalEncodedBuff []byte
 	)
-	if db.DetermineJournalTypeForReader() == JournalFileType {
+	if journalTypeForReader == JournalFileType {
 		if err := r.Decode(&journalEncodedBuff); err != nil {
 			return nil, fmt.Errorf("load disk journal: %v", err)
 		}
@@ -310,7 +313,7 @@ func (db *Database) loadDiskLayer(r *rlp.Stream) (layer, error) {
 		nodes[entry.Owner] = subset
 	}
 
-	if db.DetermineJournalTypeForReader() == JournalFileType {
+	if journalTypeForReader == JournalFileType {
 		var shaSum [32]byte
 		if err := r.Decode(&shaSum); err != nil {
 			return nil, fmt.Errorf("load shasum: %v", err)
@@ -329,14 +332,14 @@ func (db *Database) loadDiskLayer(r *rlp.Stream) (layer, error) {
 
 // loadDiffLayer reads the next sections of a layer journal, reconstructing a new
 // diff and verifying that it can be linked to the requested parent.
-func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream) (layer, error) {
+func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream, journalTypeForReader JournalType) (layer, error) {
 	// Read the next diff journal entry
 	var (
 		root               common.Hash
 		journalBuf         *rlp.Stream
 		journalEncodedBuff []byte
 	)
-	if db.DetermineJournalTypeForReader() == JournalFileType {
+	if journalTypeForReader == JournalFileType {
 		if err := r.Decode(&journalEncodedBuff); err != nil {
 			// The first read may fail with EOF, marking the end of the journal
 			if err == io.EOF {
@@ -409,7 +412,7 @@ func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream) (layer, error) {
 		storages[entry.Account] = set
 	}
 
-	if db.DetermineJournalTypeForReader() == JournalFileType {
+	if journalTypeForReader == JournalFileType {
 		var shaSum [32]byte
 		if err := r.Decode(&shaSum); err != nil {
 			return nil, fmt.Errorf("load shasum: %v", err)
@@ -423,7 +426,7 @@ func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream) (layer, error) {
 
 	log.Debug("Loaded diff layer journal", "root", root, "parent", parent.rootHash(), "id", parent.stateID()+1, "block", block)
 
-	return db.loadDiffLayer(newDiffLayer(parent, root, parent.stateID()+1, block, nodes, triestate.New(accounts, storages, incomplete)), r)
+	return db.loadDiffLayer(newDiffLayer(parent, root, parent.stateID()+1, block, nodes, triestate.New(accounts, storages, incomplete)), r, journalTypeForReader)
 }
 
 // journal implements the layer interface, marshaling the un-flushed trie nodes