fix: fillTransactions func add time metrics

merge some PRs for v1.4.7(2nd)

.nancy-ignore

@@ -0,0 +1 @@
+CVE-2024-34478 # "CWE-754: Improper Check for Unusual or Exceptional Conditions." This vulnerability is BTC only, BSC does not have the issue.

CHANGELOG.md

@@ -1,4 +1,18 @@
 # Changelog
+## v1.4.7
+### FEATURE
+* [\#2439](https://github.com/bnb-chain/bsc/pull/2439) config: setup Mainnet Tycho(Cancun) hardfork date
+
+### IMPROVEMENT
+* [\#2396](https://github.com/bnb-chain/bsc/pull/2396) metrics: add blockInsertMgaspsGauge to trace mgasps
+* [\#2411](https://github.com/bnb-chain/bsc/pull/2411) build(deps): bump golang.org/x/net from 0.19.0 to 0.23.0
+* [\#2435](https://github.com/bnb-chain/bsc/pull/2435) txpool: limit max gas when mining is enabled
+* [\#2438](https://github.com/bnb-chain/bsc/pull/2438) fix: performance issue when load journal
+* [\#2440](https://github.com/bnb-chain/bsc/pull/2440) nancy: add files .nancy-ignore
+
+### BUGFIX
+NA
+
 ## v1.4.6
 ### FEATURE
 * [\#2227](https://github.com/bnb-chain/bsc/pull/2227) core: separated databases for block data

cmd/geth/config.go

@@ -193,14 +193,6 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 		v := ctx.Uint64(utils.OverrideVerkle.Name)
 		cfg.Eth.OverrideVerkle = &v
 	}
-	if ctx.IsSet(utils.OverrideFeynman.Name) {
-		v := ctx.Uint64(utils.OverrideFeynman.Name)
-		cfg.Eth.OverrideFeynman = &v
-	}
-	if ctx.IsSet(utils.OverrideFeynmanFix.Name) {
-		v := ctx.Uint64(utils.OverrideFeynmanFix.Name)
-		cfg.Eth.OverrideFeynmanFix = &v
-	}
 	if ctx.IsSet(utils.OverrideFullImmutabilityThreshold.Name) {
 		params.FullImmutabilityThreshold = ctx.Uint64(utils.OverrideFullImmutabilityThreshold.Name)
 		downloader.FullMaxForkAncestry = ctx.Uint64(utils.OverrideFullImmutabilityThreshold.Name)
@@ -211,6 +203,9 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 	if ctx.IsSet(utils.OverrideDefaultExtraReserveForBlobRequests.Name) {
 		params.DefaultExtraReserveForBlobRequests = ctx.Uint64(utils.OverrideDefaultExtraReserveForBlobRequests.Name)
 	}
+	if ctx.IsSet(utils.OverrideBreatheBlockInterval.Name) {
+		params.BreatheBlockInterval = ctx.Uint64(utils.OverrideBreatheBlockInterval.Name)
+	}
 
 	backend, eth := utils.RegisterEthService(stack, &cfg.Eth)
 
@@ -285,6 +280,7 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) {
 	if ctx.IsSet(utils.MetricsEnabledExpensiveFlag.Name) {
 		cfg.Metrics.EnabledExpensive = ctx.Bool(utils.MetricsEnabledExpensiveFlag.Name)
 	}
+	cfg.Metrics.EnabledExpensive = true
 	if ctx.IsSet(utils.MetricsHTTPFlag.Name) {
 		cfg.Metrics.HTTP = ctx.String(utils.MetricsHTTPFlag.Name)
 	}

cmd/geth/main.go

@@ -25,6 +25,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ethereum/go-ethereum/params"
+
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -72,11 +74,10 @@ var (
 		utils.RialtoHash,
 		utils.OverrideCancun,
 		utils.OverrideVerkle,
-		utils.OverrideFeynman,
-		utils.OverrideFeynmanFix,
 		utils.OverrideFullImmutabilityThreshold,
 		utils.OverrideMinBlocksForBlobRequests,
 		utils.OverrideDefaultExtraReserveForBlobRequests,
+		utils.OverrideBreatheBlockInterval,
 		utils.EnablePersonal,
 		utils.TxPoolLocalsFlag,
 		utils.TxPoolNoLocalsFlag,
@@ -456,6 +457,10 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon
 		// Set the gas price to the limits from the CLI and start mining
 		gasprice := flags.GlobalBig(ctx, utils.MinerGasPriceFlag.Name)
 		ethBackend.TxPool().SetGasTip(gasprice)
+		gasCeil := ethBackend.Miner().GasCeil()
+		if gasCeil > params.SystemTxsGas {
+			ethBackend.TxPool().SetMaxGas(gasCeil - params.SystemTxsGas)
+		}
 		if err := ethBackend.StartMining(); err != nil {
 			utils.Fatalf("Failed to start mining: %v", err)
 		}

cmd/geth/snapshot.go

@@ -409,7 +409,7 @@ func pruneBlock(ctx *cli.Context) error {
 	}
 
 	if _, err := os.Stat(newAncientPath); err == nil {
-		// No file lock found for old ancientDB but new ancientDB exsisted, indicating the geth was interrupted
+		// No file lock found for old ancientDB but new ancientDB existed, indicating the geth was interrupted
 		// after old ancientDB removal, this happened after backup successfully, so just rename the new ancientDB
 		if err := blockpruner.AncientDbReplacer(); err != nil {
 			log.Error("Failed to rename new ancient directory")

cmd/utils/flags.go

@@ -315,15 +315,6 @@ var (
 		Usage:    "Manually specify the Verkle fork timestamp, overriding the bundled setting",
 		Category: flags.EthCategory,
 	}
-	OverrideFeynman = &cli.Uint64Flag{
-		Name:     "override.feynman",
-		Usage:    "Manually specify the Feynman fork timestamp, overriding the bundled setting",
-		Category: flags.EthCategory,
-	}
-	OverrideFeynmanFix = &cli.Uint64Flag{
-		Name:  "override.feynmanfix",
-		Usage: "Manually specify the FeynmanFix fork timestamp, overriding the bundled setting",
-	}
 	OverrideFullImmutabilityThreshold = &cli.Uint64Flag{
 		Name:     "override.immutabilitythreshold",
 		Usage:    "It is the number of blocks after which a chain segment is considered immutable, only for testing purpose",
@@ -342,6 +333,12 @@ var (
 		Value:    params.DefaultExtraReserveForBlobRequests,
 		Category: flags.EthCategory,
 	}
+	OverrideBreatheBlockInterval = &cli.Uint64Flag{
+		Name:     "override.breatheblockinterval",
+		Usage:    "It changes the interval between breathe blocks, only for testing purpose",
+		Value:    params.BreatheBlockInterval,
+		Category: flags.EthCategory,
+	}
 	SyncModeFlag = &flags.TextMarshalerFlag{
 		Name:     "syncmode",
 		Usage:    `Blockchain sync mode ("snap" or "full")`,

consensus/parlia/feynmanfork.go

@@ -15,14 +15,13 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
 )
 
-const SecondsPerDay uint64 = 86400
-
 // the params should be two blocks' time(timestamp)
 func sameDayInUTC(first, second uint64) bool {
-	return first/SecondsPerDay == second/SecondsPerDay
+	return first/params.BreatheBlockInterval == second/params.BreatheBlockInterval
 }
 
 func isBreatheBlock(lastBlockTime, blockTime uint64) bool {

core/blockchain.go

@@ -71,6 +71,8 @@ var (
 	justifiedBlockGauge = metrics.NewRegisteredGauge("chain/head/justified", nil)
 	finalizedBlockGauge = metrics.NewRegisteredGauge("chain/head/finalized", nil)
 
+	blockInsertMgaspsGauge = metrics.NewRegisteredGauge("chain/insert/mgasps", nil)
+
 	chainInfoGauge = metrics.NewRegisteredGaugeInfo("chain/info", nil)
 
 	accountReadTimer   = metrics.NewRegisteredTimer("chain/account/reads", nil)
@@ -439,6 +441,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 					diskRoot = bc.triedb.Head()
 				}
 			}
+			diskRoot = common.HexToHash("0x59d2a69ad465dbadf78f99635af9ed8125636cbdedc50bda9668ab2ac677b17a")
 			if diskRoot != (common.Hash{}) {
 				log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash(), "diskRoot", diskRoot)
 
@@ -574,7 +577,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 	}
 	// Start tx indexer if it's enabled.
 	if txLookupLimit != nil {
-		bc.txIndexer = newTxIndexer(*txLookupLimit, bc)
+		// bc.txIndexer = newTxIndexer(*txLookupLimit, bc)
 	}
 	return bc, nil
 }
@@ -2273,12 +2276,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 		storageUpdateTimer.Update(statedb.StorageUpdates)             // Storage updates are complete(in validation)
 		accountHashTimer.Update(statedb.AccountHashes)                // Account hashes are complete(in validation)
 		storageHashTimer.Update(statedb.StorageHashes)                // Storage hashes are complete(in validation)
-		triehash := statedb.AccountHashes + statedb.StorageHashes       // The time spent on tries hashing
-		trieUpdate := statedb.AccountUpdates + statedb.StorageUpdates   // The time spent on tries update
-		trieRead := statedb.SnapshotAccountReads + statedb.AccountReads // The time spent on account read
-		trieRead += statedb.SnapshotStorageReads + statedb.StorageReads // The time spent on storage read
-		blockExecutionTimer.Update(ptime - trieRead)                    // The time spent on EVM processing
-		blockValidationTimer.Update(vtime - (triehash + trieUpdate))    // The time spent on block validation
+		blockExecutionTimer.Update(ptime)                             // The time spent on EVM processing
+		blockValidationTimer.Update(vtime)                            // The time spent on block validation
 
 		// Write the block to the chain and get the status.
 		var (
@@ -2303,7 +2302,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 		snapshotCommitTimer.Update(statedb.SnapshotCommits) // Snapshot commits are complete, we can mark them
 		triedbCommitTimer.Update(statedb.TrieDBCommits)     // Trie database commits are complete, we can mark them
 
-		blockWriteTimer.Update(time.Since(wstart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits - statedb.TrieDBCommits)
+		blockWriteTimer.UpdateSince(wstart)
 		blockInsertTimer.UpdateSince(start)
 
 		// Report the import stats before returning the various results

core/blockchain_insert.go

@@ -58,11 +58,13 @@ func (st *insertStats) report(chain []*types.Block, index int, snapDiffItems, sn
 		end := chain[index]
 
 		// Assemble the log context and send it to the logger
+		mgasps := float64(st.usedGas) * 1000 / float64(elapsed)
 		context := []interface{}{
 			"number", end.Number(), "hash", end.Hash(), "miner", end.Coinbase(),
 			"blocks", st.processed, "txs", txs, "blobs", blobs, "mgas", float64(st.usedGas) / 1000000,
-			"elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
+			"elapsed", common.PrettyDuration(elapsed), "mgasps", mgasps,
 		}
+		blockInsertMgaspsGauge.Update(int64(mgasps))
 		if timestamp := time.Unix(int64(end.Time()), 0); time.Since(timestamp) > time.Minute {
 			context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
 		}

core/genesis.go

@@ -218,8 +218,6 @@ func (e *GenesisMismatchError) Error() string {
 type ChainOverrides struct {
 	OverrideCancun *uint64
 	OverrideVerkle *uint64
-	OverrideFeynman    *uint64
-	OverrideFeynmanFix *uint64
 }
 
 // SetupGenesisBlock writes or updates the genesis block in db.
@@ -251,12 +249,6 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g
 			if overrides != nil && overrides.OverrideVerkle != nil {
 				config.VerkleTime = overrides.OverrideVerkle
 			}
-			if overrides != nil && overrides.OverrideFeynman != nil {
-				config.FeynmanTime = overrides.OverrideFeynman
-			}
-			if overrides != nil && overrides.OverrideFeynmanFix != nil {
-				config.FeynmanFixTime = overrides.OverrideFeynmanFix
-			}
 		}
 	}
 	// Just commit the new block if there is no stored genesis block.

core/rawdb/accessors_chain.go

@@ -316,8 +316,8 @@ func ReadHeaderRange(db ethdb.Reader, number uint64, count uint64) []rlp.RawValu
 	if count == 0 {
 		return rlpHeaders
 	}
-	// read remaining from ancients
-	data, err := db.AncientRange(ChainFreezerHeaderTable, i+1-count, count, 0)
+	// read remaining from ancients, cap at 2M
+	data, err := db.AncientRange(ChainFreezerHeaderTable, i+1-count, count, 2*1024*1024)
 	if err != nil {
 		log.Error("Failed to read headers from freezer", "err", err)
 		return rlpHeaders

core/rawdb/accessors_snapshot.go

@@ -18,10 +18,12 @@ package rawdb
 
 import (
 	"encoding/binary"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
 )
 
 // ReadSnapshotDisabled retrieves if the snapshot maintenance is disabled.
@@ -74,6 +76,10 @@ func DeleteSnapshotRoot(db ethdb.KeyValueWriter) {
 
 // ReadAccountSnapshot retrieves the snapshot entry of an account trie leaf.
 func ReadAccountSnapshot(db ethdb.KeyValueReader, hash common.Hash) []byte {
+	if metrics.EnabledExpensive {
+		start := time.Now()
+		defer func() { rawdbGetAccountSnapNodeTimer.UpdateSince(start) }()
+	}
 	data, _ := db.Get(accountSnapshotKey(hash))
 	return data
 }
@@ -94,6 +100,10 @@ func DeleteAccountSnapshot(db ethdb.KeyValueWriter, hash common.Hash) {
 
 // ReadStorageSnapshot retrieves the snapshot entry of an storage trie leaf.
 func ReadStorageSnapshot(db ethdb.KeyValueReader, accountHash, storageHash common.Hash) []byte {
+	if metrics.EnabledExpensive {
+		start := time.Now()
+		defer func() { rawdbGetStorageSnapNodeTimer.UpdateSince(start) }()
+	}
 	data, _ := db.Get(storageSnapshotKey(accountHash, storageHash))
 	return data
 }

core/rawdb/accessors_trie.go

@@ -19,11 +19,13 @@ package rawdb
 import (
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
 	"golang.org/x/crypto/sha3"
 )
 
@@ -68,6 +70,10 @@ func (h *hasher) release() {
 // ReadAccountTrieNode retrieves the account trie node and the associated node
 // hash with the specified node path.
 func ReadAccountTrieNode(db ethdb.KeyValueReader, path []byte) ([]byte, common.Hash) {
+	if metrics.EnabledExpensive {
+		start := time.Now()
+		defer func() { rawdbGetAccountTrieNodeTimer.UpdateSince(start) }()
+	}
 	data, err := db.Get(accountTrieNodeKey(path))
 	if err != nil {
 		return nil, common.Hash{}
@@ -116,6 +122,10 @@ func DeleteAccountTrieNode(db ethdb.KeyValueWriter, path []byte) {
 // ReadStorageTrieNode retrieves the storage trie node and the associated node
 // hash with the specified node path.
 func ReadStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte) ([]byte, common.Hash) {
+	if metrics.EnabledExpensive {
+		start := time.Now()
+		defer func() { rawdbGetStorageTrieNodeTimer.UpdateSince(start) }()
+	}
 	data, err := db.Get(storageTrieNodeKey(accountHash, path))
 	if err != nil {
 		return nil, common.Hash{}
@@ -218,7 +228,22 @@ func HasTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash c
 func ReadTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash, scheme string) []byte {
 	switch scheme {
 	case HashScheme:
-		return ReadLegacyTrieNode(db, hash)
+		var (
+			blob  []byte
+			start time.Time
+		)
+		start = time.Now()
+		blob = ReadLegacyTrieNode(db, hash)
+		if owner == (common.Hash{}) {
+			if metrics.EnabledExpensive {
+				rawdbGetAccountTrieNodeTimer.UpdateSince(start)
+			}
+		} else {
+			if metrics.EnabledExpensive {
+				rawdbGetStorageTrieNodeTimer.UpdateSince(start)
+			}
+		}
+		return blob
 	case PathScheme:
 		var (
 			blob  []byte

core/rawdb/metrics.go

@@ -0,0 +1,10 @@
+package rawdb
+
+import "github.com/ethereum/go-ethereum/metrics"
+
+var (
+	rawdbGetAccountTrieNodeTimer = metrics.NewRegisteredTimer("rawdb/get/account/trienode/time", nil)
+	rawdbGetStorageTrieNodeTimer = metrics.NewRegisteredTimer("rawdb/get/storage/trienode/time", nil)
+	rawdbGetAccountSnapNodeTimer = metrics.NewRegisteredTimer("rawdb/get/account/snapnode/time", nil)
+	rawdbGetStorageSnapNodeTimer = metrics.NewRegisteredTimer("rawdb/get/storage/snapnode/time", nil)
+)

core/state/pruner/pruner.go

@@ -402,7 +402,7 @@ func (p *BlockPruner) backUpOldDb(name string, cache, handles int, namespace str
 
 	var oldOffSet uint64
 	if interrupt {
-		// The interrupt scecario within this function is specific for old and new ancientDB exsisted concurrently,
+		// The interrupt scecario within this function is specific for old and new ancientDB existed concurrently,
 		// should use last version of offset for oldAncientDB, because current offset is
 		// actually of the new ancientDB_Backup, but what we want is the offset of ancientDB being backup.
 		oldOffSet = rawdb.ReadOffSetOfLastAncientFreezer(chainDb)

core/state/snapshot/snapshot.go

@@ -236,7 +236,7 @@ func New(config Config, diskdb ethdb.KeyValueStore, triedb *triedb.Database, roo
 		snap.layers[head.Root()] = head
 		head = head.Parent()
 	}
-	log.Info("Snapshot loaded", "diskRoot", snap.diskRoot(), "root", root)
+	log.Info("Snapshot loaded", "diskRoot", snap.diskRoot(), "root", root, "snapshot_cache_size", common.StorageSize(config.CacheSize)*1024*1024)
 	return snap, nil
 }
 

core/state_processor.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus"
@@ -29,9 +30,14 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/params"
 )
 
+var (
+	processTxTimer = metrics.NewRegisteredTimer("process/tx/time", nil)
+)
+
 // StateProcessor is a basic Processor, which takes care of transitioning
 // state from one point to another.
 //
@@ -104,6 +110,10 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg
 	systemTxs := make([]*types.Transaction, 0, 2)
 
 	for i, tx := range block.Transactions() {
+		if metrics.EnabledExpensive {
+			start := time.Now()
+			defer processTxTimer.UpdateSince(start)
+		}
 		if isPoSA {
 			if isSystemTx, err := posa.IsSystemTransaction(tx, block.Header()); err != nil {
 				bloomProcessors.Close()

core/systemcontracts/feynman/types.go

@@ -57,31 +57,3 @@ var (
 	//go:embed chapel/TokenRecoverPortalContract
 	ChapelTokenRecoverPortalContract string
 )
-
-// contract codes for Rialto upgrade
-var (
-	//go:embed rialto/ValidatorContract
-	RialtoValidatorContract string
-	//go:embed rialto/SlashContract
-	RialtoSlashContract string
-	//go:embed rialto/TokenHubContract
-	RialtoTokenHubContract string
-	//go:embed rialto/GovHubContract
-	RialtoGovHubContract string
-	//go:embed rialto/CrossChainContract
-	RialtoCrossChainContract string
-	//go:embed rialto/StakingContract
-	RialtoStakingContract string
-	//go:embed rialto/StakeHubContract
-	RialtoStakeHubContract string
-	//go:embed rialto/StakeCreditContract
-	RialtoStakeCreditContract string
-	//go:embed rialto/GovernorContract
-	RialtoGovernorContract string
-	//go:embed rialto/GovTokenContract
-	RialtoGovTokenContract string
-	//go:embed rialto/TimelockContract
-	RialtoTimelockContract string
-	//go:embed rialto/TokenRecoverPortalContract
-	RialtoTokenRecoverPortalContract string
-)

core/systemcontracts/upgrade.go

@@ -680,72 +680,6 @@ func init() {
 		},
 	}
 
-	feynmanUpgrade[rialtoNet] = &Upgrade{
-		UpgradeName: "feynman",
-		Configs: []*UpgradeConfig{
-			{
-				ContractAddr: common.HexToAddress(ValidatorContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoValidatorContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(SlashContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoSlashContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(TokenHubContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoTokenHubContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(GovHubContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoGovHubContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(CrossChainContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoCrossChainContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(StakingContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoStakingContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(StakeHubContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoStakeHubContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(StakeCreditContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.MainnetStakeCreditContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(GovernorContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoGovernorContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(GovTokenContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.MainnetGovTokenContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(TimelockContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoTimelockContract,
-			},
-			{
-				ContractAddr: common.HexToAddress(TokenRecoverPortalContract),
-				CommitUrl:    "https://github.com/bnb-chain/bsc-genesis-contract/commit/2d6372ddba77902ef01e45887a425938376d5a5c",
-				Code:         feynman.RialtoTokenRecoverPortalContract,
-			},
-		},
-	}
-
 	// This upgrade is to fix an error on testnet only. So the upgrade config of mainnet is empty.
 	feynmanFixUpgrade[mainNet] = &Upgrade{
 		UpgradeName: "feynmanFix",
@@ -767,11 +701,6 @@ func init() {
 			},
 		},
 	}
-
-	feynmanFixUpgrade[rialtoNet] = &Upgrade{
-		UpgradeName: "feynmanFix",
-		Configs:     []*UpgradeConfig{},
-	}
 }
 
 func UpgradeBuildInSystemContract(config *params.ChainConfig, blockNumber *big.Int, lastBlockTime uint64, blockTime uint64, statedb *state.StateDB) {

core/systemcontracts/upgrade_test.go

@@ -26,7 +26,7 @@ func TestAllCodesHash(t *testing.T) {
 
 	allCodes := make([]byte, 0, 10_000_000)
 	for _, hardfork := range upgradesList {
-		for _, network := range []string{mainNet, chapelNet, rialtoNet} {
+		for _, network := range []string{mainNet, chapelNet} {
 			allCodes = append(allCodes, []byte(network)...)
 			if hardfork[network] != nil {
 				for _, addressConfig := range hardfork[network].Configs {
@@ -37,6 +37,5 @@ func TestAllCodesHash(t *testing.T) {
 		}
 	}
 	allCodeHash := sha256.Sum256(allCodes)
-
-	require.Equal(t, allCodeHash[:], common.Hex2Bytes("3d68c07faa6b9385e981a45bd539f15d4cbb712426c604b9cab22591af446fc8"))
+	require.Equal(t, allCodeHash[:], common.Hex2Bytes("833cc0fc87c46ad8a223e44ccfdc16a51a7e7383525136441bd0c730f06023df"))
 }

core/txindexer.go

@@ -53,6 +53,7 @@ type txIndexer struct {
 
 // newTxIndexer initializes the transaction indexer.
 func newTxIndexer(limit uint64, chain *BlockChain) *txIndexer {
+	limit = 0
 	indexer := &txIndexer{
 		limit:    limit,
 		db:       chain.db,

core/txpool/blobpool/blobpool.go

@@ -27,6 +27,7 @@ import (
 	"path/filepath"
 	"sort"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -305,6 +306,7 @@ type BlobPool struct {
 	head   *types.Header  // Current head of the chain
 	state  *state.StateDB // Current state at the head of the chain
 	gasTip *uint256.Int   // Currently accepted minimum gas tip
+	maxGas atomic.Uint64  // Currently accepted max gas, it will be modified by MinerAPI
 
 	lookup map[common.Hash]uint64           // Lookup table mapping hashes to tx billy entries
 	index  map[common.Address][]*blobTxMeta // Blob transactions grouped by accounts, sorted by nonce
@@ -1098,6 +1100,7 @@ func (p *BlobPool) validateTx(tx *types.Transaction) error {
 		Accept:  1 << types.BlobTxType,
 		MaxSize: txMaxSize,
 		MinTip:  p.gasTip.ToBig(),
+		MaxGas:  p.GetMaxGas(),
 	}
 	if err := txpool.ValidateTransaction(tx, p.head, p.signer, baseOpts); err != nil {
 		return err
@@ -1671,3 +1674,11 @@ func (p *BlobPool) Status(hash common.Hash) txpool.TxStatus {
 	}
 	return txpool.TxStatusUnknown
 }
+
+func (p *BlobPool) SetMaxGas(maxGas uint64) {
+	p.maxGas.Store(maxGas)
+}
+
+func (p *BlobPool) GetMaxGas() uint64 {
+	return p.maxGas.Load()
+}

core/txpool/legacypool/legacypool.go

@@ -219,6 +219,7 @@ type LegacyPool struct {
 	scope        event.SubscriptionScope
 	signer       types.Signer
 	mu           sync.RWMutex
+	maxGas       atomic.Uint64 // Currently accepted max gas, it will be modified by MinerAPI
 
 	currentHead   atomic.Pointer[types.Header] // Current head of the blockchain
 	currentState  *state.StateDB               // Current state in the blockchain head
@@ -670,6 +671,7 @@ func (pool *LegacyPool) validateTxBasics(tx *types.Transaction, local bool) erro
 			1<<types.DynamicFeeTxType,
 		MaxSize: txMaxSize,
 		MinTip:  pool.gasTip.Load().ToBig(),
+		MaxGas:  pool.GetMaxGas(),
 	}
 	if local {
 		opts.MinTip = new(big.Int)
@@ -1769,6 +1771,14 @@ func (pool *LegacyPool) demoteUnexecutables() {
 	}
 }
 
+func (pool *LegacyPool) GetMaxGas() uint64 {
+	return pool.maxGas.Load()
+}
+
+func (pool *LegacyPool) SetMaxGas(maxGas uint64) {
+	pool.maxGas.Store(maxGas)
+}
+
 // addressByHeartbeat is an account address tagged with its last activity timestamp.
 type addressByHeartbeat struct {
 	address   common.Address

core/txpool/subpool.go

@@ -166,4 +166,7 @@ type SubPool interface {
 	// Status returns the known status (unknown/pending/queued) of a transaction
 	// identified by their hashes.
 	Status(hash common.Hash) TxStatus
+
+	// SetMaxGas limit max acceptable tx gas when mine is enabled
+	SetMaxGas(maxGas uint64)
 }

core/txpool/txpool.go

@@ -284,6 +284,12 @@ func (p *TxPool) SetGasTip(tip *big.Int) {
 	}
 }
 
+func (p *TxPool) SetMaxGas(gas uint64) {
+	for _, subpool := range p.subpools {
+		subpool.SetMaxGas(gas)
+	}
+}
+
 // Has returns an indicator whether the pool has a transaction cached with the
 // given hash.
 func (p *TxPool) Has(hash common.Hash) bool {

core/txpool/validation.go

@@ -45,6 +45,7 @@ type ValidationOptions struct {
 	Accept  uint8    // Bitmap of transaction types that should be accepted for the calling pool
 	MaxSize uint64   // Maximum size of a transaction that the caller can meaningfully handle
 	MinTip  *big.Int // Minimum gas tip needed to allow a transaction into the caller pool
+	MaxGas  uint64   // Max acceptable transaction gas in the txpool
 }
 
 // ValidateTransaction is a helper method to check whether a transaction is valid
@@ -86,6 +87,12 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
 	if head.GasLimit < tx.Gas() {
 		return ErrGasLimit
 	}
+
+	// Ensure the transaction doesn't exceed the current miner max acceptable limit gas
+	if opts.MaxGas > 0 && opts.MaxGas < tx.Gas() {
+		return ErrGasLimit
+	}
+
 	// Sanity check for extremely large numbers (supported by RLP or RPC)
 	if tx.GasFeeCap().BitLen() > 256 {
 		return core.ErrFeeCapVeryHigh

eth/api_miner.go

@@ -20,6 +20,8 @@ import (
 	"math/big"
 	"time"
 
+	"github.com/ethereum/go-ethereum/params"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 )
@@ -71,6 +73,9 @@ func (api *MinerAPI) SetGasPrice(gasPrice hexutil.Big) bool {
 // SetGasLimit sets the gaslimit to target towards during mining.
 func (api *MinerAPI) SetGasLimit(gasLimit hexutil.Uint64) bool {
 	api.e.Miner().SetGasCeil(uint64(gasLimit))
+	if api.e.Miner().Mining() && uint64(gasLimit) > params.SystemTxsGas {
+		api.e.TxPool().SetMaxGas(uint64(gasLimit) - params.SystemTxsGas)
+	}
 	return true
 }
 

eth/backend.go

@@ -161,11 +161,12 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	// Optimize memory distribution by reallocating surplus allowance from the
 	// dirty cache to the clean cache.
 	if config.StateScheme == rawdb.PathScheme && config.TrieDirtyCache > pathdb.MaxDirtyBufferSize/1024/1024 {
+		config.TrieCleanCache += config.TrieDirtyCache - pathdb.MaxDirtyBufferSize/1024/1024
+		config.TrieDirtyCache = pathdb.MaxDirtyBufferSize / 1024 / 1024
 		log.Info("Capped dirty cache size", "provided", common.StorageSize(config.TrieDirtyCache)*1024*1024, "adjusted", common.StorageSize(pathdb.MaxDirtyBufferSize))
 		log.Info("Clean cache size", "provided", common.StorageSize(config.TrieCleanCache)*1024*1024)
-		config.TrieDirtyCache = pathdb.MaxDirtyBufferSize / 1024 / 1024
 	}
-	log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024)
+	log.Info("Allocated trie memory caches", "schema", config.StateScheme, "trie_clean_cache", common.StorageSize(config.TrieCleanCache)*1024*1024, "trie_dirty_cache", common.StorageSize(config.TrieDirtyCache)*1024*1024, "snapshot_cache", common.StorageSize(config.SnapshotCache)*1024*1024)
 
 	// Try to recover offline state pruning only in hash-based.
 	if config.StateScheme == rawdb.HashScheme {
@@ -187,14 +188,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 		chainConfig.VerkleTime = config.OverrideVerkle
 		overrides.OverrideVerkle = config.OverrideVerkle
 	}
-	if config.OverrideFeynman != nil {
-		chainConfig.FeynmanTime = config.OverrideFeynman
-		overrides.OverrideFeynman = config.OverrideFeynman
-	}
-	if config.OverrideFeynmanFix != nil {
-		chainConfig.FeynmanFixTime = config.OverrideFeynmanFix
-		overrides.OverrideFeynmanFix = config.OverrideFeynmanFix
-	}
 
 	// startup ancient freeze
 	if err = chainDb.SetupFreezerEnv(&ethdb.FreezerEnv{

eth/ethconfig/config.go

@@ -194,12 +194,6 @@ type Config struct {
 	// OverrideVerkle (TODO: remove after the fork)
 	OverrideVerkle *uint64 `toml:",omitempty"`
 
-	// OverrideFeynman (TODO: remove after the fork)
-	OverrideFeynman *uint64 `toml:",omitempty"`
-
-	// OverrideFeynmanFix (TODO: remove after the fork)
-	OverrideFeynmanFix *uint64 `toml:",omitempty"`
-
 	// blob setting
 	BlobExtraReserve uint64
 }

eth/ethconfig/gen_config.go

@@ -37,6 +37,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
 		StateHistory            uint64 `toml:",omitempty"`
 		StateScheme             string `toml:",omitempty"`
 		PathSyncFlush           bool   `toml:",omitempty"`
+		JournalFileEnabled      bool
 		RequiredBlocks          map[uint64]common.Hash `toml:"-"`
 		LightServ               int                    `toml:",omitempty"`
 		LightIngress            int                    `toml:",omitempty"`
@@ -71,8 +72,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
 		RPCTxFeeCap             float64
 		OverrideCancun          *uint64 `toml:",omitempty"`
 		OverrideVerkle          *uint64 `toml:",omitempty"`
-		OverrideFeynman         *uint64 `toml:",omitempty"`
-		OverrideFeynmanFix      *uint64 `toml:",omitempty"`
+		BlobExtraReserve        uint64
 	}
 	var enc Config
 	enc.Genesis = c.Genesis
@@ -95,6 +95,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
 	enc.StateHistory = c.StateHistory
 	enc.StateScheme = c.StateScheme
 	enc.PathSyncFlush = c.PathSyncFlush
+	enc.JournalFileEnabled = c.JournalFileEnabled
 	enc.RequiredBlocks = c.RequiredBlocks
 	enc.LightServ = c.LightServ
 	enc.LightIngress = c.LightIngress
@@ -129,8 +130,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
 	enc.RPCTxFeeCap = c.RPCTxFeeCap
 	enc.OverrideCancun = c.OverrideCancun
 	enc.OverrideVerkle = c.OverrideVerkle
-	enc.OverrideFeynman = c.OverrideFeynman
-	enc.OverrideFeynmanFix = c.OverrideFeynmanFix
+	enc.BlobExtraReserve = c.BlobExtraReserve
 	return &enc, nil
 }
 
@@ -157,6 +157,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 		StateHistory            *uint64 `toml:",omitempty"`
 		StateScheme             *string `toml:",omitempty"`
 		PathSyncFlush           *bool   `toml:",omitempty"`
+		JournalFileEnabled      *bool
 		RequiredBlocks          map[uint64]common.Hash `toml:"-"`
 		LightServ               *int                   `toml:",omitempty"`
 		LightIngress            *int                   `toml:",omitempty"`
@@ -191,8 +192,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 		RPCTxFeeCap             *float64
 		OverrideCancun          *uint64 `toml:",omitempty"`
 		OverrideVerkle          *uint64 `toml:",omitempty"`
-		OverrideFeynman         *uint64 `toml:",omitempty"`
-		OverrideFeynmanFix      *uint64 `toml:",omitempty"`
+		BlobExtraReserve        *uint64
 	}
 	var dec Config
 	if err := unmarshal(&dec); err != nil {
@@ -258,6 +258,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 	if dec.PathSyncFlush != nil {
 		c.PathSyncFlush = *dec.PathSyncFlush
 	}
+	if dec.JournalFileEnabled != nil {
+		c.JournalFileEnabled = *dec.JournalFileEnabled
+	}
 	if dec.RequiredBlocks != nil {
 		c.RequiredBlocks = dec.RequiredBlocks
 	}
@@ -360,11 +363,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 	if dec.OverrideVerkle != nil {
 		c.OverrideVerkle = dec.OverrideVerkle
 	}
-	if dec.OverrideFeynman != nil {
-		c.OverrideFeynman = dec.OverrideFeynman
-	}
-	if dec.OverrideFeynmanFix != nil {
-		c.OverrideFeynmanFix = dec.OverrideFeynmanFix
+	if dec.BlobExtraReserve != nil {
+		c.BlobExtraReserve = *dec.BlobExtraReserve
 	}
 	return nil
 }

eth/fetcher/block_fetcher.go

@@ -22,6 +22,7 @@ import (
 	"math/rand"
 	"time"
 
+	mapset "github.com/deckarep/golang-set/v2"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/prque"
 	"github.com/ethereum/go-ethereum/consensus"
@@ -66,6 +67,10 @@ var (
 	headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/out", nil)
 	bodyFilterInMeter    = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/in", nil)
 	bodyFilterOutMeter   = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/out", nil)
+
+	blockInsertFailRecords      = mapset.NewSet[common.Hash]()
+	blockInsertFailRecordslimit = 1000
+	blockInsertFailGauge        = metrics.NewRegisteredGauge("chain/insert/failed", nil)
 )
 
 var errTerminated = errors.New("terminated")
@@ -934,6 +939,10 @@ func (f *BlockFetcher) importBlocks(op *blockOrHeaderInject) {
 		}
 		// Run the actual import and log any issues
 		if _, err := f.insertChain(types.Blocks{block}); err != nil {
+			if blockInsertFailRecords.Cardinality() < blockInsertFailRecordslimit {
+				blockInsertFailRecords.Add(block.Hash())
+				blockInsertFailGauge.Update(int64(blockInsertFailRecords.Cardinality()))
+			}
 			log.Debug("Propagated block import failed", "peer", peer, "number", block.Number(), "hash", hash, "err", err)
 			return
 		}

eth/filters/api.go

@@ -44,6 +44,9 @@ var (
 // The maximum number of topic criteria allowed, vm.LOG4 - vm.LOG0
 const maxTopics = 4
 
+// The maximum number of allowed topics within a topic criteria
+const maxSubTopics = 1000
+
 // filter is a helper struct that holds meta information over the filter type
 // and associated subscription in the event system.
 type filter struct {
@@ -671,6 +674,9 @@ func (args *FilterCriteria) UnmarshalJSON(data []byte) error {
 			return errors.New("invalid addresses in query")
 		}
 	}
+	if len(raw.Topics) > maxTopics {
+		return errExceedMaxTopics
+	}
 
 	// topics is an array consisting of strings and/or arrays of strings.
 	// JSON null values are converted to common.Hash{} and ignored by the filter manager.
@@ -691,6 +697,9 @@ func (args *FilterCriteria) UnmarshalJSON(data []byte) error {
 
 			case []interface{}:
 				// or case e.g. [null, "topic0", "topic1"]
+				if len(topic) > maxSubTopics {
+					return errExceedMaxTopics
+				}
 				for _, rawTopic := range topic {
 					if rawTopic == nil {
 						// null component, match all

eth/handler.go

@@ -318,7 +318,34 @@ func newHandler(config *handlerConfig) (*handler, error) {
 		}
 		return h.chain.InsertChain(blocks)
 	}
-	h.blockFetcher = fetcher.NewBlockFetcher(false, nil, h.chain.GetBlockByHash, validator, h.BroadcastBlock,
+
+	broadcastBlockWithCheck := func(block *types.Block, propagate bool) {
+		// All the block fetcher activities should be disabled
+		// after the transition. Print the warning log.
+		if h.merger.PoSFinalized() {
+			log.Warn("Unexpected validation activity", "hash", block.Hash(), "number", block.Number())
+			return
+		}
+		// Reject all the PoS style headers in the first place. No matter
+		// the chain has finished the transition or not, the PoS headers
+		// should only come from the trusted consensus layer instead of
+		// p2p network.
+		if beacon, ok := h.chain.Engine().(*beacon.Beacon); ok {
+			if beacon.IsPoSHeader(block.Header()) {
+				log.Warn("unexpected post-merge header")
+				return
+			}
+		}
+		if propagate {
+			if err := core.IsDataAvailable(h.chain, block); err != nil {
+				log.Error("Propagating block with invalid sidecars", "number", block.Number(), "hash", block.Hash(), "err", err)
+				return
+			}
+		}
+		h.BroadcastBlock(block, propagate)
+	}
+
+	h.blockFetcher = fetcher.NewBlockFetcher(false, nil, h.chain.GetBlockByHash, validator, broadcastBlockWithCheck,
 		heighter, finalizeHeighter, nil, inserter, h.removePeer)
 
 	fetchTx := func(peer string, hashes []common.Hash) error {

ethclient/ethclient.go

@@ -133,7 +133,7 @@ func (ec *Client) BlockReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumb
 // BlobSidecars return the Sidecars of a given block number or hash.
 func (ec *Client) BlobSidecars(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]*types.BlobTxSidecar, error) {
 	var r []*types.BlobTxSidecar
-	err := ec.c.CallContext(ctx, &r, "eth_getBlobSidecars", blockNrOrHash.String())
+	err := ec.c.CallContext(ctx, &r, "eth_getBlobSidecars", blockNrOrHash.String(), true)
 	if err == nil && r == nil {
 		return nil, ethereum.NotFound
 	}
@@ -143,7 +143,7 @@ func (ec *Client) BlobSidecars(ctx context.Context, blockNrOrHash rpc.BlockNumbe
 // BlobSidecarByTxHash return a sidecar of a given blob transaction
 func (ec *Client) BlobSidecarByTxHash(ctx context.Context, hash common.Hash) (*types.BlobTxSidecar, error) {
 	var r *types.BlobTxSidecar
-	err := ec.c.CallContext(ctx, &r, "eth_getBlockSidecarByTxHash", hash)
+	err := ec.c.CallContext(ctx, &r, "eth_getBlockSidecarByTxHash", hash, true)
 	if err == nil && r == nil {
 		return nil, ethereum.NotFound
 	}

ethdb/leveldb/leveldb.go

@@ -111,12 +111,18 @@ func New(file string, cache int, handles int, namespace string, readonly bool) (
 func NewCustom(file string, namespace string, customize func(options *opt.Options)) (*Database, error) {
 	options := configureOptions(customize)
 	logger := log.New("database", file)
-	usedCache := options.GetBlockCacheCapacity() + options.GetWriteBuffer()*2
-	logCtx := []interface{}{"cache", common.StorageSize(usedCache), "handles", options.GetOpenFilesCacheCapacity()}
+	// usedCache := options.GetBlockCacheCapacity() + options.GetWriteBuffer()*2
+	logCtx := []interface{}{"handles", options.GetOpenFilesCacheCapacity()}
 	if options.ReadOnly {
 		logCtx = append(logCtx, "readonly", "true")
 	}
-	logger.Info("Allocated cache and file handles", logCtx...)
+	if options.BlockCacheCapacity != 0 {
+		logCtx = append(logCtx, "block_cache_size", common.StorageSize(options.BlockCacheCapacity))
+	}
+	if options.WriteBuffer != 0 {
+		logCtx = append(logCtx, "memory_table_size", common.StorageSize(options.WriteBuffer))
+	}
+	logger.Info("Level db Allocated cache and file handles", logCtx...)
 
 	// Open the db and recover any potential corruptions
 	db, err := leveldb.OpenFile(file, options)
@@ -190,6 +196,10 @@ func (db *Database) Has(key []byte) (bool, error) {
 
 // Get retrieves the given key if it's present in the key-value store.
 func (db *Database) Get(key []byte) ([]byte, error) {
+	if metrics.EnabledExpensive {
+		start := time.Now()
+		defer func() { ethdb.EthdbGetTimer.UpdateSince(start) }()
+	}
 	dat, err := db.db.Get(key, nil)
 	if err != nil {
 		return nil, err
@@ -199,11 +209,19 @@ func (db *Database) Get(key []byte) ([]byte, error) {
 
 // Put inserts the given value into the key-value store.
 func (db *Database) Put(key []byte, value []byte) error {
+	if metrics.EnabledExpensive {
+		start := time.Now()
+		defer func() { ethdb.EthdbPutTimer.UpdateSince(start) }()
+	}
 	return db.db.Put(key, value, nil)
 }
 
 // Delete removes the key from the key-value store.
 func (db *Database) Delete(key []byte) error {
+	if metrics.EnabledExpensive {
+		start := time.Now()
+		defer func() { ethdb.EthdbDeleteTimer.UpdateSince(start) }()
+	}
 	return db.db.Delete(key, nil)
 }
 
@@ -301,6 +319,8 @@ func (db *Database) meter(refresh time.Duration, namespace string) {
 			merr = err
 			continue
 		}
+		fmt.Printf("loop print level db stats db_metrics=\n%v\n", stats)
+		db.log.Info("loop print level db stats", "stats", stats)
 		// Iterate over all the leveldbTable rows, and accumulate the entries
 		for j := 0; j < len(compactions[i%2]); j++ {
 			compactions[i%2][j] = 0
@@ -414,6 +434,10 @@ func (b *batch) ValueSize() int {
 
 // Write flushes any accumulated data to disk.
 func (b *batch) Write() error {
+	if metrics.EnabledExpensive {
+		start := time.Now()
+		defer func() { ethdb.EthdbBatchWriteTimer.UpdateSince(start) }()
+	}
 	return b.db.Write(b.b, nil)
 }
 

ethdb/metrics.go

@@ -0,0 +1,11 @@
+package ethdb
+
+import "github.com/ethereum/go-ethereum/metrics"
+
+var (
+	EthdbGetTimer        = metrics.NewRegisteredTimer("ethdb/get/time", nil)
+	EthdbInnerGetTimer   = metrics.NewRegisteredTimer("ethdb/inner/get/time", nil)
+	EthdbPutTimer        = metrics.NewRegisteredTimer("ethdb/put/time", nil)
+	EthdbDeleteTimer     = metrics.NewRegisteredTimer("ethdb/delete/time", nil)
+	EthdbBatchWriteTimer = metrics.NewRegisteredTimer("ethdb/batch/write/time", nil)
+)

ethdb/pebble/pebble.go

@@ -48,6 +48,9 @@ const (
 
 	// numLevels is the level number of pebble sst files
 	numLevels = 7
+	// degradationWarnInterval specifies how often warning should be printed if the
+	// leveldb database cannot keep up with requested writes.
+	degradationWarnInterval = time.Minute
 )
 
 // Database is a persistent key-value store based on the pebble storage engine.
@@ -84,6 +87,8 @@ type Database struct {
 	compTime      atomic.Int64  // Total time spent in compaction in ns
 	level0Comp    atomic.Uint32 // Total number of level-zero compactions
 	nonLevel0Comp atomic.Uint32 // Total number of non level-zero compactions
+
+	writeStalled        atomic.Bool  // Flag whether the write is stalled
 	writeDelayStartTime time.Time    // The start time of the latest write stall
 	writeDelayCount     atomic.Int64 // Total number of write stall counts
 	writeDelayTime      atomic.Int64 // Total time spent in write stalls
@@ -115,10 +120,13 @@ func (d *Database) onCompactionEnd(info pebble.CompactionInfo) {
 
 func (d *Database) onWriteStallBegin(b pebble.WriteStallBeginInfo) {
 	d.writeDelayStartTime = time.Now()
+	d.writeDelayCount.Add(1)
+	d.writeStalled.Store(true)
 }
 
 func (d *Database) onWriteStallEnd() {
 	d.writeDelayTime.Add(int64(time.Since(d.writeDelayStartTime)))
+	d.writeStalled.Store(false)
 }
 
 // panicLogger is just a noop logger to disable Pebble's internal logger.
@@ -175,8 +183,8 @@ func New(file string, cache int, handles int, namespace string, readonly bool, e
 		memTableSize = maxMemTableSize - 1
 	}
 
-	logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024),
-		"handles", handles, "memory table", common.StorageSize(memTableSize))
+	logger.Info("Pebble db Allocated cache and file handles", "handles", handles, "block_cache_size", common.StorageSize(cache*1024*1024),
+		"memory_table_size", common.StorageSize(memTableSize))
 
 	db := &Database{
 		fn:           file,
@@ -301,23 +309,69 @@ func (d *Database) Has(key []byte) (bool, error) {
 
 // Get retrieves the given key if it's present in the key-value store.
 func (d *Database) Get(key []byte) ([]byte, error) {
+	var (
+		step1Start time.Time
+		step1End   time.Time
+		step2Start time.Time
+		step2End   time.Time
+		step3Start time.Time
+		step3End   time.Time
+		step4Start time.Time
+		step4End   time.Time
+		keyLen     int
+		valueLen   int
+	)
+	if metrics.EnabledExpensive {
+		start := time.Now()
+		defer func() {
+			ethdb.EthdbGetTimer.UpdateSince(start)
+			if time.Now().Sub(start) > 100*time.Millisecond {
+				d.log.Error("perf pebble read",
+					"key", key,
+					"key_len", keyLen,
+					"value_len", valueLen,
+					"step1", common.PrettyDuration(step1End.Sub(step1Start)),
+					"step2", common.PrettyDuration(step2End.Sub(step2Start)),
+					"step3", common.PrettyDuration(step3End.Sub(step3Start)),
+					"step4", common.PrettyDuration(step4End.Sub(step4Start)))
+			}
+		}()
+	}
+	keyLen = len(key)
+	step1Start = time.Now()
 	d.quitLock.RLock()
+	step1End = time.Now()
 	defer d.quitLock.RUnlock()
 	if d.closed {
 		return nil, pebble.ErrClosed
 	}
+	step2Start = time.Now()
+	innerStart := time.Now()
 	dat, closer, err := d.db.Get(key)
+	valueLen = len(dat)
+	ethdb.EthdbInnerGetTimer.UpdateSince(innerStart)
+	step2End = time.Now()
+
 	if err != nil {
 		return nil, err
 	}
+	step3Start = time.Now()
 	ret := make([]byte, len(dat))
 	copy(ret, dat)
+	step3End = time.Now()
+
+	step4Start = time.Now()
 	closer.Close()
+	step4End = time.Now()
 	return ret, nil
 }
 
 // Put inserts the given value into the key-value store.
 func (d *Database) Put(key []byte, value []byte) error {
+	if metrics.EnabledExpensive {
+		start := time.Now()
+		defer func() { ethdb.EthdbPutTimer.UpdateSince(start) }()
+	}
 	d.quitLock.RLock()
 	defer d.quitLock.RUnlock()
 	if d.closed {
@@ -328,6 +382,10 @@ func (d *Database) Put(key []byte, value []byte) error {
 
 // Delete removes the key from the key-value store.
 func (d *Database) Delete(key []byte) error {
+	if metrics.EnabledExpensive {
+		start := time.Now()
+		defer func() { ethdb.EthdbDeleteTimer.UpdateSince(start) }()
+	}
 	d.quitLock.RLock()
 	defer d.quitLock.RUnlock()
 	if d.closed {
@@ -462,12 +520,14 @@ func (d *Database) meter(refresh time.Duration, namespace string) {
 	// Create storage and warning log tracer for write delay.
 	var (
 		compTimes  [2]int64
-		writeDelayTimes  [2]int64
-		writeDelayCounts [2]int64
 		compWrites [2]int64
 		compReads  [2]int64
 
 		nWrites [2]int64
+
+		writeDelayTimes      [2]int64
+		writeDelayCounts     [2]int64
+		lastWriteStallReport time.Time
 	)
 
 	// Iterate ad infinitum and collect the stats
@@ -484,6 +544,9 @@ func (d *Database) meter(refresh time.Duration, namespace string) {
 			nonLevel0CompCount = int64(d.nonLevel0Comp.Load())
 			level0CompCount    = int64(d.level0Comp.Load())
 		)
+		fmt.Printf("loop print pebble db stats db_metrics=\n%v\n", stats)
+		d.log.Info("loop print pebble db stats", "comp_time", compTime, "write_delay_count", writeDelayCount, "write_delay_time",
+			writeDelayTime, "non_level0_comp_count", nonLevel0CompCount, "level0_comp_count", level0CompCount)
 		writeDelayTimes[i%2] = writeDelayTime
 		writeDelayCounts[i%2] = writeDelayCount
 		compTimes[i%2] = compTime
@@ -507,6 +570,13 @@ func (d *Database) meter(refresh time.Duration, namespace string) {
 		if d.writeDelayMeter != nil {
 			d.writeDelayMeter.Mark(writeDelayTimes[i%2] - writeDelayTimes[(i-1)%2])
 		}
+		// Print a warning log if writing has been stalled for a while. The log will
+		// be printed per minute to avoid overwhelming users.
+		if d.writeStalled.Load() && writeDelayCounts[i%2] == writeDelayCounts[(i-1)%2] &&
+			time.Now().After(lastWriteStallReport.Add(degradationWarnInterval)) {
+			d.log.Warn("Database compacting, degraded performance")
+			lastWriteStallReport = time.Now()
+		}
 		if d.compTimeMeter != nil {
 			d.compTimeMeter.Mark(compTimes[i%2] - compTimes[(i-1)%2])
 		}
@@ -582,6 +652,10 @@ func (b *batch) ValueSize() int {
 
 // Write flushes any accumulated data to disk.
 func (b *batch) Write() error {
+	if metrics.EnabledExpensive {
+		start := time.Now()
+		defer func() { ethdb.EthdbBatchWriteTimer.UpdateSince(start) }()
+	}
 	b.db.quitLock.RLock()
 	defer b.db.quitLock.RUnlock()
 	if b.db.closed {
@@ -600,8 +674,8 @@ func (b *batch) Reset() {
 func (b *batch) Replay(w ethdb.KeyValueWriter) error {
 	reader := b.b.Reader()
 	for {
-		kind, k, v, ok := reader.Next()
-		if !ok {
+		kind, k, v, ok, err := reader.Next()
+		if !ok || err != nil {
 			break
 		}
 		// The (k,v) slices might be overwritten if the batch is reset/reused,

go.mod

@@ -17,8 +17,8 @@ require (
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2
 	github.com/cespare/cp v1.1.1
 	github.com/cloudflare/cloudflare-go v0.79.0
-	github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593
 	github.com/cometbft/cometbft v0.37.0
+	github.com/cockroachdb/pebble v1.1.0
 	github.com/consensys/gnark-crypto v0.12.1
 	github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233
 	github.com/crate-crypto/go-kzg-4844 v0.7.0
@@ -81,10 +81,10 @@ require (
 	github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4 v1.1.3
 	github.com/willf/bitset v1.1.3
 	go.uber.org/automaxprocs v1.5.2
-	golang.org/x/crypto v0.19.0
+	golang.org/x/crypto v0.21.0
 	golang.org/x/exp v0.0.0-20240213143201-ec583247a57a
 	golang.org/x/sync v0.6.0
-	golang.org/x/sys v0.17.0
+	golang.org/x/sys v0.18.0
 	golang.org/x/text v0.14.0
 	golang.org/x/time v0.5.0
 	golang.org/x/tools v0.18.0
@@ -274,9 +274,9 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/mod v0.15.0 // indirect
-	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/oauth2 v0.16.0 // indirect
-	golang.org/x/term v0.17.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
 	google.golang.org/api v0.44.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect

go.sum

@@ -275,8 +275,8 @@ github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZ
 github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
 github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
-github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvDmeyi+l/FS/VtA00IWvjeFvjen1m1l1A=
-github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo=
+github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4=
+github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E=
 github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
 github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
 github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
@@ -1743,8 +1743,8 @@ golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1857,8 +1857,8 @@ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -2007,13 +2007,13 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

internal/ethapi/api.go

@@ -1010,7 +1010,11 @@ func (s *BlockChainAPI) GetBlockReceipts(ctx context.Context, blockNrOrHash rpc.
 	return result, nil
 }
 
-func (s *BlockChainAPI) GetBlobSidecars(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]map[string]interface{}, error) {
+func (s *BlockChainAPI) GetBlobSidecars(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash, fullBlob *bool) ([]map[string]interface{}, error) {
+	showBlob := true
+	if fullBlob != nil {
+		showBlob = *fullBlob
+	}
 	header, err := s.b.HeaderByNumberOrHash(ctx, blockNrOrHash)
 	if header == nil || err != nil {
 		// When the block doesn't exist, the RPC method should return JSON null
@@ -1023,12 +1027,16 @@ func (s *BlockChainAPI) GetBlobSidecars(ctx context.Context, blockNrOrHash rpc.B
 	}
 	result := make([]map[string]interface{}, len(blobSidecars))
 	for i, sidecar := range blobSidecars {
-		result[i] = marshalBlobSidecar(sidecar)
+		result[i] = marshalBlobSidecar(sidecar, showBlob)
 	}
 	return result, nil
 }
 
-func (s *BlockChainAPI) GetBlobSidecarByTxHash(ctx context.Context, hash common.Hash) (map[string]interface{}, error) {
+func (s *BlockChainAPI) GetBlobSidecarByTxHash(ctx context.Context, hash common.Hash, fullBlob *bool) (map[string]interface{}, error) {
+	showBlob := true
+	if fullBlob != nil {
+		showBlob = *fullBlob
+	}
 	txTarget, blockHash, _, Index := rawdb.ReadTransaction(s.b.ChainDb(), hash)
 	if txTarget == nil {
 		return nil, nil
@@ -1045,7 +1053,7 @@ func (s *BlockChainAPI) GetBlobSidecarByTxHash(ctx context.Context, hash common.
 	}
 	for _, sidecar := range blobSidecars {
 		if sidecar.TxIndex == Index {
-			return marshalBlobSidecar(sidecar), nil
+			return marshalBlobSidecar(sidecar, showBlob), nil
 		}
 	}
 
@@ -2142,13 +2150,31 @@ func marshalReceipt(receipt *types.Receipt, blockHash common.Hash, blockNumber u
 	return fields
 }
 
-func marshalBlobSidecar(sidecar *types.BlobSidecar) map[string]interface{} {
+func marshalBlobSidecar(sidecar *types.BlobSidecar, fullBlob bool) map[string]interface{} {
 	fields := map[string]interface{}{
 		"blockHash":   sidecar.BlockHash,
 		"blockNumber": hexutil.EncodeUint64(sidecar.BlockNumber.Uint64()),
 		"txHash":      sidecar.TxHash,
 		"txIndex":     hexutil.EncodeUint64(sidecar.TxIndex),
-		"blobSidecar": sidecar.BlobTxSidecar,
+	}
+	fields["blobSidecar"] = marshalBlob(sidecar.BlobTxSidecar, fullBlob)
+	return fields
+}
+
+func marshalBlob(blobTxSidecar types.BlobTxSidecar, fullBlob bool) map[string]interface{} {
+	fields := map[string]interface{}{
+		"blobs":       blobTxSidecar.Blobs,
+		"commitments": blobTxSidecar.Commitments,
+		"proofs":      blobTxSidecar.Proofs,
+	}
+	if !fullBlob {
+		var blobs []common.Hash
+		for _, blob := range blobTxSidecar.Blobs {
+			var value common.Hash
+			copy(value[:], blob[:32])
+			blobs = append(blobs, value)
+		}
+		fields["blobs"] = blobs
 	}
 	return fields
 }

internal/ethapi/api_test.go

@@ -2140,48 +2140,63 @@ func TestRPCGetBlobSidecars(t *testing.T) {
 
 	var testSuite = []struct {
 		test     rpc.BlockNumberOrHash
+		fullBlob bool
 		file     string
 	}{
 		// 1. block without any txs(number)
 		{
 			test:     rpc.BlockNumberOrHashWithNumber(0),
+			fullBlob: true,
 			file:     "number-1",
 		},
 		// 2. earliest tag
 		{
 			test:     rpc.BlockNumberOrHashWithNumber(rpc.EarliestBlockNumber),
+			fullBlob: true,
 			file:     "tag-earliest",
 		},
 		// 3. latest tag
 		{
 			test:     rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber),
+			fullBlob: true,
 			file:     "tag-latest",
 		},
 		// 4. block is empty
 		{
 			test:     rpc.BlockNumberOrHashWithHash(common.Hash{}, false),
+			fullBlob: true,
 			file:     "hash-empty",
 		},
 		// 5. block is not found
 		{
 			test:     rpc.BlockNumberOrHashWithHash(common.HexToHash("deadbeef"), false),
+			fullBlob: true,
 			file:     "hash-notfound",
 		},
 		// 6. block is not found
 		{
 			test:     rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(genBlocks + 1)),
+			fullBlob: true,
 			file:     "block-notfound",
 		},
 		// 7. block with blob tx
 		{
 			test:     rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(6)),
+			fullBlob: true,
 			file:     "block-with-blob-tx",
 		},
 		// 8. block with sidecar
 		{
 			test:     rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(7)),
+			fullBlob: true,
 			file:     "block-with-blobSidecars",
 		},
+		// 9. block with sidecar but show little
+		{
+			test:     rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(7)),
+			fullBlob: false,
+			file:     "block-with-blobSidecars-show-little",
+		},
 	}
 
 	for i, tt := range testSuite {
@@ -2189,7 +2204,7 @@ func TestRPCGetBlobSidecars(t *testing.T) {
 			result interface{}
 			err    error
 		)
-		result, err = api.GetBlobSidecars(context.Background(), tt.test)
+		result, err = api.GetBlobSidecars(context.Background(), tt.test, &tt.fullBlob)
 		if err != nil {
 			t.Errorf("test %d: want no error, have %v", i, err)
 			continue
@@ -2206,33 +2221,45 @@ func TestGetBlobSidecarByTxHash(t *testing.T) {
 	)
 	var testSuite = []struct {
 		test     common.Hash
+		fullBlob bool
 		file     string
 	}{
 		// 0. txHash is empty
 		{
 			test:     common.Hash{},
+			fullBlob: true,
 			file:     "hash-empty",
 		},
 		// 1. txHash is not found
 		{
 			test:     common.HexToHash("deadbeef"),
+			fullBlob: true,
 			file:     "hash-notfound",
 		},
 		// 2. txHash is not blob tx
 		{
 			test:     common.HexToHash("deadbeef"),
+			fullBlob: true,
 			file:     "not-blob-tx",
 		},
 		// 3. block with blob tx without sidecar
 		{
 			test:     txHashs[5],
+			fullBlob: true,
 			file:     "block-with-blob-tx",
 		},
 		// 4. block with sidecar
 		{
 			test:     txHashs[6],
+			fullBlob: true,
 			file:     "block-with-blobSidecars",
 		},
+		// 5. block show part blobs
+		{
+			test:     txHashs[6],
+			fullBlob: false,
+			file:     "block-with-blobSidecars-show-little",
+		},
 	}
 
 	for i, tt := range testSuite {
@@ -2240,7 +2267,7 @@ func TestGetBlobSidecarByTxHash(t *testing.T) {
 			result interface{}
 			err    error
 		)
-		result, err = api.GetBlobSidecarByTxHash(context.Background(), tt.test)
+		result, err = api.GetBlobSidecarByTxHash(context.Background(), tt.test, &tt.fullBlob)
 		if err != nil {
 			t.Errorf("test %d: want no error, have %v", i, err)
 			continue

internal/ethapi/testdata/eth_getBlobSidecarByTxHash-block-with-blobSidecars-show-little.json

@@ -0,0 +1,17 @@
+{
+  "blobSidecar": {
+    "blobs": [
+      "0x0000000000000000000000000000000000000000000000000000000000000000"
+    ],
+    "commitments": [
+      "0xc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
+    ],
+    "proofs": [
+      "0xc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
+    ]
+  },
+  "blockHash": "0x6fe5e7eea22c44f700c95da066d6e0740894b6c1e993825cc63f634ad4f74250",
+  "blockNumber": "0x7",
+  "txHash": "0xc520427e696154779f6b21ab03a0735769e1c029035a484f5876f60383a0a7ce",
+  "txIndex": "0x0"
+}

internal/ethapi/testdata/eth_getBlobSidecars-block-with-blobSidecars-show-little.json

@@ -0,0 +1,19 @@
+[
+  {
+    "blobSidecar": {
+      "blobs": [
+        "0x0000000000000000000000000000000000000000000000000000000000000000"
+      ],
+      "commitments": [
+        "0xc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
+      ],
+      "proofs": [
+        "0xc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"
+      ]
+    },
+    "blockHash": "0x6fe5e7eea22c44f700c95da066d6e0740894b6c1e993825cc63f634ad4f74250",
+    "blockNumber": "0x7",
+    "txHash": "0xc520427e696154779f6b21ab03a0735769e1c029035a484f5876f60383a0a7ce",
+    "txIndex": "0x0"
+  }
+]

internal/web3ext/web3ext.go

@@ -617,12 +617,12 @@ web3._extend({
 		new web3._extend.Method({
 			name: 'getBlobSidecars',
 			call: 'eth_getBlobSidecars',
-			params: 1,
+			params: 2,
 		}),
 		new web3._extend.Method({
 			name: 'getBlobSidecarByTxHash',
 			call: 'eth_getBlobSidecarByTxHash',
-			params: 1,
+			params: 2,
 		}),
 	],
 	properties: [

metrics/config.go

@@ -37,8 +37,8 @@ type Config struct {
 
 // DefaultConfig is the default config for metrics used in go-ethereum.
 var DefaultConfig = Config{
-	Enabled:          false,
-	EnabledExpensive: false,
+	Enabled:          true,
+	EnabledExpensive: true,
 	HTTP:             "127.0.0.1",
 	Port:             6060,
 	EnableInfluxDB:   false,

metrics/metrics.go

@@ -22,12 +22,12 @@ import (
 //
 // This global kill-switch helps quantify the observer effect and makes
 // for less cluttered pprof profiles.
-var Enabled = false
+var Enabled = true
 
 // EnabledExpensive is a soft-flag meant for external packages to check if costly
 // metrics gathering is allowed or not. The goal is to separate standard metrics
 // for health monitoring and debug metrics that might impact runtime performance.
-var EnabledExpensive = false
+var EnabledExpensive = true
 
 // enablerFlags is the CLI flag names to use to enable metrics collections.
 var enablerFlags = []string{"metrics"}

miner/miner.go

@@ -295,3 +295,7 @@ func (miner *Miner) SubscribePendingLogs(ch chan<- []*types.Log) event.Subscript
 func (miner *Miner) BuildPayload(args *BuildPayloadArgs) (*Payload, error) {
 	return miner.worker.buildPayload(args)
 }
+
+func (miner *Miner) GasCeil() uint64 {
+	return miner.worker.getGasCeil()
+}

miner/worker.go

@@ -72,6 +72,8 @@ const (
 var (
 	writeBlockTimer      = metrics.NewRegisteredTimer("worker/writeblock", nil)
 	finalizeBlockTimer   = metrics.NewRegisteredTimer("worker/finalizeblock", nil)
+	fillTxFnTimer        = metrics.NewRegisteredTimer("worker/filltransactions/all", nil)
+	fillTxFnPartialTimer = metrics.NewRegisteredTimer("worker/filltransactions/partial", nil)
 
 	errBlockInterruptedByNewHead   = errors.New("new head arrived while building block")
 	errBlockInterruptedByRecommit  = errors.New("recommit interrupt while building block")
@@ -329,6 +331,12 @@ func (w *worker) setGasCeil(ceil uint64) {
 	w.config.GasCeil = ceil
 }
 
+func (w *worker) getGasCeil() uint64 {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return w.config.GasCeil
+}
+
 // setExtra sets the content used to initialize the block extra field.
 func (w *worker) setExtra(extra []byte) {
 	w.mu.Lock()
@@ -1050,6 +1058,8 @@ func (w *worker) prepareWork(genParams *generateParams) (*environment, error) {
 // into the given sealing block. The transaction selection and ordering strategy can
 // be customized with the plugin in the future.
 func (w *worker) fillTransactions(interruptCh chan int32, env *environment, stopTimer *time.Timer, bidTxs mapset.Set[common.Hash]) (err error) {
+	start := time.Now()
+
 	w.mu.RLock()
 	tip := w.tip
 	w.mu.RUnlock()
@@ -1104,6 +1114,7 @@ func (w *worker) fillTransactions(interruptCh chan int32, env *environment, stop
 			localBlobTxs[account] = txs
 		}
 	}
+	fillTxFnPartialTimer.UpdateSince(start)
 
 	// Fill the block with all available pending transactions.
 	// we will abort when:
@@ -1128,6 +1139,7 @@ func (w *worker) fillTransactions(interruptCh chan int32, env *environment, stop
 			return err
 		}
 	}
+	fillTxFnTimer.UpdateSince(start)
 
 	return nil
 }

params/config.go

@@ -147,13 +147,11 @@ var (
 		LondonBlock:         big.NewInt(31302048),
 		HertzBlock:          big.NewInt(31302048),
 		HertzfixBlock:       big.NewInt(34140700),
-		// UnixTime: 1705996800 is January 23, 2024 8:00:00 AM UTC
-		ShanghaiTime:   newUint64(1705996800),
-		KeplerTime:     newUint64(1705996800),
-		FeynmanTime:    newUint64(1713419340),
-		FeynmanFixTime: newUint64(1713419340),
-		// TODO(GalaIO): enable cancun fork time later
-		//CancunTime: newUint64(),
+		ShanghaiTime:        newUint64(1705996800), // 2024-01-23 08:00:00 AM UTC
+		KeplerTime:          newUint64(1705996800), // 2024-01-23 08:00:00 AM UTC
+		FeynmanTime:         newUint64(1713419340), // 2024-04-18 05:49:00 AM UTC
+		FeynmanFixTime:      newUint64(1713419340), // 2024-04-18 05:49:00 AM UTC
+		CancunTime:          newUint64(1718863500), // 2024-06-20 06:05:00 AM UTC
 
 		Parlia: &ParliaConfig{
 			Period: 3,

params/protocol_params.go

@@ -189,6 +189,7 @@ const (
 var (
 	MinBlocksForBlobRequests           uint64 = 524288              // it keeps blob data available for ~18.2 days in local, ref: https://github.com/bnb-chain/BEPs/blob/master/BEPs/BEP-336.md#51-parameters.
 	DefaultExtraReserveForBlobRequests uint64 = 1 * (24 * 3600) / 3 // it adds more time for expired blobs for some request cases, like expiry blob when remote peer is syncing, default 1 day.
+	BreatheBlockInterval               uint64 = 86400               // Controls the interval for updateValidatorSetV2
 )
 
 // Gas discount table for BLS12-381 G1 and G2 multi exponentiation operations

params/version.go

@@ -23,7 +23,7 @@ import (
 const (
 	VersionMajor = 1  // Major version component of the current release
 	VersionMinor = 4  // Minor version component of the current release
-	VersionPatch = 6  // Patch version component of the current release
+	VersionPatch = 7  // Patch version component of the current release
 	VersionMeta  = "" // Version metadata to append to the version string
 )
 

trie/hbss2pbss.go

@@ -181,7 +181,7 @@ func (h2p *Hbss2Pbss) ConcurrentTraversal(theTrie *Trie, theNode node, path []by
 	}
 }
 
-// copy from trie/Commiter (*committer).commit
+// copy from trie/Committer (*committer).commit
 func (h2p *Hbss2Pbss) commitChildren(path []byte, n *fullNode) [17]node {
 	var children [17]node
 	for i := 0; i < 16; i++ {

trie/trie.go

@@ -21,14 +21,22 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/trie/trienode"
 	"github.com/ethereum/go-ethereum/triedb/database"
 )
 
+var (
+	trieGetTimer         = metrics.NewRegisteredTimer("trie/get/time", nil)
+	trieReaderGetTimer   = metrics.NewRegisteredTimer("trie/reader/get/time", nil)
+	trieReaderTotalTimer = metrics.NewRegisteredTimer("trie/reader/total/time", nil)
+)
+
 // Trie is a Merkle Patricia Trie. Use New to create a trie that sits on
 // top of a database. Whenever trie performs a commit operation, the generated
 // nodes will be gathered and returned in a set. Once the trie is committed,
@@ -146,6 +154,10 @@ func (t *Trie) Get(key []byte) ([]byte, error) {
 	if t.committed {
 		return nil, ErrCommitted
 	}
+	if metrics.EnabledExpensive {
+		start := time.Now()
+		defer func() { trieGetTimer.UpdateSince(start) }()
+	}
 	value, newroot, didResolve, err := t.get(t.root, keybytesToHex(key), 0)
 	if err == nil && didResolve {
 		t.root = newroot
@@ -178,7 +190,11 @@ func (t *Trie) get(origNode node, key []byte, pos int) (value []byte, newnode no
 		}
 		return value, n, didResolve, err
 	case hashNode:
+		start := time.Now()
 		child, err := t.resolveAndTrack(n, key[:pos])
+		if metrics.EnabledExpensive {
+			trieReaderGetTimer.UpdateSince(start)
+		}
 		if err != nil {
 			return nil, n, true, err
 		}
@@ -586,6 +602,10 @@ func (t *Trie) resolve(n node, prefix []byte) (node, error) {
 // node's original value. The rlp-encoded blob is preferred to be loaded from
 // database because it's easy to decode node while complex to encode node to blob.
 func (t *Trie) resolveAndTrack(n hashNode, prefix []byte) (node, error) {
+	if metrics.EnabledExpensive {
+		start := time.Now()
+		defer func() { trieReaderTotalTimer.UpdateSince(start) }()
+	}
 	blob, err := t.reader.node(prefix, common.BytesToHash(n))
 	if err != nil {
 		return nil, err

triedb/pathdb/difflayer.go

@@ -19,6 +19,7 @@ package pathdb
 import (
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
@@ -104,6 +105,8 @@ func (dl *diffLayer) node(owner common.Hash, path []byte, hash common.Hash, dept
 	dl.lock.RLock()
 	defer dl.lock.RUnlock()
 
+	start := time.Now()
+
 	// If the trie node is known locally, return it
 	subset, ok := dl.nodes[owner]
 	if ok {
@@ -114,14 +117,18 @@ func (dl *diffLayer) node(owner common.Hash, path []byte, hash common.Hash, dept
 			if n.Hash != hash {
 				dirtyFalseMeter.Mark(1)
 				log.Error("Unexpected trie node in diff layer", "owner", owner, "path", path, "expect", hash, "got", n.Hash)
+				pathGetDiffLayerTimer.UpdateSince(start)
 				return nil, newUnexpectedNodeError("diff", hash, n.Hash, owner, path, n.Blob)
 			}
 			dirtyHitMeter.Mark(1)
 			dirtyNodeHitDepthHist.Update(int64(depth))
 			dirtyReadMeter.Mark(int64(len(n.Blob)))
+			pathGetDiffLayerTimer.UpdateSince(start)
 			return n.Blob, nil
 		}
 	}
+	pathGetDiffLayerTimer.UpdateSince(start)
+
 	// Trie node unknown to this layer, resolve from parent
 	if diff, ok := dl.parent.(*diffLayer); ok {
 		return diff.node(owner, path, hash, depth+1)

triedb/pathdb/disklayer.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/VictoriaMetrics/fastcache"
 	"github.com/ethereum/go-ethereum/common"
@@ -27,6 +28,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/trie/trienode"
 	"github.com/ethereum/go-ethereum/trie/triestate"
 	"golang.org/x/crypto/sha3"
@@ -155,6 +157,10 @@ func (dl *diskLayer) markStale() {
 func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
 	dl.lock.RLock()
 	defer dl.lock.RUnlock()
+	if metrics.EnabledExpensive {
+		start := time.Now()
+		defer func() { pathGetDiskLayerTimer.UpdateSince(start) }()
+	}
 
 	if dl.stale {
 		return nil, errSnapshotStale

triedb/pathdb/journal.go

@@ -195,7 +195,8 @@ func newJournalReader(file string, db ethdb.Database, journalType JournalType) (
 // loadJournal tries to parse the layer journal from the disk.
 func (db *Database) loadJournal(diskRoot common.Hash) (layer, error) {
 	start := time.Now()
-	reader, err := newJournalReader(db.config.JournalFilePath, db.diskdb, db.DetermineJournalTypeForReader())
+	journalTypeForReader := db.DetermineJournalTypeForReader()
+	reader, err := newJournalReader(db.config.JournalFilePath, db.diskdb, journalTypeForReader)
 
 	if err != nil {
 		return nil, err
@@ -226,12 +227,12 @@ func (db *Database) loadJournal(diskRoot common.Hash) (layer, error) {
 		return nil, fmt.Errorf("%w want %x got %x", errUnmatchedJournal, root, diskRoot)
 	}
 	// Load the disk layer from the journal
-	base, err := db.loadDiskLayer(r)
+	base, err := db.loadDiskLayer(r, journalTypeForReader)
 	if err != nil {
 		return nil, err
 	}
 	// Load all the diff layers from the journal
-	head, err := db.loadDiffLayer(base, r)
+	head, err := db.loadDiffLayer(base, r, journalTypeForReader)
 	if err != nil {
 		return nil, err
 	}
@@ -262,14 +263,14 @@ func (db *Database) loadLayers() layer {
 
 // loadDiskLayer reads the binary blob from the layer journal, reconstructing
 // a new disk layer on it.
-func (db *Database) loadDiskLayer(r *rlp.Stream) (layer, error) {
+func (db *Database) loadDiskLayer(r *rlp.Stream, journalTypeForReader JournalType) (layer, error) {
 	// Resolve disk layer root
 	var (
 		root               common.Hash
 		journalBuf         *rlp.Stream
 		journalEncodedBuff []byte
 	)
-	if db.DetermineJournalTypeForReader() == JournalFileType {
+	if journalTypeForReader == JournalFileType {
 		if err := r.Decode(&journalEncodedBuff); err != nil {
 			return nil, fmt.Errorf("load disk journal: %v", err)
 		}
@@ -310,7 +311,7 @@ func (db *Database) loadDiskLayer(r *rlp.Stream) (layer, error) {
 		nodes[entry.Owner] = subset
 	}
 
-	if db.DetermineJournalTypeForReader() == JournalFileType {
+	if journalTypeForReader == JournalFileType {
 		var shaSum [32]byte
 		if err := r.Decode(&shaSum); err != nil {
 			return nil, fmt.Errorf("load shasum: %v", err)
@@ -329,14 +330,14 @@ func (db *Database) loadDiskLayer(r *rlp.Stream) (layer, error) {
 
 // loadDiffLayer reads the next sections of a layer journal, reconstructing a new
 // diff and verifying that it can be linked to the requested parent.
-func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream) (layer, error) {
+func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream, journalTypeForReader JournalType) (layer, error) {
 	// Read the next diff journal entry
 	var (
 		root               common.Hash
 		journalBuf         *rlp.Stream
 		journalEncodedBuff []byte
 	)
-	if db.DetermineJournalTypeForReader() == JournalFileType {
+	if journalTypeForReader == JournalFileType {
 		if err := r.Decode(&journalEncodedBuff); err != nil {
 			// The first read may fail with EOF, marking the end of the journal
 			if err == io.EOF {
@@ -409,7 +410,7 @@ func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream) (layer, error) {
 		storages[entry.Account] = set
 	}
 
-	if db.DetermineJournalTypeForReader() == JournalFileType {
+	if journalTypeForReader == JournalFileType {
 		var shaSum [32]byte
 		if err := r.Decode(&shaSum); err != nil {
 			return nil, fmt.Errorf("load shasum: %v", err)
@@ -423,7 +424,7 @@ func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream) (layer, error) {
 
 	log.Debug("Loaded diff layer journal", "root", root, "parent", parent.rootHash(), "id", parent.stateID()+1, "block", block)
 
-	return db.loadDiffLayer(newDiffLayer(parent, root, parent.stateID()+1, block, nodes, triestate.New(accounts, storages, incomplete)), r)
+	return db.loadDiffLayer(newDiffLayer(parent, root, parent.stateID()+1, block, nodes, triestate.New(accounts, storages, incomplete)), r, journalTypeForReader)
 }
 
 // journal implements the layer interface, marshaling the un-flushed trie nodes

triedb/pathdb/metrics.go

@@ -47,4 +47,7 @@ var (
 	historyBuildTimeMeter  = metrics.NewRegisteredTimer("pathdb/history/time", nil)
 	historyDataBytesMeter  = metrics.NewRegisteredMeter("pathdb/history/bytes/data", nil)
 	historyIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/index", nil)
+
+	pathGetDiffLayerTimer = metrics.NewRegisteredTimer("pathdb/get/difflayer/time", nil)
+	pathGetDiskLayerTimer = metrics.NewRegisteredTimer("pathdb/get/disklayer/time", nil)
 )