tornadocontrib/bsc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

eth, trie/triedb/pathdb: reallocate excess memory to clean cache

Browse Source 
eth, trie/triedb/pathdb: address comments
This commit is contained in:
Gary Rong 2023-08-22 14:10:28 +08:00 committed by joeycli
parent 56424d390f
commit 604f1608b3
4 changed files with 24 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
eth/backend.go
View File

@ -63,6 +63,7 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
)
// Config contains the configuration options of the ETH protocol.
@ -129,7 +130,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
log.Warn("Sanitizing invalid miner gas price", "provided", config.Miner.GasPrice, "updated", ethconfig.Defaults.Miner.GasPrice)
config.Miner.GasPrice = new(big.Int).Set(ethconfig.Defaults.Miner.GasPrice)
}
if config.NoPruning && config.TrieDirtyCache > 0 {
// Redistribute memory allocation from in-memory trie node garbage collection
// to other caches when an archive node is requested.
if config.StateScheme == rawdb.HashScheme && config.NoPruning && config.TrieDirtyCache > 0 {
if config.SnapshotCache > 0 {
config.TrieCleanCache += config.TrieDirtyCache * 3 / 5
config.SnapshotCache += config.TrieDirtyCache * 2 / 5
@ -138,6 +141,13 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
}
config.TrieDirtyCache = 0
}
// Optimize memory distribution by reallocating surplus allowance from the
// dirty cache to the clean cache.
if config.StateScheme == rawdb.PathScheme && config.TrieDirtyCache > pathdb.MaxDirtyBufferSize/1024/1024 {
log.Info("Capped dirty cache size", "provided", common.StorageSize(config.TrieDirtyCache)*1024*1024, "adjusted", common.StorageSize(pathdb.MaxDirtyBufferSize))
config.TrieCleanCache += config.TrieDirtyCache - pathdb.MaxDirtyBufferSize/1024/1024
config.TrieDirtyCache = pathdb.MaxDirtyBufferSize / 1024 / 1024
}
log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024)
// Assemble the Ethereum object

2
eth/handler.go
View File

@ -980,6 +980,6 @@ func (h *handler) voteBroadcastLoop() {
func (h *handler) enableSyncedFeatures() {
h.acceptTxs.Store(true)
if h.chain.TrieDB().Scheme() == rawdb.PathScheme {
h.chain.TrieDB().SetBufferSize(pathdb.DefaultBufferSize)
h.chain.TrieDB().SetBufferSize(pathdb.DefaultDirtyBufferSize)
}
}

22
trie/triedb/pathdb/database.go
View File

@ -40,18 +40,18 @@ const (
// defaultCleanSize is the default memory allowance of clean cache.
defaultCleanSize = 16 * 1024 * 1024
// maxBufferSize is the maximum memory allowance of node buffer.
// MaxDirtyBufferSize is the maximum memory allowance of node buffer.
// Too large nodebuffer will cause the system to pause for a long
// time when write happens. Also, the largest batch that pebble can
// support is 4GB, node will panic if batch size exceeds this limit.
maxBufferSize = 256 * 1024 * 1024
MaxDirtyBufferSize = 256 * 1024 * 1024
// DefaultBufferSize is the default memory allowance of node buffer
// DefaultDirtyBufferSize is the default memory allowance of node buffer
// that aggregates the writes from above until it's flushed into the
// disk. It's meant to be used once the initial sync is finished.
// Do not increase the buffer size arbitrarily, otherwise the system
// pause time will increase when the database writes happen.
DefaultBufferSize = 64 * 1024 * 1024
DefaultDirtyBufferSize = 64 * 1024 * 1024
)
// layer is the interface implemented by all state layers which includes some
@ -96,9 +96,9 @@ type Config struct {
// unreasonable or unworkable.
func (c *Config) sanitize() *Config {
conf := *c
if conf.DirtyCacheSize > maxBufferSize {
log.Warn("Sanitizing invalid node buffer size", "provided", common.StorageSize(conf.DirtyCacheSize), "updated", common.StorageSize(maxBufferSize))
conf.DirtyCacheSize = maxBufferSize
if conf.DirtyCacheSize > MaxDirtyBufferSize {
log.Warn("Sanitizing invalid node buffer size", "provided", common.StorageSize(conf.DirtyCacheSize), "updated", common.StorageSize(MaxDirtyBufferSize))
conf.DirtyCacheSize = MaxDirtyBufferSize
}
return &conf
}
@ -107,7 +107,7 @@ func (c *Config) sanitize() *Config {
var Defaults = &Config{
StateHistory: params.FullImmutabilityThreshold,
CleanCacheSize: defaultCleanSize,
DirtyCacheSize: DefaultBufferSize,
DirtyCacheSize: DefaultDirtyBufferSize,
}
// ReadOnly is the config in order to open database in read only mode.
@ -413,9 +413,9 @@ func (db *Database) SetBufferSize(size int) error {
db.lock.Lock()
defer db.lock.Unlock()
if size > maxBufferSize {
log.Info("Capped node buffer size", "provided", common.StorageSize(size), "adjusted", common.StorageSize(maxBufferSize))
size = maxBufferSize
if size > MaxDirtyBufferSize {
log.Info("Capped node buffer size", "provided", common.StorageSize(size), "adjusted", common.StorageSize(MaxDirtyBufferSize))
size = MaxDirtyBufferSize
}
db.bufferSize = size
return db.tree.bottom().setBufferSize(db.bufferSize)

2
trie/triedb/pathdb/difflayer_test.go
View File

@ -29,7 +29,7 @@ import (
func emptyLayer() *diskLayer {
return &diskLayer{
db: New(rawdb.NewMemoryDatabase(), nil),
buffer: newNodeBuffer(DefaultBufferSize, nil, 0),
buffer: newNodeBuffer(DefaultDirtyBufferSize, nil, 0),
}
}