fix: upgrade pebble and improve config (#1979)
This commit is contained in:
wayen
GitHub
parent
8b94dd6b59
commit
497fdf8358
@ -27,6 +27,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/cockroachdb/errors"
|
||||
"github.com/cockroachdb/pebble"
|
||||
"github.com/cockroachdb/pebble/bloom"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
@ -116,6 +117,16 @@ func (d *Database) onWriteStallEnd() {
|
||||
d.writeDelayTime.Add(int64(time.Since(d.writeDelayStartTime)))
|
||||
}
|
||||
|
||||
// panicLogger is just a noop logger to disable Pebble's internal logger.
|
||||
type panicLogger struct{}
|
||||
|
||||
func (l panicLogger) Infof(format string, args ...interface{}) {
|
||||
}
|
||||
|
||||
func (l panicLogger) Fatalf(format string, args ...interface{}) {
|
||||
panic(errors.Errorf("fatal: "+format, args...))
|
||||
}
|
||||
|
||||
// New returns a wrapped pebble DB object. The namespace is the prefix that the
|
||||
// metrics reporting should use for surfacing internal stats.
|
||||
func New(file string, cache int, handles int, namespace string, readonly bool) (*Database, error) {
|
||||
@ -127,7 +138,6 @@ func New(file string, cache int, handles int, namespace string, readonly bool) (
|
||||
handles = minHandles
|
||||
}
|
||||
logger := log.New("database", file)
|
||||
logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles)
|
||||
|
||||
// The max memtable size is limited by the uint32 offsets stored in
|
||||
// internal/arenaskl.node, DeferredBatchOp, and flushableBatchEntry.
|
||||
@ -141,6 +151,10 @@ func New(file string, cache int, handles int, namespace string, readonly bool) (
|
||||
if memTableSize > maxMemTableSize {
|
||||
memTableSize = maxMemTableSize
|
||||
}
|
||||
|
||||
logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024),
|
||||
"handles", handles, "memory table", common.StorageSize(memTableSize))
|
||||
|
||||
db := &Database{
|
||||
fn: file,
|
||||
log: logger,
|
||||
@ -155,7 +169,7 @@ func New(file string, cache int, handles int, namespace string, readonly bool) (
|
||||
|
||||
// The size of memory table(as well as the write buffer).
|
||||
// Note, there may have more than two memory tables in the system.
|
||||
MemTableSize: memTableSize,
|
||||
MemTableSize: uint64(memTableSize),
|
||||
|
||||
// MemTableStopWritesThreshold places a hard limit on the size
|
||||
// of the existent MemTables(including the frozen one).
|
||||
@ -170,15 +184,6 @@ func New(file string, cache int, handles int, namespace string, readonly bool) (
|
||||
|
||||
// Per-level options. Options for at least one level must be specified. The
|
||||
// options for the last level are used for all subsequent levels.
|
||||
Levels: []pebble.LevelOptions{
|
||||
{TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
|
||||
{TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
|
||||
{TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
|
||||
{TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
|
||||
{TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
|
||||
{TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
|
||||
{TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
|
||||
},
|
||||
ReadOnly: readonly,
|
||||
EventListener: &pebble.EventListener{
|
||||
CompactionBegin: db.onCompactionBegin,
|
||||
@ -186,7 +191,21 @@ func New(file string, cache int, handles int, namespace string, readonly bool) (
|
||||
WriteStallBegin: db.onWriteStallBegin,
|
||||
WriteStallEnd: db.onWriteStallEnd,
|
||||
},
|
||||
Logger: panicLogger{}, // TODO(karalabe): Delete when this is upstreamed in Pebble
|
||||
}
|
||||
|
||||
for i := 0; i < len(opt.Levels); i++ {
|
||||
l := &opt.Levels[i]
|
||||
l.BlockSize = 32 << 10 // 32 KB
|
||||
l.IndexBlockSize = 256 << 10 // 256 KB
|
||||
l.FilterPolicy = bloom.FilterPolicy(10)
|
||||
l.FilterType = pebble.TableFilter
|
||||
if i > 0 {
|
||||
l.TargetFileSize = opt.Levels[i-1].TargetFileSize * 2
|
||||
}
|
||||
l.EnsureDefaults()
|
||||
}
|
||||
|
||||
// Disable seek compaction explicitly. Check https://github.com/ethereum/go-ethereum/pull/20130
|
||||
// for more details.
|
||||
opt.Experimental.ReadSamplingMultiplier = -1
|
||||
@ -305,9 +324,9 @@ func (d *Database) NewBatch() ethdb.Batch {
|
||||
// It's not supported by pebble, but pebble has better memory allocation strategy
|
||||
// which turns out a lot faster than leveldb. It's performant enough to construct
|
||||
// batch object without any pre-allocated space.
|
||||
func (d *Database) NewBatchWithSize(_ int) ethdb.Batch {
|
||||
func (d *Database) NewBatchWithSize(size int) ethdb.Batch {
|
||||
return &batch{
|
||||
b: d.db.NewBatch(),
|
||||
b: d.db.NewBatchWithSize(size),
|
||||
db: d,
|
||||
}
|
||||
}
|
||||
@ -576,7 +595,7 @@ type pebbleIterator struct {
|
||||
// of database content with a particular key prefix, starting at a particular
|
||||
// initial key (or after, if it does not exist).
|
||||
func (d *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
|
||||
iter := d.db.NewIter(&pebble.IterOptions{
|
||||
iter, _ := d.db.NewIter(&pebble.IterOptions{
|
||||
LowerBound: append(prefix, start...),
|
||||
UpperBound: upperBound(prefix),
|
||||
})
|
||||
|
||||
5
go.mod
5
go.mod
@ -13,7 +13,8 @@ require (
|
||||
github.com/btcsuite/btcd/btcec/v2 v2.3.2
|
||||
github.com/cespare/cp v1.1.1
|
||||
github.com/cloudflare/cloudflare-go v0.14.0
|
||||
github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06
|
||||
github.com/cockroachdb/errors v1.9.1
|
||||
github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593
|
||||
github.com/cometbft/cometbft v0.37.0
|
||||
github.com/consensys/gnark-crypto v0.10.0
|
||||
github.com/crate-crypto/go-kzg-4844 v0.3.0
|
||||
@ -109,9 +110,9 @@ require (
|
||||
github.com/cespare/xxhash v1.1.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/chzyer/readline v1.5.1 // indirect
|
||||
github.com/cockroachdb/errors v1.9.1 // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
|
||||
github.com/cockroachdb/redact v1.1.3 // indirect
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
|
||||
github.com/consensys/bavard v0.1.13 // indirect
|
||||
github.com/containerd/cgroups v1.1.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
|
||||
|
||||
8
go.sum
8
go.sum
@ -247,16 +247,18 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht
|
||||
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
|
||||
github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
|
||||
github.com/cockroachdb/datadriven v1.0.3-0.20230801171734-e384cf455877 h1:1MLK4YpFtIEo3ZtMA5C795Wtv5VuUnrXX7mQG+aHg6o=
|
||||
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
|
||||
github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8=
|
||||
github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk=
|
||||
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
|
||||
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
|
||||
github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06 h1:T+Np/xtzIjYM/P5NAw0e2Rf1FGvzDau1h54MKvx8G7w=
|
||||
github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06/go.mod h1:bynZ3gvVyhlvjLI7PT6dmZ7g76xzJ7HpxfjgkzCGz6s=
|
||||
github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 h1:aPEJyR4rPBvDmeyi+l/FS/VtA00IWvjeFvjen1m1l1A=
|
||||
github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593/go.mod h1:6hk1eMY/u5t+Cf18q5lFMUA1Rc+Sm5I6Ra1QuPyxXCo=
|
||||
github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ=
|
||||
github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
|
||||
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
|
||||
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
|
||||
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
|
||||
github.com/cometbft/cometbft-db v0.7.0 h1:uBjbrBx4QzU0zOEnU8KxoDl18dMNgDh+zZRUE0ucsbo=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user