tornadocontrib/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

core, eth, node: break rawdb -> {leveldb, pebble} dependency (#30689)

Browse Source
This commit is contained in:
Péter Szilágyi 2024-10-29 10:31:04 +02:00 committed by GitHub
parent 98056e1ef2
commit 7180d26530
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 184 additions and 377 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
core/bench_test.go
View File

@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/pebble"
"github.com/ethereum/go-ethereum/params"
)
@ -173,18 +174,16 @@ func genUncles(i int, gen *BlockGen) {
func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
// Create the database in memory or in a temporary directory.
var db ethdb.Database
var err error
if !disk {
db = rawdb.NewMemoryDatabase()
} else {
dir := b.TempDir()
db, err = rawdb.NewLevelDBDatabase(dir, 128, 128, "", false)
pdb, err := pebble.New(b.TempDir(), 128, 128, "", false)
if err != nil {
b.Fatalf("cannot create temporary database: %v", err)
}
db = rawdb.NewDatabase(pdb)
defer db.Close()
}
// Generate a chain of b.N blocks using the supplied block
// generator function.
gspec := &Genesis{
@ -281,11 +280,11 @@ func makeChainForBench(db ethdb.Database, genesis *Genesis, full bool, count uin
func benchWriteChain(b *testing.B, full bool, count uint64) {
genesis := &Genesis{Config: params.AllEthashProtocolChanges}
for i := 0; i < b.N; i++ {
dir := b.TempDir()
db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false)
pdb, err := pebble.New(b.TempDir(), 1024, 128, "", false)
if err != nil {
b.Fatalf("error opening database at %v: %v", dir, err)
b.Fatalf("error opening database: %v", err)
}
db := rawdb.NewDatabase(pdb)
makeChainForBench(db, genesis, full, count)
db.Close()
}
@ -294,10 +293,12 @@ func benchWriteChain(b *testing.B, full bool, count uint64) {
func benchReadChain(b *testing.B, full bool, count uint64) {
dir := b.TempDir()
db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false)
pdb, err := pebble.New(dir, 1024, 128, "", false)
if err != nil {
b.Fatalf("error opening database at %v: %v", dir, err)
b.Fatalf("error opening database: %v", err)
}
db := rawdb.NewDatabase(pdb)
genesis := &Genesis{Config: params.AllEthashProtocolChanges}
makeChainForBench(db, genesis, full, count)
db.Close()
@ -308,15 +309,16 @@ func benchReadChain(b *testing.B, full bool, count uint64) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false)
pdb, err = pebble.New(dir, 1024, 128, "", false)
if err != nil {
b.Fatalf("error opening database at %v: %v", dir, err)
b.Fatalf("error opening database: %v", err)
}
db = rawdb.NewDatabase(pdb)
chain, err := NewBlockChain(db, &cacheConfig, genesis, nil, ethash.NewFaker(), vm.Config{}, nil)
if err != nil {
b.Fatalf("error creating chain: %v", err)
}
for n := uint64(0); n < count; n++ {
header := chain.GetHeaderByNumber(n)
if full {

45
core/blockchain_repair_test.go
View File

@ -31,6 +31,7 @@ import (
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/ethdb/pebble"
"github.com/ethereum/go-ethereum/params"
)
@ -1764,12 +1765,13 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
datadir := t.TempDir()
ancient := filepath.Join(datadir, "ancient")
db, err := rawdb.Open(rawdb.OpenOptions{
Directory: datadir,
AncientsDirectory: ancient,
})
pdb, err := pebble.New(datadir, 0, 0, "", false)
if err != nil {
t.Fatalf("Failed to create persistent database: %v", err)
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
defer db.Close() // Might double close, should be fine
@ -1848,12 +1850,13 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
chain.stopWithoutSaving()
// Start a new blockchain back up and see where the repair leads us
db, err = rawdb.Open(rawdb.OpenOptions{
Directory: datadir,
AncientsDirectory: ancient,
})
pdb, err = pebble.New(datadir, 0, 0, "", false)
if err != nil {
t.Fatalf("Failed to reopen persistent database: %v", err)
t.Fatalf("Failed to reopen persistent key-value database: %v", err)
}
db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
if err != nil {
t.Fatalf("Failed to reopen persistent freezer database: %v", err)
}
defer db.Close()
@ -1912,12 +1915,13 @@ func testIssue23496(t *testing.T, scheme string) {
datadir := t.TempDir()
ancient := filepath.Join(datadir, "ancient")
db, err := rawdb.Open(rawdb.OpenOptions{
Directory: datadir,
AncientsDirectory: ancient,
})
pdb, err := pebble.New(datadir, 0, 0, "", false)
if err != nil {
t.Fatalf("Failed to create persistent database: %v", err)
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
defer db.Close() // Might double close, should be fine
@ -1969,12 +1973,13 @@ func testIssue23496(t *testing.T, scheme string) {
chain.stopWithoutSaving()
// Start a new blockchain back up and see where the repair leads us
db, err = rawdb.Open(rawdb.OpenOptions{
Directory: datadir,
AncientsDirectory: ancient,
})
pdb, err = pebble.New(datadir, 0, 0, "", false)
if err != nil {
t.Fatalf("Failed to reopen persistent database: %v", err)
t.Fatalf("Failed to reopen persistent key-value database: %v", err)
}
db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
if err != nil {
t.Fatalf("Failed to reopen persistent freezer database: %v", err)
}
defer db.Close()

12
core/blockchain_sethead_test.go
View File

@ -33,6 +33,7 @@ import (
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/ethdb/pebble"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/triedb"
"github.com/ethereum/go-ethereum/triedb/hashdb"
@ -1968,12 +1969,13 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme
datadir := t.TempDir()
ancient := filepath.Join(datadir, "ancient")
db, err := rawdb.Open(rawdb.OpenOptions{
Directory: datadir,
AncientsDirectory: ancient,
})
pdb, err := pebble.New(datadir, 0, 0, "", false)
if err != nil {
t.Fatalf("Failed to create persistent database: %v", err)
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
defer db.Close()

23
core/blockchain_snapshot_test.go
View File

@ -35,6 +35,7 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/pebble"
"github.com/ethereum/go-ethereum/params"
)
@ -65,12 +66,13 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
datadir := t.TempDir()
ancient := filepath.Join(datadir, "ancient")
db, err := rawdb.Open(rawdb.OpenOptions{
Directory: datadir,
AncientsDirectory: ancient,
})
pdb, err := pebble.New(datadir, 0, 0, "", false)
if err != nil {
t.Fatalf("Failed to create persistent database: %v", err)
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
// Initialize a fresh chain
var (
@ -255,12 +257,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
chain.triedb.Close()
// Start a new blockchain back up and see where the repair leads us
newdb, err := rawdb.Open(rawdb.OpenOptions{
Directory: snaptest.datadir,
AncientsDirectory: snaptest.ancient,
})
pdb, err := pebble.New(snaptest.datadir, 0, 0, "", false)
if err != nil {
t.Fatalf("Failed to reopen persistent database: %v", err)
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
newdb, err := rawdb.NewDatabaseWithFreezer(pdb, snaptest.ancient, "", false)
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
defer newdb.Close()

46
core/blockchain_test.go
View File

@ -29,7 +29,6 @@ import (
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/math"
"github.com/ethereum/go-ethereum/consensus"
"github.com/ethereum/go-ethereum/consensus/beacon"
"github.com/ethereum/go-ethereum/consensus/ethash"
@ -40,6 +39,7 @@ import (
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/eth/tracers/logger"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/pebble"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
"github.com/holiman/uint256"
@ -2663,12 +2663,13 @@ func testSideImportPrunedBlocks(t *testing.T, scheme string) {
datadir := t.TempDir()
ancient := path.Join(datadir, "ancient")
db, err := rawdb.Open(rawdb.OpenOptions{
Directory: datadir,
AncientsDirectory: ancient,
})
pdb, err := pebble.New(datadir, 0, 0, "", false)
if err != nil {
t.Fatalf("Failed to create persistent database: %v", err)
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
defer db.Close()
@ -4231,36 +4232,3 @@ func TestPragueRequests(t *testing.T) {
t.Fatalf("block %d: failed to insert into chain: %v", n, err)
}
}
func BenchmarkReorg(b *testing.B) {
chainLength := b.N
dir := b.TempDir()
db, err := rawdb.NewLevelDBDatabase(dir, 128, 128, "", false)
if err != nil {
b.Fatalf("cannot create temporary database: %v", err)
}
defer db.Close()
gspec := &Genesis{
Config: params.TestChainConfig,
Alloc: types.GenesisAlloc{benchRootAddr: {Balance: math.BigPow(2, 254)}},
}
blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
defer blockchain.Stop()
// Insert an easy and a difficult chain afterwards
easyBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.GetBlockByHash(blockchain.CurrentBlock().Hash()), ethash.NewFaker(), db, chainLength, genValueTx(50000))
diffBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.GetBlockByHash(blockchain.CurrentBlock().Hash()), ethash.NewFaker(), db, chainLength, genValueTx(50000))
if _, err := blockchain.InsertChain(easyBlocks); err != nil {
b.Fatalf("failed to insert easy chain: %v", err)
}
b.ResetTimer()
if _, err := blockchain.InsertChain(diffBlocks); err != nil {
b.Fatalf("failed to insert difficult chain: %v", err)
}
}
// Master: BenchmarkReorg-8 10000 899591 ns/op 820154 B/op 1440 allocs/op 1549443072 bytes of heap used
// WithoutOldChain: BenchmarkReorg-8 10000 1147281 ns/op 943163 B/op 1564 allocs/op 1163870208 bytes of heap used
// WithoutNewChain: BenchmarkReorg-8 10000 1018922 ns/op 943580 B/op 1564 allocs/op 1171890176 bytes of heap used

101
core/rawdb/database.go
View File

@ -27,9 +27,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/leveldb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/ethdb/pebble"
"github.com/ethereum/go-ethereum/log"
"github.com/olekukonko/tablewriter"
)
@ -299,37 +297,9 @@ func NewMemoryDatabase() ethdb.Database {
return NewDatabase(memorydb.New())
}
// NewMemoryDatabaseWithCap creates an ephemeral in-memory key-value database
// with an initial starting capacity, but without a freezer moving immutable
// chain segments into cold storage.
func NewMemoryDatabaseWithCap(size int) ethdb.Database {
return NewDatabase(memorydb.NewWithCap(size))
}
// NewLevelDBDatabase creates a persistent key-value database without a freezer
// moving immutable chain segments into cold storage.
func NewLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
db, err := leveldb.New(file, cache, handles, namespace, readonly)
if err != nil {
return nil, err
}
log.Info("Using LevelDB as the backing database")
return NewDatabase(db), nil
}
// NewPebbleDBDatabase creates a persistent key-value database without a freezer
// moving immutable chain segments into cold storage.
func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
db, err := pebble.New(file, cache, handles, namespace, readonly)
if err != nil {
return nil, err
}
return NewDatabase(db), nil
}
const (
dbPebble = "pebble"
dbLeveldb = "leveldb"
DBPebble = "pebble"
DBLeveldb = "leveldb"
)
// PreexistingDatabase checks the given data directory whether a database is already
@ -343,72 +313,9 @@ func PreexistingDatabase(path string) string {
if err != nil {
panic(err) // only possible if the pattern is malformed
}
return dbPebble
return DBPebble
}
return dbLeveldb
}
// OpenOptions contains the options to apply when opening a database.
// OBS: If AncientsDirectory is empty, it indicates that no freezer is to be used.
type OpenOptions struct {
Type string // "leveldb" | "pebble"
Directory string // the datadir
AncientsDirectory string // the ancients-dir
Namespace string // the namespace for database relevant metrics
Cache int // the capacity(in megabytes) of the data caching
Handles int // number of files to be open simultaneously
ReadOnly bool
}
// openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble.
//
// type == null type != null
// +----------------------------------------
// db is non-existent | pebble default | specified type
// db is existent | from db | specified type (if compatible)
func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) {
// Reject any unsupported database type
if len(o.Type) != 0 && o.Type != dbLeveldb && o.Type != dbPebble {
return nil, fmt.Errorf("unknown db.engine %v", o.Type)
}
// Retrieve any pre-existing database's type and use that or the requested one
// as long as there's no conflict between the two types
existingDb := PreexistingDatabase(o.Directory)
if len(existingDb) != 0 && len(o.Type) != 0 && o.Type != existingDb {
return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb)
}
if o.Type == dbPebble || existingDb == dbPebble {
log.Info("Using pebble as the backing database")
return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
}
if o.Type == dbLeveldb || existingDb == dbLeveldb {
log.Info("Using leveldb as the backing database")
return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
}
// No pre-existing database, no user-requested one either. Default to Pebble.
log.Info("Defaulting to pebble as the backing database")
return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
}
// Open opens both a disk-based key-value database such as leveldb or pebble, but also
// integrates it with a freezer database -- if the AncientDir option has been
// set on the provided OpenOptions.
// The passed o.AncientDir indicates the path of root ancient directory where
// the chain freezer can be opened.
func Open(o OpenOptions) (ethdb.Database, error) {
kvdb, err := openKeyValueDatabase(o)
if err != nil {
return nil, err
}
if len(o.AncientsDirectory) == 0 {
return kvdb, nil
}
frdb, err := NewDatabaseWithFreezer(kvdb, o.AncientsDirectory, o.Namespace, o.ReadOnly)
if err != nil {
kvdb.Close()
return nil, err
}
return frdb, nil
return DBLeveldb
}
type counter uint64

189
eth/filters/bench_test.go
View File

@ -1,189 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package filters
import (
"context"
"fmt"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/bitutil"
"github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/node"
)
func BenchmarkBloomBits512(b *testing.B) {
benchmarkBloomBits(b, 512)
}
func BenchmarkBloomBits1k(b *testing.B) {
benchmarkBloomBits(b, 1024)
}
func BenchmarkBloomBits2k(b *testing.B) {
benchmarkBloomBits(b, 2048)
}
func BenchmarkBloomBits4k(b *testing.B) {
benchmarkBloomBits(b, 4096)
}
func BenchmarkBloomBits8k(b *testing.B) {
benchmarkBloomBits(b, 8192)
}
func BenchmarkBloomBits16k(b *testing.B) {
benchmarkBloomBits(b, 16384)
}
func BenchmarkBloomBits32k(b *testing.B) {
benchmarkBloomBits(b, 32768)
}
const benchFilterCnt = 2000
func benchmarkBloomBits(b *testing.B, sectionSize uint64) {
b.Skip("test disabled: this tests presume (and modify) an existing datadir.")
benchDataDir := node.DefaultDataDir() + "/geth/chaindata"
b.Log("Running bloombits benchmark section size:", sectionSize)
db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false)
if err != nil {
b.Fatalf("error opening database at %v: %v", benchDataDir, err)
}
head := rawdb.ReadHeadBlockHash(db)
if head == (common.Hash{}) {
b.Fatalf("chain data not found at %v", benchDataDir)
}
clearBloomBits(db)
b.Log("Generating bloombits data...")
headNum := rawdb.ReadHeaderNumber(db, head)
if headNum == nil || *headNum < sectionSize+512 {
b.Fatalf("not enough blocks for running a benchmark")
}
start := time.Now()
cnt := (*headNum - 512) / sectionSize
var dataSize, compSize uint64
for sectionIdx := uint64(0); sectionIdx < cnt; sectionIdx++ {
bc, err := bloombits.NewGenerator(uint(sectionSize))
if err != nil {
b.Fatalf("failed to create generator: %v", err)
}
var header *types.Header
for i := sectionIdx * sectionSize; i < (sectionIdx+1)*sectionSize; i++ {
hash := rawdb.ReadCanonicalHash(db, i)
if header = rawdb.ReadHeader(db, hash, i); header == nil {
b.Fatalf("Error creating bloomBits data")
return
}
bc.AddBloom(uint(i-sectionIdx*sectionSize), header.Bloom)
}
sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*sectionSize-1)
for i := 0; i < types.BloomBitLength; i++ {
data, err := bc.Bitset(uint(i))
if err != nil {
b.Fatalf("failed to retrieve bitset: %v", err)
}
comp := bitutil.CompressBytes(data)
dataSize += uint64(len(data))
compSize += uint64(len(comp))
rawdb.WriteBloomBits(db, uint(i), sectionIdx, sectionHead, comp)
}
//if sectionIdx%50 == 0 {
// b.Log(" section", sectionIdx, "/", cnt)
//}
}
d := time.Since(start)
b.Log("Finished generating bloombits data")
b.Log(" ", d, "total ", d/time.Duration(cnt*sectionSize), "per block")
b.Log(" data size:", dataSize, " compressed size:", compSize, " compression ratio:", float64(compSize)/float64(dataSize))
b.Log("Running filter benchmarks...")
start = time.Now()
var (
backend *testBackend
sys *FilterSystem
)
for i := 0; i < benchFilterCnt; i++ {
if i%20 == 0 {
db.Close()
db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false)
backend = &testBackend{db: db, sections: cnt}
sys = NewFilterSystem(backend, Config{})
}
var addr common.Address
addr[0] = byte(i)
addr[1] = byte(i / 256)
filter := sys.NewRangeFilter(0, int64(cnt*sectionSize-1), []common.Address{addr}, nil)
if _, err := filter.Logs(context.Background()); err != nil {
b.Error("filter.Logs error:", err)
}
}
d = time.Since(start)
b.Log("Finished running filter benchmarks")
b.Log(" ", d, "total ", d/time.Duration(benchFilterCnt), "per address", d*time.Duration(1000000)/time.Duration(benchFilterCnt*cnt*sectionSize), "per million blocks")
db.Close()
}
//nolint:unused
func clearBloomBits(db ethdb.Database) {
var bloomBitsPrefix = []byte("bloomBits-")
fmt.Println("Clearing bloombits data...")
it := db.NewIterator(bloomBitsPrefix, nil)
for it.Next() {
db.Delete(it.Key())
}
it.Release()
}
func BenchmarkNoBloomBits(b *testing.B) {
b.Skip("test disabled: this tests presume (and modify) an existing datadir.")
benchDataDir := node.DefaultDataDir() + "/geth/chaindata"
b.Log("Running benchmark without bloombits")
db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false)
if err != nil {
b.Fatalf("error opening database at %v: %v", benchDataDir, err)
}
head := rawdb.ReadHeadBlockHash(db)
if head == (common.Hash{}) {
b.Fatalf("chain data not found at %v", benchDataDir)
}
headNum := rawdb.ReadHeaderNumber(db, head)
clearBloomBits(db)
_, sys := newTestFilterSystem(b, db, Config{})
b.Log("Running filter benchmarks...")
start := time.Now()
filter := sys.NewRangeFilter(0, int64(*headNum), []common.Address{{}}, nil)
filter.Logs(context.Background())
d := time.Since(start)
b.Log("Finished running filter benchmarks")
b.Log(" ", d, "total ", d*time.Duration(1000000)/time.Duration(*headNum+1), "per million blocks")
db.Close()
}

2
eth/filters/filter_test.go
View File

@ -48,7 +48,7 @@ func makeReceipt(addr common.Address) *types.Receipt {
func BenchmarkFilters(b *testing.B) {
var (
db, _ = rawdb.NewLevelDBDatabase(b.TempDir(), 0, 0, "", false)
db = rawdb.NewMemoryDatabase()
_, sys = newTestFilterSystem(b, db, Config{})
key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
addr1 = crypto.PubkeyToAddress(key1.PublicKey)

111
node/database.go Normal file
View File

@ -0,0 +1,111 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package node
import (
"fmt"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/leveldb"
"github.com/ethereum/go-ethereum/ethdb/pebble"
"github.com/ethereum/go-ethereum/log"
)
// openOptions contains the options to apply when opening a database.
// OBS: If AncientsDirectory is empty, it indicates that no freezer is to be used.
type openOptions struct {
Type string // "leveldb" | "pebble"
Directory string // the datadir
AncientsDirectory string // the ancients-dir
Namespace string // the namespace for database relevant metrics
Cache int // the capacity(in megabytes) of the data caching
Handles int // number of files to be open simultaneously
ReadOnly bool
}
// openDatabase opens both a disk-based key-value database such as leveldb or pebble, but also
// integrates it with a freezer database -- if the AncientDir option has been
// set on the provided OpenOptions.
// The passed o.AncientDir indicates the path of root ancient directory where
// the chain freezer can be opened.
func openDatabase(o openOptions) (ethdb.Database, error) {
kvdb, err := openKeyValueDatabase(o)
if err != nil {
return nil, err
}
if len(o.AncientsDirectory) == 0 {
return kvdb, nil
}
frdb, err := rawdb.NewDatabaseWithFreezer(kvdb, o.AncientsDirectory, o.Namespace, o.ReadOnly)
if err != nil {
kvdb.Close()
return nil, err
}
return frdb, nil
}
// openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble.
//
// type == null type != null
// +----------------------------------------
// db is non-existent | pebble default | specified type
// db is existent | from db | specified type (if compatible)
func openKeyValueDatabase(o openOptions) (ethdb.Database, error) {
// Reject any unsupported database type
if len(o.Type) != 0 && o.Type != rawdb.DBLeveldb && o.Type != rawdb.DBPebble {
return nil, fmt.Errorf("unknown db.engine %v", o.Type)
}
// Retrieve any pre-existing database's type and use that or the requested one
// as long as there's no conflict between the two types
existingDb := rawdb.PreexistingDatabase(o.Directory)
if len(existingDb) != 0 && len(o.Type) != 0 && o.Type != existingDb {
return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb)
}
if o.Type == rawdb.DBPebble || existingDb == rawdb.DBPebble {
log.Info("Using pebble as the backing database")
return newPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
}
if o.Type == rawdb.DBLeveldb || existingDb == rawdb.DBLeveldb {
log.Info("Using leveldb as the backing database")
return newLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
}
// No pre-existing database, no user-requested one either. Default to Pebble.
log.Info("Defaulting to pebble as the backing database")
return newPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
}
// newLevelDBDatabase creates a persistent key-value database without a freezer
// moving immutable chain segments into cold storage.
func newLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
db, err := leveldb.New(file, cache, handles, namespace, readonly)
if err != nil {
return nil, err
}
log.Info("Using LevelDB as the backing database")
return rawdb.NewDatabase(db), nil
}
// newPebbleDBDatabase creates a persistent key-value database without a freezer
// moving immutable chain segments into cold storage.
func newPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
db, err := pebble.New(file, cache, handles, namespace, readonly)
if err != nil {
return nil, err
}
return rawdb.NewDatabase(db), nil
}

6
node/node.go
View File

@ -723,7 +723,7 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r
if n.config.DataDir == "" {
db = rawdb.NewMemoryDatabase()
} else {
db, err = rawdb.Open(rawdb.OpenOptions{
db, err = openDatabase(openOptions{
Type: n.config.DBEngine,
Directory: n.ResolvePath(name),
Namespace: namespace,
@ -732,7 +732,6 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r
ReadOnly: readonly,
})
}
if err == nil {
db = n.wrapDatabase(db)
}
@ -755,7 +754,7 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient
if n.config.DataDir == "" {
db, err = rawdb.NewDatabaseWithFreezer(memorydb.New(), "", namespace, readonly)
} else {
db, err = rawdb.Open(rawdb.OpenOptions{
db, err = openDatabase(openOptions{
Type: n.config.DBEngine,
Directory: n.ResolvePath(name),
AncientsDirectory: n.ResolveAncient(name, ancient),
@ -765,7 +764,6 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient
ReadOnly: readonly,
})
}
if err == nil {
db = n.wrapDatabase(db)
}