Merge pull request #19252 from karalabe/badgerdb

ethdb: tiny API tidy-up from the database rework pr
This commit is contained in:
Péter Szilágyi 2019-03-12 12:57:23 +02:00 committed by GitHub
commit b87a68407b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 81 additions and 81 deletions

@ -38,27 +38,27 @@ import (
) )
const ( const (
// leveldbDegradationWarnInterval specifies how often warning should be printed // degradationWarnInterval specifies how often warning should be printed if the
// if the leveldb database cannot keep up with requested writes. // leveldb database cannot keep up with requested writes.
leveldbDegradationWarnInterval = time.Minute degradationWarnInterval = time.Minute
// leveldbMinCache is the minimum amount of memory in megabytes to allocate to // minCache is the minimum amount of memory in megabytes to allocate to leveldb
// leveldb read and write caching, split half and half. // read and write caching, split half and half.
leveldbMinCache = 16 minCache = 16
// leveldbMinHandles is the minimum number of files handles to allocate to the // minHandles is the minimum number of files handles to allocate to the open
// open database files. // database files.
leveldbMinHandles = 16 minHandles = 16
// metricsGatheringInterval specifies the interval to retrieve leveldb database // metricsGatheringInterval specifies the interval to retrieve leveldb database
// compaction, io and pause stats to report to the user. // compaction, io and pause stats to report to the user.
metricsGatheringInterval = 3 * time.Second metricsGatheringInterval = 3 * time.Second
) )
// LevelDBDatabase is a persistent key-value store. Apart from basic data storage // Database is a persistent key-value store. Apart from basic data storage
// functionality it also supports batch writes and iterating over the keyspace in // functionality it also supports batch writes and iterating over the keyspace in
// binary-alphabetical order. // binary-alphabetical order.
type LevelDBDatabase struct { type Database struct {
fn string // filename for reporting fn string // filename for reporting
db *leveldb.DB // LevelDB instance db *leveldb.DB // LevelDB instance
@ -78,13 +78,13 @@ type LevelDBDatabase struct {
// New returns a wrapped LevelDB object. The namespace is the prefix that the // New returns a wrapped LevelDB object. The namespace is the prefix that the
// metrics reporting should use for surfacing internal stats. // metrics reporting should use for surfacing internal stats.
func New(file string, cache int, handles int, namespace string) (*LevelDBDatabase, error) { func New(file string, cache int, handles int, namespace string) (*Database, error) {
// Ensure we have some minimal caching and file guarantees // Ensure we have some minimal caching and file guarantees
if cache < leveldbMinCache { if cache < minCache {
cache = leveldbMinCache cache = minCache
} }
if handles < leveldbMinHandles { if handles < minHandles {
handles = leveldbMinHandles handles = minHandles
} }
logger := log.New("database", file) logger := log.New("database", file)
logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles) logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles)
@ -103,7 +103,7 @@ func New(file string, cache int, handles int, namespace string) (*LevelDBDatabas
return nil, err return nil, err
} }
// Assemble the wrapper with all the registered metrics // Assemble the wrapper with all the registered metrics
ldb := &LevelDBDatabase{ ldb := &Database{
fn: file, fn: file,
db: db, db: db,
log: logger, log: logger,
@ -124,7 +124,7 @@ func New(file string, cache int, handles int, namespace string) (*LevelDBDatabas
// Close stops the metrics collection, flushes any pending data to disk and closes // Close stops the metrics collection, flushes any pending data to disk and closes
// all io accesses to the underlying key-value store. // all io accesses to the underlying key-value store.
func (db *LevelDBDatabase) Close() error { func (db *Database) Close() error {
db.quitLock.Lock() db.quitLock.Lock()
defer db.quitLock.Unlock() defer db.quitLock.Unlock()
@ -140,12 +140,12 @@ func (db *LevelDBDatabase) Close() error {
} }
// Has retrieves if a key is present in the key-value store. // Has retrieves if a key is present in the key-value store.
func (db *LevelDBDatabase) Has(key []byte) (bool, error) { func (db *Database) Has(key []byte) (bool, error) {
return db.db.Has(key, nil) return db.db.Has(key, nil)
} }
// Get retrieves the given key if it's present in the key-value store. // Get retrieves the given key if it's present in the key-value store.
func (db *LevelDBDatabase) Get(key []byte) ([]byte, error) { func (db *Database) Get(key []byte) ([]byte, error) {
dat, err := db.db.Get(key, nil) dat, err := db.db.Get(key, nil)
if err != nil { if err != nil {
return nil, err return nil, err
@ -154,19 +154,19 @@ func (db *LevelDBDatabase) Get(key []byte) ([]byte, error) {
} }
// Put inserts the given value into the key-value store. // Put inserts the given value into the key-value store.
func (db *LevelDBDatabase) Put(key []byte, value []byte) error { func (db *Database) Put(key []byte, value []byte) error {
return db.db.Put(key, value, nil) return db.db.Put(key, value, nil)
} }
// Delete removes the key from the key-value store. // Delete removes the key from the key-value store.
func (db *LevelDBDatabase) Delete(key []byte) error { func (db *Database) Delete(key []byte) error {
return db.db.Delete(key, nil) return db.db.Delete(key, nil)
} }
// NewBatch creates a write-only key-value store that buffers changes to its host // NewBatch creates a write-only key-value store that buffers changes to its host
// database until a final write is called. // database until a final write is called.
func (db *LevelDBDatabase) NewBatch() ethdb.Batch { func (db *Database) NewBatch() ethdb.Batch {
return &levelDBBatch{ return &batch{
db: db.db, db: db.db,
b: new(leveldb.Batch), b: new(leveldb.Batch),
} }
@ -174,18 +174,18 @@ func (db *LevelDBDatabase) NewBatch() ethdb.Batch {
// NewIterator creates a binary-alphabetical iterator over the entire keyspace // NewIterator creates a binary-alphabetical iterator over the entire keyspace
// contained within the leveldb database. // contained within the leveldb database.
func (db *LevelDBDatabase) NewIterator() ethdb.Iterator { func (db *Database) NewIterator() ethdb.Iterator {
return db.NewIteratorWithPrefix(nil) return db.NewIteratorWithPrefix(nil)
} }
// NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset // NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset
// of database content with a particular key prefix. // of database content with a particular key prefix.
func (db *LevelDBDatabase) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator { func (db *Database) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator {
return db.db.NewIterator(util.BytesPrefix(prefix), nil) return db.db.NewIterator(util.BytesPrefix(prefix), nil)
} }
// Stat returns a particular internal stat of the database. // Stat returns a particular internal stat of the database.
func (db *LevelDBDatabase) Stat(property string) (string, error) { func (db *Database) Stat(property string) (string, error) {
return db.db.GetProperty(property) return db.db.GetProperty(property)
} }
@ -196,12 +196,12 @@ func (db *LevelDBDatabase) Stat(property string) (string, error) {
// A nil start is treated as a key before all keys in the data store; a nil limit // A nil start is treated as a key before all keys in the data store; a nil limit
// is treated as a key after all keys in the data store. If both is nil then it // is treated as a key after all keys in the data store. If both is nil then it
// will compact entire data store. // will compact entire data store.
func (db *LevelDBDatabase) Compact(start []byte, limit []byte) error { func (db *Database) Compact(start []byte, limit []byte) error {
return db.db.CompactRange(util.Range{Start: start, Limit: limit}) return db.db.CompactRange(util.Range{Start: start, Limit: limit})
} }
// Path returns the path to the database directory. // Path returns the path to the database directory.
func (db *LevelDBDatabase) Path() string { func (db *Database) Path() string {
return db.fn return db.fn
} }
@ -222,7 +222,7 @@ func (db *LevelDBDatabase) Path() string {
// //
// This is how the iostats look like (currently): // This is how the iostats look like (currently):
// Read(MB):3895.04860 Write(MB):3654.64712 // Read(MB):3895.04860 Write(MB):3654.64712
func (db *LevelDBDatabase) meter(refresh time.Duration) { func (db *Database) meter(refresh time.Duration) {
// Create the counters to store current and previous compaction values // Create the counters to store current and previous compaction values
compactions := make([][]float64, 2) compactions := make([][]float64, 2)
for i := 0; i < 2; i++ { for i := 0; i < 2; i++ {
@ -326,7 +326,7 @@ func (db *LevelDBDatabase) meter(refresh time.Duration) {
// If a warning that db is performing compaction has been displayed, any subsequent // If a warning that db is performing compaction has been displayed, any subsequent
// warnings will be withheld for one minute not to overwhelm the user. // warnings will be withheld for one minute not to overwhelm the user.
if paused && delayN-delaystats[0] == 0 && duration.Nanoseconds()-delaystats[1] == 0 && if paused && delayN-delaystats[0] == 0 && duration.Nanoseconds()-delaystats[1] == 0 &&
time.Now().After(lastWritePaused.Add(leveldbDegradationWarnInterval)) { time.Now().After(lastWritePaused.Add(degradationWarnInterval)) {
db.log.Warn("Database compacting, degraded performance") db.log.Warn("Database compacting, degraded performance")
lastWritePaused = time.Now() lastWritePaused = time.Now()
} }
@ -379,40 +379,40 @@ func (db *LevelDBDatabase) meter(refresh time.Duration) {
errc <- merr errc <- merr
} }
// levelDBBatch is a write-only leveldb batch that commits changes to its host // batch is a write-only leveldb batch that commits changes to its host database
// database when Write is called. A batch cannot be used concurrently. // when Write is called. A batch cannot be used concurrently.
type levelDBBatch struct { type batch struct {
db *leveldb.DB db *leveldb.DB
b *leveldb.Batch b *leveldb.Batch
size int size int
} }
// Put inserts the given value into the batch for later committing. // Put inserts the given value into the batch for later committing.
func (b *levelDBBatch) Put(key, value []byte) error { func (b *batch) Put(key, value []byte) error {
b.b.Put(key, value) b.b.Put(key, value)
b.size += len(value) b.size += len(value)
return nil return nil
} }
// Delete inserts the a key removal into the batch for later committing. // Delete inserts the a key removal into the batch for later committing.
func (b *levelDBBatch) Delete(key []byte) error { func (b *batch) Delete(key []byte) error {
b.b.Delete(key) b.b.Delete(key)
b.size += 1 b.size++
return nil return nil
} }
// ValueSize retrieves the amount of data queued up for writing. // ValueSize retrieves the amount of data queued up for writing.
func (b *levelDBBatch) ValueSize() int { func (b *batch) ValueSize() int {
return b.size return b.size
} }
// Write flushes any accumulated data to disk. // Write flushes any accumulated data to disk.
func (b *levelDBBatch) Write() error { func (b *batch) Write() error {
return b.db.Write(b.b, nil) return b.db.Write(b.b, nil)
} }
// Reset resets the batch for reuse. // Reset resets the batch for reuse.
func (b *levelDBBatch) Reset() { func (b *batch) Reset() {
b.b.Reset() b.b.Reset()
b.size = 0 b.size = 0
} }

@ -37,33 +37,33 @@ var (
errMemorydbNotFound = errors.New("not found") errMemorydbNotFound = errors.New("not found")
) )
// MemoryDatabase is an ephemeral key-value store. Apart from basic data storage // Database is an ephemeral key-value store. Apart from basic data storage
// functionality it also supports batch writes and iterating over the keyspace in // functionality it also supports batch writes and iterating over the keyspace in
// binary-alphabetical order. // binary-alphabetical order.
type MemoryDatabase struct { type Database struct {
db map[string][]byte db map[string][]byte
lock sync.RWMutex lock sync.RWMutex
} }
// New returns a wrapped map with all the required database interface methods // New returns a wrapped map with all the required database interface methods
// implemented. // implemented.
func New() *MemoryDatabase { func New() *Database {
return &MemoryDatabase{ return &Database{
db: make(map[string][]byte), db: make(map[string][]byte),
} }
} }
// NewWithCap returns a wrapped map pre-allocated to the provided capcity with // NewWithCap returns a wrapped map pre-allocated to the provided capcity with
// all the required database interface methods implemented. // all the required database interface methods implemented.
func NewWithCap(size int) *MemoryDatabase { func NewWithCap(size int) *Database {
return &MemoryDatabase{ return &Database{
db: make(map[string][]byte, size), db: make(map[string][]byte, size),
} }
} }
// Close deallocates the internal map and ensures any consecutive data access op // Close deallocates the internal map and ensures any consecutive data access op
// failes with an error. // failes with an error.
func (db *MemoryDatabase) Close() error { func (db *Database) Close() error {
db.lock.Lock() db.lock.Lock()
defer db.lock.Unlock() defer db.lock.Unlock()
@ -72,7 +72,7 @@ func (db *MemoryDatabase) Close() error {
} }
// Has retrieves if a key is present in the key-value store. // Has retrieves if a key is present in the key-value store.
func (db *MemoryDatabase) Has(key []byte) (bool, error) { func (db *Database) Has(key []byte) (bool, error) {
db.lock.RLock() db.lock.RLock()
defer db.lock.RUnlock() defer db.lock.RUnlock()
@ -84,7 +84,7 @@ func (db *MemoryDatabase) Has(key []byte) (bool, error) {
} }
// Get retrieves the given key if it's present in the key-value store. // Get retrieves the given key if it's present in the key-value store.
func (db *MemoryDatabase) Get(key []byte) ([]byte, error) { func (db *Database) Get(key []byte) ([]byte, error) {
db.lock.RLock() db.lock.RLock()
defer db.lock.RUnlock() defer db.lock.RUnlock()
@ -98,7 +98,7 @@ func (db *MemoryDatabase) Get(key []byte) ([]byte, error) {
} }
// Put inserts the given value into the key-value store. // Put inserts the given value into the key-value store.
func (db *MemoryDatabase) Put(key []byte, value []byte) error { func (db *Database) Put(key []byte, value []byte) error {
db.lock.Lock() db.lock.Lock()
defer db.lock.Unlock() defer db.lock.Unlock()
@ -110,7 +110,7 @@ func (db *MemoryDatabase) Put(key []byte, value []byte) error {
} }
// Delete removes the key from the key-value store. // Delete removes the key from the key-value store.
func (db *MemoryDatabase) Delete(key []byte) error { func (db *Database) Delete(key []byte) error {
db.lock.Lock() db.lock.Lock()
defer db.lock.Unlock() defer db.lock.Unlock()
@ -123,21 +123,21 @@ func (db *MemoryDatabase) Delete(key []byte) error {
// NewBatch creates a write-only key-value store that buffers changes to its host // NewBatch creates a write-only key-value store that buffers changes to its host
// database until a final write is called. // database until a final write is called.
func (db *MemoryDatabase) NewBatch() ethdb.Batch { func (db *Database) NewBatch() ethdb.Batch {
return &memoryBatch{ return &batch{
db: db, db: db,
} }
} }
// NewIterator creates a binary-alphabetical iterator over the entire keyspace // NewIterator creates a binary-alphabetical iterator over the entire keyspace
// contained within the memory database. // contained within the memory database.
func (db *MemoryDatabase) NewIterator() ethdb.Iterator { func (db *Database) NewIterator() ethdb.Iterator {
return db.NewIteratorWithPrefix(nil) return db.NewIteratorWithPrefix(nil)
} }
// NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset // NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset
// of database content with a particular key prefix. // of database content with a particular key prefix.
func (db *MemoryDatabase) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator { func (db *Database) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator {
db.lock.RLock() db.lock.RLock()
defer db.lock.RUnlock() defer db.lock.RUnlock()
@ -157,19 +157,19 @@ func (db *MemoryDatabase) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator {
for _, key := range keys { for _, key := range keys {
values = append(values, db.db[key]) values = append(values, db.db[key])
} }
return &memoryIterator{ return &iterator{
keys: keys, keys: keys,
values: values, values: values,
} }
} }
// Stat returns a particular internal stat of the database. // Stat returns a particular internal stat of the database.
func (db *MemoryDatabase) Stat(property string) (string, error) { func (db *Database) Stat(property string) (string, error) {
return "", errors.New("unknown property") return "", errors.New("unknown property")
} }
// Compact is not supported on a memory database. // Compact is not supported on a memory database.
func (db *MemoryDatabase) Compact(start []byte, limit []byte) error { func (db *Database) Compact(start []byte, limit []byte) error {
return errors.New("unsupported operation") return errors.New("unsupported operation")
} }
@ -177,7 +177,7 @@ func (db *MemoryDatabase) Compact(start []byte, limit []byte) error {
// //
// Note, this method is only used for testing (i.e. not public in general) and // Note, this method is only used for testing (i.e. not public in general) and
// does not have explicit checks for closed-ness to allow simpler testing code. // does not have explicit checks for closed-ness to allow simpler testing code.
func (db *MemoryDatabase) Len() int { func (db *Database) Len() int {
db.lock.RLock() db.lock.RLock()
defer db.lock.RUnlock() defer db.lock.RUnlock()
@ -192,35 +192,35 @@ type keyvalue struct {
delete bool delete bool
} }
// memoryBatch is a write-only memory batch that commits changes to its host // batch is a write-only memory batch that commits changes to its host
// database when Write is called. A batch cannot be used concurrently. // database when Write is called. A batch cannot be used concurrently.
type memoryBatch struct { type batch struct {
db *MemoryDatabase db *Database
writes []keyvalue writes []keyvalue
size int size int
} }
// Put inserts the given value into the batch for later committing. // Put inserts the given value into the batch for later committing.
func (b *memoryBatch) Put(key, value []byte) error { func (b *batch) Put(key, value []byte) error {
b.writes = append(b.writes, keyvalue{common.CopyBytes(key), common.CopyBytes(value), false}) b.writes = append(b.writes, keyvalue{common.CopyBytes(key), common.CopyBytes(value), false})
b.size += len(value) b.size += len(value)
return nil return nil
} }
// Delete inserts the a key removal into the batch for later committing. // Delete inserts the a key removal into the batch for later committing.
func (b *memoryBatch) Delete(key []byte) error { func (b *batch) Delete(key []byte) error {
b.writes = append(b.writes, keyvalue{common.CopyBytes(key), nil, true}) b.writes = append(b.writes, keyvalue{common.CopyBytes(key), nil, true})
b.size += 1 b.size += 1
return nil return nil
} }
// ValueSize retrieves the amount of data queued up for writing. // ValueSize retrieves the amount of data queued up for writing.
func (b *memoryBatch) ValueSize() int { func (b *batch) ValueSize() int {
return b.size return b.size
} }
// Write flushes any accumulated data to the memory database. // Write flushes any accumulated data to the memory database.
func (b *memoryBatch) Write() error { func (b *batch) Write() error {
b.db.lock.Lock() b.db.lock.Lock()
defer b.db.lock.Unlock() defer b.db.lock.Unlock()
@ -235,15 +235,15 @@ func (b *memoryBatch) Write() error {
} }
// Reset resets the batch for reuse. // Reset resets the batch for reuse.
func (b *memoryBatch) Reset() { func (b *batch) Reset() {
b.writes = b.writes[:0] b.writes = b.writes[:0]
b.size = 0 b.size = 0
} }
// memoryIterator can walk over the (potentially partial) keyspace of a memory // iterator can walk over the (potentially partial) keyspace of a memory key
// key value store. Internally it is a deep copy of the entire iterated state, // value store. Internally it is a deep copy of the entire iterated state,
// sorted by keys. // sorted by keys.
type memoryIterator struct { type iterator struct {
inited bool inited bool
keys []string keys []string
values [][]byte values [][]byte
@ -251,7 +251,7 @@ type memoryIterator struct {
// Next moves the iterator to the next key/value pair. It returns whether the // Next moves the iterator to the next key/value pair. It returns whether the
// iterator is exhausted. // iterator is exhausted.
func (it *memoryIterator) Next() bool { func (it *iterator) Next() bool {
// If the iterator was not yet initialized, do it now // If the iterator was not yet initialized, do it now
if !it.inited { if !it.inited {
it.inited = true it.inited = true
@ -267,14 +267,14 @@ func (it *memoryIterator) Next() bool {
// Error returns any accumulated error. Exhausting all the key/value pairs // Error returns any accumulated error. Exhausting all the key/value pairs
// is not considered to be an error. A memory iterator cannot encounter errors. // is not considered to be an error. A memory iterator cannot encounter errors.
func (it *memoryIterator) Error() error { func (it *iterator) Error() error {
return nil return nil
} }
// Key returns the key of the current key/value pair, or nil if done. The caller // Key returns the key of the current key/value pair, or nil if done. The caller
// should not modify the contents of the returned slice, and its contents may // should not modify the contents of the returned slice, and its contents may
// change on the next call to Next. // change on the next call to Next.
func (it *memoryIterator) Key() []byte { func (it *iterator) Key() []byte {
if len(it.keys) > 0 { if len(it.keys) > 0 {
return []byte(it.keys[0]) return []byte(it.keys[0])
} }
@ -284,7 +284,7 @@ func (it *memoryIterator) Key() []byte {
// Value returns the value of the current key/value pair, or nil if done. The // Value returns the value of the current key/value pair, or nil if done. The
// caller should not modify the contents of the returned slice, and its contents // caller should not modify the contents of the returned slice, and its contents
// may change on the next call to Next. // may change on the next call to Next.
func (it *memoryIterator) Value() []byte { func (it *iterator) Value() []byte {
if len(it.values) > 0 { if len(it.values) > 0 {
return it.values[0] return it.values[0]
} }
@ -293,6 +293,6 @@ func (it *memoryIterator) Value() []byte {
// Release releases associated resources. Release should always succeed and can // Release releases associated resources. Release should always succeed and can
// be called multiple times without causing error. // be called multiple times without causing error.
func (it *memoryIterator) Release() { func (it *iterator) Release() {
it.keys, it.values = nil, nil it.keys, it.values = nil, nil
} }

@ -34,17 +34,17 @@ func init() {
// makeProvers creates Merkle trie provers based on different implementations to // makeProvers creates Merkle trie provers based on different implementations to
// test all variations. // test all variations.
func makeProvers(trie *Trie) []func(key []byte) *memorydb.MemoryDatabase { func makeProvers(trie *Trie) []func(key []byte) *memorydb.Database {
var provers []func(key []byte) *memorydb.MemoryDatabase var provers []func(key []byte) *memorydb.Database
// Create a direct trie based Merkle prover // Create a direct trie based Merkle prover
provers = append(provers, func(key []byte) *memorydb.MemoryDatabase { provers = append(provers, func(key []byte) *memorydb.Database {
proof := memorydb.New() proof := memorydb.New()
trie.Prove(key, 0, proof) trie.Prove(key, 0, proof)
return proof return proof
}) })
// Create a leaf iterator based Merkle prover // Create a leaf iterator based Merkle prover
provers = append(provers, func(key []byte) *memorydb.MemoryDatabase { provers = append(provers, func(key []byte) *memorydb.Database {
proof := memorydb.New() proof := memorydb.New()
if it := NewIterator(trie.NodeIterator(key)); it.Next() && bytes.Equal(key, it.Key) { if it := NewIterator(trie.NodeIterator(key)); it.Next() && bytes.Equal(key, it.Key) {
for _, p := range it.Prove() { for _, p := range it.Prove() {
@ -180,7 +180,7 @@ func BenchmarkVerifyProof(b *testing.B) {
trie, vals := randomTrie(100) trie, vals := randomTrie(100)
root := trie.Hash() root := trie.Hash()
var keys []string var keys []string
var proofs []*memorydb.MemoryDatabase var proofs []*memorydb.Database
for k := range vals { for k := range vals {
keys = append(keys, k) keys = append(keys, k)
proof := memorydb.New() proof := memorydb.New()

@ -542,7 +542,7 @@ func benchGet(b *testing.B, commit bool) {
b.StopTimer() b.StopTimer()
if commit { if commit {
ldb := trie.db.diskdb.(*leveldb.LevelDBDatabase) ldb := trie.db.diskdb.(*leveldb.Database)
ldb.Close() ldb.Close()
os.RemoveAll(ldb.Path()) os.RemoveAll(ldb.Path())
} }