tornadocontrib/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

core/rawdb, triedb, cmd: create an isolated disk namespace for verkle (#30105)

Browse Source 
* core, triedb/pathdb, cmd: define verkle state ancient store

* core/rawdb, triedb: add verkle namespace in pathdb
This commit is contained in:
rjl493456442 2024-07-16 21:17:58 +08:00 committed by GitHub
parent c54294bd41
commit f59d013e40
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 84 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/geth/dbcmd.go
View File

@ -248,7 +248,8 @@ func removeDB(ctx *cli.Context) error {
// Delete state data
statePaths := []string{
rootDir,
filepath.Join(ancientDir, rawdb.StateFreezerName),
filepath.Join(ancientDir, rawdb.MerkleStateFreezerName),
filepath.Join(ancientDir, rawdb.VerkleStateFreezerName),
}
confirmAndRemoveDB(statePaths, "state data", ctx, removeStateDataFlag.Name)

6
core/genesis_test.go
View File

@ -311,7 +311,7 @@ func TestVerkleGenesisCommit(t *testing.T) {
}
db := rawdb.NewMemoryDatabase()
triedb := triedb.NewDatabase(db, &triedb.Config{IsVerkle: true, PathDB: pathdb.Defaults})
triedb := triedb.NewDatabase(db, triedb.VerkleDefaults)
block := genesis.MustCommit(db, triedb)
if !bytes.Equal(block.Root().Bytes(), expected) {
t.Fatalf("invalid genesis state root, expected %x, got %x", expected, block.Root())
@ -321,8 +321,8 @@ func TestVerkleGenesisCommit(t *testing.T) {
if !triedb.IsVerkle() {
t.Fatalf("expected trie to be verkle")
}
if !rawdb.HasAccountTrieNode(db, nil) {
vdb := rawdb.NewTable(db, string(rawdb.VerklePrefix))
if !rawdb.HasAccountTrieNode(vdb, nil) {
t.Fatal("could not find node")
}
}

12
core/rawdb/accessors_trie.go
View File

@ -245,7 +245,7 @@ func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, has
// ReadStateScheme reads the state scheme of persistent state, or none
// if the state is not present in database.
func ReadStateScheme(db ethdb.Reader) string {
func ReadStateScheme(db ethdb.Database) string {
// Check if state in path-based scheme is present.
if HasAccountTrieNode(db, nil) {
return PathScheme
@ -255,6 +255,16 @@ func ReadStateScheme(db ethdb.Reader) string {
if id := ReadPersistentStateID(db); id != 0 {
return PathScheme
}
// Check if verkle state in path-based scheme is present.
vdb := NewTable(db, string(VerklePrefix))
if HasAccountTrieNode(vdb, nil) {
return PathScheme
}
// The root node of verkle might be deleted during the initial snap sync,
// check the persistent state id then.
if id := ReadPersistentStateID(vdb); id != 0 {
return PathScheme
}
// In a hash-based scheme, the genesis state is consistently stored
// on the disk. To assess the scheme of the persistent state, it
// suffices to inspect the scheme of the genesis state.

17
core/rawdb/ancient_scheme.go
View File

@ -72,12 +72,13 @@ var stateFreezerNoSnappy = map[string]bool{
// The list of identifiers of ancient stores.
var (
ChainFreezerName = "chain" // the folder name of chain segment ancient store.
StateFreezerName = "state" // the folder name of reverse diff ancient store.
ChainFreezerName = "chain" // the folder name of chain segment ancient store.
MerkleStateFreezerName = "state" // the folder name of state history ancient store.
VerkleStateFreezerName = "state_verkle" // the folder name of state history ancient store.
)
// freezers the collections of all builtin freezers.
var freezers = []string{ChainFreezerName, StateFreezerName}
var freezers = []string{ChainFreezerName, MerkleStateFreezerName, VerkleStateFreezerName}
// NewStateFreezer initializes the ancient store for state history.
//
@ -85,9 +86,15 @@ var freezers = []string{ChainFreezerName, StateFreezerName}
// state freezer (e.g. dev mode).
// - if non-empty directory is given, initializes the regular file-based
// state freezer.
func NewStateFreezer(ancientDir string, readOnly bool) (ethdb.ResettableAncientStore, error) {
func NewStateFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.ResettableAncientStore, error) {
if ancientDir == "" {
return NewMemoryFreezer(readOnly, stateFreezerNoSnappy), nil
}
return newResettableFreezer(filepath.Join(ancientDir, StateFreezerName), "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerNoSnappy)
var name string
if verkle {
name = filepath.Join(ancientDir, VerkleStateFreezerName)
} else {
name = filepath.Join(ancientDir, MerkleStateFreezerName)
}
return newResettableFreezer(name, "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerNoSnappy)
}

6
core/rawdb/ancient_utils.go
View File

@ -88,12 +88,12 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
}
infos = append(infos, info)
case StateFreezerName:
case MerkleStateFreezerName, VerkleStateFreezerName:
datadir, err := db.AncientDatadir()
if err != nil {
return nil, err
}
f, err := NewStateFreezer(datadir, true)
f, err := NewStateFreezer(datadir, freezer == VerkleStateFreezerName, true)
if err != nil {
continue // might be possible the state freezer is not existent
}
@ -124,7 +124,7 @@ func InspectFreezerTable(ancient string, freezerName string, tableName string, s
switch freezerName {
case ChainFreezerName:
path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
case StateFreezerName:
case MerkleStateFreezerName, VerkleStateFreezerName:
path, tables = filepath.Join(ancient, freezerName), stateFreezerNoSnappy
default:
return fmt.Errorf("unknown freezer, supported ones: %v", freezers)

24
core/rawdb/database.go
View File

@ -481,6 +481,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
beaconHeaders stat
cliqueSnaps stat
// Verkle statistics
verkleTries stat
verkleStateLookups stat
// Les statistic
chtTrieNodes stat
bloomTrieNodes stat
@ -550,6 +554,24 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
bytes.HasPrefix(key, BloomTrieIndexPrefix) ||
bytes.HasPrefix(key, BloomTriePrefix): // Bloomtrie sub
bloomTrieNodes.Add(size)
// Verkle trie data is detected, determine the sub-category
case bytes.HasPrefix(key, VerklePrefix):
remain := key[len(VerklePrefix):]
switch {
case IsAccountTrieNode(remain):
verkleTries.Add(size)
case bytes.HasPrefix(remain, stateIDPrefix) && len(remain) == len(stateIDPrefix)+common.HashLength:
verkleStateLookups.Add(size)
case bytes.Equal(remain, persistentStateIDKey):
metadata.Add(size)
case bytes.Equal(remain, trieJournalKey):
metadata.Add(size)
case bytes.Equal(remain, snapSyncStatusFlagKey):
metadata.Add(size)
default:
unaccounted.Add(size)
}
default:
var accounted bool
for _, meta := range [][]byte{
@ -590,6 +612,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
{"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()},
{"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()},
{"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()},
{"Key-Value store", "Verkle trie nodes", verkleTries.Size(), verkleTries.Count()},
{"Key-Value store", "Verkle trie state lookups", verkleStateLookups.Size(), verkleStateLookups.Count()},
{"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()},
{"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()},
{"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()},

7
core/rawdb/schema.go
View File

@ -117,6 +117,13 @@ var (
TrieNodeStoragePrefix = []byte("O") // TrieNodeStoragePrefix + accountHash + hexPath -> trie node
stateIDPrefix = []byte("L") // stateIDPrefix + state root -> state id
// VerklePrefix is the database prefix for Verkle trie data, which includes:
// (a) Trie nodes
// (b) In-memory trie node journal
// (c) Persistent state ID
// (d) State ID lookups, etc.
VerklePrefix = []byte("v")
PreimagePrefix = []byte("secure-key-") // PreimagePrefix + hash -> preimage
configPrefix = []byte("ethereum-config-") // config prefix for the db
genesisPrefix = []byte("ethereum-genesis-") // genesis state prefix for the db

11
triedb/database.go
View File

@ -42,9 +42,18 @@ type Config struct {
// default settings.
var HashDefaults = &Config{
Preimages: false,
IsVerkle: false,
HashDB: hashdb.Defaults,
}
// VerkleDefaults represents a config for holding verkle trie data
// using path-based scheme with default settings.
var VerkleDefaults = &Config{
Preimages: false,
IsVerkle: true,
PathDB: pathdb.Defaults,
}
// backend defines the methods needed to access/update trie nodes in different
// state scheme.
type backend interface {
@ -84,7 +93,6 @@ type backend interface {
// relevant with trie nodes and node preimages.
type Database struct {
config *Config // Configuration for trie database
diskdb ethdb.Database // Persistent database to store the snapshot
preimages *preimageStore // The store for caching preimages
backend backend // The backend for managing trie nodes
}
@ -102,7 +110,6 @@ func NewDatabase(diskdb ethdb.Database, config *Config) *Database {
}
db := &Database{
config: config,
diskdb: diskdb,
preimages: preimages,
}
if config.HashDB != nil && config.PathDB != nil {

10
triedb/pathdb/database.go
View File

@ -152,6 +152,14 @@ func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database {
}
config = config.sanitize()
// Establish a dedicated database namespace tailored for verkle-specific
// data, ensuring the isolation of both verkle and merkle tree data. It's
// important to note that the introduction of a prefix won't lead to
// substantial storage overhead, as the underlying database will efficiently
// compress the shared key prefix.
if isVerkle {
diskdb = rawdb.NewTable(diskdb, string(rawdb.VerklePrefix))
}
db := &Database{
readOnly: config.ReadOnly,
isVerkle: isVerkle,
@ -190,7 +198,7 @@ func (db *Database) repairHistory() error {
// all of them. Fix the tests first.
return nil
}
freezer, err := rawdb.NewStateFreezer(ancient, db.readOnly)
freezer, err := rawdb.NewStateFreezer(ancient, db.isVerkle, db.readOnly)
if err != nil {
log.Crit("Failed to open state history freezer", "err", err)
}

8
triedb/pathdb/history_test.go
View File

@ -129,7 +129,7 @@ func TestTruncateHeadHistory(t *testing.T) {
roots []common.Hash
hs = makeHistories(10)
db = rawdb.NewMemoryDatabase()
freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false)
freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false, false)
)
defer freezer.Close()
@ -157,7 +157,7 @@ func TestTruncateTailHistory(t *testing.T) {
roots []common.Hash
hs = makeHistories(10)
db = rawdb.NewMemoryDatabase()
freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false)
freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false, false)
)
defer freezer.Close()
@ -200,7 +200,7 @@ func TestTruncateTailHistories(t *testing.T) {
roots []common.Hash
hs = makeHistories(10)
db = rawdb.NewMemoryDatabase()
freezer, _ = rawdb.NewStateFreezer(t.TempDir()+fmt.Sprintf("%d", i), false)
freezer, _ = rawdb.NewStateFreezer(t.TempDir()+fmt.Sprintf("%d", i), false, false)
)
defer freezer.Close()
@ -228,7 +228,7 @@ func TestTruncateOutOfRange(t *testing.T) {
var (
hs = makeHistories(10)
db = rawdb.NewMemoryDatabase()
freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false)
freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false, false)
)
defer freezer.Close()