tornadocontrib/bsc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

all: prepare for path-based trie storage (#26603)

Browse Source 
This PR moves some trie-related db accessor methods to a different file, and also removes the schema type. Instead of the schema type, a string is used to distinguish between hashbased/pathbased db accessors.
This also moves some code from trie package to rawdb package.

This PR is intended to be a no-functionality-change prep PR for #25963 .

---------

Co-authored-by: Gary Rong <garyrong0905@gmail.com>
This commit is contained in:
Martin Holst Swende 2023-02-06 10:28:40 -05:00 committed by GitHub
parent 10c14847af
commit 8860b39754
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 336 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/geth/snapshot.go
View File

@ -399,7 +399,7 @@ func traverseRawState(ctx *cli.Context) error {
// Check the present for non-empty hash node(embedded node doesn't
// have their own hash).
if node != (common.Hash{}) {
blob := rawdb.ReadTrieNode(chaindb, node)
blob := rawdb.ReadLegacyTrieNode(chaindb, node)
if len(blob) == 0 {
log.Error("Missing trie node(account)", "hash", node)
return errors.New("missing account")
@ -436,7 +436,7 @@ func traverseRawState(ctx *cli.Context) error {
// Check the present for non-empty hash node(embedded node doesn't
// have their own hash).
if node != (common.Hash{}) {
blob := rawdb.ReadTrieNode(chaindb, node)
blob := rawdb.ReadLegacyTrieNode(chaindb, node)
if len(blob) == 0 {
log.Error("Missing trie node(storage)", "hash", node)
return errors.New("missing storage")

48
core/rawdb/accessors_state.go
View File

@ -28,6 +28,17 @@ func ReadPreimage(db ethdb.KeyValueReader, hash common.Hash) []byte {
return data
}
// WritePreimages writes the provided set of preimages to the database.
func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) {
for hash, preimage := range preimages {
if err := db.Put(preimageKey(hash), preimage); err != nil {
log.Crit("Failed to store trie preimage", "err", err)
}
}
preimageCounter.Inc(int64(len(preimages)))
preimageHitCounter.Inc(int64(len(preimages)))
}
// ReadCode retrieves the contract code of the provided code hash.
func ReadCode(db ethdb.KeyValueReader, hash common.Hash) []byte {
// Try with the prefixed code scheme first, if not then try with legacy
@ -48,12 +59,6 @@ func ReadCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) []byte {
return data
}
// ReadTrieNode retrieves the trie node of the provided hash.
func ReadTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte {
data, _ := db.Get(hash.Bytes())
return data
}
// HasCode checks if the contract code corresponding to the
// provided code hash is present in the db.
func HasCode(db ethdb.KeyValueReader, hash common.Hash) bool {
@ -74,23 +79,6 @@ func HasCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) bool {
return ok
}
// HasTrieNode checks if the trie node with the provided hash is present in db.
func HasTrieNode(db ethdb.KeyValueReader, hash common.Hash) bool {
ok, _ := db.Has(hash.Bytes())
return ok
}
// WritePreimages writes the provided set of preimages to the database.
func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) {
for hash, preimage := range preimages {
if err := db.Put(preimageKey(hash), preimage); err != nil {
log.Crit("Failed to store trie preimage", "err", err)
}
}
preimageCounter.Inc(int64(len(preimages)))
preimageHitCounter.Inc(int64(len(preimages)))
}
// WriteCode writes the provided contract code database.
func WriteCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) {
if err := db.Put(codeKey(hash), code); err != nil {
@ -98,23 +86,9 @@ func WriteCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) {
}
}
// WriteTrieNode writes the provided trie node database.
func WriteTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) {
if err := db.Put(hash.Bytes(), node); err != nil {
log.Crit("Failed to store trie node", "err", err)
}
}
// DeleteCode deletes the specified contract code from the database.
func DeleteCode(db ethdb.KeyValueWriter, hash common.Hash) {
if err := db.Delete(codeKey(hash)); err != nil {
log.Crit("Failed to delete contract code", "err", err)
}
}
// DeleteTrieNode deletes the specified trie node from the database.
func DeleteTrieNode(db ethdb.KeyValueWriter, hash common.Hash) {
if err := db.Delete(hash.Bytes()); err != nil {
log.Crit("Failed to delete trie node", "err", err)
}
}

263
core/rawdb/accessors_trie.go Normal file
View File

@ -0,0 +1,263 @@
// Copyright 2022 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
package rawdb
import (
"fmt"
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"golang.org/x/crypto/sha3"
)
// HashScheme is the legacy hash-based state scheme with which trie nodes are
// stored in the disk with node hash as the database key. The advantage of this
// scheme is that different versions of trie nodes can be stored in disk, which
// is very beneficial for constructing archive nodes. The drawback is it will
// store different trie nodes on the same path to different locations on the disk
// with no data locality, and it's unfriendly for designing state pruning.
//
// Now this scheme is still kept for backward compatibility, and it will be used
// for archive node and some other tries(e.g. light trie).
const HashScheme = "hashScheme"
// PathScheme is the new path-based state scheme with which trie nodes are stored
// in the disk with node path as the database key. This scheme will only store one
// version of state data in the disk, which means that the state pruning operation
// is native. At the same time, this scheme will put adjacent trie nodes in the same
// area of the disk with good data locality property. But this scheme needs to rely
// on extra state diffs to survive deep reorg.
const PathScheme = "pathScheme"
// nodeHasher used to derive the hash of trie node.
type nodeHasher struct{ sha crypto.KeccakState }
var hasherPool = sync.Pool{
New: func() interface{} { return &nodeHasher{sha: sha3.NewLegacyKeccak256().(crypto.KeccakState)} },
}
func newNodeHasher() *nodeHasher { return hasherPool.Get().(*nodeHasher) }
func returnHasherToPool(h *nodeHasher) { hasherPool.Put(h) }
func (h *nodeHasher) hashData(data []byte) (n common.Hash) {
h.sha.Reset()
h.sha.Write(data)
h.sha.Read(n[:])
return n
}
// ReadAccountTrieNode retrieves the account trie node and the associated node
// hash with the specified node path.
func ReadAccountTrieNode(db ethdb.KeyValueReader, path []byte) ([]byte, common.Hash) {
data, err := db.Get(accountTrieNodeKey(path))
if err != nil {
return nil, common.Hash{}
}
hasher := newNodeHasher()
defer returnHasherToPool(hasher)
return data, hasher.hashData(data)
}
// HasAccountTrieNode checks the account trie node presence with the specified
// node path and the associated node hash.
func HasAccountTrieNode(db ethdb.KeyValueReader, path []byte, hash common.Hash) bool {
data, err := db.Get(accountTrieNodeKey(path))
if err != nil {
return false
}
hasher := newNodeHasher()
defer returnHasherToPool(hasher)
return hasher.hashData(data) == hash
}
// WriteAccountTrieNode writes the provided account trie node into database.
func WriteAccountTrieNode(db ethdb.KeyValueWriter, path []byte, node []byte) {
if err := db.Put(accountTrieNodeKey(path), node); err != nil {
log.Crit("Failed to store account trie node", "err", err)
}
}
// DeleteAccountTrieNode deletes the specified account trie node from the database.
func DeleteAccountTrieNode(db ethdb.KeyValueWriter, path []byte) {
if err := db.Delete(accountTrieNodeKey(path)); err != nil {
log.Crit("Failed to delete account trie node", "err", err)
}
}
// ReadStorageTrieNode retrieves the storage trie node and the associated node
// hash with the specified node path.
func ReadStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte) ([]byte, common.Hash) {
data, err := db.Get(storageTrieNodeKey(accountHash, path))
if err != nil {
return nil, common.Hash{}
}
hasher := newNodeHasher()
defer returnHasherToPool(hasher)
return data, hasher.hashData(data)
}
// HasStorageTrieNode checks the storage trie node presence with the provided
// node path and the associated node hash.
func HasStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte, hash common.Hash) bool {
data, err := db.Get(storageTrieNodeKey(accountHash, path))
if err != nil {
return false
}
hasher := newNodeHasher()
defer returnHasherToPool(hasher)
return hasher.hashData(data) == hash
}
// WriteStorageTrieNode writes the provided storage trie node into database.
func WriteStorageTrieNode(db ethdb.KeyValueWriter, accountHash common.Hash, path []byte, node []byte) {
if err := db.Put(storageTrieNodeKey(accountHash, path), node); err != nil {
log.Crit("Failed to store storage trie node", "err", err)
}
}
// DeleteStorageTrieNode deletes the specified storage trie node from the database.
func DeleteStorageTrieNode(db ethdb.KeyValueWriter, accountHash common.Hash, path []byte) {
if err := db.Delete(storageTrieNodeKey(accountHash, path)); err != nil {
log.Crit("Failed to delete storage trie node", "err", err)
}
}
// ReadLegacyTrieNode retrieves the legacy trie node with the given
// associated node hash.
func ReadLegacyTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte {
data, err := db.Get(hash.Bytes())
if err != nil {
return nil
}
return data
}
// HasLegacyTrieNode checks if the trie node with the provided hash is present in db.
func HasLegacyTrieNode(db ethdb.KeyValueReader, hash common.Hash) bool {
ok, _ := db.Has(hash.Bytes())
return ok
}
// WriteLegacyTrieNode writes the provided legacy trie node to database.
func WriteLegacyTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) {
if err := db.Put(hash.Bytes(), node); err != nil {
log.Crit("Failed to store legacy trie node", "err", err)
}
}
// DeleteLegacyTrieNode deletes the specified legacy trie node from database.
func DeleteLegacyTrieNode(db ethdb.KeyValueWriter, hash common.Hash) {
if err := db.Delete(hash.Bytes()); err != nil {
log.Crit("Failed to delete legacy trie node", "err", err)
}
}
// HasTrieNode checks the trie node presence with the provided node info and
// the associated node hash.
func HasTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash, scheme string) bool {
switch scheme {
case HashScheme:
return HasLegacyTrieNode(db, hash)
case PathScheme:
if owner == (common.Hash{}) {
return HasAccountTrieNode(db, path, hash)
}
return HasStorageTrieNode(db, owner, path, hash)
default:
panic(fmt.Sprintf("Unknown scheme %v", scheme))
}
}
// ReadTrieNode retrieves the trie node from database with the provided node info
// and associated node hash.
// hashScheme-based lookup requires the following:
// - hash
//
// pathScheme-based lookup requires the following:
// - owner
// - path
func ReadTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash, scheme string) []byte {
switch scheme {
case HashScheme:
return ReadLegacyTrieNode(db, hash)
case PathScheme:
var (
blob []byte
nHash common.Hash
)
if owner == (common.Hash{}) {
blob, nHash = ReadAccountTrieNode(db, path)
} else {
blob, nHash = ReadStorageTrieNode(db, owner, path)
}
if nHash != hash {
return nil
}
return blob
default:
panic(fmt.Sprintf("Unknown scheme %v", scheme))
}
}
// WriteTrieNode writes the trie node into database with the provided node info
// and associated node hash.
// hashScheme-based lookup requires the following:
// - hash
//
// pathScheme-based lookup requires the following:
// - owner
// - path
func WriteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash, node []byte, scheme string) {
switch scheme {
case HashScheme:
WriteLegacyTrieNode(db, hash, node)
case PathScheme:
if owner == (common.Hash{}) {
WriteAccountTrieNode(db, path, node)
} else {
WriteStorageTrieNode(db, owner, path, node)
}
default:
panic(fmt.Sprintf("Unknown scheme %v", scheme))
}
}
// DeleteTrieNode deletes the trie node from database with the provided node info
// and associated node hash.
// hashScheme-based lookup requires the following:
// - hash
//
// pathScheme-based lookup requires the following:
// - owner
// - path
func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash, scheme string) {
switch scheme {
case HashScheme:
DeleteLegacyTrieNode(db, hash)
case PathScheme:
if owner == (common.Hash{}) {
DeleteAccountTrieNode(db, path)
} else {
DeleteStorageTrieNode(db, owner, path)
}
default:
panic(fmt.Sprintf("Unknown scheme %v", scheme))
}
}

16
core/rawdb/schema.go
View File

@ -100,12 +100,14 @@ var (
CodePrefix = []byte("c") // CodePrefix + code hash -> account code
skeletonHeaderPrefix = []byte("S") // skeletonHeaderPrefix + num (uint64 big endian) -> header
// Path-based trie node scheme.
trieNodeAccountPrefix = []byte("A") // trieNodeAccountPrefix + hexPath -> trie node
trieNodeStoragePrefix = []byte("O") // trieNodeStoragePrefix + accountHash + hexPath -> trie node
PreimagePrefix = []byte("secure-key-") // PreimagePrefix + hash -> preimage
configPrefix = []byte("ethereum-config-") // config prefix for the db
genesisPrefix = []byte("ethereum-genesis-") // genesis state prefix for the db
// Chain index prefixes (use `i` + single byte to avoid mixing data types).
// BloomBitsIndexPrefix is the data table of a chain indexer to track its progress
BloomBitsIndexPrefix = []byte("iB")
@ -236,3 +238,13 @@ func configKey(hash common.Hash) []byte {
func genesisStateSpecKey(hash common.Hash) []byte {
return append(genesisPrefix, hash.Bytes()...)
}
// accountTrieNodeKey = trieNodeAccountPrefix + nodePath.
func accountTrieNodeKey(path []byte) []byte {
return append(trieNodeAccountPrefix, path...)
}
// storageTrieNodeKey = trieNodeStoragePrefix + accountHash + nodePath.
func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte {
return append(append(trieNodeStoragePrefix, accountHash.Bytes()...), path...)
}

4
core/state/pruner/pruner.go
View File

@ -276,7 +276,7 @@ func (p *Pruner) Prune(root common.Hash) error {
// Ensure the root is really present. The weak assumption
// is the presence of root can indicate the presence of the
// entire trie.
if !rawdb.HasTrieNode(p.db, root) {
if !rawdb.HasLegacyTrieNode(p.db, root) {
// The special case is for clique based networks(rinkeby, goerli
// and some other private networks), it's possible that two
// consecutive blocks will have same root. In this case snapshot
@ -290,7 +290,7 @@ func (p *Pruner) Prune(root common.Hash) error {
// as the pruning target.
var found bool
for i := len(layers) - 2; i >= 2; i-- {
if rawdb.HasTrieNode(p.db, layers[i].Root()) {
if rawdb.HasLegacyTrieNode(p.db, layers[i].Root()) {
root = layers[i].Root()
found = true
log.Info("Selecting middle-layer as the pruning target", "root", root, "depth", i)

12
core/state/snapshot/conversion.go
View File

@ -43,7 +43,7 @@ type trieKV struct {
type (
// trieGeneratorFn is the interface of trie generation which can
// be implemented by different trie algorithm.
trieGeneratorFn func(db ethdb.KeyValueWriter, scheme trie.NodeScheme, owner common.Hash, in chan (trieKV), out chan (common.Hash))
trieGeneratorFn func(db ethdb.KeyValueWriter, scheme string, owner common.Hash, in chan (trieKV), out chan (common.Hash))
// leafCallbackFn is the callback invoked at the leaves of the trie,
// returns the subtrie root with the specified subtrie identifier.
@ -52,12 +52,12 @@ type (
// GenerateAccountTrieRoot takes an account iterator and reproduces the root hash.
func GenerateAccountTrieRoot(it AccountIterator) (common.Hash, error) {
return generateTrieRoot(nil, nil, it, common.Hash{}, stackTrieGenerate, nil, newGenerateStats(), true)
return generateTrieRoot(nil, "", it, common.Hash{}, stackTrieGenerate, nil, newGenerateStats(), true)
}
// GenerateStorageTrieRoot takes a storage iterator and reproduces the root hash.
func GenerateStorageTrieRoot(account common.Hash, it StorageIterator) (common.Hash, error) {
return generateTrieRoot(nil, nil, it, account, stackTrieGenerate, nil, newGenerateStats(), true)
return generateTrieRoot(nil, "", it, account, stackTrieGenerate, nil, newGenerateStats(), true)
}
// GenerateTrie takes the whole snapshot tree as the input, traverses all the
@ -243,7 +243,7 @@ func runReport(stats *generateStats, stop chan bool) {
// generateTrieRoot generates the trie hash based on the snapshot iterator.
// It can be used for generating account trie, storage trie or even the
// whole state which connects the accounts and the corresponding storages.
func generateTrieRoot(db ethdb.KeyValueWriter, scheme trie.NodeScheme, it Iterator, account common.Hash, generatorFn trieGeneratorFn, leafCallback leafCallbackFn, stats *generateStats, report bool) (common.Hash, error) {
func generateTrieRoot(db ethdb.KeyValueWriter, scheme string, it Iterator, account common.Hash, generatorFn trieGeneratorFn, leafCallback leafCallbackFn, stats *generateStats, report bool) (common.Hash, error) {
var (
in = make(chan trieKV) // chan to pass leaves
out = make(chan common.Hash, 1) // chan to collect result
@ -361,11 +361,11 @@ func generateTrieRoot(db ethdb.KeyValueWriter, scheme trie.NodeScheme, it Iterat
return stop(nil)
}
func stackTrieGenerate(db ethdb.KeyValueWriter, scheme trie.NodeScheme, owner common.Hash, in chan trieKV, out chan common.Hash) {
func stackTrieGenerate(db ethdb.KeyValueWriter, scheme string, owner common.Hash, in chan trieKV, out chan common.Hash) {
var nodeWriter trie.NodeWriteFunc
if db != nil {
nodeWriter = func(owner common.Hash, path []byte, hash common.Hash, blob []byte) {
scheme.WriteTrieNode(db, owner, path, hash, blob)
rawdb.WriteTrieNode(db, owner, path, hash, blob, scheme)
}
}
t := trie.NewStackTrieWithOwner(nodeWriter, owner)

4
core/state/snapshot/generate_test.go
View File

@ -117,12 +117,12 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) {
accIt := snap.AccountIterator(common.Hash{})
defer accIt.Release()
snapRoot, err := generateTrieRoot(nil, nil, accIt, common.Hash{}, stackTrieGenerate,
snapRoot, err := generateTrieRoot(nil, "", accIt, common.Hash{}, stackTrieGenerate,
func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) {
storageIt, _ := snap.StorageIterator(accountHash, common.Hash{})
defer storageIt.Release()
hash, err := generateTrieRoot(nil, nil, storageIt, accountHash, stackTrieGenerate, nil, stat, false)
hash, err := generateTrieRoot(nil, "", storageIt, accountHash, stackTrieGenerate, nil, stat, false)
if err != nil {
return common.Hash{}, err
}

4
core/state/snapshot/snapshot.go
View File

@ -776,14 +776,14 @@ func (t *Tree) Verify(root common.Hash) error {
}
defer acctIt.Release()
got, err := generateTrieRoot(nil, nil, acctIt, common.Hash{}, stackTrieGenerate, func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) {
got, err := generateTrieRoot(nil, "", acctIt, common.Hash{}, stackTrieGenerate, func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) {
storageIt, err := t.StorageIterator(root, accountHash, common.Hash{})
if err != nil {
return common.Hash{}, err
}
defer storageIt.Release()
hash, err := generateTrieRoot(nil, nil, storageIt, accountHash, stackTrieGenerate, nil, stat, false)
hash, err := generateTrieRoot(nil, "", storageIt, accountHash, stackTrieGenerate, nil, stat, false)
if err != nil {
return common.Hash{}, err
}

2
core/state/sync.go
View File

@ -27,7 +27,7 @@ import (
)
// NewStateSync create a new state trie download scheduler.
func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(keys [][]byte, leaf []byte) error, scheme trie.NodeScheme) *trie.Sync {
func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(keys [][]byte, leaf []byte) error, scheme string) *trie.Sync {
// Register the storage slot callback if the external callback is specified.
var onSlot func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error
if onLeaf != nil {

6
core/state/sync_test.go
View File

@ -663,14 +663,14 @@ func TestIncompleteStateSync(t *testing.T) {
for i, path := range addedPaths {
owner, inner := trie.ResolvePath([]byte(path))
hash := addedHashes[i]
val := scheme.ReadTrieNode(dstDb, owner, inner, hash)
val := rawdb.ReadTrieNode(dstDb, owner, inner, hash, scheme)
if val == nil {
t.Error("missing trie node")
}
scheme.DeleteTrieNode(dstDb, owner, inner, hash)
rawdb.DeleteTrieNode(dstDb, owner, inner, hash, scheme)
if err := checkStateConsistency(dstDb, srcRoot); err == nil {
t.Errorf("trie inconsistency not caught, missing: %v", path)
}
scheme.WriteTrieNode(dstDb, owner, inner, hash, val)
rawdb.WriteTrieNode(dstDb, owner, inner, hash, val, scheme)
}
}

2
eth/protocols/eth/handler_test.go
View File

@ -524,7 +524,7 @@ func testGetNodeData(t *testing.T, protocol uint, drop bool) {
// Reconstruct state tree from the received data.
reconstructDB := rawdb.NewMemoryDatabase()
for i := 0; i < len(data); i++ {
rawdb.WriteTrieNode(reconstructDB, hashes[i], data[i])
rawdb.WriteLegacyTrieNode(reconstructDB, hashes[i], data[i])
}
// Sanity check whether all state matches.

18
eth/protocols/snap/sync.go
View File

@ -418,7 +418,7 @@ type SyncPeer interface {
// - The peer delivers a refusal to serve the requested state
type Syncer struct {
db ethdb.KeyValueStore // Database to store the trie nodes into (and dedup)
scheme trie.NodeScheme // Node scheme used in node database
scheme string // Node scheme used in node database
root common.Hash // Current state trie root being synced
tasks []*accountTask // Current account task set being synced
@ -486,7 +486,7 @@ type Syncer struct {
// NewSyncer creates a new snapshot syncer to download the Ethereum state over the
// snap protocol.
func NewSyncer(db ethdb.KeyValueStore, scheme trie.NodeScheme) *Syncer {
func NewSyncer(db ethdb.KeyValueStore, scheme string) *Syncer {
return &Syncer{
db: db,
scheme: scheme,
@ -746,7 +746,7 @@ func (s *Syncer) loadSyncStatus() {
},
}
task.genTrie = trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, val []byte) {
s.scheme.WriteTrieNode(task.genBatch, owner, path, hash, val)
rawdb.WriteTrieNode(task.genBatch, owner, path, hash, val, s.scheme)
})
for accountHash, subtasks := range task.SubTasks {
for _, subtask := range subtasks {
@ -757,7 +757,7 @@ func (s *Syncer) loadSyncStatus() {
},
}
subtask.genTrie = trie.NewStackTrieWithOwner(func(owner common.Hash, path []byte, hash common.Hash, val []byte) {
s.scheme.WriteTrieNode(subtask.genBatch, owner, path, hash, val)
rawdb.WriteTrieNode(subtask.genBatch, owner, path, hash, val, s.scheme)
}, accountHash)
}
}
@ -816,7 +816,7 @@ func (s *Syncer) loadSyncStatus() {
SubTasks: make(map[common.Hash][]*storageTask),
genBatch: batch,
genTrie: trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, val []byte) {
s.scheme.WriteTrieNode(batch, owner, path, hash, val)
rawdb.WriteTrieNode(batch, owner, path, hash, val, s.scheme)
}),
})
log.Debug("Created account sync task", "from", next, "last", last)
@ -1842,7 +1842,7 @@ func (s *Syncer) processAccountResponse(res *accountResponse) {
}
// Check if the account is a contract with an unknown storage trie
if account.Root != emptyRoot {
if !s.scheme.HasTrieNode(s.db, res.hashes[i], nil, account.Root) {
if !rawdb.HasTrieNode(s.db, res.hashes[i], nil, account.Root, s.scheme) {
// If there was a previous large state retrieval in progress,
// don't restart it from scratch. This happens if a sync cycle
// is interrupted and resumed later. However, *do* update the
@ -2015,7 +2015,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) {
root: acc.Root,
genBatch: batch,
genTrie: trie.NewStackTrieWithOwner(func(owner common.Hash, path []byte, hash common.Hash, val []byte) {
s.scheme.WriteTrieNode(batch, owner, path, hash, val)
rawdb.WriteTrieNode(batch, owner, path, hash, val, s.scheme)
}, account),
})
for r.Next() {
@ -2031,7 +2031,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) {
root: acc.Root,
genBatch: batch,
genTrie: trie.NewStackTrieWithOwner(func(owner common.Hash, path []byte, hash common.Hash, val []byte) {
s.scheme.WriteTrieNode(batch, owner, path, hash, val)
rawdb.WriteTrieNode(batch, owner, path, hash, val, s.scheme)
}, account),
})
}
@ -2078,7 +2078,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) {
if i < len(res.hashes)-1 || res.subTask == nil {
tr := trie.NewStackTrieWithOwner(func(owner common.Hash, path []byte, hash common.Hash, val []byte) {
s.scheme.WriteTrieNode(batch, owner, path, hash, val)
rawdb.WriteTrieNode(batch, owner, path, hash, val, s.scheme)
}, account)
for j := 0; j < len(res.hashes[i]); j++ {
tr.Update(res.hashes[i][j][:], res.slots[i][j])

10
eth/protocols/snap/sync_test.go
View File

@ -623,7 +623,7 @@ func TestSyncBloatedProof(t *testing.T) {
}
}
func setupSyncer(scheme trie.NodeScheme, peers ...*testPeer) *Syncer {
func setupSyncer(scheme string, peers ...*testPeer) *Syncer {
stateDb := rawdb.NewMemoryDatabase()
syncer := NewSyncer(stateDb, scheme)
for _, peer := range peers {
@ -1366,7 +1366,7 @@ func getCodeByHash(hash common.Hash) []byte {
}
// makeAccountTrieNoStorage spits out a trie, along with the leafs
func makeAccountTrieNoStorage(n int) (trie.NodeScheme, *trie.Trie, entrySlice) {
func makeAccountTrieNoStorage(n int) (string, *trie.Trie, entrySlice) {
var (
db = trie.NewDatabase(rawdb.NewMemoryDatabase())
accTrie = trie.NewEmpty(db)
@ -1398,7 +1398,7 @@ func makeAccountTrieNoStorage(n int) (trie.NodeScheme, *trie.Trie, entrySlice) {
// makeBoundaryAccountTrie constructs an account trie. Instead of filling
// accounts normally, this function will fill a few accounts which have
// boundary hash.
func makeBoundaryAccountTrie(n int) (trie.NodeScheme, *trie.Trie, entrySlice) {
func makeBoundaryAccountTrie(n int) (string, *trie.Trie, entrySlice) {
var (
entries entrySlice
boundaries []common.Hash
@ -1459,7 +1459,7 @@ func makeBoundaryAccountTrie(n int) (trie.NodeScheme, *trie.Trie, entrySlice) {
// makeAccountTrieWithStorageWithUniqueStorage creates an account trie where each accounts
// has a unique storage set.
func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (trie.NodeScheme, *trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) {
func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (string, *trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) {
var (
db = trie.NewDatabase(rawdb.NewMemoryDatabase())
accTrie = trie.NewEmpty(db)
@ -1514,7 +1514,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool)
}
// makeAccountTrieWithStorage spits out a trie, along with the leafs
func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (trie.NodeScheme, *trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) {
func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (string, *trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) {
var (
db = trie.NewDatabase(rawdb.NewMemoryDatabase())
accTrie = trie.NewEmpty(db)

2
les/downloader/downloader.go
View File

@ -226,7 +226,7 @@ func New(checkpoint uint64, stateDb ethdb.Database, mux *event.TypeMux, chain Bl
headerProcCh: make(chan []*types.Header, 1),
quitCh: make(chan struct{}),
stateCh: make(chan dataPack),
SnapSyncer: snap.NewSyncer(stateDb, nil),
SnapSyncer: snap.NewSyncer(stateDb, rawdb.HashScheme),
stateSyncStart: make(chan *stateSync),
//syncStatsState: stateSyncStats{
// processed: rawdb.ReadFastTrieProgress(stateDb),

3
les/downloader/statesync.go
View File

@ -298,11 +298,10 @@ type codeTask struct {
func newStateSync(d *Downloader, root common.Hash) *stateSync {
// Hack the node scheme here. It's a dead code is not used
// by light client at all. Just aim for passing tests.
scheme := trie.NewDatabase(rawdb.NewMemoryDatabase()).Scheme()
return &stateSync{
d: d,
root: root,
sched: state.NewStateSync(root, d.stateDB, nil, scheme),
sched: state.NewStateSync(root, d.stateDB, nil, rawdb.HashScheme),
keccak: sha3.NewLegacyKeccak256().(crypto.KeccakState),
trieTasks: make(map[string]*trieTask),
codeTasks: make(map[common.Hash]*codeTask),

2
tests/fuzzers/stacktrie/trie_fuzzer.go
View File

@ -152,7 +152,7 @@ func (f *fuzzer) fuzz() int {
spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()}
dbB = trie.NewDatabase(rawdb.NewDatabase(spongeB))
trieB = trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) {
dbB.Scheme().WriteTrieNode(spongeB, owner, path, hash, blob)
rawdb.WriteTrieNode(spongeB, owner, path, hash, blob, dbB.Scheme())
})
vals kvs
useful bool

10
trie/database.go
View File

@ -405,7 +405,7 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) {
memcacheDirtyMissMeter.Mark(1)
// Content unavailable in memory, attempt to retrieve from disk
enc := rawdb.ReadTrieNode(db.diskdb, hash)
enc := rawdb.ReadLegacyTrieNode(db.diskdb, hash)
if len(enc) != 0 {
if db.cleans != nil {
db.cleans.Set(hash[:], enc)
@ -571,7 +571,7 @@ func (db *Database) Cap(limit common.StorageSize) error {
for size > limit && oldest != (common.Hash{}) {
// Fetch the oldest referenced node and push into the batch
node := db.dirties[oldest]
rawdb.WriteTrieNode(batch, oldest, node.rlp())
rawdb.WriteLegacyTrieNode(batch, oldest, node.rlp())
// If we exceeded the ideal batch size, commit and reset
if batch.ValueSize() >= ethdb.IdealBatchSize {
@ -703,7 +703,7 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch, uncacher *cleane
return err
}
// If we've reached an optimal batch size, commit and start over
rawdb.WriteTrieNode(batch, hash, node.rlp())
rawdb.WriteLegacyTrieNode(batch, hash, node.rlp())
if callback != nil {
callback(hash)
}
@ -919,6 +919,6 @@ func (db *Database) CommitPreimages() error {
}
// Scheme returns the node scheme used in the database.
func (db *Database) Scheme() NodeScheme {
return &hashScheme{}
func (db *Database) Scheme() string {
return rawdb.HashScheme
}

96
trie/schema.go
View File

@ -1,96 +0,0 @@
// Copyright 2021 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package trie
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
)
const (
HashScheme = "hashScheme" // Identifier of hash based node scheme
// Path-based scheme will be introduced in the following PRs.
// PathScheme = "pathScheme" // Identifier of path based node scheme
)
// NodeScheme describes the scheme for interacting nodes in disk.
type NodeScheme interface {
// Name returns the identifier of node scheme.
Name() string
// HasTrieNode checks the trie node presence with the provided node info and
// the associated node hash.
HasTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash) bool
// ReadTrieNode retrieves the trie node from database with the provided node
// info and the associated node hash.
ReadTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash) []byte
// WriteTrieNode writes the trie node into database with the provided node
// info and associated node hash.
WriteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash, node []byte)
// DeleteTrieNode deletes the trie node from database with the provided node
// info and associated node hash.
DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash)
// IsTrieNode returns an indicator if the given database key is the key of
// trie node according to the scheme.
IsTrieNode(key []byte) (bool, []byte)
}
type hashScheme struct{}
// Name returns the identifier of hash based scheme.
func (scheme *hashScheme) Name() string {
return HashScheme
}
// HasTrieNode checks the trie node presence with the provided node info and
// the associated node hash.
func (scheme *hashScheme) HasTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash) bool {
return rawdb.HasTrieNode(db, hash)
}
// ReadTrieNode retrieves the trie node from database with the provided node info
// and associated node hash.
func (scheme *hashScheme) ReadTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash) []byte {
return rawdb.ReadTrieNode(db, hash)
}
// WriteTrieNode writes the trie node into database with the provided node info
// and associated node hash.
func (scheme *hashScheme) WriteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash, node []byte) {
rawdb.WriteTrieNode(db, hash, node)
}
// DeleteTrieNode deletes the trie node from database with the provided node info
// and associated node hash.
func (scheme *hashScheme) DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash) {
rawdb.DeleteTrieNode(db, hash)
}
// IsTrieNode returns an indicator if the given database key is the key of trie
// node according to the scheme.
func (scheme *hashScheme) IsTrieNode(key []byte) (bool, []byte) {
if len(key) == common.HashLength {
return true, key
}
return false, nil
}

10
trie/sync.go
View File

@ -155,7 +155,7 @@ func (batch *syncMemBatch) hasCode(hash common.Hash) bool {
// unknown trie hashes to retrieve, accepts node data associated with said hashes
// and reconstructs the trie step by step until all is done.
type Sync struct {
scheme NodeScheme // Node scheme descriptor used in database.
scheme string // Node scheme descriptor used in database.
database ethdb.KeyValueReader // Persistent database to check for existing entries
membatch *syncMemBatch // Memory buffer to avoid frequent database writes
nodeReqs map[string]*nodeRequest // Pending requests pertaining to a trie node path
@ -165,7 +165,7 @@ type Sync struct {
}
// NewSync creates a new trie data download scheduler.
func NewSync(root common.Hash, database ethdb.KeyValueReader, callback LeafCallback, scheme NodeScheme) *Sync {
func NewSync(root common.Hash, database ethdb.KeyValueReader, callback LeafCallback, scheme string) *Sync {
ts := &Sync{
scheme: scheme,
database: database,
@ -191,7 +191,7 @@ func (s *Sync) AddSubTrie(root common.Hash, path []byte, parent common.Hash, par
return
}
owner, inner := ResolvePath(path)
if s.scheme.HasTrieNode(s.database, owner, inner, root) {
if rawdb.HasTrieNode(s.database, owner, inner, root, s.scheme) {
return
}
// Assemble the new sub-trie sync request
@ -349,7 +349,7 @@ func (s *Sync) Commit(dbw ethdb.Batch) error {
// Dump the membatch into a database dbw
for path, value := range s.membatch.nodes {
owner, inner := ResolvePath([]byte(path))
s.scheme.WriteTrieNode(dbw, owner, inner, s.membatch.hashes[path], value)
rawdb.WriteTrieNode(dbw, owner, inner, s.membatch.hashes[path], value, s.scheme)
}
for hash, value := range s.membatch.codes {
rawdb.WriteCode(dbw, hash, value)
@ -474,7 +474,7 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) {
chash = common.BytesToHash(node)
owner, inner = ResolvePath(child.path)
)
if s.scheme.HasTrieNode(s.database, owner, inner, chash) {
if rawdb.HasTrieNode(s.database, owner, inner, chash, s.scheme) {
return
}
// Locally unknown node, schedule for retrieval

4
trie/trie_test.go
View File

@ -898,7 +898,7 @@ func TestCommitSequenceStackTrie(t *testing.T) {
// Another sponge is used for the stacktrie commits
stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"}
stTrie := NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) {
db.Scheme().WriteTrieNode(stackTrieSponge, owner, path, hash, blob)
rawdb.WriteTrieNode(stackTrieSponge, owner, path, hash, blob, db.Scheme())
})
// Fill the trie with elements
for i := 0; i < count; i++ {
@ -957,7 +957,7 @@ func TestCommitSequenceSmallRoot(t *testing.T) {
// Another sponge is used for the stacktrie commits
stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"}
stTrie := NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) {
db.Scheme().WriteTrieNode(stackTrieSponge, owner, path, hash, blob)
rawdb.WriteTrieNode(stackTrieSponge, owner, path, hash, blob, db.Scheme())
})
// Add a single small-element to the trie(s)
key := make([]byte, 5)