tornadocontrib/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

core: add an end-to-end verkle test (#29262)

Browse Source 
core: add a simple verkle test

triedb, core: skip hash comparison in verkle

core: remove legacy daoFork logic in verkle chain maker

fix: nil pointer in tests

triedb/pathdb: add blob hex

core: less defensive

Co-authored-by: Ignacio Hagopian <jsign.uy@gmail.com>
Co-authored-by: Martin HS <martin@swende.se>
Co-authored-by: Gary Rong <garyrong0905@gmail.com>
This commit is contained in:
Guillaume Ballet 2024-03-26 21:25:41 +01:00 committed by GitHub
parent 723b1e36ad
commit da7469e5c4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 358 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
core/blockchain.go
View File

@ -147,8 +147,11 @@ type CacheConfig struct {
}
// triedbConfig derives the configures for trie database.
func (c *CacheConfig) triedbConfig() *triedb.Config {
config := &triedb.Config{Preimages: c.Preimages}
func (c *CacheConfig) triedbConfig(isVerkle bool) *triedb.Config {
config := &triedb.Config{
Preimages: c.Preimages,
IsVerkle: isVerkle,
}
if c.StateScheme == rawdb.HashScheme {
config.HashDB = &hashdb.Config{
CleanCacheSize: c.TrieCleanLimit * 1024 * 1024,
@ -265,7 +268,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
cacheConfig = defaultCacheConfig
}
// Open trie database with provided config
triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig())
triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(genesis != nil && genesis.IsVerkle()))
// Setup the genesis block, commit the provided genesis specification
// to database if the genesis block is not present yet, or load the

107
core/chain_makers.go
View File

@ -32,6 +32,7 @@ import (
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/triedb"
"github.com/gballet/go-verkle"
"github.com/holiman/uint256"
)
@ -418,6 +419,112 @@ func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int,
return db, blocks, receipts
}
func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, trdb *triedb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) {
if config == nil {
config = params.TestChainConfig
}
proofs := make([]*verkle.VerkleProof, 0, n)
keyvals := make([]verkle.StateDiff, 0, n)
cm := newChainMaker(parent, config, engine)
genblock := func(i int, parent *types.Block, triedb *triedb.Database, statedb *state.StateDB) (*types.Block, types.Receipts) {
b := &BlockGen{i: i, cm: cm, parent: parent, statedb: statedb, engine: engine}
b.header = cm.makeHeader(parent, statedb, b.engine)
// TODO uncomment when proof generation is merged
// Save pre state for proof generation
// preState := statedb.Copy()
// TODO uncomment when the 2935 PR is merged
// if config.IsPrague(b.header.Number, b.header.Time) {
// if !config.IsPrague(b.parent.Number(), b.parent.Time()) {
// Transition case: insert all 256 ancestors
// InsertBlockHashHistoryAtEip2935Fork(statedb, b.header.Number.Uint64()-1, b.header.ParentHash, chainreader)
// } else {
// ProcessParentBlockHash(statedb, b.header.Number.Uint64()-1, b.header.ParentHash)
// }
// }
// Execute any user modifications to the block
if gen != nil {
gen(i, b)
}
body := &types.Body{
Transactions: b.txs,
Uncles: b.uncles,
Withdrawals: b.withdrawals,
}
block, err := b.engine.FinalizeAndAssemble(cm, b.header, statedb, body, b.receipts)
if err != nil {
panic(err)
}
// Write state changes to db
root, err := statedb.Commit(b.header.Number.Uint64(), config.IsEIP158(b.header.Number))
if err != nil {
panic(fmt.Sprintf("state write error: %v", err))
}
if err = triedb.Commit(root, false); err != nil {
panic(fmt.Sprintf("trie write error: %v", err))
}
// TODO uncomment when proof generation is merged
// proofs = append(proofs, block.ExecutionWitness().VerkleProof)
// keyvals = append(keyvals, block.ExecutionWitness().StateDiff)
return block, b.receipts
}
for i := 0; i < n; i++ {
statedb, err := state.New(parent.Root(), state.NewDatabaseWithNodeDB(db, trdb), nil)
if err != nil {
panic(err)
}
block, receipts := genblock(i, parent, trdb, statedb)
// Post-process the receipts.
// Here we assign the final block hash and other info into the receipt.
// In order for DeriveFields to work, the transaction and receipt lists need to be
// of equal length. If AddUncheckedTx or AddUncheckedReceipt are used, there will be
// extra ones, so we just trim the lists here.
receiptsCount := len(receipts)
txs := block.Transactions()
if len(receipts) > len(txs) {
receipts = receipts[:len(txs)]
} else if len(receipts) < len(txs) {
txs = txs[:len(receipts)]
}
var blobGasPrice *big.Int
if block.ExcessBlobGas() != nil {
blobGasPrice = eip4844.CalcBlobFee(*block.ExcessBlobGas())
}
if err := receipts.DeriveFields(config, block.Hash(), block.NumberU64(), block.Time(), block.BaseFee(), blobGasPrice, txs); err != nil {
panic(err)
}
// Re-expand to ensure all receipts are returned.
receipts = receipts[:receiptsCount]
// Advance the chain.
cm.add(block, receipts)
parent = block
}
return cm.chain, cm.receipts, proofs, keyvals
}
func GenerateVerkleChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) {
db := rawdb.NewMemoryDatabase()
cacheConfig := DefaultCacheConfigWithScheme(rawdb.PathScheme)
cacheConfig.SnapshotLimit = 0
triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(true))
defer triedb.Close()
genesisBlock, err := genesis.Commit(db, triedb)
if err != nil {
panic(err)
}
blocks, receipts, proofs, keyvals := GenerateVerkleChain(genesis.Config, genesisBlock, engine, db, triedb, n, gen)
return db, blocks, receipts, proofs, keyvals
}
func (cm *chainMaker) makeHeader(parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header {
time := parent.Time() + 10 // block time is fixed at 10 seconds
header := &types.Header{

2
core/state/database.go
View File

@ -209,6 +209,8 @@ func (db *cachingDB) CopyTrie(t Trie) Trie {
switch t := t.(type) {
case *trie.StateTrie:
return t.Copy()
case *trie.VerkleTrie:
return t.Copy()
default:
panic(fmt.Errorf("unknown trie type %T", t))
}

5
core/state/statedb.go
View File

@ -1156,6 +1156,11 @@ func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) error {
return nil
}
// GetTrie returns the account trie.
func (s *StateDB) GetTrie() Trie {
return s.trie
}
// Commit writes the state to the underlying in-memory trie database.
// Once the state is committed, tries cached in stateDB (including account
// trie, storage tries) will no longer be functional. A new state instance

105
core/state_processor_test.go
View File

@ -422,3 +422,108 @@ func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Tr
}
return types.NewBlock(header, txs, nil, receipts, trie.NewStackTrie(nil))
}
var (
code = common.FromHex(`6060604052600a8060106000396000f360606040526008565b00`)
intrinsicContractCreationGas, _ = IntrinsicGas(code, nil, true, true, true, true)
// A contract creation that calls EXTCODECOPY in the constructor. Used to ensure that the witness
// will not contain that copied data.
// Source: https://gist.github.com/gballet/a23db1e1cb4ed105616b5920feb75985
codeWithExtCodeCopy = common.FromHex(`0x60806040526040516100109061017b565b604051809103906000f08015801561002c573d6000803e3d6000fd5b506000806101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff16021790555034801561007857600080fd5b5060008067ffffffffffffffff8111156100955761009461024a565b5b6040519080825280601f01601f1916602001820160405280156100c75781602001600182028036833780820191505090505b50905060008060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1690506020600083833c81610101906101e3565b60405161010d90610187565b61011791906101a3565b604051809103906000f080158015610133573d6000803e3d6000fd5b50600160006101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff160217905550505061029b565b60d58061046783390190565b6102068061053c83390190565b61019d816101d9565b82525050565b60006020820190506101b86000830184610194565b92915050565b6000819050602082019050919050565b600081519050919050565b6000819050919050565b60006101ee826101ce565b826101f8846101be565b905061020381610279565b925060208210156102435761023e7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8360200360080261028e565b831692505b5050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b600061028582516101d9565b80915050919050565b600082821b905092915050565b6101bd806102aa6000396000f3fe608060405234801561001057600080fd5b506004361061002b5760003560e01c8063f566852414610030575b600080fd5b61003861004e565b6040516100459190610146565b60405180910390f35b6000600160009054906101000a900473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff166381ca91d36040518163ffffffff1660e01b815260040160206040518083038186803b1580156100b857600080fd5b505afa1580156100cc573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906100f0919061010a565b905090565b60008151905061010481610170565b92915050565b6000602082840312156101205761011f61016b565b5b600061012e848285016100f5565b91505092915050565b61014081610161565b82525050565b600060208201905061015b6000830184610137565b92915050565b6000819050919050565b600080fd5b61017981610161565b811461018457600080fd5b5056fea2646970667358221220a6a0e11af79f176f9c421b7b12f441356b25f6489b83d38cc828a701720b41f164736f6c63430008070033608060405234801561001057600080fd5b5060b68061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063ab5ed15014602d575b600080fd5b60336047565b604051603e9190605d565b60405180910390f35b60006001905090565b6057816076565b82525050565b6000602082019050607060008301846050565b92915050565b600081905091905056fea26469706673582212203a14eb0d5cd07c277d3e24912f110ddda3e553245a99afc4eeefb2fbae5327aa64736f6c63430008070033608060405234801561001057600080fd5b5060405161020638038061020683398181016040528101906100329190610063565b60018160001c6100429190610090565b60008190555050610145565b60008151905061005d8161012e565b92915050565b60006020828403121561007957610078610129565b5b60006100878482850161004e565b91505092915050565b600061009b826100f0565b91506100a6836100f0565b9250827fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff038211156100db576100da6100fa565b5b828201905092915050565b6000819050919050565b6000819050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b600080fd5b610137816100e6565b811461014257600080fd5b50565b60b3806101536000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c806381ca91d314602d575b600080fd5b60336047565b604051603e9190605a565b60405180910390f35b60005481565b6054816073565b82525050565b6000602082019050606d6000830184604d565b92915050565b600081905091905056fea26469706673582212209bff7098a2f526de1ad499866f27d6d0d6f17b74a413036d6063ca6a0998ca4264736f6c63430008070033`)
intrinsicCodeWithExtCodeCopyGas, _ = IntrinsicGas(codeWithExtCodeCopy, nil, true, true, true, true)
)
func TestProcessVerkle(t *testing.T) {
var (
config = &params.ChainConfig{
ChainID: big.NewInt(1),
HomesteadBlock: big.NewInt(0),
EIP150Block: big.NewInt(0),
EIP155Block: big.NewInt(0),
EIP158Block: big.NewInt(0),
ByzantiumBlock: big.NewInt(0),
ConstantinopleBlock: big.NewInt(0),
PetersburgBlock: big.NewInt(0),
IstanbulBlock: big.NewInt(0),
MuirGlacierBlock: big.NewInt(0),
BerlinBlock: big.NewInt(0),
LondonBlock: big.NewInt(0),
Ethash: new(params.EthashConfig),
ShanghaiTime: u64(0),
VerkleTime: u64(0),
TerminalTotalDifficulty: common.Big0,
TerminalTotalDifficultyPassed: true,
// TODO uncomment when proof generation is merged
// ProofInBlocks: true,
}
signer = types.LatestSigner(config)
testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
bcdb = rawdb.NewMemoryDatabase() // Database for the blockchain
coinbase = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7")
gspec = &Genesis{
Config: config,
Alloc: GenesisAlloc{
coinbase: GenesisAccount{
Balance: big.NewInt(1000000000000000000), // 1 ether
Nonce: 0,
},
},
}
)
// Verkle trees use the snapshot, which must be enabled before the
// data is saved into the tree+database.
// genesis := gspec.MustCommit(bcdb, triedb)
cacheConfig := DefaultCacheConfigWithScheme("path")
cacheConfig.SnapshotLimit = 0
blockchain, _ := NewBlockChain(bcdb, cacheConfig, gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil, nil)
defer blockchain.Stop()
txCost1 := params.TxGas
txCost2 := params.TxGas
contractCreationCost := intrinsicContractCreationGas + uint64(2039 /* execution costs */)
codeWithExtCodeCopyGas := intrinsicCodeWithExtCodeCopyGas + uint64(293644 /* execution costs */)
blockGasUsagesExpected := []uint64{
txCost1*2 + txCost2,
txCost1*2 + txCost2 + contractCreationCost + codeWithExtCodeCopyGas,
}
_, chain, _, _, _ := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) {
gen.SetPoS()
// TODO need to check that the tx cost provided is the exact amount used (no remaining left-over)
tx, _ := types.SignTx(types.NewTransaction(uint64(i)*3, common.Address{byte(i), 2, 3}, big.NewInt(999), txCost1, big.NewInt(875000000), nil), signer, testKey)
gen.AddTx(tx)
tx, _ = types.SignTx(types.NewTransaction(uint64(i)*3+1, common.Address{}, big.NewInt(999), txCost1, big.NewInt(875000000), nil), signer, testKey)
gen.AddTx(tx)
tx, _ = types.SignTx(types.NewTransaction(uint64(i)*3+2, common.Address{}, big.NewInt(0), txCost2, big.NewInt(875000000), nil), signer, testKey)
gen.AddTx(tx)
// Add two contract creations in block #2
if i == 1 {
tx, _ = types.SignTx(types.NewContractCreation(6, big.NewInt(16), 3000000, big.NewInt(875000000), code), signer, testKey)
gen.AddTx(tx)
tx, _ = types.SignTx(types.NewContractCreation(7, big.NewInt(0), 3000000, big.NewInt(875000000), codeWithExtCodeCopy), signer, testKey)
gen.AddTx(tx)
}
})
t.Log("inserting blocks into the chain")
endnum, err := blockchain.InsertChain(chain)
if err != nil {
t.Fatalf("block %d imported with error: %v", endnum, err)
}
for i := 0; i < 2; i++ {
b := blockchain.GetBlockByNumber(uint64(i) + 1)
if b == nil {
t.Fatalf("expected block %d to be present in chain", i+1)
}
if b.Hash() != chain[i].Hash() {
t.Fatalf("block #%d not found at expected height", b.NumberU64())
}
if b.GasUsed() != blockGasUsagesExpected[i] {
t.Fatalf("expected block #%d txs to use %d, got %d\n", b.NumberU64(), blockGasUsagesExpected[i], b.GasUsed())
}
}
}

4
triedb/database.go
View File

@ -108,12 +108,12 @@ func NewDatabase(diskdb ethdb.Database, config *Config) *Database {
log.Crit("Both 'hash' and 'path' mode are configured")
}
if config.PathDB != nil {
db.backend = pathdb.New(diskdb, config.PathDB)
db.backend = pathdb.New(diskdb, config.PathDB, config.IsVerkle)
} else {
var resolver hashdb.ChildResolver
if config.IsVerkle {
// TODO define verkle resolver
log.Crit("Verkle node resolver is not defined")
log.Crit("verkle does not use a hash db")
} else {
resolver = trie.MerkleResolver{}
}

24
triedb/pathdb/database.go
View File

@ -59,11 +59,12 @@ var (
// layer is the interface implemented by all state layers which includes some
// public methods and some additional methods for internal usage.
type layer interface {
// Node retrieves the trie node with the node info. An error will be returned
// if the read operation exits abnormally. For example, if the layer is already
// stale, or the associated state is regarded as corrupted. Notably, no error
// will be returned if the requested node is not found in database.
Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error)
// node retrieves the trie node with the node info. An error will be returned
// if the read operation exits abnormally. Specifically, if the layer is
// already stale.
//
// Note, no error will be returned if the requested node is not found in database.
node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error)
// rootHash returns the root hash for which this layer was made.
rootHash() common.Hash
@ -132,6 +133,7 @@ type Database struct {
// the shutdown to reject all following unexpected mutations.
readOnly bool // Flag if database is opened in read only mode
waitSync bool // Flag if database is deactivated due to initial state sync
isVerkle bool // Flag if database is used for verkle tree
bufferSize int // Memory allowance (in bytes) for caching dirty nodes
config *Config // Configuration for database
diskdb ethdb.Database // Persistent storage for matured trie nodes
@ -143,7 +145,7 @@ type Database struct {
// New attempts to load an already existing layer from a persistent key-value
// store (with a number of memory layers from a journal). If the journal is not
// matched with the base persistent layer, all the recorded diff layers are discarded.
func New(diskdb ethdb.Database, config *Config) *Database {
func New(diskdb ethdb.Database, config *Config, isVerkle bool) *Database {
if config == nil {
config = Defaults
}
@ -151,6 +153,7 @@ func New(diskdb ethdb.Database, config *Config) *Database {
db := &Database{
readOnly: config.ReadOnly,
isVerkle: isVerkle,
bufferSize: config.DirtyCacheSize,
config: config,
diskdb: diskdb,
@ -208,15 +211,6 @@ func New(diskdb ethdb.Database, config *Config) *Database {
return db
}
// Reader retrieves a layer belonging to the given state root.
func (db *Database) Reader(root common.Hash) (layer, error) {
l := db.tree.get(root)
if l == nil {
return nil, fmt.Errorf("state %#x is not available", root)
}
return l, nil
}
// Update adds a new layer into the tree, if that can be linked to an existing
// old parent. It is disallowed to insert a disk layer (the origin of all). Apart
// from that this function will flatten the extra diff layers at bottom into disk

8
triedb/pathdb/database_test.go
View File

@ -106,7 +106,7 @@ func newTester(t *testing.T, historyLimit uint64) *tester {
StateHistory: historyLimit,
CleanCacheSize: 16 * 1024,
DirtyCacheSize: 16 * 1024,
})
}, false)
obj = &tester{
db: db,
preimages: make(map[common.Hash]common.Address),
@ -550,7 +550,7 @@ func TestJournal(t *testing.T) {
t.Errorf("Failed to journal, err: %v", err)
}
tester.db.Close()
tester.db = New(tester.db.diskdb, nil)
tester.db = New(tester.db.diskdb, nil, false)
// Verify states including disk layer and all diff on top.
for i := 0; i < len(tester.roots); i++ {
@ -588,7 +588,7 @@ func TestCorruptedJournal(t *testing.T) {
rawdb.WriteTrieJournal(tester.db.diskdb, blob)
// Verify states, all not-yet-written states should be discarded
tester.db = New(tester.db.diskdb, nil)
tester.db = New(tester.db.diskdb, nil, false)
for i := 0; i < len(tester.roots); i++ {
if tester.roots[i] == root {
if err := tester.verifyState(root); err != nil {
@ -625,7 +625,7 @@ func TestTailTruncateHistory(t *testing.T) {
defer tester.release()
tester.db.Close()
tester.db = New(tester.db.diskdb, &Config{StateHistory: 10})
tester.db = New(tester.db.diskdb, &Config{StateHistory: 10}, false)
head, err := tester.db.freezer.Ancients()
if err != nil {

28
triedb/pathdb/difflayer.go
View File

@ -95,10 +95,9 @@ func (dl *diffLayer) parentLayer() layer {
return dl.parent
}
// node retrieves the node with provided node information. It's the internal
// version of Node function with additional accessed layer tracked. No error
// will be returned if node is not found.
func (dl *diffLayer) node(owner common.Hash, path []byte, hash common.Hash, depth int) ([]byte, error) {
// node implements the layer interface, retrieving the trie node blob with the
// provided node information. No error will be returned if the node is not found.
func (dl *diffLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error) {
// Hold the lock, ensure the parent won't be changed during the
// state accessing.
dl.lock.RLock()
@ -109,31 +108,14 @@ func (dl *diffLayer) node(owner common.Hash, path []byte, hash common.Hash, dept
if ok {
n, ok := subset[string(path)]
if ok {
// If the trie node is not hash matched, or marked as removed,
// bubble up an error here. It shouldn't happen at all.
if n.Hash != hash {
dirtyFalseMeter.Mark(1)
log.Error("Unexpected trie node in diff layer", "owner", owner, "path", path, "expect", hash, "got", n.Hash)
return nil, newUnexpectedNodeError("diff", hash, n.Hash, owner, path, n.Blob)
}
dirtyHitMeter.Mark(1)
dirtyNodeHitDepthHist.Update(int64(depth))
dirtyReadMeter.Mark(int64(len(n.Blob)))
return n.Blob, nil
return n.Blob, n.Hash, &nodeLoc{loc: locDiffLayer, depth: depth}, nil
}
}
// Trie node unknown to this layer, resolve from parent
if diff, ok := dl.parent.(*diffLayer); ok {
return diff.node(owner, path, hash, depth+1)
}
// Failed to resolve through diff layers, fallback to disk layer
return dl.parent.Node(owner, path, hash)
}
// Node implements the layer interface, retrieving the trie node blob with the
// provided node information. No error will be returned if the node is not found.
func (dl *diffLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
return dl.node(owner, path, hash, 0)
return dl.parent.node(owner, path, depth+1)
}
// update implements the layer interface, creating a new layer on top of the

6
triedb/pathdb/difflayer_test.go
View File

@ -29,7 +29,7 @@ import (
func emptyLayer() *diskLayer {
return &diskLayer{
db: New(rawdb.NewMemoryDatabase(), nil),
db: New(rawdb.NewMemoryDatabase(), nil, false),
buffer: newNodeBuffer(DefaultBufferSize, nil, 0),
}
}
@ -58,7 +58,6 @@ func BenchmarkSearch1Layer(b *testing.B) { benchmarkSearch(b, 127, 128) }
func benchmarkSearch(b *testing.B, depth int, total int) {
var (
npath []byte
nhash common.Hash
nblob []byte
)
// First, we set up 128 diff layers, with 3K items each
@ -75,7 +74,6 @@ func benchmarkSearch(b *testing.B, depth int, total int) {
if npath == nil && depth == index {
npath = common.CopyBytes(path)
nblob = common.CopyBytes(node.Blob)
nhash = node.Hash
}
}
return newDiffLayer(parent, common.Hash{}, 0, 0, nodes, nil)
@ -92,7 +90,7 @@ func benchmarkSearch(b *testing.B, depth int, total int) {
err error
)
for i := 0; i < b.N; i++ {
have, err = layer.Node(common.Hash{}, npath, nhash)
have, _, _, err = layer.node(common.Hash{}, npath, 0)
if err != nil {
b.Fatal(err)
}

34
triedb/pathdb/disklayer.go
View File

@ -94,27 +94,25 @@ func (dl *diskLayer) markStale() {
dl.stale = true
}
// Node implements the layer interface, retrieving the trie node with the
// node implements the layer interface, retrieving the trie node with the
// provided node info. No error will be returned if the node is not found.
func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error) {
dl.lock.RLock()
defer dl.lock.RUnlock()
if dl.stale {
return nil, errSnapshotStale
return nil, common.Hash{}, nil, errSnapshotStale
}
// Try to retrieve the trie node from the not-yet-written
// node buffer first. Note the buffer is lock free since
// it's impossible to mutate the buffer before tagging the
// layer as stale.
n, err := dl.buffer.node(owner, path, hash)
if err != nil {
return nil, err
}
if n != nil {
n, found := dl.buffer.node(owner, path)
if found {
dirtyHitMeter.Mark(1)
dirtyReadMeter.Mark(int64(len(n.Blob)))
return n.Blob, nil
dirtyNodeHitDepthHist.Update(int64(depth))
return n.Blob, n.Hash, &nodeLoc{loc: locDirtyCache, depth: depth}, nil
}
dirtyMissMeter.Mark(1)
@ -125,14 +123,9 @@ func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
h := newHasher()
defer h.release()
got := h.hash(blob)
if got == hash {
cleanHitMeter.Mark(1)
cleanReadMeter.Mark(int64(len(blob)))
return blob, nil
}
cleanFalseMeter.Mark(1)
log.Error("Unexpected trie node in clean cache", "owner", owner, "path", path, "expect", hash, "got", got)
cleanHitMeter.Mark(1)
cleanReadMeter.Mark(int64(len(blob)))
return blob, h.hash(blob), &nodeLoc{loc: locCleanCache, depth: depth}, nil
}
cleanMissMeter.Mark(1)
}
@ -146,16 +139,11 @@ func (dl *diskLayer) Node(owner common.Hash, path []byte, hash common.Hash) ([]b
} else {
nBlob, nHash = rawdb.ReadStorageTrieNode(dl.db.diskdb, owner, path)
}
if nHash != hash {
diskFalseMeter.Mark(1)
log.Error("Unexpected trie node in disk", "owner", owner, "path", path, "expect", hash, "got", nHash)
return nil, newUnexpectedNodeError("disk", hash, nHash, owner, path, nBlob)
}
if dl.cleans != nil && len(nBlob) > 0 {
dl.cleans.Set(key, nBlob)
cleanWriteMeter.Mark(int64(len(nBlob)))
}
return nBlob, nil
return nBlob, nHash, &nodeLoc{loc: locDiskLayer, depth: depth}, nil
}
// update implements the layer interface, returning a new diff layer on top

20
triedb/pathdb/errors.go
View File

@ -16,13 +16,7 @@
package pathdb
import (
"errors"
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
)
import "errors"
var (
// errDatabaseReadOnly is returned if the database is opened in read only mode
@ -45,16 +39,4 @@ var (
// errStateUnrecoverable is returned if state is required to be reverted to
// a destination without associated state history available.
errStateUnrecoverable = errors.New("state is unrecoverable")
// errUnexpectedNode is returned if the requested node with specified path is
// not hash matched with expectation.
errUnexpectedNode = errors.New("unexpected node")
)
func newUnexpectedNodeError(loc string, expHash common.Hash, gotHash common.Hash, owner common.Hash, path []byte, blob []byte) error {
blobHex := "nil"
if len(blob) > 0 {
blobHex = hexutil.Encode(blob)
}
return fmt.Errorf("%w, loc: %s, node: (%x %v), %x!=%x, blob: %s", errUnexpectedNode, loc, owner, path, expHash, gotHash, blobHex)
}

1
triedb/pathdb/metrics.go
View File

@ -33,6 +33,7 @@ var (
cleanFalseMeter = metrics.NewRegisteredMeter("pathdb/clean/false", nil)
dirtyFalseMeter = metrics.NewRegisteredMeter("pathdb/dirty/false", nil)
diskFalseMeter = metrics.NewRegisteredMeter("pathdb/disk/false", nil)
diffFalseMeter = metrics.NewRegisteredMeter("pathdb/diff/false", nil)
commitTimeTimer = metrics.NewRegisteredTimer("pathdb/commit/time", nil)
commitNodesMeter = metrics.NewRegisteredMeter("pathdb/commit/nodes", nil)

13
triedb/pathdb/nodebuffer.go
View File

@ -59,21 +59,16 @@ func newNodeBuffer(limit int, nodes map[common.Hash]map[string]*trienode.Node, l
}
// node retrieves the trie node with given node info.
func (b *nodebuffer) node(owner common.Hash, path []byte, hash common.Hash) (*trienode.Node, error) {
func (b *nodebuffer) node(owner common.Hash, path []byte) (*trienode.Node, bool) {
subset, ok := b.nodes[owner]
if !ok {
return nil, nil
return nil, false
}
n, ok := subset[string(path)]
if !ok {
return nil, nil
return nil, false
}
if n.Hash != hash {
dirtyFalseMeter.Mark(1)
log.Error("Unexpected trie node in node buffer", "owner", owner, "path", path, "expect", hash, "got", n.Hash)
return nil, newUnexpectedNodeError("dirty", hash, n.Hash, owner, path, n.Blob)
}
return n, nil
return n, true
}
// commit merges the dirty nodes into the nodebuffer. This operation won't take

94
triedb/pathdb/reader.go Normal file
View File

@ -0,0 +1,94 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
package pathdb
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/triedb/database"
)
// The types of locations where the node is found.
const (
locDirtyCache = "dirty" // dirty cache
locCleanCache = "clean" // clean cache
locDiskLayer = "disk" // persistent state
locDiffLayer = "diff" // diff layers
)
// nodeLoc is a helpful structure that contains the location where the node
// is found, as it's useful for debugging purposes.
type nodeLoc struct {
loc string
depth int
}
// string returns the string representation of node location.
func (loc *nodeLoc) string() string {
return fmt.Sprintf("loc: %s, depth: %d", loc.loc, loc.depth)
}
// reader implements the database.Reader interface, providing the functionalities to
// retrieve trie nodes by wrapping the internal state layer.
type reader struct {
layer layer
noHashCheck bool
}
// Node implements database.Reader interface, retrieving the node with specified
// node info. Don't modify the returned byte slice since it's not deep-copied
// and still be referenced by database.
func (r *reader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
blob, got, loc, err := r.layer.node(owner, path, 0)
if err != nil {
return nil, err
}
// Error out if the local one is inconsistent with the target.
if !r.noHashCheck && got != hash {
// Location is always available even if the node
// is not found.
switch loc.loc {
case locCleanCache:
cleanFalseMeter.Mark(1)
case locDirtyCache:
dirtyFalseMeter.Mark(1)
case locDiffLayer:
diffFalseMeter.Mark(1)
case locDiskLayer:
diskFalseMeter.Mark(1)
}
blobHex := "nil"
if len(blob) > 0 {
blobHex = hexutil.Encode(blob)
}
log.Error("Unexpected trie node", "location", loc.loc, "owner", owner, "path", path, "expect", hash, "got", got, "blob", blobHex)
return nil, fmt.Errorf("unexpected node: (%x %v), %x!=%x, %s, blob: %s", owner, path, hash, got, loc.string(), blobHex)
}
return blob, nil
}
// Reader retrieves a layer belonging to the given state root.
func (db *Database) Reader(root common.Hash) (database.Reader, error) {
layer := db.tree.get(root)
if layer == nil {
return nil, fmt.Errorf("state %#x is not available", root)
}
return &reader{layer: layer, noHashCheck: db.isVerkle}, nil
}