From 6e05ccd845da26774774185956b3fd67966894ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 3 Mar 2020 09:10:23 +0200 Subject: [PATCH] core/state/snapshot, tests: sync snap gen + snaps in consensus tests --- cmd/evm/staterunner.go | 2 +- core/blockchain.go | 4 +++- core/state/snapshot/disklayer.go | 5 +++-- core/state/snapshot/generate.go | 14 ++++++------ core/state/snapshot/journal.go | 1 + core/state/snapshot/snapshot.go | 37 ++++++++++++++++++++++++++------ eth/tracers/tracers_test.go | 4 ++-- tests/block_test.go | 8 ++++--- tests/block_test_util.go | 9 ++++++-- tests/state_test.go | 13 ++++++++--- tests/state_test_util.go | 19 ++++++++++------ tests/transaction_test_util.go | 1 - tests/vm_test.go | 5 ++++- tests/vm_test_util.go | 4 ++-- 14 files changed, 90 insertions(+), 36 deletions(-) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index cef2aedb5..52c1eca71 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -96,7 +96,7 @@ func stateTestCmd(ctx *cli.Context) error { for _, st := range test.Subtests() { // Run the test and aggregate the result result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} - state, err := test.Run(st, cfg) + state, err := test.Run(st, cfg, false) // print state root for evmlab tracing if ctx.GlobalBool(MachineFlag.Name) && state != nil { fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false)) diff --git a/core/blockchain.go b/core/blockchain.go index 491eccecd..b0309ef70 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -121,6 +121,8 @@ type CacheConfig struct { TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node) TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory + + SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it } // BlockChain represents the canonical chain given a database with a genesis @@ -303,7 +305,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par } // Load any existing snapshot, regenerating it if loading failed if bc.cacheConfig.SnapshotLimit > 0 { - bc.snaps = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, bc.CurrentBlock().Root()) + bc.snaps = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, bc.CurrentBlock().Root(), !bc.cacheConfig.SnapshotWait) } // Take ownership of this particular state go bc.update() diff --git a/core/state/snapshot/disklayer.go b/core/state/snapshot/disklayer.go index 0c4c3deb1..3266424a8 100644 --- a/core/state/snapshot/disklayer.go +++ b/core/state/snapshot/disklayer.go @@ -37,8 +37,9 @@ type diskLayer struct { root common.Hash // Root hash of the base snapshot stale bool // Signals that the layer became stale (state progressed) - genMarker []byte // Marker for the state that's indexed during initial layer generation - genAbort chan chan *generatorStats // Notification channel to abort generating the snapshot in this layer + genMarker []byte // Marker for the state that's indexed during initial layer generation + genPending chan struct{} // Notification channel when generation is done (test synchronicity) + genAbort chan chan *generatorStats // Notification channel to abort generating the snapshot in this layer lock sync.RWMutex } diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 8a407e30d..4b017fe69 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -101,12 +101,13 @@ func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache i rawdb.WriteSnapshotRoot(diskdb, root) base := &diskLayer{ - diskdb: diskdb, - triedb: triedb, - root: root, - cache: fastcache.New(cache * 1024 * 1024), - genMarker: []byte{}, // Initialized but empty! - genAbort: make(chan chan *generatorStats), + diskdb: diskdb, + triedb: triedb, + root: root, + cache: fastcache.New(cache * 1024 * 1024), + genMarker: []byte{}, // Initialized but empty! + genPending: make(chan struct{}), + genAbort: make(chan chan *generatorStats), } go base.generate(&generatorStats{wiping: wiper, start: time.Now()}) return base @@ -252,6 +253,7 @@ func (dl *diskLayer) generate(stats *generatorStats) { dl.lock.Lock() dl.genMarker = nil + close(dl.genPending) dl.lock.Unlock() // Someone will be looking for us, wait it out diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 8e039606f..c42a26d21 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -108,6 +108,7 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, if base.genMarker == nil { base.genMarker = []byte{} } + base.genPending = make(chan struct{}) base.genAbort = make(chan chan *generatorStats) var origin uint64 diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index ad602bbbb..d031dd2c1 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -164,7 +164,7 @@ type Tree struct { // If the snapshot is missing or inconsistent, the entirety is deleted and will // be reconstructed from scratch based on the tries in the key-value store, on a // background thread. -func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash) *Tree { +func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, async bool) *Tree { // Create a new, empty snapshot tree snap := &Tree{ diskdb: diskdb, @@ -172,6 +172,9 @@ func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root comm cache: cache, layers: make(map[common.Hash]snapshot), } + if !async { + defer snap.waitBuild() + } // Attempt to load a previously persisted snapshot and rebuild one if failed head, err := loadSnapshot(diskdb, triedb, cache, root) if err != nil { @@ -187,6 +190,27 @@ func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root comm return snap } +// waitBuild blocks until the snapshot finishes rebuilding. This method is meant +// to be used by tests to ensure we're testing what we believe we are. +func (t *Tree) waitBuild() { + // Find the rebuild termination channel + var done chan struct{} + + t.lock.RLock() + for _, layer := range t.layers { + if layer, ok := layer.(*diskLayer); ok { + done = layer.genPending + break + } + } + t.lock.RUnlock() + + // Wait until the snapshot is generated + if done != nil { + <-done + } +} + // Snapshot retrieves a snapshot belonging to the given block root, or nil if no // snapshot is maintained for that block. func (t *Tree) Snapshot(blockRoot common.Hash) Snapshot { @@ -477,11 +501,12 @@ func diffToDisk(bottom *diffLayer) *diskLayer { log.Crit("Failed to write leftover snapshot", "err", err) } res := &diskLayer{ - root: bottom.root, - cache: base.cache, - diskdb: base.diskdb, - triedb: base.triedb, - genMarker: base.genMarker, + root: bottom.root, + cache: base.cache, + diskdb: base.diskdb, + triedb: base.triedb, + genMarker: base.genMarker, + genPending: base.genPending, } // If snapshot generation hasn't finished yet, port over all the starts and // continue where the previous round left off. diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 69eb80a5c..289c6c5bb 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -168,7 +168,7 @@ func TestPrestateTracerCreate2(t *testing.T) { Code: []byte{}, Balance: big.NewInt(500000000000000), } - statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc) + statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false) // Create the tracer, the EVM environment and run it tracer, err := New("prestateTracer") @@ -242,7 +242,7 @@ func TestCallTracer(t *testing.T) { GasLimit: uint64(test.Context.GasLimit), GasPrice: tx.GasPrice(), } - statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc) + statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) // Create the tracer, the EVM environment and run it tracer, err := New("callTracer") diff --git a/tests/block_test.go b/tests/block_test.go index 3a55e4c34..8fa90e3e3 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -45,11 +45,13 @@ func TestBlockchain(t *testing.T) { bt.skipLoad(`.*randomStatetest94.json.*`) bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { - if err := bt.checkFailure(t, name, test.Run()); err != nil { - t.Error(err) + if err := bt.checkFailure(t, name+"/trie", test.Run(false)); err != nil { + t.Errorf("test without snapshotter failed: %v", err) + } + if err := bt.checkFailure(t, name+"/snap", test.Run(true)); err != nil { + t.Errorf("test with snapshotter failed: %v", err) } }) - // There is also a LegacyTests folder, containing blockchain tests generated // prior to Istanbul. However, they are all derived from GeneralStateTests, // which run natively, so there's no reason to run them here. diff --git a/tests/block_test_util.go b/tests/block_test_util.go index b5f1de3ef..1ae986e3c 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -94,7 +94,7 @@ type btHeaderMarshaling struct { Timestamp math.HexOrDecimal64 } -func (t *BlockTest) Run() error { +func (t *BlockTest) Run(snapshotter bool) error { config, ok := Forks[t.json.Network] if !ok { return UnsupportedForkError{t.json.Network} @@ -118,7 +118,12 @@ func (t *BlockTest) Run() error { } else { engine = ethash.NewShared() } - chain, err := core.NewBlockChain(db, &core.CacheConfig{TrieCleanLimit: 0}, config, engine, vm.Config{}, nil) + cache := &core.CacheConfig{TrieCleanLimit: 0} + if snapshotter { + cache.SnapshotLimit = 1 + cache.SnapshotWait = true + } + chain, err := core.NewBlockChain(db, cache, config, engine, vm.Config{}, nil) if err != nil { return err } diff --git a/tests/state_test.go b/tests/state_test.go index f9499d4a8..c0a90b3a4 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -63,10 +63,17 @@ func TestState(t *testing.T) { subtest := subtest key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) name := name + "/" + key - t.Run(key, func(t *testing.T) { + + t.Run(key+"/trie", func(t *testing.T) { withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { - _, err := test.Run(subtest, vmconfig) - return st.checkFailure(t, name, err) + _, err := test.Run(subtest, vmconfig, false) + return st.checkFailure(t, name+"/trie", err) + }) + }) + t.Run(key+"/snap", func(t *testing.T) { + withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { + _, err := test.Run(subtest, vmconfig, true) + return st.checkFailure(t, name+"/snap", err) }) }) } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index a10d044cd..5e5b96d52 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -24,6 +24,8 @@ import ( "strconv" "strings" + "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" @@ -145,8 +147,8 @@ func (t *StateTest) Subtests() []StateSubtest { } // Run executes a specific subtest and verifies the post-state and logs -func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) (*state.StateDB, error) { - statedb, root, err := t.RunNoVerify(subtest, vmconfig) +func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*state.StateDB, error) { + statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter) if err != nil { return statedb, err } @@ -163,14 +165,14 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) (*state.StateD } // RunNoVerify runs a specific subtest and returns the statedb and post-state root -func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config) (*state.StateDB, common.Hash, error) { +func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*state.StateDB, common.Hash, error) { config, eips, err := getVMConfig(subtest.Fork) if err != nil { return nil, common.Hash{}, UnsupportedForkError{subtest.Fork} } vmconfig.ExtraEips = eips block := t.genesis(config).ToBlock(nil) - statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre) + statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter) post := t.json.Post[subtest.Fork][subtest.Index] msg, err := t.json.Tx.toMessage(post) @@ -204,7 +206,7 @@ func (t *StateTest) gasLimit(subtest StateSubtest) uint64 { return t.json.Tx.GasLimit[t.json.Post[subtest.Fork][subtest.Index].Indexes.Gas] } -func MakePreState(db ethdb.Database, accounts core.GenesisAlloc) *state.StateDB { +func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter bool) *state.StateDB { sdb := state.NewDatabase(db) statedb, _ := state.New(common.Hash{}, sdb, nil) for addr, a := range accounts { @@ -217,7 +219,12 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc) *state.StateDB } // Commit and re-open to start with a clean state. root, _ := statedb.Commit(false) - statedb, _ = state.New(root, sdb, nil) + + var snaps *snapshot.Tree + if snapshotter { + snaps = snapshot.New(db, sdb.TrieDB(), 1, root, false) + } + statedb, _ = state.New(root, sdb, snaps) return statedb } diff --git a/tests/transaction_test_util.go b/tests/transaction_test_util.go index 43debae83..aea90535c 100644 --- a/tests/transaction_test_util.go +++ b/tests/transaction_test_util.go @@ -45,7 +45,6 @@ type ttFork struct { } func (tt *TransactionTest) Run(config *params.ChainConfig) error { - validateTx := func(rlpData hexutil.Bytes, signer types.Signer, isHomestead bool, isIstanbul bool) (*common.Address, *common.Hash, error) { tx := new(types.Transaction) if err := rlp.DecodeBytes(rlpData, tx); err != nil { diff --git a/tests/vm_test.go b/tests/vm_test.go index 441483dff..fb839827a 100644 --- a/tests/vm_test.go +++ b/tests/vm_test.go @@ -30,7 +30,10 @@ func TestVM(t *testing.T) { vmt.walk(t, vmTestDir, func(t *testing.T, name string, test *VMTest) { withTrace(t, test.json.Exec.GasLimit, func(vmconfig vm.Config) error { - return vmt.checkFailure(t, name, test.Run(vmconfig)) + return vmt.checkFailure(t, name+"/trie", test.Run(vmconfig, false)) + }) + withTrace(t, test.json.Exec.GasLimit, func(vmconfig vm.Config) error { + return vmt.checkFailure(t, name+"/snap", test.Run(vmconfig, true)) }) }) } diff --git a/tests/vm_test_util.go b/tests/vm_test_util.go index 91566c47e..9acbe59f4 100644 --- a/tests/vm_test_util.go +++ b/tests/vm_test_util.go @@ -78,8 +78,8 @@ type vmExecMarshaling struct { GasPrice *math.HexOrDecimal256 } -func (t *VMTest) Run(vmconfig vm.Config) error { - statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre) +func (t *VMTest) Run(vmconfig vm.Config, snapshotter bool) error { + statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter) ret, gasRemaining, err := t.exec(statedb, vmconfig) if t.json.GasRemaining == nil {