26d271dfbb
* core/state/snapshot: implement storage iterator * core/state/snapshot, tests: implement helper function * core/state/snapshot: fix storage issue If an account is deleted in the tx_1 but recreated in the tx_2, the it can happen that in this diff layer, both destructedSet and storageData records this account. In this case, the storage iterator should be able to iterate the slots belong to new account but disable further iteration in deeper layers(belong to old account) * core/state/snapshot: address peter and martin's comment * core/state: address comments * core/state/snapshot: fix test
401 lines
12 KiB
Go
401 lines
12 KiB
Go
// Copyright 2019 The go-ethereum Authors
|
|
// This file is part of the go-ethereum library.
|
|
//
|
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Lesser General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Lesser General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Lesser General Public License
|
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
package snapshot
|
|
|
|
import (
|
|
"bytes"
|
|
"math/rand"
|
|
"testing"
|
|
|
|
"github.com/VictoriaMetrics/fastcache"
|
|
"github.com/ethereum/go-ethereum/common"
|
|
"github.com/ethereum/go-ethereum/crypto"
|
|
"github.com/ethereum/go-ethereum/ethdb/memorydb"
|
|
)
|
|
|
|
func copyDestructs(destructs map[common.Hash]struct{}) map[common.Hash]struct{} {
|
|
copy := make(map[common.Hash]struct{})
|
|
for hash := range destructs {
|
|
copy[hash] = struct{}{}
|
|
}
|
|
return copy
|
|
}
|
|
|
|
func copyAccounts(accounts map[common.Hash][]byte) map[common.Hash][]byte {
|
|
copy := make(map[common.Hash][]byte)
|
|
for hash, blob := range accounts {
|
|
copy[hash] = blob
|
|
}
|
|
return copy
|
|
}
|
|
|
|
func copyStorage(storage map[common.Hash]map[common.Hash][]byte) map[common.Hash]map[common.Hash][]byte {
|
|
copy := make(map[common.Hash]map[common.Hash][]byte)
|
|
for accHash, slots := range storage {
|
|
copy[accHash] = make(map[common.Hash][]byte)
|
|
for slotHash, blob := range slots {
|
|
copy[accHash][slotHash] = blob
|
|
}
|
|
}
|
|
return copy
|
|
}
|
|
|
|
// TestMergeBasics tests some simple merges
|
|
func TestMergeBasics(t *testing.T) {
|
|
var (
|
|
destructs = make(map[common.Hash]struct{})
|
|
accounts = make(map[common.Hash][]byte)
|
|
storage = make(map[common.Hash]map[common.Hash][]byte)
|
|
)
|
|
// Fill up a parent
|
|
for i := 0; i < 100; i++ {
|
|
h := randomHash()
|
|
data := randomAccount()
|
|
|
|
accounts[h] = data
|
|
if rand.Intn(4) == 0 {
|
|
destructs[h] = struct{}{}
|
|
}
|
|
if rand.Intn(2) == 0 {
|
|
accStorage := make(map[common.Hash][]byte)
|
|
value := make([]byte, 32)
|
|
rand.Read(value)
|
|
accStorage[randomHash()] = value
|
|
storage[h] = accStorage
|
|
}
|
|
}
|
|
// Add some (identical) layers on top
|
|
parent := newDiffLayer(emptyLayer(), common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
|
|
child := newDiffLayer(parent, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
|
|
child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
|
|
child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
|
|
child = newDiffLayer(child, common.Hash{}, copyDestructs(destructs), copyAccounts(accounts), copyStorage(storage))
|
|
// And flatten
|
|
merged := (child.flatten()).(*diffLayer)
|
|
|
|
{ // Check account lists
|
|
if have, want := len(merged.accountList), 0; have != want {
|
|
t.Errorf("accountList wrong: have %v, want %v", have, want)
|
|
}
|
|
if have, want := len(merged.AccountList()), len(accounts); have != want {
|
|
t.Errorf("AccountList() wrong: have %v, want %v", have, want)
|
|
}
|
|
if have, want := len(merged.accountList), len(accounts); have != want {
|
|
t.Errorf("accountList [2] wrong: have %v, want %v", have, want)
|
|
}
|
|
}
|
|
{ // Check account drops
|
|
if have, want := len(merged.destructSet), len(destructs); have != want {
|
|
t.Errorf("accountDrop wrong: have %v, want %v", have, want)
|
|
}
|
|
}
|
|
{ // Check storage lists
|
|
i := 0
|
|
for aHash, sMap := range storage {
|
|
if have, want := len(merged.storageList), i; have != want {
|
|
t.Errorf("[1] storageList wrong: have %v, want %v", have, want)
|
|
}
|
|
list, _ := merged.StorageList(aHash)
|
|
if have, want := len(list), len(sMap); have != want {
|
|
t.Errorf("[2] StorageList() wrong: have %v, want %v", have, want)
|
|
}
|
|
if have, want := len(merged.storageList[aHash]), len(sMap); have != want {
|
|
t.Errorf("storageList wrong: have %v, want %v", have, want)
|
|
}
|
|
i++
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestMergeDelete tests some deletion
|
|
func TestMergeDelete(t *testing.T) {
|
|
var (
|
|
storage = make(map[common.Hash]map[common.Hash][]byte)
|
|
)
|
|
// Fill up a parent
|
|
h1 := common.HexToHash("0x01")
|
|
h2 := common.HexToHash("0x02")
|
|
|
|
flipDrops := func() map[common.Hash]struct{} {
|
|
return map[common.Hash]struct{}{
|
|
h2: {},
|
|
}
|
|
}
|
|
flipAccs := func() map[common.Hash][]byte {
|
|
return map[common.Hash][]byte{
|
|
h1: randomAccount(),
|
|
}
|
|
}
|
|
flopDrops := func() map[common.Hash]struct{} {
|
|
return map[common.Hash]struct{}{
|
|
h1: {},
|
|
}
|
|
}
|
|
flopAccs := func() map[common.Hash][]byte {
|
|
return map[common.Hash][]byte{
|
|
h2: randomAccount(),
|
|
}
|
|
}
|
|
// Add some flipAccs-flopping layers on top
|
|
parent := newDiffLayer(emptyLayer(), common.Hash{}, flipDrops(), flipAccs(), storage)
|
|
child := parent.Update(common.Hash{}, flopDrops(), flopAccs(), storage)
|
|
child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage)
|
|
child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage)
|
|
child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage)
|
|
child = child.Update(common.Hash{}, flopDrops(), flopAccs(), storage)
|
|
child = child.Update(common.Hash{}, flipDrops(), flipAccs(), storage)
|
|
|
|
if data, _ := child.Account(h1); data == nil {
|
|
t.Errorf("last diff layer: expected %x account to be non-nil", h1)
|
|
}
|
|
if data, _ := child.Account(h2); data != nil {
|
|
t.Errorf("last diff layer: expected %x account to be nil", h2)
|
|
}
|
|
if _, ok := child.destructSet[h1]; ok {
|
|
t.Errorf("last diff layer: expected %x drop to be missing", h1)
|
|
}
|
|
if _, ok := child.destructSet[h2]; !ok {
|
|
t.Errorf("last diff layer: expected %x drop to be present", h1)
|
|
}
|
|
// And flatten
|
|
merged := (child.flatten()).(*diffLayer)
|
|
|
|
if data, _ := merged.Account(h1); data == nil {
|
|
t.Errorf("merged layer: expected %x account to be non-nil", h1)
|
|
}
|
|
if data, _ := merged.Account(h2); data != nil {
|
|
t.Errorf("merged layer: expected %x account to be nil", h2)
|
|
}
|
|
if _, ok := merged.destructSet[h1]; !ok { // Note, drops stay alive until persisted to disk!
|
|
t.Errorf("merged diff layer: expected %x drop to be present", h1)
|
|
}
|
|
if _, ok := merged.destructSet[h2]; !ok { // Note, drops stay alive until persisted to disk!
|
|
t.Errorf("merged diff layer: expected %x drop to be present", h1)
|
|
}
|
|
// If we add more granular metering of memory, we can enable this again,
|
|
// but it's not implemented for now
|
|
//if have, want := merged.memory, child.memory; have != want {
|
|
// t.Errorf("mem wrong: have %d, want %d", have, want)
|
|
//}
|
|
}
|
|
|
|
// This tests that if we create a new account, and set a slot, and then merge
|
|
// it, the lists will be correct.
|
|
func TestInsertAndMerge(t *testing.T) {
|
|
// Fill up a parent
|
|
var (
|
|
acc = common.HexToHash("0x01")
|
|
slot = common.HexToHash("0x02")
|
|
parent *diffLayer
|
|
child *diffLayer
|
|
)
|
|
{
|
|
var (
|
|
destructs = make(map[common.Hash]struct{})
|
|
accounts = make(map[common.Hash][]byte)
|
|
storage = make(map[common.Hash]map[common.Hash][]byte)
|
|
)
|
|
parent = newDiffLayer(emptyLayer(), common.Hash{}, destructs, accounts, storage)
|
|
}
|
|
{
|
|
var (
|
|
destructs = make(map[common.Hash]struct{})
|
|
accounts = make(map[common.Hash][]byte)
|
|
storage = make(map[common.Hash]map[common.Hash][]byte)
|
|
)
|
|
accounts[acc] = randomAccount()
|
|
storage[acc] = make(map[common.Hash][]byte)
|
|
storage[acc][slot] = []byte{0x01}
|
|
child = newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
|
|
}
|
|
// And flatten
|
|
merged := (child.flatten()).(*diffLayer)
|
|
{ // Check that slot value is present
|
|
have, _ := merged.Storage(acc, slot)
|
|
if want := []byte{0x01}; !bytes.Equal(have, want) {
|
|
t.Errorf("merged slot value wrong: have %x, want %x", have, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func emptyLayer() *diskLayer {
|
|
return &diskLayer{
|
|
diskdb: memorydb.New(),
|
|
cache: fastcache.New(500 * 1024),
|
|
}
|
|
}
|
|
|
|
// BenchmarkSearch checks how long it takes to find a non-existing key
|
|
// BenchmarkSearch-6 200000 10481 ns/op (1K per layer)
|
|
// BenchmarkSearch-6 200000 10760 ns/op (10K per layer)
|
|
// BenchmarkSearch-6 100000 17866 ns/op
|
|
//
|
|
// BenchmarkSearch-6 500000 3723 ns/op (10k per layer, only top-level RLock()
|
|
func BenchmarkSearch(b *testing.B) {
|
|
// First, we set up 128 diff layers, with 1K items each
|
|
fill := func(parent snapshot) *diffLayer {
|
|
var (
|
|
destructs = make(map[common.Hash]struct{})
|
|
accounts = make(map[common.Hash][]byte)
|
|
storage = make(map[common.Hash]map[common.Hash][]byte)
|
|
)
|
|
for i := 0; i < 10000; i++ {
|
|
accounts[randomHash()] = randomAccount()
|
|
}
|
|
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
|
|
}
|
|
var layer snapshot
|
|
layer = emptyLayer()
|
|
for i := 0; i < 128; i++ {
|
|
layer = fill(layer)
|
|
}
|
|
key := crypto.Keccak256Hash([]byte{0x13, 0x38})
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
layer.AccountRLP(key)
|
|
}
|
|
}
|
|
|
|
// BenchmarkSearchSlot checks how long it takes to find a non-existing key
|
|
// - Number of layers: 128
|
|
// - Each layers contains the account, with a couple of storage slots
|
|
// BenchmarkSearchSlot-6 100000 14554 ns/op
|
|
// BenchmarkSearchSlot-6 100000 22254 ns/op (when checking parent root using mutex)
|
|
// BenchmarkSearchSlot-6 100000 14551 ns/op (when checking parent number using atomic)
|
|
// With bloom filter:
|
|
// BenchmarkSearchSlot-6 3467835 351 ns/op
|
|
func BenchmarkSearchSlot(b *testing.B) {
|
|
// First, we set up 128 diff layers, with 1K items each
|
|
accountKey := crypto.Keccak256Hash([]byte{0x13, 0x37})
|
|
storageKey := crypto.Keccak256Hash([]byte{0x13, 0x37})
|
|
accountRLP := randomAccount()
|
|
fill := func(parent snapshot) *diffLayer {
|
|
var (
|
|
destructs = make(map[common.Hash]struct{})
|
|
accounts = make(map[common.Hash][]byte)
|
|
storage = make(map[common.Hash]map[common.Hash][]byte)
|
|
)
|
|
accounts[accountKey] = accountRLP
|
|
|
|
accStorage := make(map[common.Hash][]byte)
|
|
for i := 0; i < 5; i++ {
|
|
value := make([]byte, 32)
|
|
rand.Read(value)
|
|
accStorage[randomHash()] = value
|
|
storage[accountKey] = accStorage
|
|
}
|
|
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
|
|
}
|
|
var layer snapshot
|
|
layer = emptyLayer()
|
|
for i := 0; i < 128; i++ {
|
|
layer = fill(layer)
|
|
}
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
layer.Storage(accountKey, storageKey)
|
|
}
|
|
}
|
|
|
|
// With accountList and sorting
|
|
// BenchmarkFlatten-6 50 29890856 ns/op
|
|
//
|
|
// Without sorting and tracking accountlist
|
|
// BenchmarkFlatten-6 300 5511511 ns/op
|
|
func BenchmarkFlatten(b *testing.B) {
|
|
fill := func(parent snapshot) *diffLayer {
|
|
var (
|
|
destructs = make(map[common.Hash]struct{})
|
|
accounts = make(map[common.Hash][]byte)
|
|
storage = make(map[common.Hash]map[common.Hash][]byte)
|
|
)
|
|
for i := 0; i < 100; i++ {
|
|
accountKey := randomHash()
|
|
accounts[accountKey] = randomAccount()
|
|
|
|
accStorage := make(map[common.Hash][]byte)
|
|
for i := 0; i < 20; i++ {
|
|
value := make([]byte, 32)
|
|
rand.Read(value)
|
|
accStorage[randomHash()] = value
|
|
|
|
}
|
|
storage[accountKey] = accStorage
|
|
}
|
|
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
|
|
}
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
b.StopTimer()
|
|
var layer snapshot
|
|
layer = emptyLayer()
|
|
for i := 1; i < 128; i++ {
|
|
layer = fill(layer)
|
|
}
|
|
b.StartTimer()
|
|
|
|
for i := 1; i < 128; i++ {
|
|
dl, ok := layer.(*diffLayer)
|
|
if !ok {
|
|
break
|
|
}
|
|
layer = dl.flatten()
|
|
}
|
|
b.StopTimer()
|
|
}
|
|
}
|
|
|
|
// This test writes ~324M of diff layers to disk, spread over
|
|
// - 128 individual layers,
|
|
// - each with 200 accounts
|
|
// - containing 200 slots
|
|
//
|
|
// BenchmarkJournal-6 1 1471373923 ns/ops
|
|
// BenchmarkJournal-6 1 1208083335 ns/op // bufio writer
|
|
func BenchmarkJournal(b *testing.B) {
|
|
fill := func(parent snapshot) *diffLayer {
|
|
var (
|
|
destructs = make(map[common.Hash]struct{})
|
|
accounts = make(map[common.Hash][]byte)
|
|
storage = make(map[common.Hash]map[common.Hash][]byte)
|
|
)
|
|
for i := 0; i < 200; i++ {
|
|
accountKey := randomHash()
|
|
accounts[accountKey] = randomAccount()
|
|
|
|
accStorage := make(map[common.Hash][]byte)
|
|
for i := 0; i < 200; i++ {
|
|
value := make([]byte, 32)
|
|
rand.Read(value)
|
|
accStorage[randomHash()] = value
|
|
|
|
}
|
|
storage[accountKey] = accStorage
|
|
}
|
|
return newDiffLayer(parent, common.Hash{}, destructs, accounts, storage)
|
|
}
|
|
layer := snapshot(new(diskLayer))
|
|
for i := 1; i < 128; i++ {
|
|
layer = fill(layer)
|
|
}
|
|
b.ResetTimer()
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
layer.Journal(new(bytes.Buffer))
|
|
}
|
|
}
|