tornadocontrib/go-ethereum
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

core/state, cmd/geth: streaming json output for dump command (#15475)

Browse Source 
* core/state, cmd/geth: streaming json output dump cmd + optional code+storage

* dump: add option to continue even if preimages are missing

* core, evm: lint nits

* cmd: use local flags for dump, omit empty code/storage

* core/state: fix state dump test
This commit is contained in:
Martin Holst Swende 2019-06-24 16:16:44 +02:00 committed by Péter Szilágyi
parent e4a1488b2f
commit 1da5e0ebb0
7 changed files with 143 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/evm/runner.go
View File

@ -209,7 +209,7 @@ func runCmd(ctx *cli.Context) error {
if ctx.GlobalBool(DumpFlag.Name) {
statedb.Commit(true)
statedb.IntermediateRoot(true)
fmt.Println(string(statedb.Dump()))
fmt.Println(string(statedb.Dump(false, false, true)))
}
if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" {

2
cmd/evm/staterunner.go
View File

@ -105,7 +105,7 @@ func stateTestCmd(ctx *cli.Context) error {
// Test failed, mark as so and dump any state to aid debugging
result.Pass, result.Error = false, err.Error()
if ctx.GlobalBool(DumpFlag.Name) && state != nil {
dump := state.RawDump()
dump := state.RawDump(false, false, true)
result.State = &dump
}
}

21
cmd/geth/chaincmd.go
View File

@ -162,6 +162,10 @@ Remove blockchain and state databases`,
utils.DataDirFlag,
utils.CacheFlag,
utils.SyncModeFlag,
utils.IterativeOutputFlag,
utils.ExcludeCodeFlag,
utils.ExcludeStorageFlag,
utils.IncludeIncompletesFlag,
},
Category: "BLOCKCHAIN COMMANDS",
Description: `
@ -287,7 +291,7 @@ func importChain(ctx *cli.Context) error {
fmt.Printf("Allocations: %.3f million\n", float64(mem.Mallocs)/1000000)
fmt.Printf("GC pause: %v\n\n", time.Duration(mem.PauseTotalNs))
if ctx.GlobalIsSet(utils.NoCompactionFlag.Name) {
if ctx.GlobalBool(utils.NoCompactionFlag.Name) {
return nil
}
@ -504,6 +508,7 @@ func dump(ctx *cli.Context) error {
defer stack.Close()
chain, chainDb := utils.MakeChain(ctx, stack)
defer chainDb.Close()
for _, arg := range ctx.Args() {
var block *types.Block
if hashish(arg) {
@ -520,10 +525,20 @@ func dump(ctx *cli.Context) error {
if err != nil {
utils.Fatalf("could not create new state: %v", err)
}
fmt.Printf("%s\n", state.Dump())
excludeCode := ctx.Bool(utils.ExcludeCodeFlag.Name)
excludeStorage := ctx.Bool(utils.ExcludeStorageFlag.Name)
includeMissing := ctx.Bool(utils.IncludeIncompletesFlag.Name)
if ctx.Bool(utils.IterativeOutputFlag.Name) {
state.IterativeDump(excludeCode, excludeStorage, !includeMissing, json.NewEncoder(os.Stdout))
} else {
if includeMissing {
fmt.Printf("If you want to include accounts with missing preimages, you need iterative output, since" +
" otherwise the accounts will overwrite each other in the resulting mapping.")
}
fmt.Printf("%v %s\n", includeMissing, state.Dump(excludeCode, excludeStorage, false))
}
}
}
chainDb.Close()
return nil
}

16
cmd/utils/flags.go
View File

@ -196,6 +196,22 @@ var (
Name: "ulc.trusted",
Usage: "List of trusted ULC servers",
}
IterativeOutputFlag = cli.BoolFlag{
Name: "iterative",
Usage: "Print streaming JSON iteratively, delimited by newlines",
}
ExcludeStorageFlag = cli.BoolFlag{
Name: "nostorage",
Usage: "Exclude storage entries (save db lookups)",
}
IncludeIncompletesFlag = cli.BoolFlag{
Name: "incompletes",
Usage: "Include accounts for which we don't have the address (missing preimage)",
}
ExcludeCodeFlag = cli.BoolFlag{
Name: "nocode",
Usage: "Exclude contract code (save db lookups)",
}
defaultSyncMode = eth.DefaultConfig.SyncMode
SyncModeFlag = TextMarshalerFlag{
Name: "syncmode",

124
core/state/dump.go
View File

@ -21,61 +21,133 @@ import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
// DumpAccount represents an account in the state
type DumpAccount struct {
Balance string `json:"balance"`
Nonce uint64 `json:"nonce"`
Root string `json:"root"`
CodeHash string `json:"codeHash"`
Code string `json:"code"`
Storage map[string]string `json:"storage"`
Balance string `json:"balance"`
Nonce uint64 `json:"nonce"`
Root string `json:"root"`
CodeHash string `json:"codeHash"`
Code string `json:"code,omitempty"`
Storage map[common.Hash]string `json:"storage,omitempty"`
Address *common.Address `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode
SecureKey hexutil.Bytes `json:"key,omitempty"` // If we don't have address, we can output the key
}
// Dump represents the full dump in a collected format, as one large map
type Dump struct {
Root string `json:"root"`
Accounts map[string]DumpAccount `json:"accounts"`
Root string `json:"root"`
Accounts map[common.Address]DumpAccount `json:"accounts"`
}
func (self *StateDB) RawDump() Dump {
dump := Dump{
Root: fmt.Sprintf("%x", self.trie.Hash()),
Accounts: make(map[string]DumpAccount),
}
// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively
type iterativeDump json.Encoder
// Collector interface which the state trie calls during iteration
type collector interface {
onRoot(common.Hash)
onAccount(common.Address, DumpAccount)
}
func (self *Dump) onRoot(root common.Hash) {
self.Root = fmt.Sprintf("%x", root)
}
func (self *Dump) onAccount(addr common.Address, account DumpAccount) {
self.Accounts[addr] = account
}
func (self iterativeDump) onAccount(addr common.Address, account DumpAccount) {
dumpAccount := &DumpAccount{
Balance: account.Balance,
Nonce: account.Nonce,
Root: account.Root,
CodeHash: account.CodeHash,
Code: account.Code,
Storage: account.Storage,
SecureKey: account.SecureKey,
Address: nil,
}
if addr != (common.Address{}) {
dumpAccount.Address = &addr
}
(*json.Encoder)(&self).Encode(dumpAccount)
}
func (self iterativeDump) onRoot(root common.Hash) {
(*json.Encoder)(&self).Encode(struct {
Root common.Hash `json:"root"`
}{root})
}
func (self *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool) {
emptyAddress := (common.Address{})
missingPreimages := 0
c.onRoot(self.trie.Hash())
it := trie.NewIterator(self.trie.NodeIterator(nil))
for it.Next() {
addr := self.trie.GetKey(it.Key)
var data Account
if err := rlp.DecodeBytes(it.Value, &data); err != nil {
panic(err)
}
obj := newObject(nil, common.BytesToAddress(addr), data)
addr := common.BytesToAddress(self.trie.GetKey(it.Key))
obj := newObject(nil, addr, data)
account := DumpAccount{
Balance: data.Balance.String(),
Nonce: data.Nonce,
Root: common.Bytes2Hex(data.Root[:]),
CodeHash: common.Bytes2Hex(data.CodeHash),
Code: common.Bytes2Hex(obj.Code(self.db)),
Storage: make(map[string]string),
}
storageIt := trie.NewIterator(obj.getTrie(self.db).NodeIterator(nil))
for storageIt.Next() {
account.Storage[common.Bytes2Hex(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
if emptyAddress == addr {
// Preimage missing
missingPreimages++
if excludeMissingPreimages {
continue
}
account.SecureKey = it.Key
}
dump.Accounts[common.Bytes2Hex(addr)] = account
if !excludeCode {
account.Code = common.Bytes2Hex(obj.Code(self.db))
}
if !excludeStorage {
account.Storage = make(map[common.Hash]string)
storageIt := trie.NewIterator(obj.getTrie(self.db).NodeIterator(nil))
for storageIt.Next() {
account.Storage[common.BytesToHash(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
}
}
c.onAccount(addr, account)
}
if missingPreimages > 0 {
log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages)
}
return dump
}
func (self *StateDB) Dump() []byte {
json, err := json.MarshalIndent(self.RawDump(), "", " ")
// RawDump returns the entire state an a single large object
func (self *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages bool) Dump {
dump := &Dump{
Accounts: make(map[common.Address]DumpAccount),
}
self.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages)
return *dump
}
// Dump returns a JSON string representing the entire state as a single json-object
func (self *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool) []byte {
dump := self.RawDump(excludeCode, excludeStorage, excludeMissingPreimages)
json, err := json.MarshalIndent(dump, "", " ")
if err != nil {
fmt.Println("dump err", err)
}
return json
}
// IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout
func (self *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) {
self.dump(iterativeDump(*output), excludeCode, excludeStorage, excludeMissingPreimages)
}

19
core/state/state_test.go
View File

@ -52,33 +52,28 @@ func (s *StateSuite) TestDump(c *checker.C) {
s.state.Commit(false)
// check that dump contains the state objects that are in trie
got := string(s.state.Dump())
got := string(s.state.Dump(false, false, true))
want := `{
"root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2",
"accounts": {
"0000000000000000000000000000000000000001": {
"0x0000000000000000000000000000000000000001": {
"balance": "22",
"nonce": 0,
"root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
"code": "",
"storage": {}
"codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
},
"0000000000000000000000000000000000000002": {
"0x0000000000000000000000000000000000000002": {
"balance": "44",
"nonce": 0,
"root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
"code": "",
"storage": {}
"codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
},
"0000000000000000000000000000000000000102": {
"0x0000000000000000000000000000000000000102": {
"balance": "0",
"nonce": 0,
"root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"codeHash": "87874902497a5bb968da31a2998d8f22e949d1ef6214bcdedd8bae24cca4b9e3",
"code": "03030303030303",
"storage": {}
"code": "03030303030303"
}
}
}`

4
eth/api.go
View File

@ -266,7 +266,7 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error
// both the pending block as well as the pending state from
// the miner and operate on those
_, stateDb := api.eth.miner.Pending()
return stateDb.RawDump(), nil
return stateDb.RawDump(false, false, true), nil
}
var block *types.Block
if blockNr == rpc.LatestBlockNumber {
@ -281,7 +281,7 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error
if err != nil {
return state.Dump{}, err
}
return stateDb.RawDump(), nil
return stateDb.RawDump(false, false, true), nil
}
// PrivateDebugAPI is the collection of Ethereum full node APIs exposed over