From fecd2bfafe4a6c69a2c8ed64a2167198f5cba6d6 Mon Sep 17 00:00:00 2001 From: VM <112189277+sysvm@users.noreply.github.com> Date: Thu, 25 Jan 2024 22:07:44 +0800 Subject: [PATCH] cmd: fix dump cli cannot work in path mode (#2160) --- cmd/geth/chaincmd.go | 76 ++++++++++++++++++++++++--- cmd/geth/main.go | 1 + cmd/utils/flags.go | 9 ++-- core/state/dump.go | 11 +++- eth/api_debug.go | 3 ++ trie/database.go | 14 ++++- trie/triedb/pathdb/asyncnodebuffer.go | 2 +- trie/triedb/pathdb/database.go | 23 ++++++++ 8 files changed, 125 insertions(+), 14 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index c8055ae83..6ce1e5255 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -30,6 +30,9 @@ import ( "sync/atomic" "time" + "github.com/olekukonko/tablewriter" + "github.com/urfave/cli/v2" + "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -44,7 +47,8 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/urfave/cli/v2" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" ) var ( @@ -191,6 +195,21 @@ It's deprecated, please use "geth db export" instead. }, utils.DatabasePathFlags), Description: ` This command dumps out the state for a given block (or latest, if none provided). +If you use "dump" command in path mode, please firstly use "dump-roothash" command to get all available state root hash. +`, + } + dumpRootHashCommand = &cli.Command{ + Action: dumpAllRootHashInPath, + Name: "dump-roothash", + Usage: "Dump all available state root hash in path mode", + Flags: flags.Merge([]cli.Flag{ + utils.StateSchemeFlag, + }, utils.DatabasePathFlags), + Description: ` +The dump-roothash command dump all available state root hash in path mode. +If you use "dump" command in path mode, please note that it only keeps at most 129 blocks which belongs to diffLayer or diskLayer. +Therefore, you must specify the blockNumber or blockHash that locates in diffLayer or diskLayer. +"geth" will print all available blockNumber and related block state root hash, and you can query block hash by block number. `, } ) @@ -590,11 +609,20 @@ func exportPreimages(ctx *cli.Context) error { } func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) { - db := utils.MakeChainDatabase(ctx, stack, true, false) - var header *types.Header if ctx.NArg() > 1 { return nil, nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg()) } + + db := utils.MakeChainDatabase(ctx, stack, true, false) + scheme, err := rawdb.ParseStateScheme(ctx.String(utils.StateSchemeFlag.Name), db) + if err != nil { + return nil, nil, common.Hash{}, err + } + if scheme == rawdb.PathScheme { + fmt.Println("You are using geth dump in path mode, please use `geth dump-roothash` command to get all available blocks.") + } + + header := &types.Header{} if ctx.NArg() == 1 { arg := ctx.Args().First() if hashish(arg) { @@ -617,11 +645,22 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth } } else { // Use latest - header = rawdb.ReadHeadHeader(db) + if scheme == rawdb.PathScheme { + triedb := trie.NewDatabase(db, &trie.Config{PathDB: pathdb.ReadOnly}) + defer triedb.Close() + if stateRoot := triedb.Head(); stateRoot != (common.Hash{}) { + header.Root = stateRoot + } else { + return nil, nil, common.Hash{}, fmt.Errorf("no top state root hash in path db") + } + } else { + header = rawdb.ReadHeadHeader(db) + } } if header == nil { return nil, nil, common.Hash{}, errors.New("no head block found") } + startArg := common.FromHex(ctx.String(utils.StartKeyFlag.Name)) var start common.Hash switch len(startArg) { @@ -634,6 +673,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth default: return nil, nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg) } + var conf = &state.DumpConfig{ SkipCode: ctx.Bool(utils.ExcludeCodeFlag.Name), SkipStorage: ctx.Bool(utils.ExcludeStorageFlag.Name), @@ -641,9 +681,10 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth Start: start.Bytes(), Max: ctx.Uint64(utils.DumpLimitFlag.Name), } + conf.StateScheme = scheme log.Info("State dump configured", "block", header.Number, "hash", header.Hash().Hex(), - "skipcode", conf.SkipCode, "skipstorage", conf.SkipStorage, - "start", hexutil.Encode(conf.Start), "limit", conf.Max) + "skipcode", conf.SkipCode, "skipstorage", conf.SkipStorage, "start", hexutil.Encode(conf.Start), + "limit", conf.Max, "state scheme", conf.StateScheme) return conf, db, header.Root, nil } @@ -675,6 +716,29 @@ func dump(ctx *cli.Context) error { return nil } +func dumpAllRootHashInPath(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + db := utils.MakeChainDatabase(ctx, stack, true, false) + defer db.Close() + triedb := trie.NewDatabase(db, &trie.Config{PathDB: pathdb.ReadOnly}) + defer triedb.Close() + + scheme, err := rawdb.ParseStateScheme(ctx.String(utils.StateSchemeFlag.Name), db) + if err != nil { + return err + } + if scheme == rawdb.HashScheme { + return errors.New("incorrect state scheme, you should use it in path mode") + } + + table := tablewriter.NewWriter(os.Stdout) + table.SetHeader([]string{"Block Number", "Block State Root Hash"}) + table.AppendBulk(triedb.GetAllRooHash()) + table.Render() + return nil +} + // hashish returns true for strings that look like hashes. func hashish(x string) bool { _, err := strconv.Atoi(x) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 11acc9b3c..f11d56d53 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -239,6 +239,7 @@ func init() { removedbCommand, dumpCommand, dumpGenesisCommand, + dumpRootHashCommand, // See accountcmd.go: accountCommand, walletCommand, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index a360b0f9f..3b79a5402 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1884,7 +1884,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(StateHistoryFlag.Name) { cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name) } - scheme, err := compareCLIWithConfig(ctx) + scheme, err := CompareStateSchemeCLIWithConfig(ctx) if err != nil { Fatalf("%v", err) } @@ -2353,7 +2353,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } - provided, err := compareCLIWithConfig(ctx) + provided, err := CompareStateSchemeCLIWithConfig(ctx) if err != nil { Fatalf("%v", err) } @@ -2425,7 +2425,7 @@ func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, read config := &trie.Config{ Preimages: preimage, } - provided, err := compareCLIWithConfig(ctx) + provided, err := CompareStateSchemeCLIWithConfig(ctx) if err != nil { Fatalf("%v", err) } @@ -2448,7 +2448,8 @@ func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, read return trie.NewDatabase(disk, config) } -func compareCLIWithConfig(ctx *cli.Context) (string, error) { +// CompareStateSchemeCLIWithConfig compare state scheme in CLI with config whether are equal. +func CompareStateSchemeCLIWithConfig(ctx *cli.Context) (string, error) { var ( cfgScheme string err error diff --git a/core/state/dump.go b/core/state/dump.go index 9ce6cd394..cf206030b 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -37,6 +38,7 @@ type DumpConfig struct { OnlyWithAddresses bool Start []byte Max uint64 + StateScheme string } // DumpCollector interface which the state trie calls during iteration @@ -57,7 +59,6 @@ type DumpAccount struct { Storage map[common.Hash]string `json:"storage,omitempty"` Address *common.Address `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode SecureKey hexutil.Bytes `json:"key,omitempty"` // If we don't have address, we can output the key - } // Dump represents the full dump in a collected format, as one large map. @@ -177,7 +178,13 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] } if !conf.SkipStorage { account.Storage = make(map[common.Hash]string) - tr, err := obj.getTrie() + var tr Trie + if conf.StateScheme == rawdb.PathScheme { + tr, err = trie.NewStateTrie(trie.StorageTrieID(obj.db.originalRoot, common.BytesToHash(it.Key), + obj.data.Root), obj.db.db.TrieDB()) + } else { + tr, err = obj.getTrie() + } if err != nil { log.Error("Failed to load storage trie", "err", err) continue diff --git a/eth/api_debug.go b/eth/api_debug.go index 6afa04678..8d6f45463 100644 --- a/eth/api_debug.go +++ b/eth/api_debug.go @@ -59,6 +59,7 @@ func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { if stateDb == nil { return state.Dump{}, errors.New("pending state is not available") } + opts.StateScheme = stateDb.Database().TrieDB().Scheme() return stateDb.RawDump(opts), nil } var header *types.Header @@ -83,6 +84,7 @@ func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { if err != nil { return state.Dump{}, err } + opts.StateScheme = stateDb.Database().TrieDB().Scheme() return stateDb.RawDump(opts), nil } @@ -188,6 +190,7 @@ func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hex OnlyWithAddresses: !incompletes, Start: start, Max: uint64(maxResults), + StateScheme: stateDb.Database().TrieDB().Scheme(), } if maxResults > AccountRangeMaxResults || maxResults <= 0 { opts.Max = AccountRangeMaxResults diff --git a/trie/database.go b/trie/database.go index 8581b8bca..df83dd081 100644 --- a/trie/database.go +++ b/trie/database.go @@ -355,7 +355,7 @@ func (db *Database) SetBufferSize(size int) error { } // Head return the top non-fork difflayer/disklayer root hash for rewinding. -// It's only supported by path-based database and will return an error for +// It's only supported by path-based database and will return empty hash for // others. func (db *Database) Head() common.Hash { pdb, ok := db.backend.(*pathdb.Database) @@ -364,3 +364,15 @@ func (db *Database) Head() common.Hash { } return pdb.Head() } + +// GetAllHash returns all MPT root hash in diffLayer and diskLayer. +// It's only supported by path-based database and will return nil for +// others. +func (db *Database) GetAllRooHash() [][]string { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + log.Error("Not supported") + return nil + } + return pdb.GetAllRooHash() +} diff --git a/trie/triedb/pathdb/asyncnodebuffer.go b/trie/triedb/pathdb/asyncnodebuffer.go index 5efb46a91..2c96fedc0 100644 --- a/trie/triedb/pathdb/asyncnodebuffer.go +++ b/trie/triedb/pathdb/asyncnodebuffer.go @@ -226,7 +226,7 @@ func (nc *nodecache) node(owner common.Hash, path []byte, hash common.Hash) (*tr } if n.Hash != hash { dirtyFalseMeter.Mark(1) - log.Error("Unexpected trie node in node buffer", "owner", owner, "path", path, "expect", hash, "got", n.Hash) + log.Error("Unexpected trie node in async node buffer", "owner", owner, "path", path, "expect", hash, "got", n.Hash) return nil, newUnexpectedNodeError("dirty", hash, n.Hash, owner, path, n.Blob) } return n, nil diff --git a/trie/triedb/pathdb/database.go b/trie/triedb/pathdb/database.go index 2da971798..aacd6b932 100644 --- a/trie/triedb/pathdb/database.go +++ b/trie/triedb/pathdb/database.go @@ -20,6 +20,8 @@ import ( "errors" "fmt" "io" + "sort" + "strconv" "sync" "time" @@ -441,3 +443,24 @@ func (db *Database) Head() common.Hash { defer db.lock.Unlock() return db.tree.front() } + +// GetAllRooHash returns all diffLayer and diskLayer root hash +func (db *Database) GetAllRooHash() [][]string { + db.lock.Lock() + defer db.lock.Unlock() + + data := make([][]string, 0, len(db.tree.layers)) + for _, v := range db.tree.layers { + if dl, ok := v.(*diffLayer); ok { + data = append(data, []string{fmt.Sprintf("%d", dl.block), dl.rootHash().String()}) + } + } + sort.Slice(data, func(i, j int) bool { + block1, _ := strconv.Atoi(data[i][0]) + block2, _ := strconv.Atoi(data[j][0]) + return block1 > block2 + }) + + data = append(data, []string{"-1", db.tree.bottom().rootHash().String()}) + return data +}