2022-10-28 16:23:49 +08:00
|
|
|
// Copyright 2022 The go-ethereum Authors
|
|
|
|
// This file is part of the go-ethereum library.
|
|
|
|
//
|
|
|
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
|
|
|
// it under the terms of the GNU Lesser General Public License as published by
|
|
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
|
|
// (at your option) any later version.
|
|
|
|
//
|
|
|
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
|
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
// GNU Lesser General Public License for more details.
|
|
|
|
//
|
|
|
|
// You should have received a copy of the GNU Lesser General Public License
|
|
|
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
|
|
package rawdb
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
2023-09-27 15:01:49 +08:00
|
|
|
"path/filepath"
|
2022-10-28 16:23:49 +08:00
|
|
|
|
|
|
|
"github.com/ethereum/go-ethereum/common"
|
|
|
|
"github.com/ethereum/go-ethereum/ethdb"
|
2023-09-27 15:01:49 +08:00
|
|
|
"github.com/ethereum/go-ethereum/log"
|
|
|
|
"github.com/ethereum/go-ethereum/metrics"
|
2022-10-28 16:23:49 +08:00
|
|
|
)
|
|
|
|
|
|
|
|
type tableSize struct {
|
|
|
|
name string
|
|
|
|
size common.StorageSize
|
|
|
|
}
|
|
|
|
|
|
|
|
// freezerInfo contains the basic information of the freezer.
|
|
|
|
type freezerInfo struct {
|
|
|
|
name string // The identifier of freezer
|
|
|
|
head uint64 // The number of last stored item in the freezer
|
|
|
|
tail uint64 // The number of first stored item in the freezer
|
|
|
|
sizes []tableSize // The storage size per table
|
|
|
|
}
|
|
|
|
|
|
|
|
// count returns the number of stored items in the freezer.
|
|
|
|
func (info *freezerInfo) count() uint64 {
|
|
|
|
return info.head - info.tail + 1
|
|
|
|
}
|
|
|
|
|
|
|
|
// size returns the storage size of the entire freezer.
|
|
|
|
func (info *freezerInfo) size() common.StorageSize {
|
|
|
|
var total common.StorageSize
|
|
|
|
for _, table := range info.sizes {
|
|
|
|
total += table.size
|
|
|
|
}
|
|
|
|
return total
|
|
|
|
}
|
|
|
|
|
all: activate pbss as experimental feature from eth (#26274)
* all: activate pbss
* core/rawdb: fix compilation error
* cma, core, eth, les, trie: address comments
* cmd, core, eth, trie: polish code
* core, cmd, eth: address comments
* cmd, core, eth, les, light, tests: address comment
* cmd/utils: shorten log message
* trie/triedb/pathdb: limit node buffer size to 1gb
* cmd/utils: fix opening non-existing db
* cmd/utils: rename flag name
* cmd, core: group chain history flags and fix tests
* core, eth, trie: fix memory leak in snapshot generation
* cmd, eth, internal: deprecate flags
* all: enable state tests for pathdb, fixes
* cmd, core: polish code
* trie/triedb/pathdb: limit the node buffer size to 256mb
---------
Co-authored-by: Martin Holst Swende <martin@swende.se>
Co-authored-by: Péter Szilágyi <peterke@gmail.com>
2023-08-11 03:21:36 +08:00
|
|
|
func inspect(name string, order map[string]bool, reader ethdb.AncientReader) (freezerInfo, error) {
|
|
|
|
info := freezerInfo{name: name}
|
|
|
|
for t := range order {
|
|
|
|
size, err := reader.AncientSize(t)
|
|
|
|
if err != nil {
|
|
|
|
return freezerInfo{}, err
|
|
|
|
}
|
|
|
|
info.sizes = append(info.sizes, tableSize{name: t, size: common.StorageSize(size)})
|
|
|
|
}
|
|
|
|
// Retrieve the number of last stored item
|
|
|
|
ancients, err := reader.Ancients()
|
|
|
|
if err != nil {
|
|
|
|
return freezerInfo{}, err
|
|
|
|
}
|
|
|
|
info.head = ancients - 1
|
|
|
|
|
|
|
|
// Retrieve the number of first stored item
|
|
|
|
tail, err := reader.Tail()
|
|
|
|
if err != nil {
|
|
|
|
return freezerInfo{}, err
|
|
|
|
}
|
|
|
|
info.tail = tail
|
|
|
|
return info, nil
|
|
|
|
}
|
|
|
|
|
2022-10-28 16:23:49 +08:00
|
|
|
// inspectFreezers inspects all freezers registered in the system.
|
|
|
|
func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
|
|
|
|
var infos []freezerInfo
|
|
|
|
for _, freezer := range freezers {
|
|
|
|
switch freezer {
|
core, cmd, trie: fix the condition of pathdb initialization (#28718)
Original problem was caused by #28595, where we made it so that as soon as we start to sync, the root of the disk layer is deleted. That is not wrong per se, but another part of the code uses the "presence of the root" as an init-check for the pathdb. And, since the init-check now failed, the code tried to re-initialize it which failed since a sync was already ongoing.
The total impact being: after a state-sync has begun, if the node for some reason is is shut down, it will refuse to start up again, with the error message: `Fatal: Failed to register the Ethereum service: waiting for sync.`.
This change also modifies how `geth removedb` works, so that the user is prompted for two things: `state data` and `ancient chain`. The former includes both the chaindb aswell as any state history stored in ancients.
---------
Co-authored-by: Martin HS <martin@swende.se>
2023-12-22 03:28:32 +08:00
|
|
|
case ChainFreezerName:
|
|
|
|
info, err := inspect(ChainFreezerName, chainFreezerNoSnappy, db)
|
all: activate pbss as experimental feature from eth (#26274)
* all: activate pbss
* core/rawdb: fix compilation error
* cma, core, eth, les, trie: address comments
* cmd, core, eth, trie: polish code
* core, cmd, eth: address comments
* cmd, core, eth, les, light, tests: address comment
* cmd/utils: shorten log message
* trie/triedb/pathdb: limit node buffer size to 1gb
* cmd/utils: fix opening non-existing db
* cmd/utils: rename flag name
* cmd, core: group chain history flags and fix tests
* core, eth, trie: fix memory leak in snapshot generation
* cmd, eth, internal: deprecate flags
* all: enable state tests for pathdb, fixes
* cmd, core: polish code
* trie/triedb/pathdb: limit the node buffer size to 256mb
---------
Co-authored-by: Martin Holst Swende <martin@swende.se>
Co-authored-by: Péter Szilágyi <peterke@gmail.com>
2023-08-11 03:21:36 +08:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
infos = append(infos, info)
|
|
|
|
|
core, cmd, trie: fix the condition of pathdb initialization (#28718)
Original problem was caused by #28595, where we made it so that as soon as we start to sync, the root of the disk layer is deleted. That is not wrong per se, but another part of the code uses the "presence of the root" as an init-check for the pathdb. And, since the init-check now failed, the code tried to re-initialize it which failed since a sync was already ongoing.
The total impact being: after a state-sync has begun, if the node for some reason is is shut down, it will refuse to start up again, with the error message: `Fatal: Failed to register the Ethereum service: waiting for sync.`.
This change also modifies how `geth removedb` works, so that the user is prompted for two things: `state data` and `ancient chain`. The former includes both the chaindb aswell as any state history stored in ancients.
---------
Co-authored-by: Martin HS <martin@swende.se>
2023-12-22 03:28:32 +08:00
|
|
|
case StateFreezerName:
|
2024-03-08 16:07:29 +08:00
|
|
|
if ReadStateScheme(db) != PathScheme || db.StateStore() != nil {
|
2023-09-24 12:21:00 +08:00
|
|
|
continue
|
|
|
|
}
|
all: activate pbss as experimental feature from eth (#26274)
* all: activate pbss
* core/rawdb: fix compilation error
* cma, core, eth, les, trie: address comments
* cmd, core, eth, trie: polish code
* core, cmd, eth: address comments
* cmd, core, eth, les, light, tests: address comment
* cmd/utils: shorten log message
* trie/triedb/pathdb: limit node buffer size to 1gb
* cmd/utils: fix opening non-existing db
* cmd/utils: rename flag name
* cmd, core: group chain history flags and fix tests
* core, eth, trie: fix memory leak in snapshot generation
* cmd, eth, internal: deprecate flags
* all: enable state tests for pathdb, fixes
* cmd, core: polish code
* trie/triedb/pathdb: limit the node buffer size to 256mb
---------
Co-authored-by: Martin Holst Swende <martin@swende.se>
Co-authored-by: Péter Szilágyi <peterke@gmail.com>
2023-08-11 03:21:36 +08:00
|
|
|
datadir, err := db.AncientDatadir()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2022-10-28 16:23:49 +08:00
|
|
|
}
|
all: activate pbss as experimental feature from eth (#26274)
* all: activate pbss
* core/rawdb: fix compilation error
* cma, core, eth, les, trie: address comments
* cmd, core, eth, trie: polish code
* core, cmd, eth: address comments
* cmd, core, eth, les, light, tests: address comment
* cmd/utils: shorten log message
* trie/triedb/pathdb: limit node buffer size to 1gb
* cmd/utils: fix opening non-existing db
* cmd/utils: rename flag name
* cmd, core: group chain history flags and fix tests
* core, eth, trie: fix memory leak in snapshot generation
* cmd, eth, internal: deprecate flags
* all: enable state tests for pathdb, fixes
* cmd, core: polish code
* trie/triedb/pathdb: limit the node buffer size to 256mb
---------
Co-authored-by: Martin Holst Swende <martin@swende.se>
Co-authored-by: Péter Szilágyi <peterke@gmail.com>
2023-08-11 03:21:36 +08:00
|
|
|
f, err := NewStateFreezer(datadir, true, 0)
|
2022-10-28 16:23:49 +08:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
all: activate pbss as experimental feature from eth (#26274)
* all: activate pbss
* core/rawdb: fix compilation error
* cma, core, eth, les, trie: address comments
* cmd, core, eth, trie: polish code
* core, cmd, eth: address comments
* cmd, core, eth, les, light, tests: address comment
* cmd/utils: shorten log message
* trie/triedb/pathdb: limit node buffer size to 1gb
* cmd/utils: fix opening non-existing db
* cmd/utils: rename flag name
* cmd, core: group chain history flags and fix tests
* core, eth, trie: fix memory leak in snapshot generation
* cmd, eth, internal: deprecate flags
* all: enable state tests for pathdb, fixes
* cmd, core: polish code
* trie/triedb/pathdb: limit the node buffer size to 256mb
---------
Co-authored-by: Martin Holst Swende <martin@swende.se>
Co-authored-by: Péter Szilágyi <peterke@gmail.com>
2023-08-11 03:21:36 +08:00
|
|
|
defer f.Close()
|
2022-10-28 16:23:49 +08:00
|
|
|
|
core, cmd, trie: fix the condition of pathdb initialization (#28718)
Original problem was caused by #28595, where we made it so that as soon as we start to sync, the root of the disk layer is deleted. That is not wrong per se, but another part of the code uses the "presence of the root" as an init-check for the pathdb. And, since the init-check now failed, the code tried to re-initialize it which failed since a sync was already ongoing.
The total impact being: after a state-sync has begun, if the node for some reason is is shut down, it will refuse to start up again, with the error message: `Fatal: Failed to register the Ethereum service: waiting for sync.`.
This change also modifies how `geth removedb` works, so that the user is prompted for two things: `state data` and `ancient chain`. The former includes both the chaindb aswell as any state history stored in ancients.
---------
Co-authored-by: Martin HS <martin@swende.se>
2023-12-22 03:28:32 +08:00
|
|
|
info, err := inspect(StateFreezerName, stateFreezerNoSnappy, f)
|
2022-10-28 16:23:49 +08:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
infos = append(infos, info)
|
|
|
|
|
|
|
|
default:
|
|
|
|
return nil, fmt.Errorf("unknown freezer, supported ones: %v", freezers)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return infos, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// InspectFreezerTable dumps out the index of a specific freezer table. The passed
|
|
|
|
// ancient indicates the path of root ancient directory where the chain freezer can
|
|
|
|
// be opened. Start and end specify the range for dumping out indexes.
|
|
|
|
// Note this function can only be used for debugging purposes.
|
2024-04-18 15:12:05 +08:00
|
|
|
func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64, multiDatabase bool) error {
|
2022-10-28 16:23:49 +08:00
|
|
|
var (
|
|
|
|
path string
|
|
|
|
tables map[string]bool
|
|
|
|
)
|
|
|
|
switch freezerName {
|
core, cmd, trie: fix the condition of pathdb initialization (#28718)
Original problem was caused by #28595, where we made it so that as soon as we start to sync, the root of the disk layer is deleted. That is not wrong per se, but another part of the code uses the "presence of the root" as an init-check for the pathdb. And, since the init-check now failed, the code tried to re-initialize it which failed since a sync was already ongoing.
The total impact being: after a state-sync has begun, if the node for some reason is is shut down, it will refuse to start up again, with the error message: `Fatal: Failed to register the Ethereum service: waiting for sync.`.
This change also modifies how `geth removedb` works, so that the user is prompted for two things: `state data` and `ancient chain`. The former includes both the chaindb aswell as any state history stored in ancients.
---------
Co-authored-by: Martin HS <martin@swende.se>
2023-12-22 03:28:32 +08:00
|
|
|
case ChainFreezerName:
|
2024-04-18 15:12:05 +08:00
|
|
|
if multiDatabase {
|
|
|
|
path, tables = resolveChainFreezerDir(filepath.Dir(ancient)+"/block/ancient"), chainFreezerNoSnappy
|
|
|
|
} else {
|
|
|
|
path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
|
|
|
|
}
|
|
|
|
|
core, cmd, trie: fix the condition of pathdb initialization (#28718)
Original problem was caused by #28595, where we made it so that as soon as we start to sync, the root of the disk layer is deleted. That is not wrong per se, but another part of the code uses the "presence of the root" as an init-check for the pathdb. And, since the init-check now failed, the code tried to re-initialize it which failed since a sync was already ongoing.
The total impact being: after a state-sync has begun, if the node for some reason is is shut down, it will refuse to start up again, with the error message: `Fatal: Failed to register the Ethereum service: waiting for sync.`.
This change also modifies how `geth removedb` works, so that the user is prompted for two things: `state data` and `ancient chain`. The former includes both the chaindb aswell as any state history stored in ancients.
---------
Co-authored-by: Martin HS <martin@swende.se>
2023-12-22 03:28:32 +08:00
|
|
|
case StateFreezerName:
|
2024-04-18 15:12:05 +08:00
|
|
|
if multiDatabase {
|
|
|
|
path, tables = filepath.Join(filepath.Dir(ancient)+"/state/ancient", freezerName), stateFreezerNoSnappy
|
|
|
|
} else {
|
|
|
|
path, tables = filepath.Join(ancient, freezerName), stateFreezerNoSnappy
|
|
|
|
}
|
2022-10-28 16:23:49 +08:00
|
|
|
default:
|
|
|
|
return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
|
|
|
|
}
|
|
|
|
noSnappy, exist := tables[tableName]
|
|
|
|
if !exist {
|
|
|
|
var names []string
|
|
|
|
for name := range tables {
|
|
|
|
names = append(names, name)
|
|
|
|
}
|
|
|
|
return fmt.Errorf("unknown table, supported ones: %v", names)
|
|
|
|
}
|
|
|
|
table, err := newFreezerTable(path, tableName, noSnappy, true)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
table.dumpIndexStdout(start, end)
|
|
|
|
return nil
|
|
|
|
}
|
2023-09-27 15:01:49 +08:00
|
|
|
|
|
|
|
func ResetStateFreezerTableOffset(ancient string, virtualTail uint64) error {
|
2024-02-02 15:43:33 +08:00
|
|
|
path, tables := filepath.Join(ancient, StateFreezerName), stateFreezerNoSnappy
|
2023-09-27 15:01:49 +08:00
|
|
|
|
|
|
|
for name, disableSnappy := range tables {
|
|
|
|
log.Info("Handle table", "name", name, "disableSnappy", disableSnappy)
|
|
|
|
table, err := newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy, false)
|
|
|
|
if err != nil {
|
|
|
|
log.Error("New table failed", "error", err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
// Reset the metadata of the freezer table
|
|
|
|
err = table.ResetItemsOffset(virtualTail)
|
|
|
|
if err != nil {
|
|
|
|
log.Error("Reset items offset of the table", "name", name, "error", err)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|