internal/build: parse buildbot tag better

core, eth, les, rpc: polish catalyst errors, add context

.github/CODEOWNERS

@@ -10,11 +10,12 @@ consensus                       @karalabe
 core/                           @karalabe @holiman @rjl493456442
 eth/                            @karalabe @holiman @rjl493456442
 eth/catalyst/                   @gballet
-graphql/                        @gballet
+eth/tracers/                    @s1na
+graphql/                        @gballet @s1na
 les/                            @zsfelfoldi @rjl493456442
 light/                          @zsfelfoldi @rjl493456442
 mobile/                         @karalabe @ligi
-node/                           @fjl @renaynay
+node/                           @fjl
 p2p/                            @fjl @zsfelfoldi
 rpc/                            @fjl @holiman
 p2p/simulations                 @fjl

.travis.yml

@@ -16,7 +16,7 @@ jobs:
     - stage: lint
       os: linux
       dist: bionic
-      go: 1.17.x
+      go: 1.18.x
       env:
         - lint
       git:
@@ -31,7 +31,7 @@ jobs:
       os: linux
       arch: amd64
       dist: bionic
-      go: 1.17.x
+      go: 1.18.x
       env:
         - docker
       services:
@@ -48,7 +48,7 @@ jobs:
       os: linux
       arch: arm64
       dist: bionic
-      go: 1.17.x
+      go: 1.18.x
       env:
         - docker
       services:
@@ -65,7 +65,7 @@ jobs:
       if: type = push
       os: linux
       dist: bionic
-      go: 1.17.x
+      go: 1.18.x
       env:
         - ubuntu-ppa
         - GO111MODULE=on
@@ -90,7 +90,7 @@ jobs:
       os: linux
       dist: bionic
       sudo: required
-      go: 1.17.x
+      go: 1.18.x
       env:
         - azure-linux
         - GO111MODULE=on
@@ -148,7 +148,7 @@ jobs:
         - sdkmanager "platform-tools" "platforms;android-15" "platforms;android-19" "platforms;android-24" "ndk-bundle"
 
         # Install Go to allow building with
-        - curl https://dl.google.com/go/go1.16.linux-amd64.tar.gz | tar -xz
+        - curl https://dl.google.com/go/go1.18.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go
@@ -162,7 +162,7 @@ jobs:
     - stage: build
       if: type = push
       os: osx
-      go: 1.17.x
+      go: 1.18.x
       env:
         - azure-osx
         - azure-ios
@@ -194,7 +194,7 @@ jobs:
       os: linux
       arch: amd64
       dist: bionic
-      go: 1.17.x
+      go: 1.18.x
       env:
         - GO111MODULE=on
       script:
@@ -205,7 +205,7 @@ jobs:
       os: linux
       arch: arm64
       dist: bionic
-      go: 1.17.x
+      go: 1.18.x
       env:
         - GO111MODULE=on
       script:
@@ -214,7 +214,7 @@ jobs:
     - stage: build
       os: linux
       dist: bionic
-      go: 1.16.x
+      go: 1.17.x
       env:
         - GO111MODULE=on
       script:
@@ -225,7 +225,7 @@ jobs:
       if: type = cron
       os: linux
       dist: bionic
-      go: 1.17.x
+      go: 1.18.x
       env:
         - azure-purge
         - GO111MODULE=on
@@ -239,7 +239,7 @@ jobs:
       if: type = cron
       os: linux
       dist: bionic
-      go: 1.17.x
+      go: 1.18.x
       env:
         - GO111MODULE=on
       script:

Dockerfile

@@ -4,10 +4,15 @@ ARG VERSION=""
 ARG BUILDNUM=""
 
 # Build Geth in a stock Go builder container
-FROM golang:1.17-alpine as builder
+FROM golang:1.18-alpine as builder
 
 RUN apk add --no-cache gcc musl-dev linux-headers git
 
+# Get dependencies - will also be cached if we won't change go.mod/go.sum
+COPY go.mod /go-ethereum/
+COPY go.sum /go-ethereum/
+RUN cd /go-ethereum && go mod download
+
 ADD . /go-ethereum
 RUN cd /go-ethereum && go run build/ci.go install ./cmd/geth
 

Dockerfile.alltools

@@ -4,10 +4,15 @@ ARG VERSION=""
 ARG BUILDNUM=""
 
 # Build Geth in a stock Go builder container
-FROM golang:1.17-alpine as builder
+FROM golang:1.18-alpine as builder
 
 RUN apk add --no-cache gcc musl-dev linux-headers git
 
+# Get dependencies - will also be cached if we won't change go.mod/go.sum
+COPY go.mod /go-ethereum/
+COPY go.sum /go-ethereum/
+RUN cd /go-ethereum && go mod download
+
 ADD . /go-ethereum
 RUN cd /go-ethereum && go run build/ci.go install
 

Makefile

@@ -43,7 +43,6 @@ clean:
 
 devtools:
 	env GOBIN= go install golang.org/x/tools/cmd/stringer@latest
-	env GOBIN= go install github.com/kevinburke/go-bindata/go-bindata@latest
 	env GOBIN= go install github.com/fjl/gencodec@latest
 	env GOBIN= go install github.com/golang/protobuf/protoc-gen-go@latest
 	env GOBIN= go install ./cmd/abigen

README.md

@@ -16,7 +16,7 @@ archives are published at https://geth.ethereum.org/downloads/.
 
 For prerequisites and detailed build instructions please read the [Installation Instructions](https://geth.ethereum.org/docs/install-and-build/installing-geth).
 
-Building `geth` requires both a Go (version 1.14 or later) and a C compiler. You can install
+Building `geth` requires both a Go (version 1.16 or later) and a C compiler. You can install
 them using your favourite package manager. Once the dependencies are installed, run
 
 ```shell
@@ -52,6 +52,22 @@ Going through all the possible command line flags is out of scope here (please c
 but we've enumerated a few common parameter combos to get you up to speed quickly
 on how you can run your own `geth` instance.
 
+### Hardware Requirements
+
+Minimum:
+
+* CPU with 2+ cores
+* 4GB RAM
+* 1TB free storage space to sync the Mainnet
+* 8 MBit/sec download Internet service
+
+Recommended:
+
+* Fast CPU with 4+ cores
+* 16GB+ RAM
+* High Performance SSD with at least 1TB free space
+* 25+ MBit/sec download Internet service
+
 ### Full node on the main Ethereum network
 
 By far the most common scenario is people wanting to simply interact with the Ethereum

accounts/abi/argument.go

@@ -78,7 +78,7 @@ func (arguments Arguments) isTuple() bool {
 // Unpack performs the operation hexdata -> Go format.
 func (arguments Arguments) Unpack(data []byte) ([]interface{}, error) {
 	if len(data) == 0 {
-		if len(arguments) != 0 {
+		if len(arguments.NonIndexed()) != 0 {
 			return nil, fmt.Errorf("abi: attempting to unmarshall an empty string while arguments are expected")
 		}
 		return make([]interface{}, 0), nil
@@ -93,7 +93,7 @@ func (arguments Arguments) UnpackIntoMap(v map[string]interface{}, data []byte)
 		return fmt.Errorf("abi: cannot unpack into a nil map")
 	}
 	if len(data) == 0 {
-		if len(arguments) != 0 {
+		if len(arguments.NonIndexed()) != 0 {
 			return fmt.Errorf("abi: attempting to unmarshall an empty string while arguments are expected")
 		}
 		return nil // Nothing to unmarshal, return
@@ -115,8 +115,8 @@ func (arguments Arguments) Copy(v interface{}, values []interface{}) error {
 		return fmt.Errorf("abi: Unpack(non-pointer %T)", v)
 	}
 	if len(values) == 0 {
-		if len(arguments) != 0 {
-			return fmt.Errorf("abi: attempting to copy no values while %d arguments are expected", len(arguments))
+		if len(arguments.NonIndexed()) != 0 {
+			return fmt.Errorf("abi: attempting to copy no values while arguments are expected")
 		}
 		return nil // Nothing to copy, return
 	}

accounts/abi/bind/auth.go

@@ -21,7 +21,6 @@ import (
 	"crypto/ecdsa"
 	"errors"
 	"io"
-	"io/ioutil"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/accounts"
@@ -45,7 +44,7 @@ var ErrNotAuthorized = errors.New("not authorized to sign this account")
 // Deprecated: Use NewTransactorWithChainID instead.
 func NewTransactor(keyin io.Reader, passphrase string) (*TransactOpts, error) {
 	log.Warn("WARNING: NewTransactor has been deprecated in favour of NewTransactorWithChainID")
-	json, err := ioutil.ReadAll(keyin)
+	json, err := io.ReadAll(keyin)
 	if err != nil {
 		return nil, err
 	}
@@ -106,7 +105,7 @@ func NewKeyedTransactor(key *ecdsa.PrivateKey) *TransactOpts {
 // NewTransactorWithChainID is a utility method to easily create a transaction signer from
 // an encrypted json key stream and the associated passphrase.
 func NewTransactorWithChainID(keyin io.Reader, passphrase string, chainID *big.Int) (*TransactOpts, error) {
-	json, err := ioutil.ReadAll(keyin)
+	json, err := io.ReadAll(keyin)
 	if err != nil {
 		return nil, err
 	}

accounts/abi/bind/base.go

@@ -171,7 +171,10 @@ func (c *BoundContract) Call(opts *CallOpts, results *[]interface{}, method stri
 			return ErrNoPendingState
 		}
 		output, err = pb.PendingCallContract(ctx, msg)
-		if err == nil && len(output) == 0 {
+		if err != nil {
+			return err
+		}
+		if len(output) == 0 {
 			// Make sure we have a contract to operate on, and bail out otherwise.
 			if code, err = pb.PendingCodeAt(ctx, c.address); err != nil {
 				return err

accounts/abi/bind/base_test.go

@@ -18,6 +18,7 @@ package bind_test
 
 import (
 	"context"
+	"errors"
 	"math/big"
 	"reflect"
 	"strings"
@@ -75,34 +76,51 @@ func (mt *mockTransactor) SendTransaction(ctx context.Context, tx *types.Transac
 }
 
 type mockCaller struct {
-	codeAtBlockNumber         *big.Int
-	callContractBlockNumber   *big.Int
-	pendingCodeAtCalled       bool
-	pendingCallContractCalled bool
+	codeAtBlockNumber       *big.Int
+	callContractBlockNumber *big.Int
+	callContractBytes       []byte
+	callContractErr         error
+	codeAtBytes             []byte
+	codeAtErr               error
 }
 
 func (mc *mockCaller) CodeAt(ctx context.Context, contract common.Address, blockNumber *big.Int) ([]byte, error) {
 	mc.codeAtBlockNumber = blockNumber
-	return []byte{1, 2, 3}, nil
+	return mc.codeAtBytes, mc.codeAtErr
 }
 
 func (mc *mockCaller) CallContract(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) {
 	mc.callContractBlockNumber = blockNumber
-	return nil, nil
+	return mc.callContractBytes, mc.callContractErr
 }
 
-func (mc *mockCaller) PendingCodeAt(ctx context.Context, contract common.Address) ([]byte, error) {
+type mockPendingCaller struct {
+	*mockCaller
+	pendingCodeAtBytes        []byte
+	pendingCodeAtErr          error
+	pendingCodeAtCalled       bool
+	pendingCallContractCalled bool
+	pendingCallContractBytes  []byte
+	pendingCallContractErr    error
+}
+
+func (mc *mockPendingCaller) PendingCodeAt(ctx context.Context, contract common.Address) ([]byte, error) {
 	mc.pendingCodeAtCalled = true
-	return nil, nil
+	return mc.pendingCodeAtBytes, mc.pendingCodeAtErr
 }
 
-func (mc *mockCaller) PendingCallContract(ctx context.Context, call ethereum.CallMsg) ([]byte, error) {
+func (mc *mockPendingCaller) PendingCallContract(ctx context.Context, call ethereum.CallMsg) ([]byte, error) {
 	mc.pendingCallContractCalled = true
-	return nil, nil
+	return mc.pendingCallContractBytes, mc.pendingCallContractErr
 }
+
 func TestPassingBlockNumber(t *testing.T) {
 
-	mc := &mockCaller{}
+	mc := &mockPendingCaller{
+		mockCaller: &mockCaller{
+			codeAtBytes: []byte{1, 2, 3},
+		},
+	}
 
 	bc := bind.NewBoundContract(common.HexToAddress("0x0"), abi.ABI{
 		Methods: map[string]abi.Method{
@@ -341,3 +359,132 @@ func newMockLog(topics []common.Hash, txHash common.Hash) types.Log {
 		Removed:     false,
 	}
 }
+
+func TestCall(t *testing.T) {
+	var method, methodWithArg = "something", "somethingArrrrg"
+	tests := []struct {
+		name, method string
+		opts         *bind.CallOpts
+		mc           bind.ContractCaller
+		results      *[]interface{}
+		wantErr      bool
+		wantErrExact error
+	}{{
+		name: "ok not pending",
+		mc: &mockCaller{
+			codeAtBytes: []byte{0},
+		},
+		method: method,
+	}, {
+		name: "ok pending",
+		mc: &mockPendingCaller{
+			pendingCodeAtBytes: []byte{0},
+		},
+		opts: &bind.CallOpts{
+			Pending: true,
+		},
+		method: method,
+	}, {
+		name:    "pack error, no method",
+		mc:      new(mockCaller),
+		method:  "else",
+		wantErr: true,
+	}, {
+		name: "interface error, pending but not a PendingContractCaller",
+		mc:   new(mockCaller),
+		opts: &bind.CallOpts{
+			Pending: true,
+		},
+		method:       method,
+		wantErrExact: bind.ErrNoPendingState,
+	}, {
+		name: "pending call canceled",
+		mc: &mockPendingCaller{
+			pendingCallContractErr: context.DeadlineExceeded,
+		},
+		opts: &bind.CallOpts{
+			Pending: true,
+		},
+		method:       method,
+		wantErrExact: context.DeadlineExceeded,
+	}, {
+		name: "pending code at error",
+		mc: &mockPendingCaller{
+			pendingCodeAtErr: errors.New(""),
+		},
+		opts: &bind.CallOpts{
+			Pending: true,
+		},
+		method:  method,
+		wantErr: true,
+	}, {
+		name: "no pending code at",
+		mc:   new(mockPendingCaller),
+		opts: &bind.CallOpts{
+			Pending: true,
+		},
+		method:       method,
+		wantErrExact: bind.ErrNoCode,
+	}, {
+		name: "call contract error",
+		mc: &mockCaller{
+			callContractErr: context.DeadlineExceeded,
+		},
+		method:       method,
+		wantErrExact: context.DeadlineExceeded,
+	}, {
+		name: "code at error",
+		mc: &mockCaller{
+			codeAtErr: errors.New(""),
+		},
+		method:  method,
+		wantErr: true,
+	}, {
+		name:         "no code at",
+		mc:           new(mockCaller),
+		method:       method,
+		wantErrExact: bind.ErrNoCode,
+	}, {
+		name: "unpack error missing arg",
+		mc: &mockCaller{
+			codeAtBytes: []byte{0},
+		},
+		method:  methodWithArg,
+		wantErr: true,
+	}, {
+		name: "interface unpack error",
+		mc: &mockCaller{
+			codeAtBytes: []byte{0},
+		},
+		method:  method,
+		results: &[]interface{}{0},
+		wantErr: true,
+	}}
+	for _, test := range tests {
+		bc := bind.NewBoundContract(common.HexToAddress("0x0"), abi.ABI{
+			Methods: map[string]abi.Method{
+				method: {
+					Name:    method,
+					Outputs: abi.Arguments{},
+				},
+				methodWithArg: {
+					Name:    methodWithArg,
+					Outputs: abi.Arguments{abi.Argument{}},
+				},
+			},
+		}, test.mc, nil, nil)
+		err := bc.Call(test.opts, test.results, test.method)
+		if test.wantErr || test.wantErrExact != nil {
+			if err == nil {
+				t.Fatalf("%q expected error", test.name)
+			}
+			if test.wantErrExact != nil && !errors.Is(err, test.wantErrExact) {
+				t.Fatalf("%q expected error %q but got %q", test.name, test.wantErrExact, err)
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%q unexpected error: %v", test.name, err)
+		}
+	}
+}

accounts/abi/bind/bind.go

@@ -179,7 +179,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string]
 
 		contracts[types[i]] = &tmplContract{
 			Type:        capitalise(types[i]),
-			InputABI:    strings.Replace(strippedABI, "\"", "\\\"", -1),
+			InputABI:    strings.ReplaceAll(strippedABI, "\"", "\\\""),
 			InputBin:    strings.TrimPrefix(strings.TrimSpace(bytecodes[i]), "0x"),
 			Constructor: evmABI.Constructor,
 			Calls:       calls,

accounts/abi/bind/bind_test.go

@@ -18,7 +18,6 @@ package bind
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -1966,14 +1965,10 @@ func TestGolangBindings(t *testing.T) {
 		t.Skip("go sdk not found for testing")
 	}
 	// Create a temporary workspace for the test suite
-	ws, err := ioutil.TempDir("", "binding-test")
-	if err != nil {
-		t.Fatalf("failed to create temporary workspace: %v", err)
-	}
-	//defer os.RemoveAll(ws)
+	ws := t.TempDir()
 
 	pkg := filepath.Join(ws, "bindtest")
-	if err = os.MkdirAll(pkg, 0700); err != nil {
+	if err := os.MkdirAll(pkg, 0700); err != nil {
 		t.Fatalf("failed to create package: %v", err)
 	}
 	// Generate the test suite for all the contracts
@@ -1990,7 +1985,7 @@ func TestGolangBindings(t *testing.T) {
 			if err != nil {
 				t.Fatalf("test %d: failed to generate binding: %v", i, err)
 			}
-			if err = ioutil.WriteFile(filepath.Join(pkg, strings.ToLower(tt.name)+".go"), []byte(bind), 0600); err != nil {
+			if err = os.WriteFile(filepath.Join(pkg, strings.ToLower(tt.name)+".go"), []byte(bind), 0600); err != nil {
 				t.Fatalf("test %d: failed to write binding: %v", i, err)
 			}
 			// Generate the test file with the injected test code
@@ -2006,7 +2001,7 @@ func TestGolangBindings(t *testing.T) {
 				%s
 			}
 		`, tt.imports, tt.name, tt.tester)
-			if err := ioutil.WriteFile(filepath.Join(pkg, strings.ToLower(tt.name)+"_test.go"), []byte(code), 0600); err != nil {
+			if err := os.WriteFile(filepath.Join(pkg, strings.ToLower(tt.name)+"_test.go"), []byte(code), 0600); err != nil {
 				t.Fatalf("test %d: failed to write tests: %v", i, err)
 			}
 		})

accounts/abi/bind/template.go

@@ -161,7 +161,7 @@ var (
 		  }
 		  {{range $pattern, $name := .Libraries}}
 			{{decapitalise $name}}Addr, _, _, _ := Deploy{{capitalise $name}}(auth, backend)
-			{{$contract.Type}}Bin = strings.Replace({{$contract.Type}}Bin, "__${{$pattern}}$__", {{decapitalise $name}}Addr.String()[2:], -1)
+			{{$contract.Type}}Bin = strings.ReplaceAll({{$contract.Type}}Bin, "__${{$pattern}}$__", {{decapitalise $name}}Addr.String()[2:])
 		  {{end}}
 		  address, tx, contract, err := bind.DeployContract(auth, *parsed, common.FromHex({{.Type}}Bin), backend {{range .Constructor.Inputs}}, {{.Name}}{{end}})
 		  if err != nil {

accounts/abi/selector_parser.go

@@ -0,0 +1,160 @@
+package abi
+
+import (
+	"fmt"
+)
+
+type SelectorMarshaling struct {
+	Name   string               `json:"name"`
+	Type   string               `json:"type"`
+	Inputs []ArgumentMarshaling `json:"inputs"`
+}
+
+func isDigit(c byte) bool {
+	return c >= '0' && c <= '9'
+}
+
+func isAlpha(c byte) bool {
+	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
+}
+
+func isIdentifierSymbol(c byte) bool {
+	return c == '$' || c == '_'
+}
+
+func parseToken(unescapedSelector string, isIdent bool) (string, string, error) {
+	if len(unescapedSelector) == 0 {
+		return "", "", fmt.Errorf("empty token")
+	}
+	firstChar := unescapedSelector[0]
+	position := 1
+	if !(isAlpha(firstChar) || (isIdent && isIdentifierSymbol(firstChar))) {
+		return "", "", fmt.Errorf("invalid token start: %c", firstChar)
+	}
+	for position < len(unescapedSelector) {
+		char := unescapedSelector[position]
+		if !(isAlpha(char) || isDigit(char) || (isIdent && isIdentifierSymbol(char))) {
+			break
+		}
+		position++
+	}
+	return unescapedSelector[:position], unescapedSelector[position:], nil
+}
+
+func parseIdentifier(unescapedSelector string) (string, string, error) {
+	return parseToken(unescapedSelector, true)
+}
+
+func parseElementaryType(unescapedSelector string) (string, string, error) {
+	parsedType, rest, err := parseToken(unescapedSelector, false)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to parse elementary type: %v", err)
+	}
+	// handle arrays
+	for len(rest) > 0 && rest[0] == '[' {
+		parsedType = parsedType + string(rest[0])
+		rest = rest[1:]
+		for len(rest) > 0 && isDigit(rest[0]) {
+			parsedType = parsedType + string(rest[0])
+			rest = rest[1:]
+		}
+		if len(rest) == 0 || rest[0] != ']' {
+			return "", "", fmt.Errorf("failed to parse array: expected ']', got %c", unescapedSelector[0])
+		}
+		parsedType = parsedType + string(rest[0])
+		rest = rest[1:]
+	}
+	return parsedType, rest, nil
+}
+
+func parseCompositeType(unescapedSelector string) ([]interface{}, string, error) {
+	if len(unescapedSelector) == 0 || unescapedSelector[0] != '(' {
+		return nil, "", fmt.Errorf("expected '(', got %c", unescapedSelector[0])
+	}
+	parsedType, rest, err := parseType(unescapedSelector[1:])
+	if err != nil {
+		return nil, "", fmt.Errorf("failed to parse type: %v", err)
+	}
+	result := []interface{}{parsedType}
+	for len(rest) > 0 && rest[0] != ')' {
+		parsedType, rest, err = parseType(rest[1:])
+		if err != nil {
+			return nil, "", fmt.Errorf("failed to parse type: %v", err)
+		}
+		result = append(result, parsedType)
+	}
+	if len(rest) == 0 || rest[0] != ')' {
+		return nil, "", fmt.Errorf("expected ')', got '%s'", rest)
+	}
+	if len(rest) >= 3 && rest[1] == '[' && rest[2] == ']' {
+		return append(result, "[]"), rest[3:], nil
+	}
+	return result, rest[1:], nil
+}
+
+func parseType(unescapedSelector string) (interface{}, string, error) {
+	if len(unescapedSelector) == 0 {
+		return nil, "", fmt.Errorf("empty type")
+	}
+	if unescapedSelector[0] == '(' {
+		return parseCompositeType(unescapedSelector)
+	} else {
+		return parseElementaryType(unescapedSelector)
+	}
+}
+
+func assembleArgs(args []interface{}) ([]ArgumentMarshaling, error) {
+	arguments := make([]ArgumentMarshaling, 0)
+	for i, arg := range args {
+		// generate dummy name to avoid unmarshal issues
+		name := fmt.Sprintf("name%d", i)
+		if s, ok := arg.(string); ok {
+			arguments = append(arguments, ArgumentMarshaling{name, s, s, nil, false})
+		} else if components, ok := arg.([]interface{}); ok {
+			subArgs, err := assembleArgs(components)
+			if err != nil {
+				return nil, fmt.Errorf("failed to assemble components: %v", err)
+			}
+			tupleType := "tuple"
+			if len(subArgs) != 0 && subArgs[len(subArgs)-1].Type == "[]" {
+				subArgs = subArgs[:len(subArgs)-1]
+				tupleType = "tuple[]"
+			}
+			arguments = append(arguments, ArgumentMarshaling{name, tupleType, tupleType, subArgs, false})
+		} else {
+			return nil, fmt.Errorf("failed to assemble args: unexpected type %T", arg)
+		}
+	}
+	return arguments, nil
+}
+
+// ParseSelector converts a method selector into a struct that can be JSON encoded
+// and consumed by other functions in this package.
+// Note, although uppercase letters are not part of the ABI spec, this function
+// still accepts it as the general format is valid.
+func ParseSelector(unescapedSelector string) (SelectorMarshaling, error) {
+	name, rest, err := parseIdentifier(unescapedSelector)
+	if err != nil {
+		return SelectorMarshaling{}, fmt.Errorf("failed to parse selector '%s': %v", unescapedSelector, err)
+	}
+	args := []interface{}{}
+	if len(rest) >= 2 && rest[0] == '(' && rest[1] == ')' {
+		rest = rest[2:]
+	} else {
+		args, rest, err = parseCompositeType(rest)
+		if err != nil {
+			return SelectorMarshaling{}, fmt.Errorf("failed to parse selector '%s': %v", unescapedSelector, err)
+		}
+	}
+	if len(rest) > 0 {
+		return SelectorMarshaling{}, fmt.Errorf("failed to parse selector '%s': unexpected string '%s'", unescapedSelector, rest)
+	}
+
+	// Reassemble the fake ABI and constuct the JSON
+	fakeArgs, err := assembleArgs(args)
+	if err != nil {
+		return SelectorMarshaling{}, fmt.Errorf("failed to parse selector: %v", err)
+	}
+
+	return SelectorMarshaling{name, "function", fakeArgs}, nil
+}

accounts/abi/selector_parser_test.go

@@ -0,0 +1,63 @@
+package abi
+
+import (
+	"fmt"
+	"log"
+	"reflect"
+	"testing"
+)
+
+func TestParseSelector(t *testing.T) {
+	mkType := func(types ...interface{}) []ArgumentMarshaling {
+		var result []ArgumentMarshaling
+		for i, typeOrComponents := range types {
+			name := fmt.Sprintf("name%d", i)
+			if typeName, ok := typeOrComponents.(string); ok {
+				result = append(result, ArgumentMarshaling{name, typeName, typeName, nil, false})
+			} else if components, ok := typeOrComponents.([]ArgumentMarshaling); ok {
+				result = append(result, ArgumentMarshaling{name, "tuple", "tuple", components, false})
+			} else if components, ok := typeOrComponents.([][]ArgumentMarshaling); ok {
+				result = append(result, ArgumentMarshaling{name, "tuple[]", "tuple[]", components[0], false})
+			} else {
+				log.Fatalf("unexpected type %T", typeOrComponents)
+			}
+		}
+		return result
+	}
+	tests := []struct {
+		input string
+		name  string
+		args  []ArgumentMarshaling
+	}{
+		{"noargs()", "noargs", []ArgumentMarshaling{}},
+		{"simple(uint256,uint256,uint256)", "simple", mkType("uint256", "uint256", "uint256")},
+		{"other(uint256,address)", "other", mkType("uint256", "address")},
+		{"withArray(uint256[],address[2],uint8[4][][5])", "withArray", mkType("uint256[]", "address[2]", "uint8[4][][5]")},
+		{"singleNest(bytes32,uint8,(uint256,uint256),address)", "singleNest", mkType("bytes32", "uint8", mkType("uint256", "uint256"), "address")},
+		{"multiNest(address,(uint256[],uint256),((address,bytes32),uint256))", "multiNest",
+			mkType("address", mkType("uint256[]", "uint256"), mkType(mkType("address", "bytes32"), "uint256"))},
+		{"arrayNest((uint256,uint256)[],bytes32)", "arrayNest", mkType([][]ArgumentMarshaling{mkType("uint256", "uint256")}, "bytes32")},
+		{"multiArrayNest((uint256,uint256)[],(uint256,uint256)[])", "multiArrayNest",
+			mkType([][]ArgumentMarshaling{mkType("uint256", "uint256")}, [][]ArgumentMarshaling{mkType("uint256", "uint256")})},
+		{"singleArrayNestAndArray((uint256,uint256)[],bytes32[])", "singleArrayNestAndArray",
+			mkType([][]ArgumentMarshaling{mkType("uint256", "uint256")}, "bytes32[]")},
+		{"singleArrayNestWithArrayAndArray((uint256[],address[2],uint8[4][][5])[],bytes32[])", "singleArrayNestWithArrayAndArray",
+			mkType([][]ArgumentMarshaling{mkType("uint256[]", "address[2]", "uint8[4][][5]")}, "bytes32[]")},
+	}
+	for i, tt := range tests {
+		selector, err := ParseSelector(tt.input)
+		if err != nil {
+			t.Errorf("test %d: failed to parse selector '%v': %v", i, tt.input, err)
+		}
+		if selector.Name != tt.name {
+			t.Errorf("test %d: unexpected function name: '%s' != '%s'", i, selector.Name, tt.name)
+		}
+
+		if selector.Type != "function" {
+			t.Errorf("test %d: unexpected type: '%s' != '%s'", i, selector.Type, "function")
+		}
+		if !reflect.DeepEqual(selector.Inputs, tt.args) {
+			t.Errorf("test %d: unexpected args: '%v' != '%v'", i, selector.Inputs, tt.args)
+		}
+	}
+}

accounts/abi/type.go

@@ -201,7 +201,7 @@ func NewType(t string, internalType string, components []ArgumentMarshaling) (ty
 		if internalType != "" && strings.HasPrefix(internalType, structPrefix) {
 			// Foo.Bar type definition is not allowed in golang,
 			// convert the format to FooBar
-			typ.TupleRawName = strings.Replace(internalType[len(structPrefix):], ".", "", -1)
+			typ.TupleRawName = strings.ReplaceAll(internalType[len(structPrefix):], ".", "")
 		}
 
 	case "function":

accounts/abi/unpack_test.go

@@ -201,6 +201,23 @@ var unpackTests = []unpackTest{
 			IntOne *big.Int
 		}{big.NewInt(1)},
 	},
+	{
+		def:  `[{"type":"bool"}]`,
+		enc:  "",
+		want: false,
+		err:  "abi: attempting to unmarshall an empty string while arguments are expected",
+	},
+	{
+		def:  `[{"type":"bytes32","indexed":true},{"type":"uint256","indexed":false}]`,
+		enc:  "",
+		want: false,
+		err:  "abi: attempting to unmarshall an empty string while arguments are expected",
+	},
+	{
+		def:  `[{"type":"bool","indexed":true},{"type":"uint64","indexed":true}]`,
+		enc:  "",
+		want: false,
+	},
 }
 
 // TestLocalUnpackTests runs test specially designed only for unpacking.

accounts/errors.go

@@ -41,8 +41,7 @@ var ErrInvalidPassphrase = errors.New("invalid password")
 // second time.
 var ErrWalletAlreadyOpen = errors.New("wallet already open")
 
-// ErrWalletClosed is returned if a wallet is attempted to be opened the
-// second time.
+// ErrWalletClosed is returned if a wallet is offline.
 var ErrWalletClosed = errors.New("wallet closed")
 
 // AuthNeededError is returned by backends for signing requests where the user

accounts/hd.go

@@ -41,7 +41,7 @@ var DefaultBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60,
 var LegacyLedgerBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}
 
 // DerivationPath represents the computer friendly version of a hierarchical
-// deterministic wallet account derivaion path.
+// deterministic wallet account derivation path.
 //
 // The BIP-32 spec https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki
 // defines derivation paths to be of the form:

accounts/keystore/account_cache_test.go

@@ -18,7 +18,6 @@ package keystore
 
 import (
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -55,7 +54,6 @@ func TestWatchNewFile(t *testing.T) {
 	t.Parallel()
 
 	dir, ks := tmpKeyStore(t, false)
-	defer os.RemoveAll(dir)
 
 	// Ensure the watcher is started before adding any files.
 	ks.Accounts()
@@ -381,11 +379,11 @@ func TestUpdatedKeyfileContents(t *testing.T) {
 		return
 	}
 
-	// needed so that modTime of `file` is different to its current value after ioutil.WriteFile
+	// needed so that modTime of `file` is different to its current value after os.WriteFile
 	time.Sleep(1000 * time.Millisecond)
 
 	// Now replace file contents with crap
-	if err := ioutil.WriteFile(file, []byte("foo"), 0644); err != nil {
+	if err := os.WriteFile(file, []byte("foo"), 0644); err != nil {
 		t.Fatal(err)
 		return
 	}
@@ -398,9 +396,9 @@ func TestUpdatedKeyfileContents(t *testing.T) {
 
 // forceCopyFile is like cp.CopyFile, but doesn't complain if the destination exists.
 func forceCopyFile(dst, src string) error {
-	data, err := ioutil.ReadFile(src)
+	data, err := os.ReadFile(src)
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(dst, data, 0644)
+	return os.WriteFile(dst, data, 0644)
 }

accounts/keystore/file_cache.go

@@ -17,7 +17,6 @@
 package keystore
 
 import (
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -41,7 +40,7 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er
 	t0 := time.Now()
 
 	// List all the failes from the keystore folder
-	files, err := ioutil.ReadDir(keyDir)
+	files, err := os.ReadDir(keyDir)
 	if err != nil {
 		return nil, nil, nil, err
 	}
@@ -65,7 +64,11 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er
 		// Gather the set of all and fresly modified files
 		all.Add(path)
 
-		modified := fi.ModTime()
+		info, err := fi.Info()
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		modified := info.ModTime()
 		if modified.After(fc.lastMod) {
 			mods.Add(path)
 		}
@@ -89,13 +92,13 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er
 }
 
 // nonKeyFile ignores editor backups, hidden files and folders/symlinks.
-func nonKeyFile(fi os.FileInfo) bool {
+func nonKeyFile(fi os.DirEntry) bool {
 	// Skip editor backups and UNIX-style hidden files.
 	if strings.HasSuffix(fi.Name(), "~") || strings.HasPrefix(fi.Name(), ".") {
 		return true
 	}
 	// Skip misc special files, directories (yes, symlinks too).
-	if fi.IsDir() || fi.Mode()&os.ModeType != 0 {
+	if fi.IsDir() || !fi.Type().IsRegular() {
 		return true
 	}
 	return false

accounts/keystore/key.go

@@ -23,7 +23,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -197,7 +196,7 @@ func writeTemporaryKeyFile(file string, content []byte) (string, error) {
 	}
 	// Atomic write: create a temporary hidden file first
 	// then move it into place. TempFile assigns mode 0600.
-	f, err := ioutil.TempFile(filepath.Dir(file), "."+filepath.Base(file)+".tmp")
+	f, err := os.CreateTemp(filepath.Dir(file), "."+filepath.Base(file)+".tmp")
 	if err != nil {
 		return "", err
 	}

accounts/keystore/keystore_test.go

@@ -17,7 +17,6 @@
 package keystore
 
 import (
-	"io/ioutil"
 	"math/rand"
 	"os"
 	"runtime"
@@ -38,7 +37,6 @@ var testSigData = make([]byte, 32)
 
 func TestKeyStore(t *testing.T) {
 	dir, ks := tmpKeyStore(t, true)
-	defer os.RemoveAll(dir)
 
 	a, err := ks.NewAccount("foo")
 	if err != nil {
@@ -72,8 +70,7 @@ func TestKeyStore(t *testing.T) {
 }
 
 func TestSign(t *testing.T) {
-	dir, ks := tmpKeyStore(t, true)
-	defer os.RemoveAll(dir)
+	_, ks := tmpKeyStore(t, true)
 
 	pass := "" // not used but required by API
 	a1, err := ks.NewAccount(pass)
@@ -89,8 +86,7 @@ func TestSign(t *testing.T) {
 }
 
 func TestSignWithPassphrase(t *testing.T) {
-	dir, ks := tmpKeyStore(t, true)
-	defer os.RemoveAll(dir)
+	_, ks := tmpKeyStore(t, true)
 
 	pass := "passwd"
 	acc, err := ks.NewAccount(pass)
@@ -117,8 +113,7 @@ func TestSignWithPassphrase(t *testing.T) {
 }
 
 func TestTimedUnlock(t *testing.T) {
-	dir, ks := tmpKeyStore(t, true)
-	defer os.RemoveAll(dir)
+	_, ks := tmpKeyStore(t, true)
 
 	pass := "foo"
 	a1, err := ks.NewAccount(pass)
@@ -152,8 +147,7 @@ func TestTimedUnlock(t *testing.T) {
 }
 
 func TestOverrideUnlock(t *testing.T) {
-	dir, ks := tmpKeyStore(t, false)
-	defer os.RemoveAll(dir)
+	_, ks := tmpKeyStore(t, false)
 
 	pass := "foo"
 	a1, err := ks.NewAccount(pass)
@@ -193,8 +187,7 @@ func TestOverrideUnlock(t *testing.T) {
 
 // This test should fail under -race if signing races the expiration goroutine.
 func TestSignRace(t *testing.T) {
-	dir, ks := tmpKeyStore(t, false)
-	defer os.RemoveAll(dir)
+	_, ks := tmpKeyStore(t, false)
 
 	// Create a test account.
 	a1, err := ks.NewAccount("")
@@ -222,8 +215,7 @@ func TestSignRace(t *testing.T) {
 // addition and removal of wallet event subscriptions.
 func TestWalletNotifierLifecycle(t *testing.T) {
 	// Create a temporary kesytore to test with
-	dir, ks := tmpKeyStore(t, false)
-	defer os.RemoveAll(dir)
+	_, ks := tmpKeyStore(t, false)
 
 	// Ensure that the notification updater is not running yet
 	time.Sleep(250 * time.Millisecond)
@@ -283,8 +275,7 @@ type walletEvent struct {
 // Tests that wallet notifications and correctly fired when accounts are added
 // or deleted from the keystore.
 func TestWalletNotifications(t *testing.T) {
-	dir, ks := tmpKeyStore(t, false)
-	defer os.RemoveAll(dir)
+	_, ks := tmpKeyStore(t, false)
 
 	// Subscribe to the wallet feed and collect events.
 	var (
@@ -345,8 +336,7 @@ func TestWalletNotifications(t *testing.T) {
 
 // TestImportExport tests the import functionality of a keystore.
 func TestImportECDSA(t *testing.T) {
-	dir, ks := tmpKeyStore(t, true)
-	defer os.RemoveAll(dir)
+	_, ks := tmpKeyStore(t, true)
 	key, err := crypto.GenerateKey()
 	if err != nil {
 		t.Fatalf("failed to generate key: %v", key)
@@ -364,8 +354,7 @@ func TestImportECDSA(t *testing.T) {
 
 // TestImportECDSA tests the import and export functionality of a keystore.
 func TestImportExport(t *testing.T) {
-	dir, ks := tmpKeyStore(t, true)
-	defer os.RemoveAll(dir)
+	_, ks := tmpKeyStore(t, true)
 	acc, err := ks.NewAccount("old")
 	if err != nil {
 		t.Fatalf("failed to create account: %v", acc)
@@ -374,8 +363,7 @@ func TestImportExport(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to export account: %v", acc)
 	}
-	dir2, ks2 := tmpKeyStore(t, true)
-	defer os.RemoveAll(dir2)
+	_, ks2 := tmpKeyStore(t, true)
 	if _, err = ks2.Import(json, "old", "old"); err == nil {
 		t.Errorf("importing with invalid password succeeded")
 	}
@@ -395,8 +383,7 @@ func TestImportExport(t *testing.T) {
 // TestImportRace tests the keystore on races.
 // This test should fail under -race if importing races.
 func TestImportRace(t *testing.T) {
-	dir, ks := tmpKeyStore(t, true)
-	defer os.RemoveAll(dir)
+	_, ks := tmpKeyStore(t, true)
 	acc, err := ks.NewAccount("old")
 	if err != nil {
 		t.Fatalf("failed to create account: %v", acc)
@@ -405,8 +392,7 @@ func TestImportRace(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to export account: %v", acc)
 	}
-	dir2, ks2 := tmpKeyStore(t, true)
-	defer os.RemoveAll(dir2)
+	_, ks2 := tmpKeyStore(t, true)
 	var atom uint32
 	var wg sync.WaitGroup
 	wg.Add(2)
@@ -462,10 +448,7 @@ func checkEvents(t *testing.T, want []walletEvent, have []walletEvent) {
 }
 
 func tmpKeyStore(t *testing.T, encrypted bool) (string, *KeyStore) {
-	d, err := ioutil.TempDir("", "eth-keystore-test")
-	if err != nil {
-		t.Fatal(err)
-	}
+	d := t.TempDir()
 	newKs := NewPlaintextKeyStore
 	if encrypted {
 		newKs = func(kd string) *KeyStore { return NewKeyStore(kd, veryLightScryptN, veryLightScryptP) }

accounts/keystore/passphrase.go

@@ -34,7 +34,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -82,7 +81,7 @@ type keyStorePassphrase struct {
 
 func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string) (*Key, error) {
 	// Load the key from the keystore and decrypt its contents
-	keyjson, err := ioutil.ReadFile(filename)
+	keyjson, err := os.ReadFile(filename)
 	if err != nil {
 		return nil, err
 	}

accounts/keystore/passphrase_test.go

@@ -17,7 +17,7 @@
 package keystore
 
 import (
-	"io/ioutil"
+	"os"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -30,7 +30,7 @@ const (
 
 // Tests that a json key file can be decrypted and encrypted in multiple rounds.
 func TestKeyEncryptDecrypt(t *testing.T) {
-	keyjson, err := ioutil.ReadFile("testdata/very-light-scrypt.json")
+	keyjson, err := os.ReadFile("testdata/very-light-scrypt.json")
 	if err != nil {
 		t.Fatal(err)
 	}

accounts/keystore/plain_test.go

@@ -20,8 +20,6 @@ import (
 	"crypto/rand"
 	"encoding/hex"
 	"fmt"
-	"io/ioutil"
-	"os"
 	"path/filepath"
 	"reflect"
 	"strings"
@@ -32,10 +30,7 @@ import (
 )
 
 func tmpKeyStoreIface(t *testing.T, encrypted bool) (dir string, ks keyStore) {
-	d, err := ioutil.TempDir("", "geth-keystore-test")
-	if err != nil {
-		t.Fatal(err)
-	}
+	d := t.TempDir()
 	if encrypted {
 		ks = &keyStorePassphrase{d, veryLightScryptN, veryLightScryptP, true}
 	} else {
@@ -45,8 +40,7 @@ func tmpKeyStoreIface(t *testing.T, encrypted bool) (dir string, ks keyStore) {
 }
 
 func TestKeyStorePlain(t *testing.T) {
-	dir, ks := tmpKeyStoreIface(t, false)
-	defer os.RemoveAll(dir)
+	_, ks := tmpKeyStoreIface(t, false)
 
 	pass := "" // not used but required by API
 	k1, account, err := storeNewKey(ks, rand.Reader, pass)
@@ -66,8 +60,7 @@ func TestKeyStorePlain(t *testing.T) {
 }
 
 func TestKeyStorePassphrase(t *testing.T) {
-	dir, ks := tmpKeyStoreIface(t, true)
-	defer os.RemoveAll(dir)
+	_, ks := tmpKeyStoreIface(t, true)
 
 	pass := "foo"
 	k1, account, err := storeNewKey(ks, rand.Reader, pass)
@@ -87,8 +80,7 @@ func TestKeyStorePassphrase(t *testing.T) {
 }
 
 func TestKeyStorePassphraseDecryptionFail(t *testing.T) {
-	dir, ks := tmpKeyStoreIface(t, true)
-	defer os.RemoveAll(dir)
+	_, ks := tmpKeyStoreIface(t, true)
 
 	pass := "foo"
 	k1, account, err := storeNewKey(ks, rand.Reader, pass)
@@ -102,7 +94,6 @@ func TestKeyStorePassphraseDecryptionFail(t *testing.T) {
 
 func TestImportPreSaleKey(t *testing.T) {
 	dir, ks := tmpKeyStoreIface(t, true)
-	defer os.RemoveAll(dir)
 
 	// file content of a presale key file generated with:
 	// python pyethsaletool.py genwallet

accounts/scwallet/hub.go

@@ -34,7 +34,7 @@ package scwallet
 
 import (
 	"encoding/json"
-	"io/ioutil"
+	"io"
 	"os"
 	"path/filepath"
 	"sort"
@@ -96,7 +96,7 @@ func (hub *Hub) readPairings() error {
 		return err
 	}
 
-	pairingData, err := ioutil.ReadAll(pairingFile)
+	pairingData, err := io.ReadAll(pairingFile)
 	if err != nil {
 		return err
 	}

appveyor.yml

@@ -13,7 +13,7 @@ environment:
       GETH_MINGW: 'C:\msys64\mingw32'
 
 install:
-  - git submodule update --init --depth 1
+  - git submodule update --init --depth 1 --recursive
   - go version
 
 for:

build/bot/macos-build.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e -x
+
+# -- Check XCode version
+xcodebuild -version
+# xcrun simctl list
+
+# -- Build for macOS and upload to Azure
+go run build/ci.go install -dlgo
+go run build/ci.go archive -type tar # -signer OSX_SIGNING_KEY -upload gethstore/builds
+
+# # -- CocoaPods
+# gem uninstall cocoapods -a -x
+# gem install cocoapods
+# mv ~/.cocoapods/repos/master ~/.cocoapods/repos/master.bak
+# sed -i '.bak' 's/repo.join/!repo.join/g' $(dirname `gem which cocoapods`)/cocoapods/sources_manager.rb
+# git clone --depth=1 https://github.com/CocoaPods/Specs.git ~/.cocoapods/repos/master
+# pod setup --verbose
+
+# # -- Build for iOS and upload to Azure
+# go run build/ci.go xcode -signer IOS_SIGNING_KEY -upload gethstore/builds

build/bot/ppa-build.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e -x
+
+# Note: this script is meant to be run in a Debian/Ubuntu docker container,
+# as user 'root'.
+
+# Install the required tools for creating source packages.
+apt-get -yq --no-install-suggests --no-install-recommends install\
+     devscripts debhelper dput fakeroot
+
+# Add the SSH key of ppa.launchpad.net to known_hosts.
+mkdir -p ~/.ssh
+echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
+
+# Build the source package and upload.
+go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"

build/checksums.txt

@@ -1,37 +1,58 @@
 # This file contains sha256 checksums of optional build dependencies.
 
-3defb9a09bed042403195e872dcbc8c6fae1485963332279668ec52e80a95a2d  go1.17.5.src.tar.gz
-2db6a5d25815b56072465a2cacc8ed426c18f1d5fc26c1fc8c4f5a7188658264  go1.17.5.darwin-amd64.tar.gz
-111f71166de0cb8089bb3e8f9f5b02d76e1bf1309256824d4062a47b0e5f98e0  go1.17.5.darwin-arm64.tar.gz
-443c1cd9768df02085014f1eb034ebc7dbe032ffc8a9bb9f2e6617d037eee23c  go1.17.5.freebsd-386.tar.gz
-17180bdc4126acffd0ebf86d66ef5cbc3488b6734e93374fb00eb09494e006d3  go1.17.5.freebsd-amd64.tar.gz
-4f4914303bc18f24fd137a97e595735308f5ce81323c7224c12466fd763fc59f  go1.17.5.linux-386.tar.gz
-bd78114b0d441b029c8fe0341f4910370925a4d270a6a590668840675b0c653e  go1.17.5.linux-amd64.tar.gz
-6f95ce3da40d9ce1355e48f31f4eb6508382415ca4d7413b1e7a3314e6430e7e  go1.17.5.linux-arm64.tar.gz
-aa1fb6c53b4fe72f159333362a10aca37ae938bde8adc9c6eaf2a8e87d1e47de  go1.17.5.linux-armv6l.tar.gz
-3d4be616e568f0a02cb7f7769bcaafda4b0969ed0f9bb4277619930b96847e70  go1.17.5.linux-ppc64le.tar.gz
-8087d4fe991e82804e6485c26568c2e0ee0bfde00ceb9015dc86cb6bf84ef40b  go1.17.5.linux-s390x.tar.gz
-6d7b9948ee14a906b14f5cbebdfab63cd6828b0b618160847ecd3cc3470a26fe  go1.17.5.windows-386.zip
-671faf99cd5d81cd7e40936c0a94363c64d654faa0148d2af4bbc262555620b9  go1.17.5.windows-amd64.zip
-45e88676b68e9cf364be469b5a27965397f4e339aa622c2f52c10433c56e5030  go1.17.5.windows-arm64.zip
+efd43e0f1402e083b73a03d444b7b6576bb4c539ac46208b63a916b69aca4088  go1.18.1.src.tar.gz
+3703e9a0db1000f18c0c7b524f3d378aac71219b4715a6a4c5683eb639f41a4d  go1.18.1.darwin-amd64.tar.gz
+6d5641a06edba8cd6d425fb0adad06bad80e2afe0fa91b4aa0e5aed1bc78f58e  go1.18.1.darwin-arm64.tar.gz
+b9a9063d4265d8ccc046c9b314194d6eadc47e56d0d637db81e98e68aad45035  go1.18.1.freebsd-386.tar.gz
+2bc1c138d645e37dbbc63517dd1cf1bf33fc4cb95f442a6384df0418b5134e9f  go1.18.1.freebsd-amd64.tar.gz
+9a8df5dde9058f08ac01ecfaae42534610db398e487138788c01da26a0d41ff9  go1.18.1.linux-386.tar.gz
+b3b815f47ababac13810fc6021eb73d65478e0b2db4b09d348eefad9581a2334  go1.18.1.linux-amd64.tar.gz
+56a91851c97fb4697077abbca38860f735c32b38993ff79b088dac46e4735633  go1.18.1.linux-arm64.tar.gz
+9edc01c8e7db64e9ceeffc8258359e027812886ceca3444e83c4eb96ddb068ee  go1.18.1.linux-armv6l.tar.gz
+33db623d1eecf362fe365107c12efc90eff0b9609e0b3345e258388019cb552a  go1.18.1.linux-ppc64le.tar.gz
+5d9301324148ed4dbfaa0800da43a843ffd65c834ee73fcf087255697c925f74  go1.18.1.linux-s390x.tar.gz
+49ae65551acbfaa57b52fbefa0350b2072512ae3103b8cf1a919a02626dbc743  go1.18.1.windows-386.zip
+c30bc3f1f7314a953fe208bd9cd5e24bd9403392a6c556ced3677f9f70f71fe1  go1.18.1.windows-amd64.zip
+2c4a8265030eac37f906634f5c13c22c3d0ea725f2488e1bca005c6b981653d7  go1.18.1.windows-arm64.zip
 
-d4bd25b9814eeaa2134197dd2c7671bb791eae786d42010d9d788af20dee4bfa  golangci-lint-1.42.0-darwin-amd64.tar.gz
-e56859c04a2ad5390c6a497b1acb1cc9329ecb1010260c6faae9b5a4c35b35ea  golangci-lint-1.42.0-darwin-arm64.tar.gz
-14d912a3fa856830339472fc4dc341933adf15f37bdb7130bbbfcf960ecf4809  golangci-lint-1.42.0-freebsd-386.tar.gz
-337257fccc9baeb5ee1cd7e70c153e9d9f59d3afde46d631659500048afbdf80  golangci-lint-1.42.0-freebsd-amd64.tar.gz
-6debcc266b629359fdd8eef4f4abb05a621604079d27016265afb5b4593b0eff  golangci-lint-1.42.0-freebsd-armv6.tar.gz
-878f0e190169db2ce9dde8cefbd99adc4fe28b90b68686bbfcfcc2085e6d693e  golangci-lint-1.42.0-freebsd-armv7.tar.gz
-42c78e31faf62b225363eff1b1d2aa74f9dbcb75686c8914aa3e90d6af65cece  golangci-lint-1.42.0-linux-386.tar.gz
-6937f62f8e2329e94822dc11c10b871ace5557ae1fcc4ee2f9980cd6aecbc159  golangci-lint-1.42.0-linux-amd64.tar.gz
-2cf8d23d96cd854a537b355dab2962b960b88a06b615232599f066afd233f246  golangci-lint-1.42.0-linux-arm64.tar.gz
-08b003d1ed61367473886defc957af5301066e62338e5d96a319c34dadc4c1d1  golangci-lint-1.42.0-linux-armv6.tar.gz
-c7c00ec4845e806a1f32685f5b150219e180bd6d6a9d584be8d27f0c41d7a1bf  golangci-lint-1.42.0-linux-armv7.tar.gz
-3650fcf29eb3d8ee326d77791a896b15259eb2d5bf77437dc72e7efe5af6bd40  golangci-lint-1.42.0-linux-mips64.tar.gz
-f51ae003fdbca4fef78ba73e2eb736a939c8eaa178cd452234213b489da5a420  golangci-lint-1.42.0-linux-mips64le.tar.gz
-1b0bb7b8b22cc4ea7da44fd5ad5faaf6111d0677e01cc6f961b62a96537de2c6  golangci-lint-1.42.0-linux-ppc64le.tar.gz
-8cb56927eb75e572450efbe0ff0f9cf3f56dc9faa81d9e8d30d6559fc1d06e6d  golangci-lint-1.42.0-linux-riscv64.tar.gz
-5ac41cd31825a176b21505a371a7b307cd9cdf17df0f35bbb3bf1466f9356ccc  golangci-lint-1.42.0-linux-s390x.tar.gz
-e1cebd2af621ac4b64c20937df92c3819264f2174c92f51e196db1e64ae097e0  golangci-lint-1.42.0-windows-386.zip
-7e70fcde8e87a17cae0455df07d257ebc86669f3968d568e12727fa24bbe9883  golangci-lint-1.42.0-windows-amd64.zip
-59da7ce1bda432616bfc28ae663e52c3675adee8d9bf5959fafd657c159576ab  golangci-lint-1.42.0-windows-armv6.zip
-65f62dda937bfcede0326ac77abe947ce1548931e6e13298ca036cb31f224db5  golangci-lint-1.42.0-windows-armv7.zip
+03c181fc1bb29ea3e73cbb23399c43b081063833a7cf7554b94e5a98308df53e  golangci-lint-1.45.2-linux-riscv64.deb
+08a50bbbf451ede6d5354179eb3e14a5634e156dfa92cb9a2606f855a637e35b  golangci-lint-1.45.2-linux-ppc64le.rpm
+0d12f6ec1296b5a70e392aa88cd2295cceef266165eb7028e675f455515dd1c9  golangci-lint-1.45.2-linux-armv7.deb
+10f2846e2e50e4ea8ae426ee62dcd2227b23adddd8e991aa3c065927ac948735  golangci-lint-1.45.2-linux-ppc64le.deb
+1463049b744871168095e3e8f687247d6040eeb895955b869889ea151e0603ab  golangci-lint-1.45.2-linux-arm64.tar.gz
+15720f9c4c6f9324af695f081dc189adc7751b255759e78d7b2df1d7e9192533  golangci-lint-1.45.2-linux-amd64.deb
+166d922e4d3cfe3d47786c590154a9c8ea689dff0aa92b73d2f5fc74fc570c29  golangci-lint-1.45.2-linux-arm64.rpm
+1a3754c69f7cc19ab89cbdcc2550da4cf9abb3120383c6b3bd440c1ec22da2e6  golangci-lint-1.45.2-freebsd-386.tar.gz
+1dec0aa46d4f0d241863b573f70129bdf1de9c595cf51172a840a588a4cd9fc5  golangci-lint-1.45.2-windows-amd64.zip
+3198453806517c1ad988229f5e758ef850e671203f46d6905509df5bdf4dc24b  golangci-lint-1.45.2-freebsd-armv7.tar.gz
+46a3cd1749d7b98adc2dc01510ddbe21abe42689c8a53fb0e81662713629f215  golangci-lint-1.45.2-linux-386.deb
+4e28bfb593d464b9e160f2acd5b71993836a183270bf8299b78ad31f7a168c0d  golangci-lint-1.45.2-linux-arm64.deb
+5157a58c8f9ab85c33af2e46f0d7c57a3b1e8953b81d61130e292e09f545cfab  golangci-lint-1.45.2-linux-mips64le.tar.gz
+518cd027644129fbf8ec4f02bd6f9ad7278aae826f92b63c80d4d0819ddde49a  golangci-lint-1.45.2-linux-armv6.rpm
+595ad6c6dade4c064351bc309f411703e457f8ffbb7a1806b3d8ee713333427f  golangci-lint-1.45.2-linux-amd64.tar.gz
+6994d6c80f0730751090986184a3481b4be2e6b6e84416238a2b857910045a4f  golangci-lint-1.45.2-windows-arm64.zip
+6c81652fc340118811b487f713c441fc6f527800bf5fd11b8929d08124efa015  golangci-lint-1.45.2-linux-armv7.tar.gz
+726cb045559b7518bafdd3459de70a0647c087eb1b4634627a4b2e95b1258580  golangci-lint-1.45.2-freebsd-amd64.tar.gz
+77df3774cdfda49b956d4a0e676da9a9b883f496ee37293c530770fef6b1d24e  golangci-lint-1.45.2-linux-mips64.deb
+7a9840f279a7d5d405bb434e101c2290964b3729630ac2add29280b962b7b9a5  golangci-lint-1.45.2-windows-armv6.zip
+7d4bf9a5d80ec467aaaf66e78dbdcab567bbc6ba8151334c714eee58766aae32  golangci-lint-1.45.2-windows-armv7.zip
+7e5f8821d39bb11d273b0841b34355f56bd5a45a2d5179f0d09e614e0efc0482  golangci-lint-1.45.2-linux-s390x.rpm
+828de1bde796b23d8656b17a8885fbd879ef612795d62d1e4618126b419728b5  golangci-lint-1.45.2-linux-mips64.rpm
+879a52107a797678a03c175cc7cf441411a14a01f66dc87f70bdfa304a4129a6  golangci-lint-1.45.2-windows-386.zip
+87b6c7e3a3769f7d9abeb3bb82119b3c91e3c975300f6834fdeef8b2e37c98ff  golangci-lint-1.45.2-linux-amd64.rpm
+8b605c6d686c8af53ecc4ef39544541eeb1644d34cc10f9ffc5087808210c4ff  golangci-lint-1.45.2-linux-s390x.deb
+9427dbf51d0ac6f73a0f992838bd40c817470cc5bf6c8e2e2bea6fac46d7af6e  golangci-lint-1.45.2-linux-ppc64le.tar.gz
+995e509e895ca6a64ffc7395ac884d5961bdec98423cb896b17f345a9b4a19cf  golangci-lint-1.45.2-darwin-amd64.tar.gz
+a3f36278f2ea5516341e9071a2df6e65df272be80230b5406a12b72c6d425bee  golangci-lint-1.45.2-linux-armv7.rpm
+a5e12c50c23e87ac1deffc872f92ae85427b1198604969399805ae47cfe43f08  golangci-lint-1.45.2-linux-riscv64.tar.gz
+aa8fa1be0729dbc2fbc4e01e82027097613eee74bd686ebef20f860b01fff8b3  golangci-lint-1.45.2-freebsd-armv6.tar.gz
+c2b9669decc1b638cf2ee9060571af4e255f6dfcbb225c293e3a7ee4bb2c7217  golangci-lint-1.45.2-darwin-arm64.tar.gz
+dfa8bdaf0387aec1cd5c1aa8857f67b2bbdfc2e42efce540c8fb9bbe3e8af302  golangci-lint-1.45.2-linux-armv6.tar.gz
+eb8b8539dd017eee5c131ea9b875893ab2cebeeca41e8c6624907fb02224d643  golangci-lint-1.45.2-linux-386.rpm
+ed6c7e17a857f30d715c5302fa250d95936936b277024bffea201187a257d7a7  golangci-lint-1.45.2-linux-armv6.deb
+ef4d0154ace4001f01b288baeb118176242efb4fd163e178763e3213b77ef30b  golangci-lint-1.45.2-linux-mips64le.deb
+ef7002a2229f5ff5ba201a715fcf877664ea88decbe58e69d163293913024955  golangci-lint-1.45.2-linux-s390x.tar.gz
+f13ecbd09228632e6bbe91a8324bd675c406eed22eb6d2c1e8192eed9ec4f914  golangci-lint-1.45.2-linux-386.tar.gz
+f4cd9cfb09252f51699407277512263cae8409b665dd764f55a34738d0e89edc  golangci-lint-1.45.2-linux-riscv64.rpm
+fb1945dc59d37c9d14bf0a4aea11ea8651fa0e1d582ea80c4c44d0a536c08893  golangci-lint-1.45.2-linux-mips64.tar.gz
+fe542c22738010f453c735a3c410decfd3784d1bd394b395c298ee298fc4c606  golangci-lint-1.45.2-linux-mips64le.rpm

build/ci.go

@@ -46,7 +46,6 @@ import (
 	"encoding/base64"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -59,6 +58,7 @@ import (
 	"time"
 
 	"github.com/cespare/cp"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto/signify"
 	"github.com/ethereum/go-ethereum/internal/build"
 	"github.com/ethereum/go-ethereum/params"
@@ -130,13 +130,15 @@ var (
 	// Distros for which packages are created.
 	// Note: vivid is unsupported because there is no golang-1.6 package for it.
 	// Note: the following Ubuntu releases have been officially deprecated on Launchpad:
-	//   wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy
+	//   wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite
 	debDistroGoBoots = map[string]string{
-		"trusty":  "golang-1.11",
-		"xenial":  "golang-go",
-		"bionic":  "golang-go",
-		"focal":   "golang-go",
-		"hirsute": "golang-go",
+		"trusty": "golang-1.11", // EOL: 04/2024
+		"xenial": "golang-go",   // EOL: 04/2026
+		"bionic": "golang-go",   // EOL: 04/2028
+		"focal":  "golang-go",   // EOL: 04/2030
+		"impish": "golang-go",   // EOL: 07/2022
+		"jammy":  "golang-go",   // EOL: 04/2032
+		//"kinetic": "golang-go",   //  EOL: 07/2023
 	}
 
 	debGoBootPaths = map[string]string{
@@ -147,7 +149,7 @@ var (
 	// This is the version of go that will be downloaded by
 	//
 	//     go run ci.go install -dlgo
-	dlgoVersion = "1.17.5"
+	dlgoVersion = "1.18.1"
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -162,7 +164,7 @@ func executablePath(name string) string {
 func main() {
 	log.SetFlags(log.Lshortfile)
 
-	if _, err := os.Stat(filepath.Join("build", "ci.go")); os.IsNotExist(err) {
+	if !common.FileExist(filepath.Join("build", "ci.go")) {
 		log.Fatal("this script must be run from the root of the repository")
 	}
 	if len(os.Args) < 2 {
@@ -331,16 +333,21 @@ func doLint(cmdline []string) {
 
 // downloadLinter downloads and unpacks golangci-lint.
 func downloadLinter(cachedir string) string {
-	const version = "1.42.0"
+	const version = "1.45.2"
 
 	csdb := build.MustLoadChecksums("build/checksums.txt")
 	arch := runtime.GOARCH
+	ext := ".tar.gz"
+
+	if runtime.GOOS == "windows" {
+		ext = ".zip"
+	}
 	if arch == "arm" {
 		arch += "v" + os.Getenv("GOARM")
 	}
 	base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, arch)
-	url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s.tar.gz", version, base)
-	archivePath := filepath.Join(cachedir, base+".tar.gz")
+	url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s%s", version, base, ext)
+	archivePath := filepath.Join(cachedir, base+ext)
 	if err := csdb.DownloadFile(url, archivePath); err != nil {
 		log.Fatal(err)
 	}
@@ -454,7 +461,7 @@ func maybeSkipArchive(env build.Environment) {
 		log.Printf("skipping archive creation because this is a cron job")
 		os.Exit(0)
 	}
-	if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") {
+	if env.Branch != "master" && env.Branch != "buildbot-testing" && !strings.HasPrefix(env.Tag, "v1.") {
 		log.Printf("skipping archive creation because branch %q, tag %q is not on the inclusion list", env.Branch, env.Tag)
 		os.Exit(0)
 	}
@@ -727,8 +734,8 @@ func ppaUpload(workdir, ppa, sshUser string, files []string) {
 	var idfile string
 	if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 {
 		idfile = filepath.Join(workdir, "sshkey")
-		if _, err := os.Stat(idfile); os.IsNotExist(err) {
-			ioutil.WriteFile(idfile, sshkey, 0600)
+		if !common.FileExist(idfile) {
+			os.WriteFile(idfile, sshkey, 0600)
 		}
 	}
 	// Upload
@@ -751,7 +758,7 @@ func makeWorkdir(wdflag string) string {
 	if wdflag != "" {
 		err = os.MkdirAll(wdflag, 0744)
 	} else {
-		wdflag, err = ioutil.TempDir("", "geth-build-")
+		wdflag, err = os.MkdirTemp("", "geth-build-")
 	}
 	if err != nil {
 		log.Fatal(err)
@@ -949,10 +956,10 @@ func doWindowsInstaller(cmdline []string) {
 	build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil)
 	build.Render("build/nsis.envvarupdate.nsh", filepath.Join(*workdir, "EnvVarUpdate.nsh"), 0644, nil)
 	if err := cp.CopyFile(filepath.Join(*workdir, "SimpleFC.dll"), "build/nsis.simplefc.dll"); err != nil {
-		log.Fatal("Failed to copy SimpleFC.dll: %v", err)
+		log.Fatalf("Failed to copy SimpleFC.dll: %v", err)
 	}
 	if err := cp.CopyFile(filepath.Join(*workdir, "COPYING"), "COPYING"); err != nil {
-		log.Fatal("Failed to copy copyright note: %v", err)
+		log.Fatalf("Failed to copy copyright note: %v", err)
 	}
 	// Build the installer. This assumes that all the needed files have been previously
 	// built (don't mix building and packaging to keep cross compilation complexity to a
@@ -1127,11 +1134,7 @@ func doXCodeFramework(cmdline []string) {
 	tc := new(build.GoToolchain)
 
 	// Build gomobile.
-	build.MustRun(tc.Install(GOBIN, "golang.org/x/mobile/cmd/gomobile@latest", "golang.org/x/mobile/cmd/gobind@latest"))
-
-	// Ensure all dependencies are available. This is required to make
-	// gomobile bind work because it expects go.sum to contain all checksums.
-	build.MustRun(tc.Go("mod", "download"))
+	build.MustRun(tc.Install(GOBIN, "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind"))
 
 	// Build the iOS XCode framework
 	bind := gomobileTool("bind", "-ldflags", "-s -w", "--target", "ios", "-v", "github.com/ethereum/go-ethereum/mobile")
@@ -1237,21 +1240,21 @@ func doPurge(cmdline []string) {
 
 	// Iterate over the blobs, collect and sort all unstable builds
 	for i := 0; i < len(blobs); i++ {
-		if !strings.Contains(blobs[i].Name, "unstable") {
+		if !strings.Contains(*blobs[i].Name, "unstable") {
 			blobs = append(blobs[:i], blobs[i+1:]...)
 			i--
 		}
 	}
 	for i := 0; i < len(blobs); i++ {
 		for j := i + 1; j < len(blobs); j++ {
-			if blobs[i].Properties.LastModified.After(blobs[j].Properties.LastModified) {
+			if blobs[i].Properties.LastModified.After(*blobs[j].Properties.LastModified) {
 				blobs[i], blobs[j] = blobs[j], blobs[i]
 			}
 		}
 	}
 	// Filter out all archives more recent that the given threshold
 	for i, blob := range blobs {
-		if time.Since(blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour {
+		if time.Since(*blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour {
 			blobs = blobs[:i]
 			break
 		}

build/deb/ethereum/deb.control

@@ -5,7 +5,7 @@ Maintainer: {{.Author}}
 Build-Depends: debhelper (>= 8.0.0), {{.GoBootPackage}}
 Standards-Version: 3.9.5
 Homepage: https://ethereum.org
-Vcs-Git: git://github.com/ethereum/go-ethereum.git
+Vcs-Git: https://github.com/ethereum/go-ethereum.git
 Vcs-Browser: https://github.com/ethereum/go-ethereum
 
 Package: {{.Name}}

build/tools/tools.go

@@ -0,0 +1,31 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+//go:build tools
+// +build tools
+
+package tools
+
+import (
+	// Tool imports for go:generate.
+	_ "github.com/fjl/gencodec"
+	_ "github.com/golang/protobuf/protoc-gen-go"
+	_ "golang.org/x/tools/cmd/stringer"
+
+	// Tool imports for mobile build.
+	_ "golang.org/x/mobile/cmd/gobind"
+	_ "golang.org/x/mobile/cmd/gomobile"
+)

build/update-license.go

@@ -39,7 +39,6 @@ import (
 	"bufio"
 	"bytes"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -241,7 +240,7 @@ func gitAuthors(files []string) []string {
 }
 
 func readAuthors() []string {
-	content, err := ioutil.ReadFile("AUTHORS")
+	content, err := os.ReadFile("AUTHORS")
 	if err != nil && !os.IsNotExist(err) {
 		log.Fatalln("error reading AUTHORS:", err)
 	}
@@ -305,7 +304,7 @@ func writeAuthors(files []string) {
 		content.WriteString("\n")
 	}
 	fmt.Println("writing AUTHORS")
-	if err := ioutil.WriteFile("AUTHORS", content.Bytes(), 0644); err != nil {
+	if err := os.WriteFile("AUTHORS", content.Bytes(), 0644); err != nil {
 		log.Fatalln(err)
 	}
 }
@@ -381,7 +380,7 @@ func writeLicense(info *info) {
 	if err != nil {
 		log.Fatalf("error stat'ing %s: %v\n", info.file, err)
 	}
-	content, err := ioutil.ReadFile(info.file)
+	content, err := os.ReadFile(info.file)
 	if err != nil {
 		log.Fatalf("error reading %s: %v\n", info.file, err)
 	}
@@ -400,7 +399,7 @@ func writeLicense(info *info) {
 		return
 	}
 	fmt.Println("writing", info.ShortLicense(), info.file)
-	if err := ioutil.WriteFile(info.file, buf.Bytes(), fi.Mode()); err != nil {
+	if err := os.WriteFile(info.file, buf.Bytes(), fi.Mode()); err != nil {
 		log.Fatalf("error writing %s: %v", info.file, err)
 	}
 }

cmd/abigen/main.go

@@ -19,7 +19,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -155,9 +155,9 @@ func abigen(c *cli.Context) error {
 		)
 		input := c.GlobalString(abiFlag.Name)
 		if input == "-" {
-			abi, err = ioutil.ReadAll(os.Stdin)
+			abi, err = io.ReadAll(os.Stdin)
 		} else {
-			abi, err = ioutil.ReadFile(input)
+			abi, err = os.ReadFile(input)
 		}
 		if err != nil {
 			utils.Fatalf("Failed to read input ABI: %v", err)
@@ -166,7 +166,7 @@ func abigen(c *cli.Context) error {
 
 		var bin []byte
 		if binFile := c.GlobalString(binFlag.Name); binFile != "" {
-			if bin, err = ioutil.ReadFile(binFile); err != nil {
+			if bin, err = os.ReadFile(binFile); err != nil {
 				utils.Fatalf("Failed to read input bytecode: %v", err)
 			}
 			if strings.Contains(string(bin), "//") {
@@ -213,7 +213,7 @@ func abigen(c *cli.Context) error {
 			}
 
 		case c.GlobalIsSet(jsonFlag.Name):
-			jsonOutput, err := ioutil.ReadFile(c.GlobalString(jsonFlag.Name))
+			jsonOutput, err := os.ReadFile(c.GlobalString(jsonFlag.Name))
 			if err != nil {
 				utils.Fatalf("Failed to read combined-json from compiler: %v", err)
 			}
@@ -263,7 +263,7 @@ func abigen(c *cli.Context) error {
 		fmt.Printf("%s\n", code)
 		return nil
 	}
-	if err := ioutil.WriteFile(c.GlobalString(outFlag.Name), []byte(code), 0600); err != nil {
+	if err := os.WriteFile(c.GlobalString(outFlag.Name), []byte(code), 0600); err != nil {
 		utils.Fatalf("Failed to write ABI binding: %v", err)
 	}
 	return nil

cmd/clef/main.go

@@ -25,7 +25,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/big"
 	"os"
 	"os/signal"
@@ -374,7 +373,7 @@ func initializeSecrets(c *cli.Context) error {
 		return fmt.Errorf("master key %v already exists, will not overwrite", location)
 	}
 	// Write the file and print the usual warning message
-	if err = ioutil.WriteFile(location, cipherSeed, 0400); err != nil {
+	if err = os.WriteFile(location, cipherSeed, 0400); err != nil {
 		return err
 	}
 	fmt.Printf("A master seed has been generated into %s\n", location)
@@ -593,7 +592,7 @@ func signer(c *cli.Context) error {
 
 		// Do we have a rule-file?
 		if ruleFile := c.GlobalString(ruleFlag.Name); ruleFile != "" {
-			ruleJS, err := ioutil.ReadFile(ruleFile)
+			ruleJS, err := os.ReadFile(ruleFile)
 			if err != nil {
 				log.Warn("Could not load rules, disabling", "file", ruleFile, "err", err)
 			} else {
@@ -661,7 +660,7 @@ func signer(c *cli.Context) error {
 		if err != nil {
 			utils.Fatalf("Could not register API: %w", err)
 		}
-		handler := node.NewHTTPHandlerStack(srv, cors, vhosts)
+		handler := node.NewHTTPHandlerStack(srv, cors, vhosts, nil)
 
 		// set port
 		port := c.Int(rpcPortFlag.Name)
@@ -751,7 +750,7 @@ func readMasterKey(ctx *cli.Context, ui core.UIClientAPI) ([]byte, error) {
 	if err := checkFile(file); err != nil {
 		return nil, err
 	}
-	cipherKey, err := ioutil.ReadFile(file)
+	cipherKey, err := os.ReadFile(file)
 	if err != nil {
 		return nil, err
 	}

cmd/clef/pythonsigner.py

@@ -1,177 +1,315 @@
-import os,sys, subprocess
+import sys
+import subprocess
+
 from tinyrpc.transports import ServerTransport
 from tinyrpc.protocols.jsonrpc import JSONRPCProtocol
-from tinyrpc.dispatch import public,RPCDispatcher
+from tinyrpc.dispatch import public, RPCDispatcher
 from tinyrpc.server import RPCServer
 
-""" This is a POC example of how to write a custom UI for Clef. The UI starts the
-clef process with the '--stdio-ui' option, and communicates with clef using standard input / output.
+"""
+This is a POC example of how to write a custom UI for Clef.
+The UI starts the clef process with the '--stdio-ui' option
+and communicates with clef using standard input / output.
 
-The standard input/output is a relatively secure way to communicate, as it does not require opening any ports
-or IPC files. Needless to say, it does not protect against memory inspection mechanisms where an attacker
-can access process memory."""
+The standard input/output is a relatively secure way to communicate,
+as it does not require opening any ports or IPC files. Needless to say,
+it does not protect against memory inspection mechanisms
+where an attacker can access process memory.
+
+To make this work install all the requirements:
+
+  pip install -r requirements.txt
+"""
 
 try:
     import urllib.parse as urlparse
 except ImportError:
     import urllib as urlparse
 
+
 class StdIOTransport(ServerTransport):
-    """ Uses std input/output for RPC """
+    """Uses std input/output for RPC"""
+
     def receive_message(self):
         return None, urlparse.unquote(sys.stdin.readline())
 
     def send_reply(self, context, reply):
         print(reply)
 
-class PipeTransport(ServerTransport):
-    """ Uses std a pipe for RPC """
 
-    def __init__(self,input, output):
+class PipeTransport(ServerTransport):
+    """Uses std a pipe for RPC"""
+
+    def __init__(self, input, output):
         self.input = input
         self.output = output
 
     def receive_message(self):
         data = self.input.readline()
-        print(">> {}".format( data))
+        print(">> {}".format(data))
         return None, urlparse.unquote(data)
 
     def send_reply(self, context, reply):
-        print("<< {}".format( reply))
-        self.output.write(reply)
-        self.output.write("\n")
+        reply = str(reply, "utf-8")
+        print("<< {}".format(reply))
+        self.output.write("{}\n".format(reply))
 
-class StdIOHandler():
+
+def sanitize(txt, limit=100):
+    return txt[:limit].encode("unicode_escape").decode("utf-8")
+
+
+def metaString(meta):
+    """
+    "meta":{"remote":"clef binary","local":"main","scheme":"in-proc","User-Agent":"","Origin":""}
+    """  # noqa: E501
+    message = (
+        "\tRequest context:\n"
+        "\t\t{remote} -> {scheme} -> {local}\n"
+        "\tAdditional HTTP header data, provided by the external caller:\n"
+        "\t\tUser-Agent: {user_agent}\n"
+        "\t\tOrigin: {origin}\n"
+    )
+    return message.format(
+        remote=meta.get("remote", "<missing>"),
+        scheme=meta.get("scheme", "<missing>"),
+        local=meta.get("local", "<missing>"),
+        user_agent=sanitize(meta.get("User-Agent"), 200),
+        origin=sanitize(meta.get("Origin"), 100),
+    )
+
+
+class StdIOHandler:
     def __init__(self):
         pass
 
     @public
-    def ApproveTx(self,req):
+    def approveTx(self, req):
         """
         Example request:
-        {
-            "jsonrpc": "2.0",
-            "method": "ApproveTx",
-            "params": [{
-                "transaction": {
-                    "to": "0xae967917c465db8578ca9024c205720b1a3651A9",
-                    "gas": "0x333",
-                    "gasPrice": "0x123",
-                    "value": "0x10",
-                    "data": "0xd7a5865800000000000000000000000000000000000000000000000000000000000000ff",
-                    "nonce": "0x0"
-                },
-                "from": "0xAe967917c465db8578ca9024c205720b1a3651A9",
-                "call_info": "Warning! Could not validate ABI-data against calldata\nSupplied ABI spec does not contain method signature in data: 0xd7a58658",
-                "meta": {
-                    "remote": "127.0.0.1:34572",
-                    "local": "localhost:8550",
-                    "scheme": "HTTP/1.1"
-                }
-            }],
-            "id": 1
-        }
+
+        {"jsonrpc":"2.0","id":20,"method":"ui_approveTx","params":[{"transaction":{"from":"0xDEADbEeF000000000000000000000000DeaDbeEf","to":"0xDEADbEeF000000000000000000000000DeaDbeEf","gas":"0x3e8","gasPrice":"0x5","maxFeePerGas":null,"maxPriorityFeePerGas":null,"value":"0x6","nonce":"0x1","data":"0x"},"call_info":null,"meta":{"remote":"clef binary","local":"main","scheme":"in-proc","User-Agent":"","Origin":""}}]}
 
         :param transaction: transaction info
         :param call_info: info abou the call, e.g. if ABI info could not be
         :param meta: metadata about the request, e.g. where the call comes from
         :return:
-        """
-        transaction = req.get('transaction')
-        _from       = req.get('from')
-        call_info   = req.get('call_info')
-        meta        = req.get('meta')
-
+        """  # noqa: E501
+        message = (
+            "Sign transaction request:\n"
+            "\t{meta_string}\n"
+            "\n"
+            "\tFrom: {from_}\n"
+            "\tTo: {to}\n"
+            "\n"
+            "\tAuto-rejecting request"
+        )
+        meta = req.get("meta", {})
+        transaction = req.get("transaction")
+        sys.stdout.write(
+            message.format(
+                meta_string=metaString(meta),
+                from_=transaction.get("from", "<missing>"),
+                to=transaction.get("to", "<missing>"),
+            )
+        )
         return {
-            "approved" : False,
-            #"transaction" : transaction,
-  #          "from" : _from,
-#            "password" : None,
+            "approved": False,
         }
 
     @public
-    def ApproveSignData(self, req):
-        """ Example request
-
-        """
-        return {"approved": False, "password" : None}
-
-    @public
-    def ApproveExport(self, req):
-        """ Example request
-
-        """
-        return {"approved" : False}
-
-    @public
-    def ApproveImport(self, req):
-        """ Example request
-
-        """
-        return { "approved" : False, "old_password": "", "new_password": ""}
-
-    @public
-    def ApproveListing(self, req):
-        """ Example request
-
-        """
-        return {'accounts': []}
-
-    @public
-    def ApproveNewAccount(self, req):
-        """
-        Example request
-
-        :return:
-        """
-        return {"approved": False,
-                #"password": ""
-                }
-
-    @public
-    def ShowError(self,message = {}):
+    def approveSignData(self, req):
         """
         Example request:
 
-        {"jsonrpc":"2.0","method":"ShowInfo","params":{"message":"Testing 'ShowError'"},"id":1}
+        {"jsonrpc":"2.0","id":8,"method":"ui_approveSignData","params":[{"content_type":"application/x-clique-header","address":"0x0011223344556677889900112233445566778899","raw_data":"+QIRoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlAAAAAAAAAAAAAAAAAAAAAAAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAuQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIIFOYIFOYIFOoIFOoIFOppFeHRyYSBkYXRhIEV4dHJhIGRhdGEgRXh0cqAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIgAAAAAAAAAAA==","messages":[{"name":"Clique header","value":"clique header 1337 [0x44381ab449d77774874aca34634cb53bc21bd22aef2d3d4cf40e51176cb585ec]","type":"clique"}],"call_info":null,"hash":"0xa47ab61438a12a06c81420e308c2b7aae44e9cd837a5df70dd021421c0f58643","meta":{"remote":"clef binary","local":"main","scheme":"in-proc","User-Agent":"","Origin":""}}]}
+        """  # noqa: E501
+        message = (
+            "Sign data request:\n"
+            "\t{meta_string}\n"
+            "\n"
+            "\tContent-type: {content_type}\n"
+            "\tAddress: {address}\n"
+            "\tHash: {hash_}\n"
+            "\n"
+            "\tAuto-rejecting request\n"
+        )
+        meta = req.get("meta", {})
+        sys.stdout.write(
+            message.format(
+                meta_string=metaString(meta),
+                content_type=req.get("content_type"),
+                address=req.get("address"),
+                hash_=req.get("hash"),
+            )
+        )
 
-        :param message: to show
-        :return: nothing
-        """
-        if 'text' in message.keys():
-            sys.stderr.write("Error: {}\n".format( message['text']))
-        return
+        return {
+            "approved": False,
+            "password": None,
+        }
 
     @public
-    def ShowInfo(self,message = {}):
+    def approveNewAccount(self, req):
         """
-        Example request
-        {"jsonrpc":"2.0","method":"ShowInfo","params":{"message":"Testing 'ShowInfo'"},"id":0}
+        Example request:
+
+        {"jsonrpc":"2.0","id":25,"method":"ui_approveNewAccount","params":[{"meta":{"remote":"clef binary","local":"main","scheme":"in-proc","User-Agent":"","Origin":""}}]}
+        """  # noqa: E501
+        message = (
+            "Create new account request:\n"
+            "\t{meta_string}\n"
+            "\n"
+            "\tAuto-rejecting request\n"
+        )
+        meta = req.get("meta", {})
+        sys.stdout.write(message.format(meta_string=metaString(meta)))
+        return {
+            "approved": False,
+        }
+
+    @public
+    def showError(self, req):
+        """
+        Example request:
+
+        {"jsonrpc":"2.0","method":"ui_showError","params":[{"text":"If you see this message, enter 'yes' to the next question"}]}
 
         :param message: to display
         :return:nothing
-        """
-
-        if 'text' in message.keys():
-            sys.stdout.write("Error: {}\n".format( message['text']))
+        """  # noqa: E501
+        message = (
+            "## Error\n{text}\n"
+            "Press enter to continue\n"
+        )
+        text = req.get("text")
+        sys.stdout.write(message.format(text=text))
+        input()
         return
 
+    @public
+    def showInfo(self, req):
+        """
+        Example request:
+
+        {"jsonrpc":"2.0","method":"ui_showInfo","params":[{"text":"If you see this message, enter 'yes' to next question"}]}
+
+        :param message: to display
+        :return:nothing
+        """  # noqa: E501
+        message = (
+            "## Info\n{text}\n"
+            "Press enter to continue\n"
+        )
+        text = req.get("text")
+        sys.stdout.write(message.format(text=text))
+        input()
+        return
+
+    @public
+    def onSignerStartup(self, req):
+        """
+        Example request:
+
+        {"jsonrpc":"2.0", "method":"ui_onSignerStartup", "params":[{"info":{"extapi_http":"n/a","extapi_ipc":"/home/user/.clef/clef.ipc","extapi_version":"6.1.0","intapi_version":"7.0.1"}}]}
+        """  # noqa: E501
+        message = (
+            "\n"
+            "\t\tExt api url: {extapi_http}\n"
+            "\t\tInt api ipc: {extapi_ipc}\n"
+            "\t\tExt api ver: {extapi_version}\n"
+            "\t\tInt api ver: {intapi_version}\n"
+        )
+        info = req.get("info")
+        sys.stdout.write(
+            message.format(
+                extapi_http=info.get("extapi_http"),
+                extapi_ipc=info.get("extapi_ipc"),
+                extapi_version=info.get("extapi_version"),
+                intapi_version=info.get("intapi_version"),
+            )
+        )
+
+    @public
+    def approveListing(self, req):
+        """
+        Example request:
+
+        {"jsonrpc":"2.0","id":23,"method":"ui_approveListing","params":[{"accounts":[{"address":...
+        """  # noqa: E501
+        message = (
+            "\n"
+            "## Account listing request\n"
+            "\t{meta_string}\n"
+            "\tDo you want to allow listing the following accounts?\n"
+            "\t-{addrs}\n"
+            "\n"
+            "->Auto-answering No\n"
+        )
+        meta = req.get("meta", {})
+        accounts = req.get("accounts", [])
+        addrs = [x.get("address") for x in accounts]
+        sys.stdout.write(
+            message.format(
+                addrs="\n\t-".join(addrs),
+                meta_string=metaString(meta)
+            )
+        )
+        return {}
+
+    @public
+    def onInputRequired(self, req):
+        """
+        Example request:
+
+        {"jsonrpc":"2.0","id":1,"method":"ui_onInputRequired","params":[{"title":"Master Password","prompt":"Please enter the password to decrypt the master seed","isPassword":true}]}
+
+        :param message: to display
+        :return:nothing
+        """  # noqa: E501
+        message = (
+            "\n"
+            "## {title}\n"
+            "\t{prompt}\n"
+            "\n"
+            "> "
+        )
+        sys.stdout.write(
+            message.format(
+                title=req.get("title"),
+                prompt=req.get("prompt")
+            )
+        )
+        isPassword = req.get("isPassword")
+        if not isPassword:
+            return {"text": input()}
+
+        return ""
+
+
 def main(args):
     cmd = ["clef", "--stdio-ui"]
     if len(args) > 0 and args[0] == "test":
         cmd.extend(["--stdio-ui-test"])
     print("cmd: {}".format(" ".join(cmd)))
+
     dispatcher = RPCDispatcher()
-    dispatcher.register_instance(StdIOHandler(), '')
+    dispatcher.register_instance(StdIOHandler(), "ui_")
+
     # line buffered
-    p = subprocess.Popen(cmd, bufsize=1, universal_newlines=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    p = subprocess.Popen(
+        cmd,
+        bufsize=1,
+        universal_newlines=True,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+    )
 
     rpc_server = RPCServer(
-        PipeTransport(p.stdout, p.stdin),
-        JSONRPCProtocol(),
-        dispatcher
+        PipeTransport(p.stdout, p.stdin), JSONRPCProtocol(), dispatcher
     )
     rpc_server.serve_forever()
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     main(sys.argv[1:])

cmd/clef/requirements.txt

@@ -0,0 +1 @@
+tinyrpc==1.1.4

cmd/devp2p/dnscmd.go

@@ -20,7 +20,6 @@ import (
 	"crypto/ecdsa"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"time"
@@ -253,7 +252,7 @@ func dnsNukeRoute53(ctx *cli.Context) error {
 
 // loadSigningKey loads a private key in Ethereum keystore format.
 func loadSigningKey(keyfile string) *ecdsa.PrivateKey {
-	keyjson, err := ioutil.ReadFile(keyfile)
+	keyjson, err := os.ReadFile(keyfile)
 	if err != nil {
 		exit(fmt.Errorf("failed to read the keyfile at '%s': %v", keyfile, err))
 	}
@@ -382,7 +381,7 @@ func writeTreeMetadata(directory string, def *dnsDefinition) {
 		exit(err)
 	}
 	metaFile, _ := treeDefinitionFiles(directory)
-	if err := ioutil.WriteFile(metaFile, metaJSON, 0644); err != nil {
+	if err := os.WriteFile(metaFile, metaJSON, 0644); err != nil {
 		exit(err)
 	}
 }
@@ -411,7 +410,7 @@ func writeTXTJSON(file string, txt map[string]string) {
 		fmt.Println()
 		return
 	}
-	if err := ioutil.WriteFile(file, txtJSON, 0644); err != nil {
+	if err := os.WriteFile(file, txtJSON, 0644); err != nil {
 		exit(err)
 	}
 }

cmd/devp2p/enrcmd.go

@@ -22,7 +22,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net"
 	"os"
 	"strconv"
@@ -34,27 +33,29 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var fileFlag = cli.StringFlag{Name: "file"}
+
 var enrdumpCommand = cli.Command{
 	Name:   "enrdump",
 	Usage:  "Pretty-prints node records",
 	Action: enrdump,
 	Flags: []cli.Flag{
-		cli.StringFlag{Name: "file"},
+		fileFlag,
 	},
 }
 
 func enrdump(ctx *cli.Context) error {
 	var source string
-	if file := ctx.String("file"); file != "" {
+	if file := ctx.String(fileFlag.Name); file != "" {
 		if ctx.NArg() != 0 {
 			return fmt.Errorf("can't dump record from command-line argument in -file mode")
 		}
 		var b []byte
 		var err error
 		if file == "-" {
-			b, err = ioutil.ReadAll(os.Stdin)
+			b, err = io.ReadAll(os.Stdin)
 		} else {
-			b, err = ioutil.ReadFile(file)
+			b, err = os.ReadFile(file)
 		}
 		if err != nil {
 			return err

cmd/devp2p/internal/ethtest/chain.go

@@ -21,7 +21,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/big"
 	"os"
 	"strings"
@@ -153,7 +152,7 @@ func loadChain(chainfile string, genesis string) (*Chain, error) {
 }
 
 func loadGenesis(genesisFile string) (core.Genesis, error) {
-	chainConfig, err := ioutil.ReadFile(genesisFile)
+	chainConfig, err := os.ReadFile(genesisFile)
 	if err != nil {
 		return core.Genesis{}, err
 	}

cmd/devp2p/internal/ethtest/snap.go

@@ -372,8 +372,8 @@ func (s *Suite) TestSnapTrieNodes(t *utesting.T) {
 		{
 			root: s.chain.RootAt(999),
 			paths: []snap.TrieNodePathSet{
-				snap.TrieNodePathSet{}, // zero-length pathset should 'abort' and kick us off
-				snap.TrieNodePathSet{[]byte{0}},
+				{}, // zero-length pathset should 'abort' and kick us off
+				{[]byte{0}},
 			},
 			nBytes:    5000,
 			expHashes: []common.Hash{},
@@ -382,8 +382,8 @@ func (s *Suite) TestSnapTrieNodes(t *utesting.T) {
 		{
 			root: s.chain.RootAt(999),
 			paths: []snap.TrieNodePathSet{
-				snap.TrieNodePathSet{[]byte{0}},
-				snap.TrieNodePathSet{[]byte{1}, []byte{0}},
+				{[]byte{0}},
+				{[]byte{1}, []byte{0}},
 			},
 			nBytes: 5000,
 			//0x6b3724a41b8c38b46d4d02fba2bb2074c47a507eb16a9a4b978f91d32e406faf
@@ -392,7 +392,7 @@ func (s *Suite) TestSnapTrieNodes(t *utesting.T) {
 		{ // nonsensically long path
 			root: s.chain.RootAt(999),
 			paths: []snap.TrieNodePathSet{
-				snap.TrieNodePathSet{[]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8,
+				{[]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8,
 					0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8}},
 			},
 			nBytes:    5000,
@@ -401,8 +401,8 @@ func (s *Suite) TestSnapTrieNodes(t *utesting.T) {
 		{
 			root: s.chain.RootAt(0),
 			paths: []snap.TrieNodePathSet{
-				snap.TrieNodePathSet{[]byte{0}},
-				snap.TrieNodePathSet{[]byte{1}, []byte{0}},
+				{[]byte{0}},
+				{[]byte{1}, []byte{0}},
 			},
 			nBytes:    5000,
 			expHashes: []common.Hash{},

cmd/devp2p/nodeset.go

@@ -20,7 +20,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"sort"
 	"time"
@@ -66,7 +65,7 @@ func writeNodesJSON(file string, nodes nodeSet) {
 		os.Stdout.Write(nodesJSON)
 		return
 	}
-	if err := ioutil.WriteFile(file, nodesJSON, 0644); err != nil {
+	if err := os.WriteFile(file, nodesJSON, 0644); err != nil {
 		exit(err)
 	}
 }

cmd/ethkey/changepassword.go

@@ -18,7 +18,7 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
+	"os"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/accounts/keystore"
@@ -45,7 +45,7 @@ Change the password of a keyfile.`,
 		keyfilepath := ctx.Args().First()
 
 		// Read key from file.
-		keyjson, err := ioutil.ReadFile(keyfilepath)
+		keyjson, err := os.ReadFile(keyfilepath)
 		if err != nil {
 			utils.Fatalf("Failed to read the keyfile at '%s': %v", keyfilepath, err)
 		}
@@ -61,7 +61,7 @@ Change the password of a keyfile.`,
 		fmt.Println("Please provide a new password")
 		var newPhrase string
 		if passFile := ctx.String(newPassphraseFlag.Name); passFile != "" {
-			content, err := ioutil.ReadFile(passFile)
+			content, err := os.ReadFile(passFile)
 			if err != nil {
 				utils.Fatalf("Failed to read new password file '%s': %v", passFile, err)
 			}
@@ -77,7 +77,7 @@ Change the password of a keyfile.`,
 		}
 
 		// Then write the new keyfile in place of the old one.
-		if err := ioutil.WriteFile(keyfilepath, newJson, 0600); err != nil {
+		if err := os.WriteFile(keyfilepath, newJson, 0600); err != nil {
 			utils.Fatalf("Error writing new keyfile to disk: %v", err)
 		}
 

cmd/ethkey/generate.go

@@ -19,7 +19,6 @@ package main
 import (
 	"crypto/ecdsa"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -35,6 +34,17 @@ type outputGenerate struct {
 	AddressEIP55 string
 }
 
+var (
+	privateKeyFlag = cli.StringFlag{
+		Name:  "privatekey",
+		Usage: "file containing a raw private key to encrypt",
+	}
+	lightKDFFlag = cli.BoolFlag{
+		Name:  "lightkdf",
+		Usage: "use less secure scrypt parameters",
+	}
+)
+
 var commandGenerate = cli.Command{
 	Name:      "generate",
 	Usage:     "generate new keyfile",
@@ -48,14 +58,8 @@ If you want to encrypt an existing private key, it can be specified by setting
 	Flags: []cli.Flag{
 		passphraseFlag,
 		jsonFlag,
-		cli.StringFlag{
-			Name:  "privatekey",
-			Usage: "file containing a raw private key to encrypt",
-		},
-		cli.BoolFlag{
-			Name:  "lightkdf",
-			Usage: "use less secure scrypt parameters",
-		},
+		privateKeyFlag,
+		lightKDFFlag,
 	},
 	Action: func(ctx *cli.Context) error {
 		// Check if keyfile path given and make sure it doesn't already exist.
@@ -71,7 +75,7 @@ If you want to encrypt an existing private key, it can be specified by setting
 
 		var privateKey *ecdsa.PrivateKey
 		var err error
-		if file := ctx.String("privatekey"); file != "" {
+		if file := ctx.String(privateKeyFlag.Name); file != "" {
 			// Load private key from file.
 			privateKey, err = crypto.LoadECDSA(file)
 			if err != nil {
@@ -99,7 +103,7 @@ If you want to encrypt an existing private key, it can be specified by setting
 		// Encrypt key with passphrase.
 		passphrase := getPassphrase(ctx, true)
 		scryptN, scryptP := keystore.StandardScryptN, keystore.StandardScryptP
-		if ctx.Bool("lightkdf") {
+		if ctx.Bool(lightKDFFlag.Name) {
 			scryptN, scryptP = keystore.LightScryptN, keystore.LightScryptP
 		}
 		keyjson, err := keystore.EncryptKey(key, passphrase, scryptN, scryptP)
@@ -111,7 +115,7 @@ If you want to encrypt an existing private key, it can be specified by setting
 		if err := os.MkdirAll(filepath.Dir(keyfilepath), 0700); err != nil {
 			utils.Fatalf("Could not create directory %s", filepath.Dir(keyfilepath))
 		}
-		if err := ioutil.WriteFile(keyfilepath, keyjson, 0600); err != nil {
+		if err := os.WriteFile(keyfilepath, keyjson, 0600); err != nil {
 			utils.Fatalf("Failed to write keyfile to %s: %v", keyfilepath, err)
 		}
 

cmd/ethkey/inspect.go

@@ -19,7 +19,7 @@ package main
 import (
 	"encoding/hex"
 	"fmt"
-	"io/ioutil"
+	"os"
 
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -33,6 +33,13 @@ type outputInspect struct {
 	PrivateKey string
 }
 
+var (
+	privateFlag = cli.BoolFlag{
+		Name:  "private",
+		Usage: "include the private key in the output",
+	}
+)
+
 var commandInspect = cli.Command{
 	Name:      "inspect",
 	Usage:     "inspect a keyfile",
@@ -45,16 +52,13 @@ make sure to use this feature with great caution!`,
 	Flags: []cli.Flag{
 		passphraseFlag,
 		jsonFlag,
-		cli.BoolFlag{
-			Name:  "private",
-			Usage: "include the private key in the output",
-		},
+		privateFlag,
 	},
 	Action: func(ctx *cli.Context) error {
 		keyfilepath := ctx.Args().First()
 
 		// Read key from file.
-		keyjson, err := ioutil.ReadFile(keyfilepath)
+		keyjson, err := os.ReadFile(keyfilepath)
 		if err != nil {
 			utils.Fatalf("Failed to read the keyfile at '%s': %v", keyfilepath, err)
 		}
@@ -67,7 +71,7 @@ make sure to use this feature with great caution!`,
 		}
 
 		// Output all relevant information we can retrieve.
-		showPrivate := ctx.Bool("private")
+		showPrivate := ctx.Bool(privateFlag.Name)
 		out := outputInspect{
 			Address: key.Address.Hex(),
 			PublicKey: hex.EncodeToString(

cmd/ethkey/message.go

@@ -19,7 +19,7 @@ package main
 import (
 	"encoding/hex"
 	"fmt"
-	"io/ioutil"
+	"os"
 
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -56,7 +56,7 @@ To sign a message contained in a file, use the --msgfile flag.
 
 		// Load the keyfile.
 		keyfilepath := ctx.Args().First()
-		keyjson, err := ioutil.ReadFile(keyfilepath)
+		keyjson, err := os.ReadFile(keyfilepath)
 		if err != nil {
 			utils.Fatalf("Failed to read the keyfile at '%s': %v", keyfilepath, err)
 		}
@@ -142,11 +142,11 @@ It is possible to refer to a file containing the message.`,
 }
 
 func getMessage(ctx *cli.Context, msgarg int) []byte {
-	if file := ctx.String("msgfile"); file != "" {
+	if file := ctx.String(msgfileFlag.Name); file != "" {
 		if len(ctx.Args()) > msgarg {
 			utils.Fatalf("Can't use --msgfile and message argument at the same time.")
 		}
-		msg, err := ioutil.ReadFile(file)
+		msg, err := os.ReadFile(file)
 		if err != nil {
 			utils.Fatalf("Can't read message file: %v", err)
 		}

cmd/ethkey/message_test.go

@@ -17,18 +17,12 @@
 package main
 
 import (
-	"io/ioutil"
-	"os"
 	"path/filepath"
 	"testing"
 )
 
 func TestMessageSignVerify(t *testing.T) {
-	tmpdir, err := ioutil.TempDir("", "ethkey-test")
-	if err != nil {
-		t.Fatal("Can't create temporary directory:", err)
-	}
-	defer os.RemoveAll(tmpdir)
+	tmpdir := t.TempDir()
 
 	keyfile := filepath.Join(tmpdir, "the-keyfile")
 	message := "test message"

cmd/ethkey/utils.go

@@ -19,7 +19,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -34,7 +34,7 @@ func getPassphrase(ctx *cli.Context, confirmation bool) string {
 	// Look for the --passwordfile flag.
 	passphraseFile := ctx.String(passphraseFlag.Name)
 	if passphraseFile != "" {
-		content, err := ioutil.ReadFile(passphraseFile)
+		content, err := os.ReadFile(passphraseFile)
 		if err != nil {
 			utils.Fatalf("Failed to read password file '%s': %v",
 				passphraseFile, err)

cmd/evm/compiler.go

@@ -19,7 +19,7 @@ package main
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"os"
 
 	"github.com/ethereum/go-ethereum/cmd/evm/internal/compiler"
 
@@ -41,7 +41,7 @@ func compileCmd(ctx *cli.Context) error {
 	}
 
 	fn := ctx.Args().First()
-	src, err := ioutil.ReadFile(fn)
+	src, err := os.ReadFile(fn)
 	if err != nil {
 		return err
 	}

cmd/evm/disasm.go

@@ -19,7 +19,7 @@ package main
 import (
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/core/asm"
@@ -38,7 +38,7 @@ func disasmCmd(ctx *cli.Context) error {
 	switch {
 	case len(ctx.Args().First()) > 0:
 		fn := ctx.Args().First()
-		input, err := ioutil.ReadFile(fn)
+		input, err := os.ReadFile(fn)
 		if err != nil {
 			return err
 		}

cmd/evm/internal/t8ntool/block.go

@@ -36,7 +36,7 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
-//go:generate gencodec -type header -field-override headerMarshaling -out gen_header.go
+//go:generate go run github.com/fjl/gencodec -type header -field-override headerMarshaling -out gen_header.go
 type header struct {
 	ParentHash  common.Hash       `json:"parentHash"`
 	OmmerHash   *common.Hash      `json:"sha3Uncles"`

cmd/evm/internal/t8ntool/execution.go

@@ -63,7 +63,7 @@ type ommer struct {
 	Address common.Address `json:"address"`
 }
 
-//go:generate gencodec -type stEnv -field-override stEnvMarshaling -out gen_stenv.go
+//go:generate go run github.com/fjl/gencodec -type stEnv -field-override stEnvMarshaling -out gen_stenv.go
 type stEnv struct {
 	Coinbase         common.Address                      `json:"currentCoinbase"   gencodec:"required"`
 	Difficulty       *big.Int                            `json:"currentDifficulty"`

cmd/evm/internal/t8ntool/flags.go

@@ -152,7 +152,7 @@ var (
 			"\n\tSyntax <forkname>(+ExtraEip)",
 			strings.Join(tests.AvailableForks(), "\n\t    "),
 			strings.Join(vm.ActivateableEips(), ", ")),
-		Value: "Istanbul",
+		Value: "ArrowGlacier",
 	}
 	VerbosityFlag = cli.IntFlag{
 		Name:  "verbosity",

cmd/evm/internal/t8ntool/transition.go

@@ -21,7 +21,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"math/big"
 	"os"
 	"path"
@@ -252,11 +251,21 @@ func Transition(ctx *cli.Context) error {
 			return NewError(ErrorConfig, errors.New("EIP-1559 config but missing 'currentBaseFee' in env section"))
 		}
 	}
-	// Sanity check, to not `panic` in state_transition
-	if prestate.Env.Random != nil && !chainConfig.IsLondon(big.NewInt(int64(prestate.Env.Number))) {
-		return NewError(ErrorConfig, errors.New("can only apply RANDOM on top of London chainrules"))
-	}
-	if env := prestate.Env; env.Difficulty == nil {
+	isMerged := chainConfig.TerminalTotalDifficulty != nil && chainConfig.TerminalTotalDifficulty.BitLen() == 0
+	env := prestate.Env
+	if isMerged {
+		// post-merge:
+		// - random must be supplied
+		// - difficulty must be zero
+		switch {
+		case env.Random == nil:
+			return NewError(ErrorConfig, errors.New("post-merge requires currentRandom to be defined in env"))
+		case env.Difficulty != nil && env.Difficulty.BitLen() != 0:
+			return NewError(ErrorConfig, errors.New("post-merge difficulty must be zero (or omitted) in env"))
+		}
+		prestate.Env.Difficulty = nil
+	} else if env.Difficulty == nil {
+		// pre-merge:
 		// If difficulty was not provided by caller, we need to calculate it.
 		switch {
 		case env.ParentDifficulty == nil:
@@ -391,7 +400,7 @@ func saveFile(baseDir, filename string, data interface{}) error {
 		return NewError(ErrorJson, fmt.Errorf("failed marshalling output: %v", err))
 	}
 	location := path.Join(baseDir, filename)
-	if err = ioutil.WriteFile(location, b, 0644); err != nil {
+	if err = os.WriteFile(location, b, 0644); err != nil {
 		return NewError(ErrorIO, fmt.Errorf("failed writing output: %v", err))
 	}
 	log.Info("Wrote file", "file", location)

cmd/evm/runner.go

@@ -20,7 +20,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"math/big"
 	"os"
 	goruntime "runtime"
@@ -165,13 +165,13 @@ func runCmd(ctx *cli.Context) error {
 			// If - is specified, it means that code comes from stdin
 			if codeFileFlag == "-" {
 				//Try reading from stdin
-				if hexcode, err = ioutil.ReadAll(os.Stdin); err != nil {
+				if hexcode, err = io.ReadAll(os.Stdin); err != nil {
 					fmt.Printf("Could not load code from stdin: %v\n", err)
 					os.Exit(1)
 				}
 			} else {
 				// Codefile with hex assembly
-				if hexcode, err = ioutil.ReadFile(codeFileFlag); err != nil {
+				if hexcode, err = os.ReadFile(codeFileFlag); err != nil {
 					fmt.Printf("Could not load code from file: %v\n", err)
 					os.Exit(1)
 				}
@@ -187,7 +187,7 @@ func runCmd(ctx *cli.Context) error {
 		code = common.FromHex(string(hexcode))
 	} else if fn := ctx.Args().First(); len(fn) > 0 {
 		// EASM-file to compile
-		src, err := ioutil.ReadFile(fn)
+		src, err := os.ReadFile(fn)
 		if err != nil {
 			return err
 		}
@@ -239,14 +239,19 @@ func runCmd(ctx *cli.Context) error {
 	var hexInput []byte
 	if inputFileFlag := ctx.GlobalString(InputFileFlag.Name); inputFileFlag != "" {
 		var err error
-		if hexInput, err = ioutil.ReadFile(inputFileFlag); err != nil {
+		if hexInput, err = os.ReadFile(inputFileFlag); err != nil {
 			fmt.Printf("could not load input from file: %v\n", err)
 			os.Exit(1)
 		}
 	} else {
 		hexInput = []byte(ctx.GlobalString(InputFlag.Name))
 	}
-	input := common.FromHex(string(bytes.TrimSpace(hexInput)))
+	hexInput = bytes.TrimSpace(hexInput)
+	if len(hexInput)%2 != 0 {
+		fmt.Println("input length must be even")
+		os.Exit(1)
+	}
+	input := common.FromHex(string(hexInput))
 
 	var execFunc func() ([]byte, uint64, error)
 	if ctx.GlobalBool(CreateFlag.Name) {

cmd/evm/staterunner.go

@@ -20,7 +20,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 
 	"github.com/ethereum/go-ethereum/core/state"
@@ -81,7 +80,7 @@ func stateTestCmd(ctx *cli.Context) error {
 		debugger = logger.NewStructLogger(config)
 	}
 	// Load the test content from the input file
-	src, err := ioutil.ReadFile(ctx.Args().First())
+	src, err := os.ReadFile(ctx.Args().First())
 	if err != nil {
 		return err
 	}

cmd/evm/t8n_test.go

@@ -203,6 +203,22 @@ func TestT8n(t *testing.T) {
 			output: t8nOutput{result: true},
 			expOut: "exp.json",
 		},
+		{ // Test post-merge transition
+			base: "./testdata/24",
+			input: t8nInput{
+				"alloc.json", "txs.json", "env.json", "Merged", "",
+			},
+			output: t8nOutput{alloc: true, result: true},
+			expOut: "exp.json",
+		},
+		{ // Test post-merge transition where input is missing random
+			base: "./testdata/24",
+			input: t8nInput{
+				"alloc.json", "txs.json", "env-missingrandom.json", "Merged", "",
+			},
+			output:      t8nOutput{alloc: false, result: false},
+			expExitCode: 3,
+		},
 	} {
 
 		args := []string{"t8n"}

cmd/evm/testdata/15/exp3.json

@@ -21,19 +21,19 @@
     "error": "transaction type not supported"
   },
   {
-    "error": "rlp: expected List"
+    "error": "typed transaction too short"
   },
   {
-    "error": "rlp: expected List"
+    "error": "typed transaction too short"
   },
   {
-    "error": "rlp: expected List"
+    "error": "typed transaction too short"
   },
   {
-    "error": "rlp: expected List"
+    "error": "typed transaction too short"
   },
   {
-    "error": "rlp: expected List"
+    "error": "typed transaction too short"
   },
   {
     "error": "rlp: expected input list for types.AccessListTx"

cmd/evm/testdata/24/alloc.json

@@ -0,0 +1,14 @@
+{
+  "a94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+    "balance": "0x5ffd4878be161d74",
+    "code": "0x",
+    "nonce": "0xac",
+    "storage": {}
+  },
+  "0x8a8eafb1cf62bfbeb1741769dae1a9dd47996192":{
+    "balance": "0xfeedbead",
+    "nonce" : "0x00",
+    "code" : "0x44600055",
+    "_comment": "The code is 'sstore(0, random)'"
+  }
+}

cmd/evm/testdata/24/env-missingrandom.json

@@ -0,0 +1,9 @@
+{
+  "currentCoinbase": "0xc94f5374fce5edbc8e2a8697c15331677e6ebf0b",
+  "currentDifficulty": null,
+  "currentRandom": null,
+  "currentGasLimit": "0x750a163df65e8a",
+  "currentBaseFee": "0x500",
+  "currentNumber": "1",
+  "currentTimestamp": "1000"
+}

cmd/evm/testdata/24/env.json

@@ -0,0 +1,9 @@
+{
+  "currentCoinbase": "0xc94f5374fce5edbc8e2a8697c15331677e6ebf0b",
+  "currentDifficulty": null,
+  "currentRandom": "0xdeadc0de",
+  "currentGasLimit": "0x750a163df65e8a",
+  "currentBaseFee": "0x500",
+  "currentNumber": "1",
+  "currentTimestamp": "1000"
+}

cmd/evm/testdata/24/exp.json

@@ -0,0 +1,53 @@
+{
+  "alloc": {
+    "0x8a8eafb1cf62bfbeb1741769dae1a9dd47996192": {
+      "code": "0x44600055",
+      "storage": {
+        "0x0000000000000000000000000000000000000000000000000000000000000000": "0x00000000000000000000000000000000000000000000000000000000deadc0de"
+      },
+      "balance": "0xfeedbeaf"
+    },
+    "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+      "balance": "0x5ffd4878b803f972",
+      "nonce": "0xae"
+    },
+    "0xc94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+      "balance": "0x1030600"
+    }
+  },
+  "result": {
+    "stateRoot": "0x9e4224c6bba343d5b0fdbe9200cc66a7ef2068240d901ae516e634c45a043c15",
+    "txRoot": "0x16cd3a7daa6686ceebadf53b7af2bc6919eccb730907f0e74a95a4423c209593",
+    "receiptsRoot": "0x22b85cda738345a9880260b2a71e144aab1ca9485f5db4fd251008350fc124c8",
+    "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+    "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+    "receipts": [
+      {
+        "root": "0x",
+        "status": "0x1",
+        "cumulativeGasUsed": "0xa861",
+        "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+        "logs": null,
+        "transactionHash": "0x92ea4a28224d033afb20e0cc2b290d4c7c2d61f6a4800a680e4e19ac962ee941",
+        "contractAddress": "0x0000000000000000000000000000000000000000",
+        "gasUsed": "0xa861",
+        "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+        "transactionIndex": "0x0"
+      },
+      {
+        "root": "0x",
+        "status": "0x1",
+        "cumulativeGasUsed": "0x10306",
+        "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+        "logs": null,
+        "transactionHash": "0x16b1d912f1d664f3f60f4e1b5f296f3c82a64a1a253117b4851d18bc03c4f1da",
+        "contractAddress": "0x0000000000000000000000000000000000000000",
+        "gasUsed": "0x5aa5",
+        "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+        "transactionIndex": "0x1"
+      }
+    ],
+    "currentDifficulty": null,
+    "gasUsed": "0x10306"
+  }
+}

cmd/evm/testdata/24/txs.json

@@ -0,0 +1,28 @@
+[
+  {
+    "gas": "0x186a0",
+    "gasPrice": "0x600",
+    "hash": "0x0557bacce3375c98d806609b8d5043072f0b6a8bae45ae5a67a00d3a1a18d673",
+    "input": "0x",
+    "nonce": "0xac",
+    "to": "0x8a8eafb1cf62bfbeb1741769dae1a9dd47996192",
+    "value": "0x1",
+    "v" : "0x0",
+    "r" : "0x0",
+    "s" : "0x0",
+    "secretKey" : "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8"
+  },
+  {
+    "gas": "0x186a0",
+    "gasPrice": "0x600",
+    "hash": "0x0557bacce3375c98d806609b8d5043072f0b6a8bae45ae5a67a00d3a1a18d673",
+    "input": "0x",
+    "nonce": "0xad",
+    "to": "0x8a8eafb1cf62bfbeb1741769dae1a9dd47996192",
+    "value": "0x1",
+    "v" : "0x0",
+    "r" : "0x0",
+    "s" : "0x0",
+    "secretKey" : "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8"
+  }
+]

cmd/faucet/README.md

@@ -10,9 +10,11 @@ The `faucet` is a single binary app (everything included) with all configuration
 
 First thing's first, the `faucet` needs to connect to an Ethereum network, for which it needs the necessary genesis and network infos. Each of the following flags must be set:
 
-- `--genesis` is a path to a file containin the network `genesis.json`
-- `--network` is the devp2p network id used during connection
-- `--bootnodes` is a list of `enode://` ids to join the network through
+- `-genesis` is a path to a file containin the network `genesis.json`. or using:
+  - `-goerli` with the faucet with Görli network config
+  - `-rinkeby` with the faucet with Rinkeby network config
+- `-network` is the devp2p network id used during connection
+- `-bootnodes` is a list of `enode://` ids to join the network through
 
 The `faucet` will use the `les` protocol to join the configured Ethereum network and will store its data in `$HOME/.faucet` (currently not configurable).
 
@@ -20,14 +22,14 @@ The `faucet` will use the `les` protocol to join the configured Ethereum network
 
 To be able to distribute funds, the `faucet` needs access to an already funded Ethereum account. This can be configured via:
 
-- `--account.json` is a path to the Ethereum account's JSON key file
-- `--account.pass` is a path to a text file with the decryption passphrase
+- `-account.json` is a path to the Ethereum account's JSON key file
+- `-account.pass` is a path to a text file with the decryption passphrase
 
 The faucet is able to distribute various amounts of Ether in exchange for various timeouts. These can be configured via:
 
-- `--faucet.amount` is the number of Ethers to send by default
-- `--faucet.minutes` is the time to wait before allowing a rerequest
-- `--faucet.tiers` is the funding tiers to support  (x3 time, x2.5 funds)
+- `-faucet.amount` is the number of Ethers to send by default
+- `-faucet.minutes` is the time to wait before allowing a rerequest
+- `-faucet.tiers` is the funding tiers to support  (x3 time, x2.5 funds)
 
 ## Sybil protection
 
@@ -35,13 +37,13 @@ To prevent the same user from exhausting funds in a loop, the `faucet` ties requ
 
 Captcha protection uses Google's invisible ReCaptcha, thus the `faucet` needs to run on a live domain. The domain needs to be registered in Google's systems to retrieve the captcha API token and secrets. After doing so, captcha protection may be enabled via:
 
-- `--captcha.token` is the API token for ReCaptcha
-- `--captcha.secret` is the API secret for ReCaptcha
+- `-captcha.token` is the API token for ReCaptcha
+- `-captcha.secret` is the API secret for ReCaptcha
 
 Sybil protection via Twitter requires an API key as of 15th December, 2020. To obtain it, a Twitter user must be upgraded to developer status and a new Twitter App deployed with it. The app's `Bearer` token is required by the faucet to retrieve tweet data:
 
-- `--twitter.token` is the Bearer token for `v2` API access
-- `--twitter.token.v1` is the Bearer token for `v1` API access
+- `-twitter.token` is the Bearer token for `v2` API access
+- `-twitter.token.v1` is the Bearer token for `v1` API access
 
 Sybil protection via Facebook uses the website to directly download post data thus does not currently require an API configuration. 
 

cmd/faucet/faucet.go

@@ -17,18 +17,16 @@
 // faucet is an Ether faucet backed by a light client.
 package main
 
-//go:generate go-bindata -nometadata -o website.go faucet.html
-//go:generate gofmt -w -s website.go
-
 import (
 	"bytes"
 	"context"
+	_ "embed"
 	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
 	"html/template"
-	"io/ioutil"
+	"io"
 	"math"
 	"math/big"
 	"net/http"
@@ -99,6 +97,9 @@ var (
 	gitDate   = "" // Git commit date YYYYMMDD of the release (set via linker flags)
 )
 
+//go:embed faucet.html
+var websiteTmpl string
+
 func main() {
 	// Parse the flags and set up the logger to print everything requested
 	flag.Parse()
@@ -130,13 +131,8 @@ func main() {
 			periods[i] = strings.TrimSuffix(periods[i], "s")
 		}
 	}
-	// Load up and render the faucet website
-	tmpl, err := Asset("faucet.html")
-	if err != nil {
-		log.Crit("Failed to load the faucet template", "err", err)
-	}
 	website := new(bytes.Buffer)
-	err = template.Must(template.New("").Parse(string(tmpl))).Execute(website, map[string]interface{}{
+	err := template.Must(template.New("").Parse(websiteTmpl)).Execute(website, map[string]interface{}{
 		"Network":   *netnameFlag,
 		"Amounts":   amounts,
 		"Periods":   periods,
@@ -147,7 +143,7 @@ func main() {
 		log.Crit("Failed to render the faucet template", "err", err)
 	}
 	// Load and parse the genesis block requested by the user
-	genesis, err := getGenesis(genesisFlag, *goerliFlag, *rinkebyFlag)
+	genesis, err := getGenesis(*genesisFlag, *goerliFlag, *rinkebyFlag)
 	if err != nil {
 		log.Crit("Failed to parse genesis config", "err", err)
 	}
@@ -161,14 +157,14 @@ func main() {
 		}
 	}
 	// Load up the account key and decrypt its password
-	blob, err := ioutil.ReadFile(*accPassFlag)
+	blob, err := os.ReadFile(*accPassFlag)
 	if err != nil {
 		log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err)
 	}
 	pass := strings.TrimSuffix(string(blob), "\n")
 
 	ks := keystore.NewKeyStore(filepath.Join(os.Getenv("HOME"), ".faucet", "keys"), keystore.StandardScryptN, keystore.StandardScryptP)
-	if blob, err = ioutil.ReadFile(*accJSONFlag); err != nil {
+	if blob, err = os.ReadFile(*accJSONFlag); err != nil {
 		log.Crit("Failed to read account key contents", "file", *accJSONFlag, "err", err)
 	}
 	acc, err := ks.Import(blob, pass, pass)
@@ -731,7 +727,7 @@ func authTwitter(url string, tokenV1, tokenV2 string) (string, string, string, c
 	}
 	username := parts[len(parts)-3]
 
-	body, err := ioutil.ReadAll(res.Body)
+	body, err := io.ReadAll(res.Body)
 	if err != nil {
 		return "", "", "", common.Address{}, err
 	}
@@ -857,7 +853,7 @@ func authFacebook(url string) (string, string, common.Address, error) {
 	}
 	defer res.Body.Close()
 
-	body, err := ioutil.ReadAll(res.Body)
+	body, err := io.ReadAll(res.Body)
 	if err != nil {
 		return "", "", common.Address{}, err
 	}
@@ -886,11 +882,11 @@ func authNoAuth(url string) (string, string, common.Address, error) {
 }
 
 // getGenesis returns a genesis based on input args
-func getGenesis(genesisFlag *string, goerliFlag bool, rinkebyFlag bool) (*core.Genesis, error) {
+func getGenesis(genesisFlag string, goerliFlag bool, rinkebyFlag bool) (*core.Genesis, error) {
 	switch {
-	case genesisFlag != nil:
+	case genesisFlag != "":
 		var genesis core.Genesis
-		err := common.LoadJSON(*genesisFlag, &genesis)
+		err := common.LoadJSON(genesisFlag, &genesis)
 		return &genesis, err
 	case goerliFlag:
 		return core.DefaultGoerliGenesisBlock(), nil

cmd/geth/accountcmd.go

@@ -18,7 +18,7 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
+	"os"
 
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
@@ -320,7 +320,7 @@ func importWallet(ctx *cli.Context) error {
 	if len(keyfile) == 0 {
 		utils.Fatalf("keyfile must be given as argument")
 	}
-	keyJSON, err := ioutil.ReadFile(keyfile)
+	keyJSON, err := os.ReadFile(keyfile)
 	if err != nil {
 		utils.Fatalf("Could not read wallet file: %v", err)
 	}

cmd/geth/accountcmd_test.go

@@ -17,7 +17,7 @@
 package main
 
 import (
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -33,7 +33,7 @@ import (
 // are copied into a temporary keystore directory.
 
 func tmpDatadirWithKeystore(t *testing.T) string {
-	datadir := tmpdir(t)
+	datadir := t.TempDir()
 	keystore := filepath.Join(datadir, "keystore")
 	source := filepath.Join("..", "..", "accounts", "keystore", "testdata", "keystore")
 	if err := cp.CopyAll(keystore, source); err != nil {
@@ -111,13 +111,13 @@ func TestAccountImport(t *testing.T) {
 }
 
 func importAccountWithExpect(t *testing.T, key string, expected string) {
-	dir := tmpdir(t)
+	dir := t.TempDir()
 	keyfile := filepath.Join(dir, "key.prv")
-	if err := ioutil.WriteFile(keyfile, []byte(key), 0600); err != nil {
+	if err := os.WriteFile(keyfile, []byte(key), 0600); err != nil {
 		t.Error(err)
 	}
 	passwordFile := filepath.Join(dir, "password.txt")
-	if err := ioutil.WriteFile(passwordFile, []byte("foobar"), 0600); err != nil {
+	if err := os.WriteFile(passwordFile, []byte("foobar"), 0600); err != nil {
 		t.Error(err)
 	}
 	geth := runGeth(t, "--lightkdf", "account", "import", keyfile, "-password", passwordFile)
@@ -162,7 +162,7 @@ Password: {{.InputLine "foo"}}
 Address: {d4584b5f6229b7be90727b0fc8c6b91bb427821f}
 `)
 
-	files, err := ioutil.ReadDir(filepath.Join(geth.Datadir, "keystore"))
+	files, err := os.ReadDir(filepath.Join(geth.Datadir, "keystore"))
 	if len(files) != 1 {
 		t.Errorf("expected one key file in keystore directory, found %d files (error: %v)", len(files), err)
 	}

cmd/geth/chaincmd.go

@@ -47,10 +47,8 @@ var (
 		Name:      "init",
 		Usage:     "Bootstrap and initialize a new genesis block",
 		ArgsUsage: "<genesisPath>",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
-		},
-		Category: "BLOCKCHAIN COMMANDS",
+		Flags:     utils.DatabasePathFlags,
+		Category:  "BLOCKCHAIN COMMANDS",
 		Description: `
 The init command initializes a new genesis block and definition for the network.
 This is a destructive action and changes the network in which you will be
@@ -63,14 +61,8 @@ It expects the genesis file as argument.`,
 		Name:      "dumpgenesis",
 		Usage:     "Dumps genesis block JSON configuration to stdout",
 		ArgsUsage: "",
-		Flags: []cli.Flag{
-			utils.MainnetFlag,
-			utils.RopstenFlag,
-			utils.SepoliaFlag,
-			utils.RinkebyFlag,
-			utils.GoerliFlag,
-		},
-		Category: "BLOCKCHAIN COMMANDS",
+		Flags:     utils.NetworkFlags,
+		Category:  "BLOCKCHAIN COMMANDS",
 		Description: `
 The dumpgenesis command dumps the genesis block configuration in JSON format to stdout.`,
 	}
@@ -79,8 +71,7 @@ The dumpgenesis command dumps the genesis block configuration in JSON format to
 		Name:      "import",
 		Usage:     "Import a blockchain file",
 		ArgsUsage: "<filename> (<filename 2> ... <filename N>) ",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
+		Flags: append([]cli.Flag{
 			utils.CacheFlag,
 			utils.SyncModeFlag,
 			utils.GCModeFlag,
@@ -102,7 +93,7 @@ The dumpgenesis command dumps the genesis block configuration in JSON format to
 			utils.MetricsInfluxDBBucketFlag,
 			utils.MetricsInfluxDBOrganizationFlag,
 			utils.TxLookupLimitFlag,
-		},
+		}, utils.DatabasePathFlags...),
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 The import command imports blocks from an RLP-encoded form. The form can be one file
@@ -116,11 +107,10 @@ processing will proceed even if an individual RLP-file import failure occurs.`,
 		Name:      "export",
 		Usage:     "Export blockchain into file",
 		ArgsUsage: "<filename> [<blockNumFirst> <blockNumLast>]",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
+		Flags: append([]cli.Flag{
 			utils.CacheFlag,
 			utils.SyncModeFlag,
-		},
+		}, utils.DatabasePathFlags...),
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 Requires a first argument of the file to write to.
@@ -134,11 +124,10 @@ be gzipped.`,
 		Name:      "import-preimages",
 		Usage:     "Import the preimage database from an RLP stream",
 		ArgsUsage: "<datafile>",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
+		Flags: append([]cli.Flag{
 			utils.CacheFlag,
 			utils.SyncModeFlag,
-		},
+		}, utils.DatabasePathFlags...),
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 The import-preimages command imports hash preimages from an RLP encoded stream.
@@ -150,11 +139,10 @@ It's deprecated, please use "geth db import" instead.
 		Name:      "export-preimages",
 		Usage:     "Export the preimage database into an RLP stream",
 		ArgsUsage: "<dumpfile>",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
+		Flags: append([]cli.Flag{
 			utils.CacheFlag,
 			utils.SyncModeFlag,
-		},
+		}, utils.DatabasePathFlags...),
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 The export-preimages command exports hash preimages to an RLP encoded stream.
@@ -166,8 +154,7 @@ It's deprecated, please use "geth db export" instead.
 		Name:      "dump",
 		Usage:     "Dump a specific block from storage",
 		ArgsUsage: "[? <blockHash> | <blockNum>]",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
+		Flags: append([]cli.Flag{
 			utils.CacheFlag,
 			utils.IterativeOutputFlag,
 			utils.ExcludeCodeFlag,
@@ -175,7 +162,7 @@ It's deprecated, please use "geth db export" instead.
 			utils.IncludeIncompletesFlag,
 			utils.StartKeyFlag,
 			utils.DumpLimitFlag,
-		},
+		}, utils.DatabasePathFlags...),
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 This command dumps out the state for a given block (or latest, if none provided).
@@ -204,9 +191,8 @@ func initGenesis(ctx *cli.Context) error {
 	// Open and initialise both full and light databases
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
-
 	for _, name := range []string{"chaindata", "lightchaindata"} {
-		chaindb, err := stack.OpenDatabase(name, 0, 0, "", false)
+		chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.GlobalString(utils.AncientFlag.Name), "", false)
 		if err != nil {
 			utils.Fatalf("Failed to open database: %v", err)
 		}

cmd/geth/config.go

@@ -32,6 +32,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts/scwallet"
 	"github.com/ethereum/go-ethereum/accounts/usbwallet"
 	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/log"
@@ -47,7 +48,7 @@ var (
 		Name:        "dumpconfig",
 		Usage:       "Show configuration values",
 		ArgsUsage:   "",
-		Flags:       append(nodeFlags, rpcFlags...),
+		Flags:       utils.GroupFlags(nodeFlags, rpcFlags),
 		Category:    "MISCELLANEOUS COMMANDS",
 		Description: `The dumpconfig command shows configuration values.`,
 	}
@@ -159,9 +160,25 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 		cfg.Eth.OverrideArrowGlacier = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideArrowGlacierFlag.Name))
 	}
 	if ctx.GlobalIsSet(utils.OverrideTerminalTotalDifficulty.Name) {
-		cfg.Eth.OverrideTerminalTotalDifficulty = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideTerminalTotalDifficulty.Name))
+		cfg.Eth.OverrideTerminalTotalDifficulty = utils.GlobalBig(ctx, utils.OverrideTerminalTotalDifficulty.Name)
+	}
+	backend, eth := utils.RegisterEthService(stack, &cfg.Eth)
+	// Warn users to migrate if they have a legacy freezer format.
+	if eth != nil {
+		firstIdx := uint64(0)
+		// Hack to speed up check for mainnet because we know
+		// the first non-empty block.
+		ghash := rawdb.ReadCanonicalHash(eth.ChainDb(), 0)
+		if cfg.Eth.NetworkId == 1 && ghash == params.MainnetGenesisHash {
+			firstIdx = 46147
+		}
+		isLegacy, _, err := dbHasLegacyReceipts(eth.ChainDb(), firstIdx)
+		if err != nil {
+			log.Error("Failed to check db for legacy receipts", "err", err)
+		} else if isLegacy {
+			log.Warn("Database has receipts with a legacy format. Please run `geth db freezer-migrate`.")
+		}
 	}
-	backend, _ := utils.RegisterEthService(stack, &cfg.Eth)
 
 	// Configure GraphQL if requested
 	if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) {

cmd/geth/consolecmd.go

@@ -18,11 +18,11 @@ package main
 
 import (
 	"fmt"
-	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -36,7 +36,7 @@ var (
 		Action:   utils.MigrateFlags(localConsole),
 		Name:     "console",
 		Usage:    "Start an interactive JavaScript environment",
-		Flags:    append(append(nodeFlags, rpcFlags...), consoleFlags...),
+		Flags:    utils.GroupFlags(nodeFlags, rpcFlags, consoleFlags),
 		Category: "CONSOLE COMMANDS",
 		Description: `
 The Geth console is an interactive shell for the JavaScript runtime environment
@@ -49,7 +49,7 @@ See https://geth.ethereum.org/docs/interface/javascript-console.`,
 		Name:      "attach",
 		Usage:     "Start an interactive JavaScript environment (connect to node)",
 		ArgsUsage: "[endpoint]",
-		Flags:     append(consoleFlags, utils.DataDirFlag),
+		Flags:     utils.GroupFlags([]cli.Flag{utils.DataDirFlag}, consoleFlags),
 		Category:  "CONSOLE COMMANDS",
 		Description: `
 The Geth console is an interactive shell for the JavaScript runtime environment
@@ -63,7 +63,7 @@ This command allows to open a console on a running geth node.`,
 		Name:      "js",
 		Usage:     "Execute the specified JavaScript files",
 		ArgsUsage: "<jsfile> [jsfile...]",
-		Flags:     append(nodeFlags, consoleFlags...),
+		Flags:     utils.GroupFlags(nodeFlags, consoleFlags),
 		Category:  "CONSOLE COMMANDS",
 		Description: `
 The JavaScript VM exposes a node admin interface as well as the Ðapp
@@ -130,7 +130,7 @@ func remoteConsole(ctx *cli.Context) error {
 				// Maintain compatibility with older Geth configurations storing the
 				// Ropsten database in `testnet` instead of `ropsten`.
 				legacyPath := filepath.Join(path, "testnet")
-				if _, err := os.Stat(legacyPath); !os.IsNotExist(err) {
+				if common.FileExist(legacyPath) {
 					path = legacyPath
 				} else {
 					path = filepath.Join(path, "ropsten")
@@ -141,6 +141,8 @@ func remoteConsole(ctx *cli.Context) error {
 				path = filepath.Join(path, "goerli")
 			} else if ctx.GlobalBool(utils.SepoliaFlag.Name) {
 				path = filepath.Join(path, "sepolia")
+			} else if ctx.GlobalBool(utils.KilnFlag.Name) {
+				path = filepath.Join(path, "kiln")
 			}
 		}
 		endpoint = fmt.Sprintf("%s/geth.ipc", path)

cmd/geth/consolecmd_test.go

@@ -19,7 +19,6 @@ package main
 import (
 	"crypto/rand"
 	"math/big"
-	"os"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -31,7 +30,7 @@ import (
 )
 
 const (
-	ipcAPIs  = "admin:1.0 debug:1.0 eth:1.0 ethash:1.0 miner:1.0 net:1.0 personal:1.0 rpc:1.0 txpool:1.0 web3:1.0"
+	ipcAPIs  = "admin:1.0 debug:1.0 engine:1.0 eth:1.0 ethash:1.0 miner:1.0 net:1.0 personal:1.0 rpc:1.0 txpool:1.0 web3:1.0"
 	httpAPIs = "eth:1.0 net:1.0 rpc:1.0 web3:1.0"
 )
 
@@ -43,7 +42,8 @@ func runMinimalGeth(t *testing.T, args ...string) *testgeth {
 	// --networkid=1337 to avoid cache bump
 	// --syncmode=full to avoid allocating fast sync bloom
 	allArgs := []string{"--ropsten", "--networkid", "1337", "--syncmode=full", "--port", "0",
-		"--nat", "none", "--nodiscover", "--maxpeers", "0", "--cache", "64"}
+		"--nat", "none", "--nodiscover", "--maxpeers", "0", "--cache", "64",
+		"--datadir.minfreedisk", "0"}
 	return runGeth(t, append(allArgs, args...)...)
 }
 
@@ -92,9 +92,7 @@ func TestAttachWelcome(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		ipc = `\\.\pipe\geth` + strconv.Itoa(trulyRandInt(100000, 999999))
 	} else {
-		ws := tmpdir(t)
-		defer os.RemoveAll(ws)
-		ipc = filepath.Join(ws, "geth.ipc")
+		ipc = filepath.Join(t.TempDir(), "geth.ipc")
 	}
 	// And HTTP + WS attachment
 	p := trulyRandInt(1024, 65533) // Yeah, sometimes this will fail, sorry :P
@@ -118,6 +116,7 @@ func TestAttachWelcome(t *testing.T) {
 		waitForEndpoint(t, endpoint, 3*time.Second)
 		testAttachWelcome(t, geth, endpoint, httpAPIs)
 	})
+	geth.ExpectExit()
 }
 
 func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) {

cmd/geth/dao_test.go

@@ -17,7 +17,6 @@
 package main
 
 import (
-	"io/ioutil"
 	"math/big"
 	"os"
 	"path/filepath"
@@ -106,13 +105,12 @@ func TestDAOForkBlockNewChain(t *testing.T) {
 
 func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBlock *big.Int, expectVote bool) {
 	// Create a temporary data directory to use and inspect later
-	datadir := tmpdir(t)
-	defer os.RemoveAll(datadir)
+	datadir := t.TempDir()
 
 	// Start a Geth instance with the requested flags set and immediately terminate
 	if genesis != "" {
 		json := filepath.Join(datadir, "genesis.json")
-		if err := ioutil.WriteFile(json, []byte(genesis), 0600); err != nil {
+		if err := os.WriteFile(json, []byte(genesis), 0600); err != nil {
 			t.Fatalf("test %d: failed to write genesis file: %v", test, err)
 		}
 		runGeth(t, "--datadir", datadir, "--networkid", "1337", "init", json).WaitExit()

cmd/geth/dbcmd.go

@@ -18,7 +18,6 @@ package main
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"os"
 	"os/signal"
@@ -35,6 +34,8 @@ import (
 	"github.com/ethereum/go-ethereum/console/prompt"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state/snapshot"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/trie"
@@ -48,10 +49,8 @@ var (
 		Name:      "removedb",
 		Usage:     "Remove blockchain and state databases",
 		ArgsUsage: "",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
-		},
-		Category: "DATABASE COMMANDS",
+		Flags:     utils.DatabasePathFlags,
+		Category:  "DATABASE COMMANDS",
 		Description: `
 Remove blockchain and state databases`,
 	}
@@ -72,54 +71,47 @@ Remove blockchain and state databases`,
 			dbImportCmd,
 			dbExportCmd,
 			dbMetadataCmd,
+			dbMigrateFreezerCmd,
+			dbCheckStateContentCmd,
 		},
 	}
 	dbInspectCmd = cli.Command{
 		Action:    utils.MigrateFlags(inspect),
 		Name:      "inspect",
 		ArgsUsage: "<prefix> <start>",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
-			utils.AncientFlag,
+		Flags: utils.GroupFlags([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.MainnetFlag,
-			utils.RopstenFlag,
-			utils.SepoliaFlag,
-			utils.RinkebyFlag,
-			utils.GoerliFlag,
-		},
+		}, utils.NetworkFlags, utils.DatabasePathFlags),
 		Usage:       "Inspect the storage size for each type of data in the database",
 		Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`,
 	}
+	dbCheckStateContentCmd = cli.Command{
+		Action:    utils.MigrateFlags(checkStateContent),
+		Name:      "check-state-content",
+		ArgsUsage: "<start (optional)>",
+		Flags:     utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags),
+		Usage:     "Verify that state data is cryptographically correct",
+		Description: `This command iterates the entire database for 32-byte keys, looking for rlp-encoded trie nodes.
+For each trie node encountered, it checks that the key corresponds to the keccak256(value). If this is not true, this indicates
+a data corruption.`,
+	}
 	dbStatCmd = cli.Command{
 		Action: utils.MigrateFlags(dbStats),
 		Name:   "stats",
 		Usage:  "Print leveldb statistics",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
+		Flags: utils.GroupFlags([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.MainnetFlag,
-			utils.RopstenFlag,
-			utils.SepoliaFlag,
-			utils.RinkebyFlag,
-			utils.GoerliFlag,
-		},
+		}, utils.NetworkFlags, utils.DatabasePathFlags),
 	}
 	dbCompactCmd = cli.Command{
 		Action: utils.MigrateFlags(dbCompact),
 		Name:   "compact",
 		Usage:  "Compact leveldb database. WARNING: May take a very long time",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
+		Flags: utils.GroupFlags([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.MainnetFlag,
-			utils.RopstenFlag,
-			utils.SepoliaFlag,
-			utils.RinkebyFlag,
-			utils.GoerliFlag,
 			utils.CacheFlag,
 			utils.CacheDatabaseFlag,
-		},
+		}, utils.NetworkFlags, utils.DatabasePathFlags),
 		Description: `This command performs a database compaction. 
 WARNING: This operation may take a very long time to finish, and may cause database
 corruption if it is aborted during execution'!`,
@@ -129,15 +121,9 @@ corruption if it is aborted during execution'!`,
 		Name:      "get",
 		Usage:     "Show the value of a database key",
 		ArgsUsage: "<hex-encoded key>",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
+		Flags: utils.GroupFlags([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.MainnetFlag,
-			utils.RopstenFlag,
-			utils.SepoliaFlag,
-			utils.RinkebyFlag,
-			utils.GoerliFlag,
-		},
+		}, utils.NetworkFlags, utils.DatabasePathFlags),
 		Description: "This command looks up the specified database key from the database.",
 	}
 	dbDeleteCmd = cli.Command{
@@ -145,15 +131,9 @@ corruption if it is aborted during execution'!`,
 		Name:      "delete",
 		Usage:     "Delete a database key (WARNING: may corrupt your database)",
 		ArgsUsage: "<hex-encoded key>",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
+		Flags: utils.GroupFlags([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.MainnetFlag,
-			utils.RopstenFlag,
-			utils.SepoliaFlag,
-			utils.RinkebyFlag,
-			utils.GoerliFlag,
-		},
+		}, utils.NetworkFlags, utils.DatabasePathFlags),
 		Description: `This command deletes the specified database key from the database. 
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
@@ -162,15 +142,9 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Name:      "put",
 		Usage:     "Set the value of a database key (WARNING: may corrupt your database)",
 		ArgsUsage: "<hex-encoded key> <hex-encoded value>",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
+		Flags: utils.GroupFlags([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.MainnetFlag,
-			utils.RopstenFlag,
-			utils.SepoliaFlag,
-			utils.RinkebyFlag,
-			utils.GoerliFlag,
-		},
+		}, utils.NetworkFlags, utils.DatabasePathFlags),
 		Description: `This command sets a given database key to the given value. 
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
@@ -179,15 +153,9 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Name:      "dumptrie",
 		Usage:     "Show the storage key/values of a given storage trie",
 		ArgsUsage: "<hex-encoded storage trie root> <hex-encoded start (optional)> <int max elements (optional)>",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
+		Flags: utils.GroupFlags([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.MainnetFlag,
-			utils.RopstenFlag,
-			utils.SepoliaFlag,
-			utils.RinkebyFlag,
-			utils.GoerliFlag,
-		},
+		}, utils.NetworkFlags, utils.DatabasePathFlags),
 		Description: "This command looks up the specified database key from the database.",
 	}
 	dbDumpFreezerIndex = cli.Command{
@@ -195,15 +163,9 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Name:      "freezer-index",
 		Usage:     "Dump out the index of a given freezer type",
 		ArgsUsage: "<type> <start (int)> <end (int)>",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
+		Flags: utils.GroupFlags([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.MainnetFlag,
-			utils.RopstenFlag,
-			utils.SepoliaFlag,
-			utils.RinkebyFlag,
-			utils.GoerliFlag,
-		},
+		}, utils.NetworkFlags, utils.DatabasePathFlags),
 		Description: "This command displays information about the freezer index.",
 	}
 	dbImportCmd = cli.Command{
@@ -211,14 +173,9 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Name:      "import",
 		Usage:     "Imports leveldb-data from an exported RLP dump.",
 		ArgsUsage: "<dumpfile> <start (optional)",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
+		Flags: utils.GroupFlags([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.MainnetFlag,
-			utils.RopstenFlag,
-			utils.RinkebyFlag,
-			utils.GoerliFlag,
-		},
+		}, utils.NetworkFlags, utils.DatabasePathFlags),
 		Description: "The import command imports the specific chain data from an RLP encoded stream.",
 	}
 	dbExportCmd = cli.Command{
@@ -226,31 +183,31 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Name:      "export",
 		Usage:     "Exports the chain data into an RLP dump. If the <dumpfile> has .gz suffix, gzip compression will be used.",
 		ArgsUsage: "<type> <dumpfile>",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
+		Flags: utils.GroupFlags([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.MainnetFlag,
-			utils.RopstenFlag,
-			utils.RinkebyFlag,
-			utils.GoerliFlag,
-		},
+		}, utils.NetworkFlags, utils.DatabasePathFlags),
 		Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.",
 	}
 	dbMetadataCmd = cli.Command{
 		Action: utils.MigrateFlags(showMetaData),
 		Name:   "metadata",
 		Usage:  "Shows metadata about the chain status.",
-		Flags: []cli.Flag{
-			utils.DataDirFlag,
+		Flags: utils.GroupFlags([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.MainnetFlag,
-			utils.RopstenFlag,
-			utils.SepoliaFlag,
-			utils.RinkebyFlag,
-			utils.GoerliFlag,
-		},
+		}, utils.NetworkFlags, utils.DatabasePathFlags),
 		Description: "Shows metadata about the chain status.",
 	}
+	dbMigrateFreezerCmd = cli.Command{
+		Action:    utils.MigrateFlags(freezerMigrate),
+		Name:      "freezer-migrate",
+		Usage:     "Migrate legacy parts of the freezer. (WARNING: may take a long time)",
+		ArgsUsage: "",
+		Flags: utils.GroupFlags([]cli.Flag{
+			utils.SyncModeFlag,
+		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		Description: `The freezer-migrate command checks your database for receipts in a legacy format and updates those.
+WARNING: please back-up the receipt files in your ancients before running this command.`,
+	}
 )
 
 func removeDB(ctx *cli.Context) error {
@@ -344,7 +301,61 @@ func inspect(ctx *cli.Context) error {
 	return rawdb.InspectDatabase(db, prefix, start)
 }
 
-func showLeveldbStats(db ethdb.Stater) {
+func checkStateContent(ctx *cli.Context) error {
+	var (
+		prefix []byte
+		start  []byte
+	)
+	if ctx.NArg() > 1 {
+		return fmt.Errorf("Max 1 argument: %v", ctx.Command.ArgsUsage)
+	}
+	if ctx.NArg() > 0 {
+		if d, err := hexutil.Decode(ctx.Args().First()); err != nil {
+			return fmt.Errorf("failed to hex-decode 'start': %v", err)
+		} else {
+			start = d
+		}
+	}
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+	var (
+		it        = rawdb.NewKeyLengthIterator(db.NewIterator(prefix, start), 32)
+		hasher    = crypto.NewKeccakState()
+		got       = make([]byte, 32)
+		errs      int
+		count     int
+		startTime = time.Now()
+		lastLog   = time.Now()
+	)
+	for it.Next() {
+		count++
+		v := it.Value()
+		k := it.Key()
+		hasher.Reset()
+		hasher.Write(v)
+		hasher.Read(got)
+		if !bytes.Equal(k, got) {
+			errs++
+			fmt.Printf("Error at 0x%x\n", k)
+			fmt.Printf("  Hash:  0x%x\n", got)
+			fmt.Printf("  Data:  0x%x\n", v)
+		}
+		if time.Since(lastLog) > 8*time.Second {
+			log.Info("Iterating the database", "at", fmt.Sprintf("%#x", k), "elapsed", common.PrettyDuration(time.Since(startTime)))
+			lastLog = time.Now()
+		}
+	}
+	if err := it.Error(); err != nil {
+		return err
+	}
+	log.Info("Iterated the state content", "errors", errs, "items", count)
+	return nil
+}
+
+func showLeveldbStats(db ethdb.KeyValueStater) {
 	if stats, err := db.Stat("leveldb.stats"); err != nil {
 		log.Warn("Failed to read database stats", "error", err)
 	} else {
@@ -399,7 +410,7 @@ func dbGet(ctx *cli.Context) error {
 	db := utils.MakeChainDatabase(ctx, stack, true)
 	defer db.Close()
 
-	key, err := parseHexOrString(ctx.Args().Get(0))
+	key, err := common.ParseHexOrString(ctx.Args().Get(0))
 	if err != nil {
 		log.Info("Could not decode the key", "error", err)
 		return err
@@ -425,7 +436,7 @@ func dbDelete(ctx *cli.Context) error {
 	db := utils.MakeChainDatabase(ctx, stack, false)
 	defer db.Close()
 
-	key, err := parseHexOrString(ctx.Args().Get(0))
+	key, err := common.ParseHexOrString(ctx.Args().Get(0))
 	if err != nil {
 		log.Info("Could not decode the key", "error", err)
 		return err
@@ -458,7 +469,7 @@ func dbPut(ctx *cli.Context) error {
 		data  []byte
 		err   error
 	)
-	key, err = parseHexOrString(ctx.Args().Get(0))
+	key, err = common.ParseHexOrString(ctx.Args().Get(0))
 	if err != nil {
 		log.Info("Could not decode the key", "error", err)
 		return err
@@ -565,15 +576,6 @@ func freezerInspect(ctx *cli.Context) error {
 	return nil
 }
 
-// ParseHexOrString tries to hexdecode b, but if the prefix is missing, it instead just returns the raw bytes
-func parseHexOrString(str string) ([]byte, error) {
-	b, err := hexutil.Decode(str)
-	if errors.Is(err, hexutil.ErrMissingPrefix) {
-		return []byte(str), nil
-	}
-	return b, err
-}
-
 func importLDBdata(ctx *cli.Context) error {
 	start := 0
 	switch ctx.NArg() {
@@ -728,6 +730,9 @@ func showMetaData(ctx *cli.Context) error {
 		data = append(data, []string{"headBlock.Root", fmt.Sprintf("%v", b.Root())})
 		data = append(data, []string{"headBlock.Number", fmt.Sprintf("%d (0x%x)", b.Number(), b.Number())})
 	}
+	if b := rawdb.ReadSkeletonSyncStatus(db); b != nil {
+		data = append(data, []string{"SkeletonSyncStatus", string(b)})
+	}
 	if h := rawdb.ReadHeadHeader(db); h != nil {
 		data = append(data, []string{"headHeader.Hash", fmt.Sprintf("%v", h.Hash())})
 		data = append(data, []string{"headHeader.Root", fmt.Sprintf("%v", h.Root)})
@@ -750,3 +755,88 @@ func showMetaData(ctx *cli.Context) error {
 	table.Render()
 	return nil
 }
+
+func freezerMigrate(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	db := utils.MakeChainDatabase(ctx, stack, false)
+	defer db.Close()
+
+	// Check first block for legacy receipt format
+	numAncients, err := db.Ancients()
+	if err != nil {
+		return err
+	}
+	if numAncients < 1 {
+		log.Info("No receipts in freezer to migrate")
+		return nil
+	}
+
+	isFirstLegacy, firstIdx, err := dbHasLegacyReceipts(db, 0)
+	if err != nil {
+		return err
+	}
+	if !isFirstLegacy {
+		log.Info("No legacy receipts to migrate")
+		return nil
+	}
+
+	log.Info("Starting migration", "ancients", numAncients, "firstLegacy", firstIdx)
+	start := time.Now()
+	if err := db.MigrateTable("receipts", types.ConvertLegacyStoredReceipts); err != nil {
+		return err
+	}
+	if err := db.Close(); err != nil {
+		return err
+	}
+	log.Info("Migration finished", "duration", time.Since(start))
+
+	return nil
+}
+
+// dbHasLegacyReceipts checks freezer entries for legacy receipts. It stops at the first
+// non-empty receipt and checks its format. The index of this first non-empty element is
+// the second return parameter.
+func dbHasLegacyReceipts(db ethdb.Database, firstIdx uint64) (bool, uint64, error) {
+	// Check first block for legacy receipt format
+	numAncients, err := db.Ancients()
+	if err != nil {
+		return false, 0, err
+	}
+	if numAncients < 1 {
+		return false, 0, nil
+	}
+	if firstIdx >= numAncients {
+		return false, firstIdx, nil
+	}
+	var (
+		legacy       bool
+		blob         []byte
+		emptyRLPList = []byte{192}
+	)
+	// Find first block with non-empty receipt, only if
+	// the index is not already provided.
+	if firstIdx == 0 {
+		for i := uint64(0); i < numAncients; i++ {
+			blob, err = db.Ancient("receipts", i)
+			if err != nil {
+				return false, 0, err
+			}
+			if len(blob) == 0 {
+				continue
+			}
+			if !bytes.Equal(blob, emptyRLPList) {
+				firstIdx = i
+				break
+			}
+		}
+	}
+	// Is first non-empty receipt legacy?
+	first, err := db.Ancient("receipts", firstIdx)
+	if err != nil {
+		return false, 0, err
+	}
+	legacy, err = types.IsLegacyStoredReceipts(first)
+	return legacy, firstIdx, err
+}

cmd/geth/genesis_test.go

@@ -17,7 +17,6 @@
 package main
 
 import (
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
@@ -73,12 +72,11 @@ var customGenesisTests = []struct {
 func TestCustomGenesis(t *testing.T) {
 	for i, tt := range customGenesisTests {
 		// Create a temporary data directory to use and inspect later
-		datadir := tmpdir(t)
-		defer os.RemoveAll(datadir)
+		datadir := t.TempDir()
 
 		// Initialize the data directory with the custom genesis block
 		json := filepath.Join(datadir, "genesis.json")
-		if err := ioutil.WriteFile(json, []byte(tt.genesis), 0600); err != nil {
+		if err := os.WriteFile(json, []byte(tt.genesis), 0600); err != nil {
 			t.Fatalf("test %d: failed to write genesis file: %v", i, err)
 		}
 		runGeth(t, "--datadir", datadir, "init", json).WaitExit()

cmd/geth/main.go

@@ -58,13 +58,11 @@ var (
 	// The app that holds all commands and flags.
 	app = flags.NewApp(gitCommit, gitDate, "the go-ethereum command line interface")
 	// flags that configure the node
-	nodeFlags = []cli.Flag{
+	nodeFlags = utils.GroupFlags([]cli.Flag{
 		utils.IdentityFlag,
 		utils.UnlockedAccountFlag,
 		utils.PasswordFileFlag,
 		utils.BootnodesFlag,
-		utils.DataDirFlag,
-		utils.AncientFlag,
 		utils.MinFreeDiskSpaceFlag,
 		utils.KeyStoreDirFlag,
 		utils.ExternalSignerFlag,
@@ -107,7 +105,8 @@ var (
 		utils.UltraLightFractionFlag,
 		utils.UltraLightOnlyAnnounceFlag,
 		utils.LightNoSyncServeFlag,
-		utils.WhitelistFlag,
+		utils.EthRequiredBlocksFlag,
+		utils.LegacyWhitelistFlag,
 		utils.BloomFilterSizeFlag,
 		utils.CacheFlag,
 		utils.CacheDatabaseFlag,
@@ -118,6 +117,7 @@ var (
 		utils.CacheSnapshotFlag,
 		utils.CacheNoPrefetchFlag,
 		utils.CachePreimagesFlag,
+		utils.FDLimitFlag,
 		utils.ListenPortFlag,
 		utils.MaxPeersFlag,
 		utils.MaxPendingPeersFlag,
@@ -138,14 +138,9 @@ var (
 		utils.NodeKeyFileFlag,
 		utils.NodeKeyHexFlag,
 		utils.DNSDiscoveryFlag,
-		utils.MainnetFlag,
 		utils.DeveloperFlag,
 		utils.DeveloperPeriodFlag,
 		utils.DeveloperGasLimitFlag,
-		utils.RopstenFlag,
-		utils.SepoliaFlag,
-		utils.RinkebyFlag,
-		utils.GoerliFlag,
 		utils.VMEnableDebugFlag,
 		utils.NetworkIdFlag,
 		utils.EthStatsURLFlag,
@@ -157,13 +152,17 @@ var (
 		utils.GpoIgnoreGasPriceFlag,
 		utils.MinerNotifyFullFlag,
 		configFileFlag,
-	}
+	}, utils.NetworkFlags, utils.DatabasePathFlags)
 
 	rpcFlags = []cli.Flag{
 		utils.HTTPEnabledFlag,
 		utils.HTTPListenAddrFlag,
 		utils.HTTPPortFlag,
 		utils.HTTPCORSDomainFlag,
+		utils.AuthListenFlag,
+		utils.AuthPortFlag,
+		utils.AuthVirtualHostsFlag,
+		utils.JWTSecretFlag,
 		utils.HTTPVirtualHostsFlag,
 		utils.GraphQLEnabledFlag,
 		utils.GraphQLCORSDomainFlag,
@@ -242,11 +241,11 @@ func init() {
 	}
 	sort.Sort(cli.CommandsByName(app.Commands))
 
-	app.Flags = append(app.Flags, nodeFlags...)
-	app.Flags = append(app.Flags, rpcFlags...)
-	app.Flags = append(app.Flags, consoleFlags...)
-	app.Flags = append(app.Flags, debug.Flags...)
-	app.Flags = append(app.Flags, metricsFlags...)
+	app.Flags = utils.GroupFlags(nodeFlags,
+		rpcFlags,
+		consoleFlags,
+		debug.Flags,
+		metricsFlags)
 
 	app.Before = func(ctx *cli.Context) error {
 		return debug.Setup(ctx)
@@ -273,17 +272,35 @@ func prepare(ctx *cli.Context) {
 	case ctx.GlobalIsSet(utils.RopstenFlag.Name):
 		log.Info("Starting Geth on Ropsten testnet...")
 
-	case ctx.GlobalIsSet(utils.SepoliaFlag.Name):
-		log.Info("Starting Geth on Sepolia testnet...")
-
 	case ctx.GlobalIsSet(utils.RinkebyFlag.Name):
 		log.Info("Starting Geth on Rinkeby testnet...")
 
 	case ctx.GlobalIsSet(utils.GoerliFlag.Name):
 		log.Info("Starting Geth on Görli testnet...")
 
+	case ctx.GlobalIsSet(utils.SepoliaFlag.Name):
+		log.Info("Starting Geth on Sepolia testnet...")
+
+	case ctx.GlobalIsSet(utils.KilnFlag.Name):
+		log.Info("Starting Geth on Kiln testnet...")
+
 	case ctx.GlobalIsSet(utils.DeveloperFlag.Name):
 		log.Info("Starting Geth in ephemeral dev mode...")
+		log.Warn(`You are running Geth in --dev mode. Please note the following:
+
+  1. This mode is only intended for fast, iterative development without assumptions on
+     security or persistence.
+  2. The database is created in memory unless specified otherwise. Therefore, shutting down
+     your computer or losing power will wipe your entire block data and chain state for
+     your dev environment.
+  3. A random, pre-allocated developer account will be available and unlocked as
+     eth.coinbase, which can be used for testing. The random dev account is temporary,
+     stored on a ramdisk, and will be lost if your machine is restarted.
+  4. Mining is enabled by default. However, the client will only seal blocks if transactions
+     are pending in the mempool. The miner's minimum accepted gas price is 1.
+  5. Networking is disabled; there is no listen-address, the maximum number of peers is set
+     to 0, and discovery is disabled.
+`)
 
 	case !ctx.GlobalIsSet(utils.NetworkIdFlag.Name):
 		log.Info("Starting Geth on Ethereum mainnet...")
@@ -295,6 +312,7 @@ func prepare(ctx *cli.Context) {
 			!ctx.GlobalIsSet(utils.SepoliaFlag.Name) &&
 			!ctx.GlobalIsSet(utils.RinkebyFlag.Name) &&
 			!ctx.GlobalIsSet(utils.GoerliFlag.Name) &&
+			!ctx.GlobalIsSet(utils.KilnFlag.Name) &&
 			!ctx.GlobalIsSet(utils.DeveloperFlag.Name) {
 			// Nope, we're really on mainnet. Bump that cache up!
 			log.Info("Bumping default cache on mainnet", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 4096)

cmd/geth/run_test.go

@@ -19,7 +19,6 @@ package main
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"testing"
 	"time"
@@ -29,14 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 )
 
-func tmpdir(t *testing.T) string {
-	dir, err := ioutil.TempDir("", "geth-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	return dir
-}
-
 type testgeth struct {
 	*cmdtest.TestCmd
 
@@ -82,15 +73,9 @@ func runGeth(t *testing.T, args ...string) *testgeth {
 		}
 	}
 	if tt.Datadir == "" {
-		tt.Datadir = tmpdir(t)
-		tt.Cleanup = func() { os.RemoveAll(tt.Datadir) }
+		// The temporary datadir will be removed automatically if something fails below.
+		tt.Datadir = t.TempDir()
 		args = append([]string{"--datadir", tt.Datadir}, args...)
-		// Remove the temporary datadir if something fails below.
-		defer func() {
-			if t.Failed() {
-				tt.Cleanup()
-			}
-		}()
 	}
 
 	// Boot "geth". This actually runs the test binary but the TestMain

cmd/geth/snapshot.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"os"
 	"time"
 
@@ -31,6 +32,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/state/snapshot"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
@@ -58,16 +60,10 @@ var (
 				ArgsUsage: "<root>",
 				Action:    utils.MigrateFlags(pruneState),
 				Category:  "MISCELLANEOUS COMMANDS",
-				Flags: []cli.Flag{
-					utils.DataDirFlag,
-					utils.AncientFlag,
-					utils.RopstenFlag,
-					utils.SepoliaFlag,
-					utils.RinkebyFlag,
-					utils.GoerliFlag,
+				Flags: utils.GroupFlags([]cli.Flag{
 					utils.CacheTrieJournalFlag,
 					utils.BloomFilterSizeFlag,
-				},
+				}, utils.NetworkFlags, utils.DatabasePathFlags),
 				Description: `
 geth snapshot prune-state <state-root>
 will prune historical state data with the help of the state snapshot.
@@ -89,19 +85,24 @@ the trie clean cache with default directory will be deleted.
 				ArgsUsage: "<root>",
 				Action:    utils.MigrateFlags(verifyState),
 				Category:  "MISCELLANEOUS COMMANDS",
-				Flags: []cli.Flag{
-					utils.DataDirFlag,
-					utils.AncientFlag,
-					utils.RopstenFlag,
-					utils.SepoliaFlag,
-					utils.RinkebyFlag,
-					utils.GoerliFlag,
-				},
+				Flags:     utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags),
 				Description: `
 geth snapshot verify-state <state-root>
 will traverse the whole accounts and storages set based on the specified
 snapshot and recalculate the root hash of state for verification.
 In other words, this command does the snapshot to trie conversion.
+`,
+			},
+			{
+				Name:      "check-dangling-storage",
+				Usage:     "Check that there is no 'dangling' snap storage",
+				ArgsUsage: "<root>",
+				Action:    utils.MigrateFlags(checkDanglingStorage),
+				Category:  "MISCELLANEOUS COMMANDS",
+				Flags:     utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags),
+				Description: `
+geth snapshot check-dangling-storage <state-root> traverses the snap storage 
+data, and verifies that all snapshot storage data has a corresponding account. 
 `,
 			},
 			{
@@ -110,14 +111,7 @@ In other words, this command does the snapshot to trie conversion.
 				ArgsUsage: "<root>",
 				Action:    utils.MigrateFlags(traverseState),
 				Category:  "MISCELLANEOUS COMMANDS",
-				Flags: []cli.Flag{
-					utils.DataDirFlag,
-					utils.AncientFlag,
-					utils.RopstenFlag,
-					utils.SepoliaFlag,
-					utils.RinkebyFlag,
-					utils.GoerliFlag,
-				},
+				Flags:     utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags),
 				Description: `
 geth snapshot traverse-state <state-root>
 will traverse the whole state from the given state root and will abort if any
@@ -133,14 +127,7 @@ It's also usable without snapshot enabled.
 				ArgsUsage: "<root>",
 				Action:    utils.MigrateFlags(traverseRawState),
 				Category:  "MISCELLANEOUS COMMANDS",
-				Flags: []cli.Flag{
-					utils.DataDirFlag,
-					utils.AncientFlag,
-					utils.RopstenFlag,
-					utils.SepoliaFlag,
-					utils.RinkebyFlag,
-					utils.GoerliFlag,
-				},
+				Flags:     utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags),
 				Description: `
 geth snapshot traverse-rawstate <state-root>
 will traverse the whole state from the given root and will abort if any referenced
@@ -157,18 +144,12 @@ It's also usable without snapshot enabled.
 				ArgsUsage: "[? <blockHash> | <blockNum>]",
 				Action:    utils.MigrateFlags(dumpState),
 				Category:  "MISCELLANEOUS COMMANDS",
-				Flags: []cli.Flag{
-					utils.DataDirFlag,
-					utils.AncientFlag,
-					utils.RopstenFlag,
-					utils.SepoliaFlag,
-					utils.RinkebyFlag,
-					utils.GoerliFlag,
+				Flags: utils.GroupFlags([]cli.Flag{
 					utils.ExcludeCodeFlag,
 					utils.ExcludeStorageFlag,
 					utils.StartKeyFlag,
 					utils.DumpLimitFlag,
-				},
+				}, utils.NetworkFlags, utils.DatabasePathFlags),
 				Description: `
 This command is semantically equivalent to 'geth dump', but uses the snapshots
 as the backend data source, making this command a lot faster. 
@@ -242,6 +223,72 @@ func verifyState(ctx *cli.Context) error {
 		return err
 	}
 	log.Info("Verified the state", "root", root)
+	if err := checkDanglingDiskStorage(chaindb); err != nil {
+		log.Error("Dangling snap disk-storage check failed", "root", root, "err", err)
+		return err
+	}
+	if err := checkDanglingMemStorage(chaindb); err != nil {
+		log.Error("Dangling snap mem-storage check failed", "root", root, "err", err)
+		return err
+	}
+	return nil
+}
+
+// checkDanglingStorage iterates the snap storage data, and verifies that all
+// storage also has corresponding account data.
+func checkDanglingStorage(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	chaindb := utils.MakeChainDatabase(ctx, stack, true)
+	if err := checkDanglingDiskStorage(chaindb); err != nil {
+		return err
+	}
+	return checkDanglingMemStorage(chaindb)
+
+}
+
+// checkDanglingDiskStorage checks if there is any 'dangling' storage data in the
+// disk-backed snapshot layer.
+func checkDanglingDiskStorage(chaindb ethdb.Database) error {
+	log.Info("Checking dangling snapshot disk storage")
+	var (
+		lastReport = time.Now()
+		start      = time.Now()
+		lastKey    []byte
+		it         = rawdb.NewKeyLengthIterator(chaindb.NewIterator(rawdb.SnapshotStoragePrefix, nil), 1+2*common.HashLength)
+	)
+	defer it.Release()
+	for it.Next() {
+		k := it.Key()
+		accKey := k[1:33]
+		if bytes.Equal(accKey, lastKey) {
+			// No need to look up for every slot
+			continue
+		}
+		lastKey = common.CopyBytes(accKey)
+		if time.Since(lastReport) > time.Second*8 {
+			log.Info("Iterating snap storage", "at", fmt.Sprintf("%#x", accKey), "elapsed", common.PrettyDuration(time.Since(start)))
+			lastReport = time.Now()
+		}
+		if data := rawdb.ReadAccountSnapshot(chaindb, common.BytesToHash(accKey)); len(data) == 0 {
+			log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", accKey), "storagekey", fmt.Sprintf("%#x", k))
+			return fmt.Errorf("dangling snapshot storage account %#x", accKey)
+		}
+	}
+	log.Info("Verified the snapshot disk storage", "time", common.PrettyDuration(time.Since(start)), "err", it.Error())
+	return nil
+}
+
+// checkDanglingMemStorage checks if there is any 'dangling' storage in the journalled
+// snapshot difflayers.
+func checkDanglingMemStorage(chaindb ethdb.Database) error {
+	start := time.Now()
+	log.Info("Checking dangling snapshot difflayer journalled storage")
+	if err := snapshot.CheckJournalStorage(chaindb); err != nil {
+		return err
+	}
+	log.Info("Verified the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start)))
 	return nil
 }
 
@@ -314,8 +361,7 @@ func traverseState(ctx *cli.Context) error {
 			}
 		}
 		if !bytes.Equal(acc.CodeHash, emptyCode) {
-			code := rawdb.ReadCode(chaindb, common.BytesToHash(acc.CodeHash))
-			if len(code) == 0 {
+			if !rawdb.HasCode(chaindb, common.BytesToHash(acc.CodeHash)) {
 				log.Error("Code is missing", "hash", common.BytesToHash(acc.CodeHash))
 				return errors.New("missing code")
 			}
@@ -386,11 +432,10 @@ func traverseRawState(ctx *cli.Context) error {
 		nodes += 1
 		node := accIter.Hash()
 
+		// Check the present for non-empty hash node(embedded node doesn't
+		// have their own hash).
 		if node != (common.Hash{}) {
-			// Check the present for non-empty hash node(embedded node doesn't
-			// have their own hash).
-			blob := rawdb.ReadTrieNode(chaindb, node)
-			if len(blob) == 0 {
+			if !rawdb.HasTrieNode(chaindb, node) {
 				log.Error("Missing trie node(account)", "hash", node)
 				return errors.New("missing account")
 			}
@@ -434,8 +479,7 @@ func traverseRawState(ctx *cli.Context) error {
 				}
 			}
 			if !bytes.Equal(acc.CodeHash, emptyCode) {
-				code := rawdb.ReadCode(chaindb, common.BytesToHash(acc.CodeHash))
-				if len(code) == 0 {
+				if !rawdb.HasCode(chaindb, common.BytesToHash(acc.CodeHash)) {
 					log.Error("Code is missing", "account", common.BytesToHash(accIter.LeafKey()))
 					return errors.New("missing code")
 				}

cmd/geth/testdata/vcheck/vulnerabilities.json

@@ -112,8 +112,59 @@
     ],
     "introduced": "v1.10.1",
     "fixed": "v1.10.6",
-    "published": "2020-12-10",
+    "published": "2021-07-22",
     "severity": "High",
     "check": "(Geth\\/v1\\.10\\.(1|2|3|4|5)-.*)$"
+  },
+  {
+    "name": "RETURNDATA corruption via datacopy",
+    "uid": "GETH-2021-02",
+    "summary": "A consensus-flaw in the Geth EVM could cause a node to deviate from the canonical chain.",
+    "description": "A memory-corruption bug within the EVM can cause a consensus error, where vulnerable nodes obtain a different `stateRoot` when processing a maliciously crafted transaction. This, in turn, would lead to the chain being split: mainnet splitting in two forks.\n\nAll Geth versions supporting the London hard fork are vulnerable (the bug is older than London), so all users should update.\n\nThis bug was exploited on Mainnet at block 13107518.\n\nCredits for the discovery go to @guidovranken (working for Sentnl during an audit of the Telos EVM) and reported via bounty@ethereum.org.",
+    "links": [
+      "https://github.com/ethereum/go-ethereum/blob/master/docs/postmortems/2021-08-22-split-postmortem.md",
+      "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-9856-9gg9-qcmq",
+      "https://github.com/ethereum/go-ethereum/releases/tag/v1.10.8"
+    ],
+    "introduced": "v1.10.0",
+    "fixed": "v1.10.8",
+    "published": "2021-08-24",
+    "severity": "High",
+    "CVE": "CVE-2021-39137",
+    "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7)-.*)$"
+  },
+  {
+    "name": "DoS via malicious `snap/1` request",
+    "uid": "GETH-2021-03",
+    "summary": "A vulnerable node is susceptible to crash when processing a maliciously crafted message from a peer, via the snap/1 protocol. The crash can be triggered by sending a malicious snap/1 GetTrieNodes package.",
+    "description": "The `snap/1` protocol handler contains two vulnerabilities related to the `GetTrieNodes` packet, which can be exploited to crash the node. Full details are available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-59hh-656j-3p7v)",
+    "links": [
+      "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-59hh-656j-3p7v",
+      "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities",
+      "https://github.com/ethereum/go-ethereum/pull/23657"
+    ],
+    "introduced": "v1.10.0",
+    "fixed": "v1.10.9",
+    "published": "2021-10-24",
+    "severity": "Medium",
+    "CVE": "CVE-2021-41173",
+    "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7|8)-.*)$"
+  },
+  {
+    "name": "DoS via malicious p2p message",
+    "uid": "GETH-2022-01",
+    "summary": "A vulnerable node can crash via p2p messages sent from an attacker node, if running with non-default log options.",
+    "description": "A vulnerable node, if configured to use high verbosity logging, can be made to crash when handling specially crafted p2p messages sent from an attacker node. Full details are available at the Github security [advisory](https://github.com/ethereum/go-ethereum/security/advisories/GHSA-wjxw-gh3m-7pm5)",
+    "links": [
+      "https://github.com/ethereum/go-ethereum/security/advisories/GHSA-wjxw-gh3m-7pm5",
+      "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities",
+      "https://github.com/ethereum/go-ethereum/pull/24507"
+    ],
+    "introduced": "v1.10.0",
+    "fixed": "v1.10.17",
+    "published": "2022-05-11",
+    "severity": "Low",
+    "CVE": "CVE-2022-29177",
+    "check": "(Geth\\/v1\\.10\\.(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16)-.*)$"
   }
 ]

cmd/geth/usage.go

@@ -32,20 +32,13 @@ import (
 var AppHelpFlagGroups = []flags.FlagGroup{
 	{
 		Name: "ETHEREUM",
-		Flags: []cli.Flag{
+		Flags: utils.GroupFlags([]cli.Flag{
 			configFileFlag,
-			utils.DataDirFlag,
-			utils.AncientFlag,
 			utils.MinFreeDiskSpaceFlag,
 			utils.KeyStoreDirFlag,
 			utils.USBFlag,
 			utils.SmartCardDaemonPathFlag,
 			utils.NetworkIdFlag,
-			utils.MainnetFlag,
-			utils.GoerliFlag,
-			utils.RinkebyFlag,
-			utils.RopstenFlag,
-			utils.SepoliaFlag,
 			utils.SyncModeFlag,
 			utils.ExitWhenSyncedFlag,
 			utils.GCModeFlag,
@@ -53,8 +46,8 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.EthStatsURLFlag,
 			utils.IdentityFlag,
 			utils.LightKDFFlag,
-			utils.WhitelistFlag,
-		},
+			utils.EthRequiredBlocksFlag,
+		}, utils.NetworkFlags, utils.DatabasePathFlags),
 	},
 	{
 		Name: "LIGHT CLIENT",
@@ -119,6 +112,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.CacheSnapshotFlag,
 			utils.CacheNoPrefetchFlag,
 			utils.CachePreimagesFlag,
+			utils.FDLimitFlag,
 		},
 	},
 	{
@@ -148,6 +142,10 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.WSApiFlag,
 			utils.WSPathPrefixFlag,
 			utils.WSAllowedOriginsFlag,
+			utils.JWTSecretFlag,
+			utils.AuthListenFlag,
+			utils.AuthPortFlag,
+			utils.AuthVirtualHostsFlag,
 			utils.GraphQLEnabledFlag,
 			utils.GraphQLCORSDomainFlag,
 			utils.GraphQLVirtualHostsFlag,
@@ -221,6 +219,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 		Name: "ALIASED (deprecated)",
 		Flags: []cli.Flag{
 			utils.NoUSBFlag,
+			utils.LegacyWhitelistFlag,
 		},
 	},
 	{

cmd/geth/version_check.go

@@ -20,8 +20,9 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
+	"os"
 	"regexp"
 	"strings"
 
@@ -112,14 +113,14 @@ func checkCurrent(url, current string) error {
 // fetch makes an HTTP request to the given url and returns the response body
 func fetch(url string) ([]byte, error) {
 	if filep := strings.TrimPrefix(url, "file://"); filep != url {
-		return ioutil.ReadFile(filep)
+		return os.ReadFile(filep)
 	}
 	res, err := http.Get(url)
 	if err != nil {
 		return nil, err
 	}
 	defer res.Body.Close()
-	body, err := ioutil.ReadAll(res.Body)
+	body, err := io.ReadAll(res.Body)
 	if err != nil {
 		return nil, err
 	}

cmd/geth/version_check_test.go

@@ -19,7 +19,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -49,17 +49,17 @@ func TestVerification(t *testing.T) {
 
 func testVerification(t *testing.T, pubkey, sigdir string) {
 	// Data to verify
-	data, err := ioutil.ReadFile("./testdata/vcheck/data.json")
+	data, err := os.ReadFile("./testdata/vcheck/data.json")
 	if err != nil {
 		t.Fatal(err)
 	}
 	// Signatures, with and without comments, both trusted and untrusted
-	files, err := ioutil.ReadDir(sigdir)
+	files, err := os.ReadDir(sigdir)
 	if err != nil {
 		t.Fatal(err)
 	}
 	for _, f := range files {
-		sig, err := ioutil.ReadFile(filepath.Join(sigdir, f.Name()))
+		sig, err := os.ReadFile(filepath.Join(sigdir, f.Name()))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -87,7 +87,7 @@ func versionUint(v string) int {
 
 // TestMatching can be used to check that the regexps are correct
 func TestMatching(t *testing.T) {
-	data, _ := ioutil.ReadFile("./testdata/vcheck/vulnerabilities.json")
+	data, _ := os.ReadFile("./testdata/vcheck/vulnerabilities.json")
 	var vulns []vulnJson
 	if err := json.Unmarshal(data, &vulns); err != nil {
 		t.Fatal(err)

cmd/p2psim/main.go

@@ -56,19 +56,58 @@ import (
 
 var client *simulations.Client
 
+var (
+	// global command flags
+	apiFlag = cli.StringFlag{
+		Name:   "api",
+		Value:  "http://localhost:8888",
+		Usage:  "simulation API URL",
+		EnvVar: "P2PSIM_API_URL",
+	}
+
+	// events subcommand flags
+	currentFlag = cli.BoolFlag{
+		Name:  "current",
+		Usage: "get existing nodes and conns first",
+	}
+	filterFlag = cli.StringFlag{
+		Name:  "filter",
+		Value: "",
+		Usage: "message filter",
+	}
+
+	// node create subcommand flags
+	nameFlag = cli.StringFlag{
+		Name:  "name",
+		Value: "",
+		Usage: "node name",
+	}
+	servicesFlag = cli.StringFlag{
+		Name:  "services",
+		Value: "",
+		Usage: "node services (comma separated)",
+	}
+	keyFlag = cli.StringFlag{
+		Name:  "key",
+		Value: "",
+		Usage: "node private key (hex encoded)",
+	}
+
+	// node rpc subcommand flags
+	subscribeFlag = cli.BoolFlag{
+		Name:  "subscribe",
+		Usage: "method is a subscription",
+	}
+)
+
 func main() {
 	app := cli.NewApp()
 	app.Usage = "devp2p simulation command-line client"
 	app.Flags = []cli.Flag{
-		cli.StringFlag{
-			Name:   "api",
-			Value:  "http://localhost:8888",
-			Usage:  "simulation API URL",
-			EnvVar: "P2PSIM_API_URL",
-		},
+		apiFlag,
 	}
 	app.Before = func(ctx *cli.Context) error {
-		client = simulations.NewClient(ctx.GlobalString("api"))
+		client = simulations.NewClient(ctx.GlobalString(apiFlag.Name))
 		return nil
 	}
 	app.Commands = []cli.Command{
@@ -82,15 +121,8 @@ func main() {
 			Usage:  "stream network events",
 			Action: streamNetwork,
 			Flags: []cli.Flag{
-				cli.BoolFlag{
-					Name:  "current",
-					Usage: "get existing nodes and conns first",
-				},
-				cli.StringFlag{
-					Name:  "filter",
-					Value: "",
-					Usage: "message filter",
-				},
+				currentFlag,
+				filterFlag,
 			},
 		},
 		{
@@ -118,21 +150,9 @@ func main() {
 					Usage:  "create a node",
 					Action: createNode,
 					Flags: []cli.Flag{
-						cli.StringFlag{
-							Name:  "name",
-							Value: "",
-							Usage: "node name",
-						},
-						cli.StringFlag{
-							Name:  "services",
-							Value: "",
-							Usage: "node services (comma separated)",
-						},
-						cli.StringFlag{
-							Name:  "key",
-							Value: "",
-							Usage: "node private key (hex encoded)",
-						},
+						nameFlag,
+						servicesFlag,
+						keyFlag,
 					},
 				},
 				{
@@ -171,10 +191,7 @@ func main() {
 					Usage:     "call a node RPC method",
 					Action:    rpcNode,
 					Flags: []cli.Flag{
-						cli.BoolFlag{
-							Name:  "subscribe",
-							Usage: "method is a subscription",
-						},
+						subscribeFlag,
 					},
 				},
 			},
@@ -207,8 +224,8 @@ func streamNetwork(ctx *cli.Context) error {
 	}
 	events := make(chan *simulations.Event)
 	sub, err := client.SubscribeNetwork(events, simulations.SubscribeOpts{
-		Current: ctx.Bool("current"),
-		Filter:  ctx.String("filter"),
+		Current: ctx.Bool(currentFlag.Name),
+		Filter:  ctx.String(filterFlag.Name),
 	})
 	if err != nil {
 		return err
@@ -279,8 +296,8 @@ func createNode(ctx *cli.Context) error {
 		return cli.ShowCommandHelp(ctx, ctx.Command.Name)
 	}
 	config := adapters.RandomNodeConfig()
-	config.Name = ctx.String("name")
-	if key := ctx.String("key"); key != "" {
+	config.Name = ctx.String(nameFlag.Name)
+	if key := ctx.String(keyFlag.Name); key != "" {
 		privKey, err := crypto.HexToECDSA(key)
 		if err != nil {
 			return err
@@ -288,7 +305,7 @@ func createNode(ctx *cli.Context) error {
 		config.ID = enode.PubkeyToIDV4(&privKey.PublicKey)
 		config.PrivateKey = privKey
 	}
-	if services := ctx.String("services"); services != "" {
+	if services := ctx.String(servicesFlag.Name); services != "" {
 		config.Lifecycles = strings.Split(services, ",")
 	}
 	node, err := client.CreateNode(config)
@@ -389,7 +406,7 @@ func rpcNode(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	if ctx.Bool("subscribe") {
+	if ctx.Bool(subscribeFlag.Name) {
 		return rpcSubscribe(rpcClient, ctx.App.Writer, method, args[3:]...)
 	}
 	var result interface{}

cmd/puppeth/genesis_test.go

@@ -19,7 +19,7 @@ package main
 import (
 	"bytes"
 	"encoding/json"
-	"io/ioutil"
+	"os"
 	"reflect"
 	"strings"
 	"testing"
@@ -30,7 +30,7 @@ import (
 
 // Tests the go-ethereum to Aleth chainspec conversion for the Stureby testnet.
 func TestAlethSturebyConverter(t *testing.T) {
-	blob, err := ioutil.ReadFile("testdata/stureby_geth.json")
+	blob, err := os.ReadFile("testdata/stureby_geth.json")
 	if err != nil {
 		t.Fatalf("could not read file: %v", err)
 	}
@@ -43,7 +43,7 @@ func TestAlethSturebyConverter(t *testing.T) {
 		t.Fatalf("failed creating chainspec: %v", err)
 	}
 
-	expBlob, err := ioutil.ReadFile("testdata/stureby_aleth.json")
+	expBlob, err := os.ReadFile("testdata/stureby_aleth.json")
 	if err != nil {
 		t.Fatalf("could not read file: %v", err)
 	}
@@ -69,7 +69,7 @@ func TestAlethSturebyConverter(t *testing.T) {
 
 // Tests the go-ethereum to Parity chainspec conversion for the Stureby testnet.
 func TestParitySturebyConverter(t *testing.T) {
-	blob, err := ioutil.ReadFile("testdata/stureby_geth.json")
+	blob, err := os.ReadFile("testdata/stureby_geth.json")
 	if err != nil {
 		t.Fatalf("could not read file: %v", err)
 	}
@@ -85,7 +85,7 @@ func TestParitySturebyConverter(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed encoding chainspec: %v", err)
 	}
-	expBlob, err := ioutil.ReadFile("testdata/stureby_parity.json")
+	expBlob, err := os.ReadFile("testdata/stureby_parity.json")
 	if err != nil {
 		t.Fatalf("could not read file: %v", err)
 	}

cmd/puppeth/ssh.go

@@ -21,7 +21,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"os"
 	"os/user"
@@ -96,7 +95,7 @@ func dial(server string, pubkey []byte) (*sshClient, error) {
 	}
 	if err != nil {
 		path := filepath.Join(user.HomeDir, ".ssh", identity)
-		if buf, err := ioutil.ReadFile(path); err != nil {
+		if buf, err := os.ReadFile(path); err != nil {
 			log.Warn("No SSH key, falling back to passwords", "path", path, "err", err)
 		} else {
 			key, err := ssh.ParsePrivateKey(buf)

cmd/puppeth/wizard.go

@@ -19,7 +19,6 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"math/big"
 	"net"
 	"net/url"
@@ -65,7 +64,7 @@ func (c config) flush() {
 	os.MkdirAll(filepath.Dir(c.path), 0755)
 
 	out, _ := json.MarshalIndent(c, "", "  ")
-	if err := ioutil.WriteFile(c.path, out, 0644); err != nil {
+	if err := os.WriteFile(c.path, out, 0644); err != nil {
 		log.Warn("Failed to save puppeth configs", "file", c.path, "err", err)
 	}
 }

cmd/puppeth/wizard_genesis.go

@@ -21,7 +21,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/big"
 	"math/rand"
 	"net/http"
@@ -263,7 +262,7 @@ func (w *wizard) manageGenesis() {
 
 		// Export the native genesis spec used by puppeth and Geth
 		gethJson := filepath.Join(folder, fmt.Sprintf("%s.json", w.network))
-		if err := ioutil.WriteFile(gethJson, out, 0644); err != nil {
+		if err := os.WriteFile(gethJson, out, 0644); err != nil {
 			log.Error("Failed to save genesis file", "err", err)
 			return
 		}
@@ -305,7 +304,7 @@ func saveGenesis(folder, network, client string, spec interface{}) {
 	path := filepath.Join(folder, fmt.Sprintf("%s-%s.json", network, client))
 
 	out, _ := json.MarshalIndent(spec, "", "  ")
-	if err := ioutil.WriteFile(path, out, 0644); err != nil {
+	if err := os.WriteFile(path, out, 0644); err != nil {
 		log.Error("Failed to save genesis file", "client", client, "err", err)
 		return
 	}

cmd/puppeth/wizard_intro.go

@@ -19,7 +19,6 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -73,7 +72,7 @@ func (w *wizard) run() {
 	// Load initial configurations and connect to all live servers
 	w.conf.path = filepath.Join(os.Getenv("HOME"), ".puppeth", w.network)
 
-	blob, err := ioutil.ReadFile(w.conf.path)
+	blob, err := os.ReadFile(w.conf.path)
 	if err != nil {
 		log.Warn("No previous configurations found", "path", w.conf.path)
 	} else if err := json.Unmarshal(blob, &w.conf); err != nil {

cmd/utils/cmd.go

@@ -77,11 +77,11 @@ func StartNode(ctx *cli.Context, stack *node.Node, isConsole bool) {
 		signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM)
 		defer signal.Stop(sigc)
 
-		minFreeDiskSpace := ethconfig.Defaults.TrieDirtyCache
+		minFreeDiskSpace := 2 * ethconfig.Defaults.TrieDirtyCache // Default 2 * 256Mb
 		if ctx.GlobalIsSet(MinFreeDiskSpaceFlag.Name) {
 			minFreeDiskSpace = ctx.GlobalInt(MinFreeDiskSpaceFlag.Name)
 		} else if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
-			minFreeDiskSpace = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
+			minFreeDiskSpace = 2 * ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
 		}
 		if minFreeDiskSpace > 0 {
 			go monitorFreeDiskSpace(sigc, stack.InstanceDir(), uint64(minFreeDiskSpace)*1024*1024)
@@ -131,7 +131,7 @@ func monitorFreeDiskSpace(sigc chan os.Signal, path string, freeDiskSpaceCritica
 		} else if freeSpace < 2*freeDiskSpaceCritical {
 			log.Warn("Disk space is running low. Geth will shutdown if disk space runs below critical level.", "available", common.StorageSize(freeSpace), "critical_level", common.StorageSize(freeDiskSpaceCritical))
 		}
-		time.Sleep(60 * time.Second)
+		time.Sleep(30 * time.Second)
 	}
 }
 

cmd/utils/customflags.go

@@ -154,11 +154,11 @@ func (b *bigValue) String() string {
 }
 
 func (b *bigValue) Set(s string) error {
-	int, ok := math.ParseBig256(s)
+	intVal, ok := math.ParseBig256(s)
 	if !ok {
 		return errors.New("invalid integer syntax")
 	}
-	*b = (bigValue)(*int)
+	*b = (bigValue)(*intVal)
 	return nil
 }
 
@@ -172,6 +172,7 @@ func (f BigFlag) String() string {
 
 func (f BigFlag) Apply(set *flag.FlagSet) {
 	eachName(f.Name, func(name string) {
+		f.Value = new(big.Int)
 		set.Var((*bigValue)(f.Value), f.Name, f.Usage)
 	})
 }

cmd/utils/flags.go

@@ -21,7 +21,6 @@ import (
 	"crypto/ecdsa"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math"
 	"math/big"
 	"os"
@@ -38,7 +37,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/fdlimit"
 	"github.com/ethereum/go-ethereum/consensus"
-	"github.com/ethereum/go-ethereum/consensus/clique"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
@@ -51,6 +49,7 @@ import (
 	"github.com/ethereum/go-ethereum/eth/gasprice"
 	"github.com/ethereum/go-ethereum/eth/tracers"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/ethdb/remotedb"
 	"github.com/ethereum/go-ethereum/ethstats"
 	"github.com/ethereum/go-ethereum/graphql"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
@@ -115,6 +114,10 @@ var (
 		Usage: "Data directory for the databases and keystore",
 		Value: DirectoryString(node.DefaultDataDir()),
 	}
+	RemoteDBFlag = cli.StringFlag{
+		Name:  "remotedb",
+		Usage: "URL for remote database",
+	}
 	AncientFlag = DirectoryFlag{
 		Name:  "datadir.ancient",
 		Usage: "Data directory for ancient chain segments (default = inside chaindata)",
@@ -145,22 +148,26 @@ var (
 		Name:  "mainnet",
 		Usage: "Ethereum mainnet",
 	}
-	GoerliFlag = cli.BoolFlag{
-		Name:  "goerli",
-		Usage: "Görli network: pre-configured proof-of-authority test network",
+	RopstenFlag = cli.BoolFlag{
+		Name:  "ropsten",
+		Usage: "Ropsten network: pre-configured proof-of-work test network",
 	}
 	RinkebyFlag = cli.BoolFlag{
 		Name:  "rinkeby",
 		Usage: "Rinkeby network: pre-configured proof-of-authority test network",
 	}
-	RopstenFlag = cli.BoolFlag{
-		Name:  "ropsten",
-		Usage: "Ropsten network: pre-configured proof-of-work test network",
+	GoerliFlag = cli.BoolFlag{
+		Name:  "goerli",
+		Usage: "Görli network: pre-configured proof-of-authority test network",
 	}
 	SepoliaFlag = cli.BoolFlag{
 		Name:  "sepolia",
 		Usage: "Sepolia network: pre-configured proof-of-work test network",
 	}
+	KilnFlag = cli.BoolFlag{
+		Name:  "kiln",
+		Usage: "Kiln network: pre-configured proof-of-work to proof-of-stake test network",
+	}
 	DeveloperFlag = cli.BoolFlag{
 		Name:  "dev",
 		Usage: "Ephemeral proof-of-authority network with a pre-funded developer account, mining enabled",
@@ -237,9 +244,13 @@ var (
 		Name:  "lightkdf",
 		Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
 	}
-	WhitelistFlag = cli.StringFlag{
+	EthRequiredBlocksFlag = cli.StringFlag{
+		Name:  "eth.requiredblocks",
+		Usage: "Comma separated block number-to-hash mappings to require for peering (<number>=<hash>)",
+	}
+	LegacyWhitelistFlag = cli.StringFlag{
 		Name:  "whitelist",
-		Usage: "Comma separated block number-to-hash mappings to enforce (<number>=<hash>)",
+		Usage: "Comma separated block number-to-hash mappings to enforce (<number>=<hash>) (deprecated in favor of --eth.requiredblocks)",
 	}
 	BloomFilterSizeFlag = cli.Uint64Flag{
 		Name:  "bloomfilter.size",
@@ -250,7 +261,7 @@ var (
 		Name:  "override.arrowglacier",
 		Usage: "Manually specify Arrow Glacier fork-block, overriding the bundled setting",
 	}
-	OverrideTerminalTotalDifficulty = cli.Uint64Flag{
+	OverrideTerminalTotalDifficulty = BigFlag{
 		Name:  "override.terminaltotaldifficulty",
 		Usage: "Manually specify TerminalTotalDifficulty, overriding the bundled setting",
 	}
@@ -433,6 +444,10 @@ var (
 		Name:  "cache.preimages",
 		Usage: "Enable recording the SHA3/keccak preimages of trie keys",
 	}
+	FDLimitFlag = cli.IntFlag{
+		Name:  "fdlimit",
+		Usage: "Raise the open file descriptor resource limit (default = system fd limit)",
+	}
 	// Miner settings
 	MiningEnabledFlag = cli.BoolFlag{
 		Name:  "mine",
@@ -518,6 +533,26 @@ var (
 		Usage: "Sets a cap on transaction fee (in ether) that can be sent via the RPC APIs (0 = no cap)",
 		Value: ethconfig.Defaults.RPCTxFeeCap,
 	}
+	// Authenticated RPC HTTP settings
+	AuthListenFlag = cli.StringFlag{
+		Name:  "authrpc.addr",
+		Usage: "Listening address for authenticated APIs",
+		Value: node.DefaultConfig.AuthAddr,
+	}
+	AuthPortFlag = cli.IntFlag{
+		Name:  "authrpc.port",
+		Usage: "Listening port for authenticated APIs",
+		Value: node.DefaultConfig.AuthPort,
+	}
+	AuthVirtualHostsFlag = cli.StringFlag{
+		Name:  "authrpc.vhosts",
+		Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.",
+		Value: strings.Join(node.DefaultConfig.AuthVirtualHosts, ","),
+	}
+	JWTSecretFlag = cli.StringFlag{
+		Name:  "authrpc.jwtsecret",
+		Usage: "Path to a JWT secret to use for authenticated RPC endpoints",
+	}
 	// Logging and debug settings
 	EthStatsURLFlag = cli.StringFlag{
 		Name:  "ethstats",
@@ -792,6 +827,37 @@ var (
 	}
 )
 
+var (
+	// TestnetFlags is the flag group of all built-in supported testnets.
+	TestnetFlags = []cli.Flag{
+		RopstenFlag,
+		RinkebyFlag,
+		GoerliFlag,
+		SepoliaFlag,
+		KilnFlag,
+	}
+	// NetworkFlags is the flag group of all built-in supported networks.
+	NetworkFlags = append([]cli.Flag{
+		MainnetFlag,
+	}, TestnetFlags...)
+
+	// DatabasePathFlags is the flag group of all database path flags.
+	DatabasePathFlags = []cli.Flag{
+		DataDirFlag,
+		AncientFlag,
+		RemoteDBFlag,
+	}
+)
+
+// GroupFlags combines the given flag slices together and returns the merged one.
+func GroupFlags(groups ...[]cli.Flag) []cli.Flag {
+	var ret []cli.Flag
+	for _, group := range groups {
+		ret = append(ret, group...)
+	}
+	return ret
+}
+
 // MakeDataDir retrieves the currently requested data directory, terminating
 // if none (or the empty string) is specified. If the node is starting a testnet,
 // then a subdirectory of the specified datadir will be used.
@@ -811,6 +877,9 @@ func MakeDataDir(ctx *cli.Context) string {
 		if ctx.GlobalBool(SepoliaFlag.Name) {
 			return filepath.Join(path, "sepolia")
 		}
+		if ctx.GlobalBool(KilnFlag.Name) {
+			return filepath.Join(path, "kiln")
+		}
 		return path
 	}
 	Fatalf("Cannot determine default data directory, please set manually (--datadir)")
@@ -865,6 +934,8 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) {
 		urls = params.RinkebyBootnodes
 	case ctx.GlobalBool(GoerliFlag.Name):
 		urls = params.GoerliBootnodes
+	case ctx.GlobalBool(KilnFlag.Name):
+		urls = params.KilnBootnodes
 	case cfg.BootstrapNodes != nil:
 		return // already set, don't apply defaults.
 	}
@@ -951,6 +1022,18 @@ func setHTTP(ctx *cli.Context, cfg *node.Config) {
 		cfg.HTTPPort = ctx.GlobalInt(HTTPPortFlag.Name)
 	}
 
+	if ctx.GlobalIsSet(AuthListenFlag.Name) {
+		cfg.AuthAddr = ctx.GlobalString(AuthListenFlag.Name)
+	}
+
+	if ctx.GlobalIsSet(AuthPortFlag.Name) {
+		cfg.AuthPort = ctx.GlobalInt(AuthPortFlag.Name)
+	}
+
+	if ctx.GlobalIsSet(AuthVirtualHostsFlag.Name) {
+		cfg.AuthVirtualHosts = SplitAndTrim(ctx.GlobalString(AuthVirtualHostsFlag.Name))
+	}
+
 	if ctx.GlobalIsSet(HTTPCORSDomainFlag.Name) {
 		cfg.HTTPCors = SplitAndTrim(ctx.GlobalString(HTTPCORSDomainFlag.Name))
 	}
@@ -1057,11 +1140,24 @@ func setLes(ctx *cli.Context, cfg *ethconfig.Config) {
 
 // MakeDatabaseHandles raises out the number of allowed file handles per process
 // for Geth and returns half of the allowance to assign to the database.
-func MakeDatabaseHandles() int {
+func MakeDatabaseHandles(max int) int {
 	limit, err := fdlimit.Maximum()
 	if err != nil {
 		Fatalf("Failed to retrieve file descriptor allowance: %v", err)
 	}
+	switch {
+	case max == 0:
+		// User didn't specify a meaningful value, use system limits
+	case max < 128:
+		// User specified something unhealthy, just use system defaults
+		log.Error("File descriptor limit invalid (<128)", "had", max, "updated", limit)
+	case max > limit:
+		// User requested more than the OS allows, notify that we can't allocate it
+		log.Warn("Requested file descriptors denied by OS", "req", max, "limit", limit)
+	default:
+		// User limit is meaningful and within allowed range, use that
+		limit = max
+	}
 	raised, err := fdlimit.Raise(uint64(limit))
 	if err != nil {
 		Fatalf("Failed to raise file descriptor allowance: %v", err)
@@ -1122,7 +1218,7 @@ func MakePasswordList(ctx *cli.Context) []string {
 	if path == "" {
 		return nil
 	}
-	text, err := ioutil.ReadFile(path)
+	text, err := os.ReadFile(path)
 	if err != nil {
 		Fatalf("Failed to read password file: %v", err)
 	}
@@ -1218,6 +1314,10 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	setDataDir(ctx, cfg)
 	setSmartCard(ctx, cfg)
 
+	if ctx.GlobalIsSet(JWTSecretFlag.Name) {
+		cfg.JWTSecret = ctx.GlobalString(JWTSecretFlag.Name)
+	}
+
 	if ctx.GlobalIsSet(ExternalSignerFlag.Name) {
 		cfg.ExternalSigner = ctx.GlobalString(ExternalSignerFlag.Name)
 	}
@@ -1272,7 +1372,7 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) {
 		// Maintain compatibility with older Geth configurations storing the
 		// Ropsten database in `testnet` instead of `ropsten`.
 		legacyPath := filepath.Join(node.DefaultDataDir(), "testnet")
-		if _, err := os.Stat(legacyPath); !os.IsNotExist(err) {
+		if common.FileExist(legacyPath) {
 			log.Warn("Using the deprecated `testnet` datadir. Future versions will store the Ropsten chain in `ropsten`.")
 			cfg.DataDir = legacyPath
 		} else {
@@ -1286,6 +1386,8 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) {
 		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "goerli")
 	case ctx.GlobalBool(SepoliaFlag.Name) && cfg.DataDir == node.DefaultDataDir():
 		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "sepolia")
+	case ctx.GlobalBool(KilnFlag.Name) && cfg.DataDir == node.DefaultDataDir():
+		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "kiln")
 	}
 }
 
@@ -1404,26 +1506,31 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) {
 	}
 }
 
-func setWhitelist(ctx *cli.Context, cfg *ethconfig.Config) {
-	whitelist := ctx.GlobalString(WhitelistFlag.Name)
-	if whitelist == "" {
-		return
+func setRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) {
+	requiredBlocks := ctx.GlobalString(EthRequiredBlocksFlag.Name)
+	if requiredBlocks == "" {
+		if ctx.GlobalIsSet(LegacyWhitelistFlag.Name) {
+			log.Warn("The flag --whitelist is deprecated and will be removed, please use --eth.requiredblocks")
+			requiredBlocks = ctx.GlobalString(LegacyWhitelistFlag.Name)
+		} else {
+			return
+		}
 	}
-	cfg.Whitelist = make(map[uint64]common.Hash)
-	for _, entry := range strings.Split(whitelist, ",") {
+	cfg.RequiredBlocks = make(map[uint64]common.Hash)
+	for _, entry := range strings.Split(requiredBlocks, ",") {
 		parts := strings.Split(entry, "=")
 		if len(parts) != 2 {
-			Fatalf("Invalid whitelist entry: %s", entry)
+			Fatalf("Invalid required block entry: %s", entry)
 		}
 		number, err := strconv.ParseUint(parts[0], 0, 64)
 		if err != nil {
-			Fatalf("Invalid whitelist block number %s: %v", parts[0], err)
+			Fatalf("Invalid required block number %s: %v", parts[0], err)
 		}
 		var hash common.Hash
 		if err = hash.UnmarshalText([]byte(parts[1])); err != nil {
-			Fatalf("Invalid whitelist hash %s: %v", parts[1], err)
+			Fatalf("Invalid required block hash %s: %v", parts[1], err)
 		}
-		cfg.Whitelist[number] = hash
+		cfg.RequiredBlocks[number] = hash
 	}
 }
 
@@ -1471,7 +1578,7 @@ func CheckExclusive(ctx *cli.Context, args ...interface{}) {
 // SetEthConfig applies eth-related command line flags to the config.
 func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	// Avoid conflicting network flags
-	CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, SepoliaFlag)
+	CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, SepoliaFlag, KilnFlag)
 	CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light")
 	CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer
 	if ctx.GlobalString(GCModeFlag.Name) == "archive" && ctx.GlobalUint64(TxLookupLimitFlag.Name) != 0 {
@@ -1490,7 +1597,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	setTxPool(ctx, &cfg.TxPool)
 	setEthash(ctx, cfg)
 	setMiner(ctx, &cfg.Miner)
-	setWhitelist(ctx, cfg)
+	setRequiredBlocks(ctx, cfg)
 	setLes(ctx, cfg)
 
 	// Cap the cache allowance and tune the garbage collector
@@ -1522,7 +1629,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheDatabaseFlag.Name) {
 		cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100
 	}
-	cfg.DatabaseHandles = MakeDatabaseHandles()
+	cfg.DatabaseHandles = MakeDatabaseHandles(ctx.GlobalInt(FDLimitFlag.Name))
 	if ctx.GlobalIsSet(AncientFlag.Name) {
 		cfg.DatabaseFreezer = ctx.GlobalString(AncientFlag.Name)
 	}
@@ -1622,6 +1729,16 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		cfg.Genesis = core.DefaultSepoliaGenesisBlock()
 		SetDNSDiscoveryDefaults(cfg, params.SepoliaGenesisHash)
 	case ctx.GlobalBool(RinkebyFlag.Name):
+		log.Warn("")
+		log.Warn("--------------------------------------------------------------------------------")
+		log.Warn("Please note, Rinkeby has been deprecated. It will still work for the time being,")
+		log.Warn("but there will be no further hard-forks shipped for it. Eventually the network")
+		log.Warn("will be permanently halted after the other networks transition through the merge")
+		log.Warn("and prove stable enough. For the most future proof testnet, choose Sepolia as")
+		log.Warn("your replacement environment (--sepolia instead of --rinkeby).")
+		log.Warn("--------------------------------------------------------------------------------")
+		log.Warn("")
+
 		if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
 			cfg.NetworkId = 4
 		}
@@ -1633,6 +1750,12 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		}
 		cfg.Genesis = core.DefaultGoerliGenesisBlock()
 		SetDNSDiscoveryDefaults(cfg, params.GoerliGenesisHash)
+	case ctx.GlobalBool(KilnFlag.Name):
+		if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
+			cfg.NetworkId = 1337802
+		}
+		cfg.Genesis = core.DefaultKilnGenesisBlock()
+		SetDNSDiscoveryDefaults(cfg, params.KilnGenesisHash)
 	case ctx.GlobalBool(DeveloperFlag.Name):
 		if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
 			cfg.NetworkId = 1337
@@ -1840,17 +1963,19 @@ func SplitTagsFlag(tagsFlag string) map[string]string {
 func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.Database {
 	var (
 		cache   = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100
-		handles = MakeDatabaseHandles()
+		handles = MakeDatabaseHandles(ctx.GlobalInt(FDLimitFlag.Name))
 
 		err     error
 		chainDb ethdb.Database
 	)
-	if ctx.GlobalString(SyncModeFlag.Name) == "light" {
-		name := "lightchaindata"
-		chainDb, err = stack.OpenDatabase(name, cache, handles, "", readonly)
-	} else {
-		name := "chaindata"
-		chainDb, err = stack.OpenDatabaseWithFreezer(name, cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly)
+	switch {
+	case ctx.GlobalIsSet(RemoteDBFlag.Name):
+		log.Info("Using remote db", "url", ctx.GlobalString(RemoteDBFlag.Name))
+		chainDb, err = remotedb.New(ctx.GlobalString(RemoteDBFlag.Name))
+	case ctx.GlobalString(SyncModeFlag.Name) == "light":
+		chainDb, err = stack.OpenDatabase("lightchaindata", cache, handles, "", readonly)
+	default:
+		chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly)
 	}
 	if err != nil {
 		Fatalf("Could not open database: %v", err)
@@ -1871,6 +1996,8 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis {
 		genesis = core.DefaultRinkebyGenesisBlock()
 	case ctx.GlobalBool(GoerliFlag.Name):
 		genesis = core.DefaultGoerliGenesisBlock()
+	case ctx.GlobalBool(KilnFlag.Name):
+		genesis = core.DefaultKilnGenesisBlock()
 	case ctx.GlobalBool(DeveloperFlag.Name):
 		Fatalf("Developer chains are ephemeral")
 	}
@@ -1885,24 +2012,13 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 	if err != nil {
 		Fatalf("%v", err)
 	}
+
 	var engine consensus.Engine
-	if config.Clique != nil {
-		engine = clique.New(config.Clique, chainDb)
-	} else {
-		engine = ethash.NewFaker()
-		if !ctx.GlobalBool(FakePoWFlag.Name) {
-			engine = ethash.New(ethash.Config{
-				CacheDir:         stack.ResolvePath(ethconfig.Defaults.Ethash.CacheDir),
-				CachesInMem:      ethconfig.Defaults.Ethash.CachesInMem,
-				CachesOnDisk:     ethconfig.Defaults.Ethash.CachesOnDisk,
-				CachesLockMmap:   ethconfig.Defaults.Ethash.CachesLockMmap,
-				DatasetDir:       stack.ResolvePath(ethconfig.Defaults.Ethash.DatasetDir),
-				DatasetsInMem:    ethconfig.Defaults.Ethash.DatasetsInMem,
-				DatasetsOnDisk:   ethconfig.Defaults.Ethash.DatasetsOnDisk,
-				DatasetsLockMmap: ethconfig.Defaults.Ethash.DatasetsLockMmap,
-			}, nil, false)
-		}
+	ethashConf := ethconfig.Defaults.Ethash
+	if ctx.GlobalBool(FakePoWFlag.Name) {
+		ethashConf.PowMode = ethash.ModeFake
 	}
+	engine = ethconfig.CreateConsensusEngine(stack, config, &ethashConf, nil, false, chainDb)
 	if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
 		Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
 	}

common/bytes.go

@@ -19,6 +19,9 @@ package common
 
 import (
 	"encoding/hex"
+	"errors"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
 )
 
 // FromHex returns the bytes represented by the hexadecimal string s.
@@ -92,6 +95,15 @@ func Hex2BytesFixed(str string, flen int) []byte {
 	return hh
 }
 
+// ParseHexOrString tries to hexdecode b, but if the prefix is missing, it instead just returns the raw bytes
+func ParseHexOrString(str string) ([]byte, error) {
+	b, err := hexutil.Decode(str)
+	if errors.Is(err, hexutil.ErrMissingPrefix) {
+		return []byte(str), nil
+	}
+	return b, err
+}
+
 // RightPadBytes zero-pads slice to the right up to length l.
 func RightPadBytes(slice []byte, l int) []byte {
 	if l <= len(slice) {

common/compiler/helpers.go

@@ -19,7 +19,7 @@ package compiler
 
 import (
 	"bytes"
-	"io/ioutil"
+	"os"
 	"regexp"
 )
 
@@ -55,7 +55,7 @@ type ContractInfo struct {
 func slurpFiles(files []string) (string, error) {
 	var concat bytes.Buffer
 	for _, file := range files {
-		content, err := ioutil.ReadFile(file)
+		content, err := os.ReadFile(file)
 		if err != nil {
 			return "", err
 		}

common/compiler/solidity.go

@@ -31,6 +31,7 @@ import (
 type Solidity struct {
 	Path, Version, FullVersion string
 	Major, Minor, Patch        int
+	ExtraAllowedPath           []string
 }
 
 // --combined-output format
@@ -58,11 +59,19 @@ type solcOutputV8 struct {
 	Version string
 }
 
+func (s *Solidity) allowedPaths() string {
+	paths := []string{".", "./", "../"} // default to support relative paths
+	if len(s.ExtraAllowedPath) > 0 {
+		paths = append(paths, s.ExtraAllowedPath...)
+	}
+	return strings.Join(paths, ", ")
+}
+
 func (s *Solidity) makeArgs() []string {
 	p := []string{
 		"--combined-json", "bin,bin-runtime,srcmap,srcmap-runtime,abi,userdoc,devdoc",
-		"--optimize",                  // code optimizer switched on
-		"--allow-paths", "., ./, ../", // default to support relative paths
+		"--optimize", // code optimizer switched on
+		"--allow-paths", s.allowedPaths(),
 	}
 	if s.Major > 0 || s.Minor > 4 || s.Patch > 6 {
 		p[1] += ",metadata,hashes"
@@ -108,10 +117,7 @@ func CompileSolidityString(solc, source string) (map[string]*Contract, error) {
 	if err != nil {
 		return nil, err
 	}
-	args := append(s.makeArgs(), "--")
-	cmd := exec.Command(s.Path, append(args, "-")...)
-	cmd.Stdin = strings.NewReader(source)
-	return s.run(cmd, source)
+	return s.CompileSource(source)
 }
 
 // CompileSolidity compiles all given Solidity source files.
@@ -119,11 +125,25 @@ func CompileSolidity(solc string, sourcefiles ...string) (map[string]*Contract,
 	if len(sourcefiles) == 0 {
 		return nil, errors.New("solc: no source files")
 	}
-	source, err := slurpFiles(sourcefiles)
+	s, err := SolidityVersion(solc)
 	if err != nil {
 		return nil, err
 	}
-	s, err := SolidityVersion(solc)
+
+	return s.CompileFiles(sourcefiles...)
+}
+
+// CompileSource builds and returns all the contracts contained within a source string.
+func (s *Solidity) CompileSource(source string) (map[string]*Contract, error) {
+	args := append(s.makeArgs(), "--")
+	cmd := exec.Command(s.Path, append(args, "-")...)
+	cmd.Stdin = strings.NewReader(source)
+	return s.run(cmd, source)
+}
+
+// CompileFiles compiles all given Solidity source files.
+func (s *Solidity) CompileFiles(sourcefiles ...string) (map[string]*Contract, error) {
+	source, err := slurpFiles(sourcefiles)
 	if err != nil {
 		return nil, err
 	}

common/test_utils.go

@@ -19,12 +19,12 @@ package common
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"os"
 )
 
 // LoadJSON reads the given file and unmarshals its content.
 func LoadJSON(file string, val interface{}) error {
-	content, err := ioutil.ReadFile(file)
+	content, err := os.ReadFile(file)
 	if err != nil {
 		return err
 	}

consensus/ethash/algorithm_test.go

@@ -19,7 +19,6 @@ package ethash
 import (
 	"bytes"
 	"encoding/binary"
-	"io/ioutil"
 	"math/big"
 	"os"
 	"reflect"
@@ -698,7 +697,9 @@ func TestHashimoto(t *testing.T) {
 // Tests that caches generated on disk may be done concurrently.
 func TestConcurrentDiskCacheGeneration(t *testing.T) {
 	// Create a temp folder to generate the caches into
-	cachedir, err := ioutil.TempDir("", "")
+	// TODO: t.TempDir fails to remove the directory on Windows
+	// \AppData\Local\Temp\1\TestConcurrentDiskCacheGeneration2382060137\001\cache-R23-1dca8a85e74aa763: Access is denied.
+	cachedir, err := os.MkdirTemp("", "")
 	if err != nil {
 		t.Fatalf("Failed to create temporary cache dir: %v", err)
 	}
@@ -794,11 +795,7 @@ func BenchmarkHashimotoFullSmall(b *testing.B) {
 
 func benchmarkHashimotoFullMmap(b *testing.B, name string, lock bool) {
 	b.Run(name, func(b *testing.B) {
-		tmpdir, err := ioutil.TempDir("", "ethash-test")
-		if err != nil {
-			b.Fatal(err)
-		}
-		defer os.RemoveAll(tmpdir)
+		tmpdir := b.TempDir()
 
 		d := &dataset{epoch: 0}
 		d.generate(tmpdir, 1, lock, false)

consensus/ethash/consensus.go

@@ -507,8 +507,8 @@ func calcDifficultyFrontier(time uint64, parent *types.Header) *big.Int {
 }
 
 // Exported for fuzzing
-var FrontierDifficultyCalulator = calcDifficultyFrontier
-var HomesteadDifficultyCalulator = calcDifficultyHomestead
+var FrontierDifficultyCalculator = calcDifficultyFrontier
+var HomesteadDifficultyCalculator = calcDifficultyHomestead
 var DynamicDifficultyCalculator = makeDifficultyCalculator
 
 // verifySeal checks whether a block satisfies the PoW difficulty requirements,

consensus/ethash/consensus_test.go

@@ -121,8 +121,8 @@ func TestDifficultyCalculators(t *testing.T) {
 			bigFn  func(time uint64, parent *types.Header) *big.Int
 			u256Fn func(time uint64, parent *types.Header) *big.Int
 		}{
-			{FrontierDifficultyCalulator, CalcDifficultyFrontierU256},
-			{HomesteadDifficultyCalulator, CalcDifficultyHomesteadU256},
+			{FrontierDifficultyCalculator, CalcDifficultyFrontierU256},
+			{HomesteadDifficultyCalculator, CalcDifficultyHomesteadU256},
 			{DynamicDifficultyCalculator(bombDelay), MakeDifficultyCalculatorU256(bombDelay)},
 		} {
 			time := header.Time + timeDelta