params, swarm: release Geth v1.8.20 and Swarm v0.3.8

* geth/core/eth: implement constantinople override flag

* les: implemnent constantinople override flag for les clients

* cmd/geth, eth, les: fix typo, move flag to experimentals

.github/CODEOWNERS

@@ -1,12 +1,35 @@
 # Lines starting with '#' are comments.
 # Each line is a file pattern followed by one or more owners.
 
-accounts/usbwallet @karalabe
+accounts/usbwallet              @karalabe
-consensus          @karalabe
+consensus                       @karalabe
-core/              @karalabe @holiman
+core/                           @karalabe @holiman
-eth/               @karalabe
+eth/                            @karalabe
-les/               @zsfelfoldi
+les/                            @zsfelfoldi
-light/             @zsfelfoldi
+light/                          @zsfelfoldi
-mobile/            @karalabe
+mobile/                         @karalabe
-p2p/               @fjl @zsfelfoldi
+p2p/                            @fjl @zsfelfoldi
-whisper/           @gballet @gluk256
+p2p/simulations                 @lmars
+p2p/protocols                   @zelig
+swarm/api/http                  @justelad
+swarm/bmt                       @zelig
+swarm/dev                       @lmars
+swarm/fuse                      @jmozah @holisticode
+swarm/grafana_dashboards        @nonsense
+swarm/metrics                   @nonsense @holisticode
+swarm/multihash                 @nolash
+swarm/network/bitvector         @zelig @janos 
+swarm/network/priorityqueue     @zelig @janos 
+swarm/network/simulations       @zelig @janos
+swarm/network/stream            @janos @zelig @holisticode @justelad
+swarm/network/stream/intervals  @janos
+swarm/network/stream/testing    @zelig
+swarm/pot                       @zelig
+swarm/pss                       @nolash @zelig @nonsense
+swarm/services                  @zelig
+swarm/state                     @justelad
+swarm/storage/encryption        @zelig @nagydani
+swarm/storage/mock              @janos
+swarm/storage/feed              @nolash @jpeletier
+swarm/testutil                  @lmars
+whisper/                        @gballet @gluk256

.github/CONTRIBUTING.md

@@ -1,16 +1,40 @@
+# Contributing
+
+Thank you for considering to help out with the source code! We welcome 
+contributions from anyone on the internet, and are grateful for even the 
+smallest of fixes!
+
+If you'd like to contribute to go-ethereum, please fork, fix, commit and send a 
+pull request for the maintainers to review and merge into the main code base. If
+you wish to submit more complex changes though, please check up with the core 
+devs first on [our gitter channel](https://gitter.im/ethereum/go-ethereum) to 
+ensure those changes are in line with the general philosophy of the project 
+and/or get some early feedback which can make both your efforts much lighter as
+well as our review and merge procedures quick and simple.
+
+## Coding guidelines
+
+Please make sure your contributions adhere to our coding guidelines:
+
+ * Code must adhere to the official Go 
+[formatting](https://golang.org/doc/effective_go.html#formatting) guidelines 
+(i.e. uses [gofmt](https://golang.org/cmd/gofmt/)).
+ * Code must be documented adhering to the official Go 
+[commentary](https://golang.org/doc/effective_go.html#commentary) guidelines.
+ * Pull requests need to be based on and opened against the `master` branch.
+ * Commit messages should be prefixed with the package(s) they modify.
+   * E.g. "eth, rpc: make trace configs optional"
+
 ## Can I have feature X
 
-Before you do a feature request please check and make sure that it isn't possible
+Before you submit a feature request, please check and make sure that it isn't 
-through some other means. The JavaScript enabled console is a powerful feature
+possible through some other means. The JavaScript-enabled console is a powerful 
-in the right hands. Please check our [Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info
+feature in the right hands. Please check our 
+[Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info
 and help.
 
-## Contributing
+## Configuration, dependencies, and tests
 
-If you'd like to contribute to go-ethereum please fork, fix, commit and
+Please see the [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
-send a pull request. Commits which do not comply with the coding standards
+for more details on configuring your environment, managing project dependencies
-are ignored (use gofmt!).
+and testing procedures.
-
-See [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
-for more details on configuring your environment, testing, and
-dependency management.

.github/no-response.yml

@@ -7,5 +7,5 @@ closeComment: >
   This issue has been automatically closed because there has been no response
   to our request for more information from the original author. With only the
   information that is currently in the issue, we don't have enough information
-  to take action. Please reach out if you have or find the answers we need so
+  to take action. Please reach out if you have more relevant information or
-  that we can investigate further.
+  answers to our questions so that we can investigate further.

.travis.yml

@@ -6,19 +6,19 @@ matrix:
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.9.x
+      go: 1.10.x
       script:
-        - sudo modprobe fuse
+      - sudo modprobe fuse
-        - sudo chmod 666 /dev/fuse
+      - sudo chmod 666 /dev/fuse
-        - sudo chown root:$USER /etc/fuse.conf
+      - sudo chown root:$USER /etc/fuse.conf
-        - go run build/ci.go install
+      - go run build/ci.go install
-        - go run build/ci.go test -coverage $TEST_PACKAGES
+      - go run build/ci.go test -coverage $TEST_PACKAGES
 
     # These are the latest Go versions.
     - os: linux
       dist: trusty
       sudo: required
-      go: 1.10.x
+      go: 1.11.x
       script:
         - sudo modprobe fuse
         - sudo chmod 666 /dev/fuse
@@ -27,18 +27,24 @@ matrix:
         - go run build/ci.go test -coverage $TEST_PACKAGES
 
     - os: osx
-      go: 1.10.x
+      go: 1.11.x
       script:
+        - echo "Increase the maximum number of open file descriptors on macOS"
+        - NOFILE=20480
+        - sudo sysctl -w kern.maxfiles=$NOFILE
+        - sudo sysctl -w kern.maxfilesperproc=$NOFILE
+        - sudo launchctl limit maxfiles $NOFILE $NOFILE
+        - sudo launchctl limit maxfiles
+        - ulimit -S -n $NOFILE
+        - ulimit -n
         - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
-        - brew update
-        - brew cask install osxfuse
         - go run build/ci.go install
         - go run build/ci.go test -coverage $TEST_PACKAGES
 
     # This builder only tests code linters on latest version of Go
     - os: linux
       dist: trusty
-      go: 1.10.x
+      go: 1.11.x
       env:
         - lint
       git:
@@ -47,9 +53,10 @@ matrix:
         - go run build/ci.go lint
 
     # This builder does the Ubuntu PPA upload
-    - os: linux
+    - if: type = push
+      os: linux
       dist: trusty
-      go: 1.10.x
+      go: 1.11.x
       env:
         - ubuntu-ppa
       git:
@@ -65,10 +72,11 @@ matrix:
         - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
 
     # This builder does the Linux Azure uploads
-    - os: linux
+    - if: type = push
+      os: linux
       dist: trusty
       sudo: required
-      go: 1.10.x
+      go: 1.11.x
       env:
         - azure-linux
       git:
@@ -98,11 +106,12 @@ matrix:
         - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
     # This builder does the Linux Azure MIPS xgo uploads
-    - os: linux
+    - if: type = push
+      os: linux
       dist: trusty
       services:
         - docker
-      go: 1.10.x
+      go: 1.11.x
       env:
         - azure-linux-mips
       git:
@@ -125,7 +134,8 @@ matrix:
         - go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
     # This builder does the Android Maven and Azure uploads
-    - os: linux
+    - if: type = push
+      os: linux
       dist: trusty
       addons:
         apt:
@@ -146,24 +156,25 @@ matrix:
       git:
         submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.10.2.linux-amd64.tar.gz | tar -xz
+        - curl https://storage.googleapis.com/golang/go1.11.2.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go
       script:
         # Build the Android archive and upload it to Maven Central and Azure
-        - curl https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip -o android-ndk-r16b.zip
+        - curl https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip -o android-ndk-r17b.zip
-        - unzip -q android-ndk-r16b.zip && rm android-ndk-r16b.zip
+        - unzip -q android-ndk-r17b.zip && rm android-ndk-r17b.zip
-        - mv android-ndk-r16b $HOME
+        - mv android-ndk-r17b $HOME
-        - export ANDROID_NDK=$HOME/android-ndk-r16b
+        - export ANDROID_NDK=$HOME/android-ndk-r17b
 
         - mkdir -p $GOPATH/src/github.com/ethereum
         - ln -s `pwd` $GOPATH/src/github.com/ethereum
         - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds
 
     # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
-    - os: osx
+    - if: type = push
-      go: 1.10.x
+      os: osx
+      go: 1.11.x
       env:
         - azure-osx
         - azure-ios
@@ -190,19 +201,13 @@ matrix:
         - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
 
     # This builder does the Azure archive purges to avoid accumulating junk
-    - os: linux
+    - if: type = cron
+      os: linux
       dist: trusty
-      go: 1.10.x
+      go: 1.11.x
       env:
         - azure-purge
       git:
         submodules: false # avoid cloning ethereum/tests
       script:
         - go run build/ci.go purge -store gethstore/builds -days 14
-
-notifications:
-  webhooks:
-    urls:
-      - https://webhooks.gitter.im/e/e09ccdce1048c5e03445
-    on_success: change
-    on_failure: always

AUTHORS

@@ -171,3 +171,4 @@ xiekeyang <xiekeyang@users.noreply.github.com>
 yoza <yoza.is12s@gmail.com>
 ΞTHΞЯSPHΞЯΞ <{viktor.tron,nagydani,zsfelfoldi}@gmail.com>
 Максим Чусовлянов <mchusovlianov@gmail.com>
+Ralph Caraveo <deckarep@gmail.com>

Dockerfile

@@ -1,5 +1,5 @@
 # Build Geth in a stock Go builder container
-FROM golang:1.10-alpine as builder
+FROM golang:1.11-alpine as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers
 

Dockerfile.alltools

@@ -1,5 +1,5 @@
 # Build Geth in a stock Go builder container
-FROM golang:1.10-alpine as builder
+FROM golang:1.11-alpine as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers
 

Makefile

@@ -41,6 +41,7 @@ lint: ## Run linters.
 	build/env.sh go run build/ci.go lint
 
 clean:
+	./build/clean_go_build_cache.sh
 	rm -fr build/_workspace/pkg/ $(GOBIN)/*
 
 # The devtools target installs tools required for 'go generate'.
@@ -56,6 +57,9 @@ devtools:
 	@type "solc" 2> /dev/null || echo 'Please install solc'
 	@type "protoc" 2> /dev/null || echo 'Please install protoc'
 
+swarm-devtools:
+	env GOBIN= go install ./cmd/swarm/mimegen
+
 # Cross Compilation Targets (xgo)
 
 geth-cross: geth-linux geth-darwin geth-windows geth-android geth-ios

README.md

@@ -7,7 +7,7 @@ https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/6874
 )](https://godoc.org/github.com/ethereum/go-ethereum)
 [![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum)
 [![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum)
-[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethereum/go-ethereum?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
+[![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv)
 
 Automated builds are available for stable releases and the unstable master branch.
 Binary archives are published at https://geth.ethereum.org/downloads/.
@@ -18,7 +18,7 @@ For prerequisites and detailed build instructions please read the
 [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum)
 on the wiki.
 
-Building geth requires both a Go (version 1.7 or later) and a C compiler.
+Building geth requires both a Go (version 1.9 or later) and a C compiler.
 You can install them using your favourite package manager.
 Once the dependencies are installed, run
 
@@ -34,13 +34,13 @@ The go-ethereum project comes with several wrappers/executables found in the `cm
 
 | Command    | Description |
 |:----------:|-------------|
-| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default) archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options. |
+| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options. |
 | `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. |
 | `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. |
 | `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). |
 | `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details. |
 | `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). |
-| `swarm`    | swarm daemon and tools. This is the entrypoint for the swarm network. `swarm --help` for command line options and subcommands. See https://swarm-guide.readthedocs.io for swarm documentation. |
+| `swarm`    | Swarm daemon and tools. This is the entrypoint for the Swarm network. `swarm --help` for command line options and subcommands. See [Swarm README](https://github.com/ethereum/go-ethereum/tree/master/swarm) for more information. |
 | `puppeth`    | a CLI wizard that aids in creating a new Ethereum network. |
 
 ## Running geth
@@ -69,7 +69,7 @@ This command will:
  * Start up Geth's built-in interactive [JavaScript console](https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console),
    (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://github.com/ethereum/wiki/wiki/JavaScript-API)
    as well as Geth's own [management APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs).
-   This too is optional and if you leave it out you can always attach to an already running Geth instance
+   This tool is optional and if you leave it out you can always attach to an already running Geth instance
    with `geth attach`.
 
 ### Full node on the Ethereum test network
@@ -168,7 +168,7 @@ HTTP based JSON-RPC API options:
   * `--ipcpath` Filename for IPC socket/pipe within the datadir (explicit paths escape it)
 
 You'll need to use your own programming environments' capabilities (libraries, tools, etc) to connect
-via HTTP, WS or IPC to a Geth node configured with the above flags and you'll need to speak [JSON-RPC](http://www.jsonrpc.org/specification)
+via HTTP, WS or IPC to a Geth node configured with the above flags and you'll need to speak [JSON-RPC](https://www.jsonrpc.org/specification)
 on all transports. You can reuse the same connection for multiple requests!
 
 **Note: Please understand the security implications of opening up an HTTP/WS based transport before

VERSION

@@ -1 +0,0 @@
-1.8.11

accounts/abi/abi.go

@@ -137,6 +137,9 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
 // MethodById looks up a method by the 4-byte id
 // returns nil if none found
 func (abi *ABI) MethodById(sigdata []byte) (*Method, error) {
+	if len(sigdata) < 4 {
+		return nil, fmt.Errorf("data too short (% bytes) for abi method lookup", len(sigdata))
+	}
 	for _, method := range abi.Methods {
 		if bytes.Equal(method.Id(), sigdata[:4]) {
 			return &method, nil

accounts/abi/abi_test.go

@@ -711,5 +711,14 @@ func TestABI_MethodById(t *testing.T) {
 			t.Errorf("Method %v (id %v) not 'findable' by id in ABI", name, common.ToHex(m.Id()))
 		}
 	}
-
+	// Also test empty
+	if _, err := abi.MethodById([]byte{0x00}); err == nil {
+		t.Errorf("Expected error, too short to decode data")
+	}
+	if _, err := abi.MethodById([]byte{}); err == nil {
+		t.Errorf("Expected error, too short to decode data")
+	}
+	if _, err := abi.MethodById(nil); err == nil {
+		t.Errorf("Expected error, nil is short to decode data")
+	}
 }

accounts/abi/argument.go

@@ -243,11 +243,7 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) {
 	// input offset is the bytes offset for packed output
 	inputOffset := 0
 	for _, abiArg := range abiArgs {
-		if abiArg.Type.T == ArrayTy {
+		inputOffset += getDynamicTypeOffset(abiArg.Type)
-			inputOffset += 32 * abiArg.Type.Size
-		} else {
-			inputOffset += 32
-		}
 	}
 	var ret []byte
 	for i, a := range args {
@@ -257,14 +253,13 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) {
 		if err != nil {
 			return nil, err
 		}
-		// check for a slice type (string, bytes, slice)
+		// check for dynamic types
-		if input.Type.requiresLengthPrefix() {
+		if isDynamicType(input.Type) {
-			// calculate the offset
-			offset := inputOffset + len(variableInput)
 			// set the offset
-			ret = append(ret, packNum(reflect.ValueOf(offset))...)
+			ret = append(ret, packNum(reflect.ValueOf(inputOffset))...)
-			// Append the packed output to the variable input. The variable input
+			// calculate next offset
-			// will be appended at the end of the input.
+			inputOffset += len(packed)
+			// append to variable input
 			variableInput = append(variableInput, packed...)
 		} else {
 			// append the packed value to the input

accounts/abi/bind/backends/simulated.go

@@ -65,11 +65,11 @@ type SimulatedBackend struct {
 
 // NewSimulatedBackend creates a new binding backend using a simulated blockchain
 // for testing purposes.
-func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend {
+func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend {
 	database := ethdb.NewMemDatabase()
-	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, Alloc: alloc}
+	genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc}
 	genesis.MustCommit(database)
-	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{})
+	blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil)
 
 	backend := &SimulatedBackend{
 		database:   database,
@@ -208,7 +208,7 @@ func (b *SimulatedBackend) PendingNonceAt(ctx context.Context, account common.Ad
 }
 
 // SuggestGasPrice implements ContractTransactor.SuggestGasPrice. Since the simulated
-// chain doens't have miners, we just return a gas price of 1 for any call.
+// chain doesn't have miners, we just return a gas price of 1 for any call.
 func (b *SimulatedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) {
 	return big.NewInt(1), nil
 }
@@ -324,18 +324,24 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
 //
 // TODO(karalabe): Deprecate when the subscription one can return past data too.
 func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.FilterQuery) ([]types.Log, error) {
-	// Initialize unset filter boundaried to run from genesis to chain head
+	var filter *filters.Filter
-	from := int64(0)
+	if query.BlockHash != nil {
-	if query.FromBlock != nil {
+		// Block filter requested, construct a single-shot filter
-		from = query.FromBlock.Int64()
+		filter = filters.NewBlockFilter(&filterBackend{b.database, b.blockchain}, *query.BlockHash, query.Addresses, query.Topics)
+	} else {
+		// Initialize unset filter boundaried to run from genesis to chain head
+		from := int64(0)
+		if query.FromBlock != nil {
+			from = query.FromBlock.Int64()
+		}
+		to := int64(-1)
+		if query.ToBlock != nil {
+			to = query.ToBlock.Int64()
+		}
+		// Construct the range filter
+		filter = filters.NewRangeFilter(&filterBackend{b.database, b.blockchain}, from, to, query.Addresses, query.Topics)
 	}
-	to := int64(-1)
+	// Run the filter and return all the logs
-	if query.ToBlock != nil {
-		to = query.ToBlock.Int64()
-	}
-	// Construct and execute the filter
-	filter := filters.New(&filterBackend{b.database, b.blockchain}, from, to, query.Addresses, query.Topics)
-
 	logs, err := filter.Logs(ctx)
 	if err != nil {
 		return nil, err
@@ -430,6 +436,10 @@ func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumb
 	return fb.bc.GetHeaderByNumber(uint64(block.Int64())), nil
 }
 
+func (fb *filterBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) {
+	return fb.bc.GetHeaderByHash(hash), nil
+}
+
 func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) {
 	number := rawdb.ReadHeaderNumber(fb.db, hash)
 	if number == nil {

accounts/abi/bind/bind.go

@@ -23,13 +23,13 @@ package bind
 import (
 	"bytes"
 	"fmt"
+	"go/format"
 	"regexp"
 	"strings"
 	"text/template"
 	"unicode"
 
 	"github.com/ethereum/go-ethereum/accounts/abi"
-	"golang.org/x/tools/imports"
 )
 
 // Lang is a target programming language selector to generate bindings for.
@@ -145,9 +145,9 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 	if err := tmpl.Execute(buffer, data); err != nil {
 		return "", err
 	}
-	// For Go bindings pass the code through goimports to clean it up and double check
+	// For Go bindings pass the code through gofmt to clean it up
 	if lang == LangGo {
-		code, err := imports.Process(".", buffer.Bytes(), nil)
+		code, err := format.Source(buffer.Bytes())
 		if err != nil {
 			return "", fmt.Errorf("%v\n%s", err, buffer)
 		}
@@ -207,7 +207,7 @@ func bindTypeGo(kind abi.Type) string {
 
 // The inner function of bindTypeGo, this finds the inner type of stringKind.
 // (Or just the type itself if it is not an array or slice)
-// The length of the matched part is returned, with the the translated type.
+// The length of the matched part is returned, with the translated type.
 func bindUnnestedTypeGo(stringKind string) (int, string) {
 
 	switch {
@@ -255,7 +255,7 @@ func bindTypeJava(kind abi.Type) string {
 
 // The inner function of bindTypeJava, this finds the inner type of stringKind.
 // (Or just the type itself if it is not an array or slice)
-// The length of the matched part is returned, with the the translated type.
+// The length of the matched part is returned, with the translated type.
 func bindUnnestedTypeJava(stringKind string) (int, string) {
 
 	switch {

accounts/abi/bind/template.go

@@ -64,6 +64,30 @@ const tmplSourceGo = `
 
 package {{.Package}}
 
+import (
+	"math/big"
+	"strings"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/event"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var (
+	_ = big.NewInt
+	_ = strings.NewReader
+	_ = ethereum.NotFound
+	_ = abi.U256
+	_ = bind.Bind
+	_ = common.Big1
+	_ = types.BloomLookup
+	_ = event.NewSubscription
+)
+
 {{range $contract := .Contracts}}
 	// {{.Type}}ABI is the input ABI used to generate the binding from.
 	const {{.Type}}ABI = "{{.InputABI}}"

accounts/abi/bind/util_test.go

@@ -53,9 +53,11 @@ var waitDeployedTests = map[string]struct {
 
 func TestWaitDeployed(t *testing.T) {
 	for name, test := range waitDeployedTests {
-		backend := backends.NewSimulatedBackend(core.GenesisAlloc{
+		backend := backends.NewSimulatedBackend(
-			crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)},
+			core.GenesisAlloc{
-		})
+				crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)},
+			}, 10000000,
+		)
 
 		// Create the transaction.
 		tx := types.NewContractCreation(0, big.NewInt(0), test.gas, big.NewInt(1), common.FromHex(test.code))

accounts/abi/method.go

@@ -47,10 +47,8 @@ type Method struct {
 // Please note that "int" is substitute for its canonical representation "int256"
 func (method Method) Sig() string {
 	types := make([]string, len(method.Inputs))
-	i := 0
+	for i, input := range method.Inputs {
-	for _, input := range method.Inputs {
 		types[i] = input.Type.String()
-		i++
 	}
 	return fmt.Sprintf("%v(%v)", method.Name, strings.Join(types, ","))
 }

accounts/abi/numbers.go

@@ -31,29 +31,14 @@ var (
 	uint16T   = reflect.TypeOf(uint16(0))
 	uint32T   = reflect.TypeOf(uint32(0))
 	uint64T   = reflect.TypeOf(uint64(0))
-	intT      = reflect.TypeOf(int(0))
 	int8T     = reflect.TypeOf(int8(0))
 	int16T    = reflect.TypeOf(int16(0))
 	int32T    = reflect.TypeOf(int32(0))
 	int64T    = reflect.TypeOf(int64(0))
 	addressT  = reflect.TypeOf(common.Address{})
-	intTS     = reflect.TypeOf([]int(nil))
-	int8TS    = reflect.TypeOf([]int8(nil))
-	int16TS   = reflect.TypeOf([]int16(nil))
-	int32TS   = reflect.TypeOf([]int32(nil))
-	int64TS   = reflect.TypeOf([]int64(nil))
 )
 
 // U256 converts a big Int into a 256bit EVM number.
 func U256(n *big.Int) []byte {
 	return math.PaddedBigBytes(math.U256(n), 32)
 }
-
-// checks whether the given reflect value is signed. This also works for slices with a number type
-func isSigned(v reflect.Value) bool {
-	switch v.Type() {
-	case intTS, int8TS, int16TS, int32TS, int64TS, intT, int8T, int16T, int32T, int64T:
-		return true
-	}
-	return false
-}

accounts/abi/numbers_test.go

@@ -19,7 +19,6 @@ package abi
 import (
 	"bytes"
 	"math/big"
-	"reflect"
 	"testing"
 )
 
@@ -32,13 +31,3 @@ func TestNumberTypes(t *testing.T) {
 		t.Errorf("expected %x got %x", ubytes, unsigned)
 	}
 }
-
-func TestSigned(t *testing.T) {
-	if isSigned(reflect.ValueOf(uint(10))) {
-		t.Error("signed")
-	}
-
-	if !isSigned(reflect.ValueOf(int(10))) {
-		t.Error("not signed")
-	}
-}

accounts/abi/pack_test.go

@@ -324,6 +324,66 @@ func TestPack(t *testing.T) {
 			"foobar",
 			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000006666f6f6261720000000000000000000000000000000000000000000000000000"),
 		},
+		{
+			"string[]",
+			[]string{"hello", "foobar"},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // len(array) = 2
+				"0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0
+				"0000000000000000000000000000000000000000000000000000000000000080" + // offset 128 to i = 1
+				"0000000000000000000000000000000000000000000000000000000000000005" + // len(str[0]) = 5
+				"68656c6c6f000000000000000000000000000000000000000000000000000000" + // str[0]
+				"0000000000000000000000000000000000000000000000000000000000000006" + // len(str[1]) = 6
+				"666f6f6261720000000000000000000000000000000000000000000000000000"), // str[1]
+		},
+		{
+			"string[2]",
+			[]string{"hello", "foobar"},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // offset to i = 0
+				"0000000000000000000000000000000000000000000000000000000000000080" + // offset to i = 1
+				"0000000000000000000000000000000000000000000000000000000000000005" + // len(str[0]) = 5
+				"68656c6c6f000000000000000000000000000000000000000000000000000000" + // str[0]
+				"0000000000000000000000000000000000000000000000000000000000000006" + // len(str[1]) = 6
+				"666f6f6261720000000000000000000000000000000000000000000000000000"), // str[1]
+		},
+		{
+			"bytes32[][]",
+			[][]common.Hash{{{1}, {2}}, {{3}, {4}, {5}}},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // len(array) = 2
+				"0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0
+				"00000000000000000000000000000000000000000000000000000000000000a0" + // offset 160 to i = 1
+				"0000000000000000000000000000000000000000000000000000000000000002" + // len(array[0]) = 2
+				"0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0]
+				"0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1]
+				"0000000000000000000000000000000000000000000000000000000000000003" + // len(array[1]) = 3
+				"0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0]
+				"0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1]
+				"0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2]
+		},
+
+		{
+			"bytes32[][2]",
+			[][]common.Hash{{{1}, {2}}, {{3}, {4}, {5}}},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0
+				"00000000000000000000000000000000000000000000000000000000000000a0" + // offset 160 to i = 1
+				"0000000000000000000000000000000000000000000000000000000000000002" + // len(array[0]) = 2
+				"0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0]
+				"0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1]
+				"0000000000000000000000000000000000000000000000000000000000000003" + // len(array[1]) = 3
+				"0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0]
+				"0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1]
+				"0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2]
+		},
+
+		{
+			"bytes32[3][2]",
+			[][]common.Hash{{{1}, {2}, {3}}, {{3}, {4}, {5}}},
+			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0]
+				"0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1]
+				"0300000000000000000000000000000000000000000000000000000000000000" + // array[0][2]
+				"0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0]
+				"0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1]
+				"0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2]
+		},
 	} {
 		typ, err := NewType(test.typ)
 		if err != nil {
@@ -336,7 +396,7 @@ func TestPack(t *testing.T) {
 		}
 
 		if !bytes.Equal(output, test.output) {
-			t.Errorf("%d failed. Expected bytes: '%x' Got: '%x'", i, test.output, output)
+			t.Errorf("input %d for typ: %v failed. Expected bytes: '%x' Got: '%x'", i, typ.String(), test.output, output)
 		}
 	}
 }

accounts/abi/type.go

@@ -103,7 +103,12 @@ func NewType(t string) (typ Type, err error) {
 		return typ, err
 	}
 	// parse the type and size of the abi-type.
-	parsedType := typeRegex.FindAllStringSubmatch(t, -1)[0]
+	matches := typeRegex.FindAllStringSubmatch(t, -1)
+	if len(matches) == 0 {
+		return Type{}, fmt.Errorf("invalid type '%v'", t)
+	}
+	parsedType := matches[0]
+
 	// varSize is the size of the variable
 	var varSize int
 	if len(parsedType[3]) > 0 {
@@ -178,23 +183,39 @@ func (t Type) pack(v reflect.Value) ([]byte, error) {
 		return nil, err
 	}
 
-	if t.T == SliceTy || t.T == ArrayTy {
+	switch t.T {
-		var packed []byte
+	case SliceTy, ArrayTy:
+		var ret []byte
 
+		if t.requiresLengthPrefix() {
+			// append length
+			ret = append(ret, packNum(reflect.ValueOf(v.Len()))...)
+		}
+
+		// calculate offset if any
+		offset := 0
+		offsetReq := isDynamicType(*t.Elem)
+		if offsetReq {
+			offset = getDynamicTypeOffset(*t.Elem) * v.Len()
+		}
+		var tail []byte
 		for i := 0; i < v.Len(); i++ {
 			val, err := t.Elem.pack(v.Index(i))
 			if err != nil {
 				return nil, err
 			}
-			packed = append(packed, val...)
+			if !offsetReq {
-		}
+				ret = append(ret, val...)
-		if t.T == SliceTy {
+				continue
-			return packBytesSlice(packed, v.Len()), nil
+			}
-		} else if t.T == ArrayTy {
+			ret = append(ret, packNum(reflect.ValueOf(offset))...)
-			return packed, nil
+			offset += len(val)
+			tail = append(tail, val...)
 		}
+		return append(ret, tail...), nil
+	default:
+		return packElement(t, v), nil
 	}
-	return packElement(t, v), nil
 }
 
 // requireLengthPrefix returns whether the type requires any sort of length
@@ -202,3 +223,27 @@ func (t Type) pack(v reflect.Value) ([]byte, error) {
 func (t Type) requiresLengthPrefix() bool {
 	return t.T == StringTy || t.T == BytesTy || t.T == SliceTy
 }
+
+// isDynamicType returns true if the type is dynamic.
+// StringTy, BytesTy, and SliceTy(irrespective of slice element type) are dynamic types
+// ArrayTy is considered dynamic if and only if the Array element is a dynamic type.
+// This function recursively checks the type for slice and array elements.
+func isDynamicType(t Type) bool {
+	// dynamic types
+	// array is also a dynamic type if the array type is dynamic
+	return t.T == StringTy || t.T == BytesTy || t.T == SliceTy || (t.T == ArrayTy && isDynamicType(*t.Elem))
+}
+
+// getDynamicTypeOffset returns the offset for the type.
+// See `isDynamicType` to know which types are considered dynamic.
+// If the type t is an array and element type is not a dynamic type, then we consider it a static type and
+// return 32 * size of array since length prefix is not required.
+// If t is a dynamic type or element type(for slices and arrays) is dynamic, then we simply return 32 as offset.
+func getDynamicTypeOffset(t Type) int {
+	// if it is an array and there are no dynamic types
+	// then the array is static type
+	if t.T == ArrayTy && !isDynamicType(*t.Elem) {
+		return 32 * t.Size
+	}
+	return 32
+}

accounts/abi/unpack.go

@@ -25,8 +25,17 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
+var (
+	maxUint256 = big.NewInt(0).Add(
+		big.NewInt(0).Exp(big.NewInt(2), big.NewInt(256), nil),
+		big.NewInt(-1))
+	maxInt256 = big.NewInt(0).Add(
+		big.NewInt(0).Exp(big.NewInt(2), big.NewInt(255), nil),
+		big.NewInt(-1))
+)
+
 // reads the integer based on its kind
-func readInteger(kind reflect.Kind, b []byte) interface{} {
+func readInteger(typ byte, kind reflect.Kind, b []byte) interface{} {
 	switch kind {
 	case reflect.Uint8:
 		return b[len(b)-1]
@@ -45,7 +54,20 @@ func readInteger(kind reflect.Kind, b []byte) interface{} {
 	case reflect.Int64:
 		return int64(binary.BigEndian.Uint64(b[len(b)-8:]))
 	default:
-		return new(big.Int).SetBytes(b)
+		// the only case lefts for integer is int256/uint256.
+		// big.SetBytes can't tell if a number is negative, positive on itself.
+		// On EVM, if the returned number > max int256, it is negative.
+		ret := new(big.Int).SetBytes(b)
+		if typ == UintTy {
+			return ret
+		}
+
+		if ret.Cmp(maxInt256) > 0 {
+			ret.Add(maxUint256, big.NewInt(0).Neg(ret))
+			ret.Add(ret, big.NewInt(1))
+			ret.Neg(ret)
+		}
+		return ret
 	}
 }
 
@@ -179,7 +201,7 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) {
 	case StringTy: // variable arrays are written at the end of the return bytes
 		return string(output[begin : begin+end]), nil
 	case IntTy, UintTy:
-		return readInteger(t.Kind, returnOutput), nil
+		return readInteger(t.T, t.Kind, returnOutput), nil
 	case BoolTy:
 		return readBool(returnOutput)
 	case AddressTy:

accounts/abi/unpack_test.go

@@ -117,6 +117,11 @@ var unpackTests = []unpackTest{
 		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
 		want: big.NewInt(1),
 	},
+	{
+		def:  `[{"type": "int256"}]`,
+		enc:  "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+		want: big.NewInt(-1),
+	},
 	{
 		def:  `[{"type": "address"}]`,
 		enc:  "0000000000000000000000000100000000000000000000000000000000000000",

accounts/accounts.go

@@ -106,7 +106,7 @@ type Wallet interface {
 	// or optionally with the aid of any location metadata from the embedded URL field.
 	//
 	// If the wallet requires additional authentication to sign the request (e.g.
-	// a password to decrypt the account, or a PIN code o verify the transaction),
+	// a password to decrypt the account, or a PIN code to verify the transaction),
 	// an AuthNeededError instance will be returned, containing infos for the user
 	// about which fields or actions are needed. The user may retry by providing
 	// the needed details via SignTxWithPassphrase, or by other means (e.g. unlock

accounts/hd.go

@@ -30,8 +30,8 @@ import (
 var DefaultRootDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}
 
 // DefaultBaseDerivationPath is the base path from which custom derivation endpoints
-// are incremented. As such, the first account will be at m/44'/60'/0'/0, the second
+// are incremented. As such, the first account will be at m/44'/60'/0'/0/0, the second
-// at m/44'/60'/0'/1, etc.
+// at m/44'/60'/0'/0/1, etc.
 var DefaultBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0}
 
 // DefaultLedgerBaseDerivationPath is the base path from which custom derivation endpoints

accounts/keystore/account_cache.go

@@ -27,10 +27,10 @@ import (
 	"sync"
 	"time"
 
+	mapset "github.com/deckarep/golang-set"
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
-	"gopkg.in/fatih/set.v0"
 )
 
 // Minimum amount of time between cache reloads. This limit applies if the platform does
@@ -79,7 +79,7 @@ func newAccountCache(keydir string) (*accountCache, chan struct{}) {
 		keydir: keydir,
 		byAddr: make(map[common.Address][]accounts.Account),
 		notify: make(chan struct{}, 1),
-		fileC:  fileCache{all: set.NewNonTS()},
+		fileC:  fileCache{all: mapset.NewThreadUnsafeSet()},
 	}
 	ac.watcher = newWatcher(ac)
 	return ac, ac.notify
@@ -237,7 +237,7 @@ func (ac *accountCache) scanAccounts() error {
 		log.Debug("Failed to reload keystore contents", "err", err)
 		return err
 	}
-	if creates.Size() == 0 && deletes.Size() == 0 && updates.Size() == 0 {
+	if creates.Cardinality() == 0 && deletes.Cardinality() == 0 && updates.Cardinality() == 0 {
 		return nil
 	}
 	// Create a helper method to scan the contents of the key files
@@ -265,22 +265,25 @@ func (ac *accountCache) scanAccounts() error {
 		case (addr == common.Address{}):
 			log.Debug("Failed to decode keystore key", "path", path, "err", "missing or zero address")
 		default:
-			return &accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}}
+			return &accounts.Account{
+				Address: addr,
+				URL:     accounts.URL{Scheme: KeyStoreScheme, Path: path},
+			}
 		}
 		return nil
 	}
 	// Process all the file diffs
 	start := time.Now()
 
-	for _, p := range creates.List() {
+	for _, p := range creates.ToSlice() {
 		if a := readAccount(p.(string)); a != nil {
 			ac.add(*a)
 		}
 	}
-	for _, p := range deletes.List() {
+	for _, p := range deletes.ToSlice() {
 		ac.deleteByFile(p.(string))
 	}
-	for _, p := range updates.List() {
+	for _, p := range updates.ToSlice() {
 		path := p.(string)
 		ac.deleteByFile(path)
 		if a := readAccount(path); a != nil {

accounts/keystore/file_cache.go

@@ -24,20 +24,20 @@ import (
 	"sync"
 	"time"
 
+	mapset "github.com/deckarep/golang-set"
 	"github.com/ethereum/go-ethereum/log"
-	set "gopkg.in/fatih/set.v0"
 )
 
 // fileCache is a cache of files seen during scan of keystore.
 type fileCache struct {
-	all     *set.SetNonTS // Set of all files from the keystore folder
+	all     mapset.Set // Set of all files from the keystore folder
-	lastMod time.Time     // Last time instance when a file was modified
+	lastMod time.Time  // Last time instance when a file was modified
 	mu      sync.RWMutex
 }
 
 // scan performs a new scan on the given directory, compares against the already
 // cached filenames, and returns file sets: creates, deletes, updates.
-func (fc *fileCache) scan(keyDir string) (set.Interface, set.Interface, set.Interface, error) {
+func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, error) {
 	t0 := time.Now()
 
 	// List all the failes from the keystore folder
@@ -51,14 +51,14 @@ func (fc *fileCache) scan(keyDir string) (set.Interface, set.Interface, set.Inte
 	defer fc.mu.Unlock()
 
 	// Iterate all the files and gather their metadata
-	all := set.NewNonTS()
+	all := mapset.NewThreadUnsafeSet()
-	mods := set.NewNonTS()
+	mods := mapset.NewThreadUnsafeSet()
 
 	var newLastMod time.Time
 	for _, fi := range files {
-		// Skip any non-key files from the folder
 		path := filepath.Join(keyDir, fi.Name())
-		if skipKeyFile(fi) {
+		// Skip any non-key files from the folder
+		if nonKeyFile(fi) {
 			log.Trace("Ignoring file on account scan", "path", path)
 			continue
 		}
@@ -76,9 +76,9 @@ func (fc *fileCache) scan(keyDir string) (set.Interface, set.Interface, set.Inte
 	t2 := time.Now()
 
 	// Update the tracked files and return the three sets
-	deletes := set.Difference(fc.all, all)   // Deletes = previous - current
+	deletes := fc.all.Difference(all)   // Deletes = previous - current
-	creates := set.Difference(all, fc.all)   // Creates = current - previous
+	creates := all.Difference(fc.all)   // Creates = current - previous
-	updates := set.Difference(mods, creates) // Updates = modified - creates
+	updates := mods.Difference(creates) // Updates = modified - creates
 
 	fc.all, fc.lastMod = all, newLastMod
 	t3 := time.Now()
@@ -88,8 +88,8 @@ func (fc *fileCache) scan(keyDir string) (set.Interface, set.Interface, set.Inte
 	return creates, deletes, updates, nil
 }
 
-// skipKeyFile ignores editor backups, hidden files and folders/symlinks.
+// nonKeyFile ignores editor backups, hidden files and folders/symlinks.
-func skipKeyFile(fi os.FileInfo) bool {
+func nonKeyFile(fi os.FileInfo) bool {
 	// Skip editor backups and UNIX-style hidden files.
 	if strings.HasSuffix(fi.Name(), "~") || strings.HasPrefix(fi.Name(), ".") {
 		return true

accounts/keystore/key.go

@@ -66,19 +66,19 @@ type plainKeyJSON struct {
 
 type encryptedKeyJSONV3 struct {
 	Address string     `json:"address"`
-	Crypto  cryptoJSON `json:"crypto"`
+	Crypto  CryptoJSON `json:"crypto"`
 	Id      string     `json:"id"`
 	Version int        `json:"version"`
 }
 
 type encryptedKeyJSONV1 struct {
 	Address string     `json:"address"`
-	Crypto  cryptoJSON `json:"crypto"`
+	Crypto  CryptoJSON `json:"crypto"`
 	Id      string     `json:"id"`
 	Version string     `json:"version"`
 }
 
-type cryptoJSON struct {
+type CryptoJSON struct {
 	Cipher       string                 `json:"cipher"`
 	CipherText   string                 `json:"ciphertext"`
 	CipherParams cipherparamsJSON       `json:"cipherparams"`
@@ -171,7 +171,10 @@ func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, accounts.Accou
 	if err != nil {
 		return nil, accounts.Account{}, err
 	}
-	a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))}}
+	a := accounts.Account{
+		Address: key.Address,
+		URL:     accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))},
+	}
 	if err := ks.StoreKey(a.URL.Path, key, auth); err != nil {
 		zeroKey(key.PrivateKey)
 		return nil, a, err
@@ -179,26 +182,34 @@ func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, accounts.Accou
 	return key, a, err
 }
 
-func writeKeyFile(file string, content []byte) error {
+func writeTemporaryKeyFile(file string, content []byte) (string, error) {
 	// Create the keystore directory with appropriate permissions
 	// in case it is not present yet.
 	const dirPerm = 0700
 	if err := os.MkdirAll(filepath.Dir(file), dirPerm); err != nil {
-		return err
+		return "", err
 	}
 	// Atomic write: create a temporary hidden file first
 	// then move it into place. TempFile assigns mode 0600.
 	f, err := ioutil.TempFile(filepath.Dir(file), "."+filepath.Base(file)+".tmp")
 	if err != nil {
-		return err
+		return "", err
 	}
 	if _, err := f.Write(content); err != nil {
 		f.Close()
 		os.Remove(f.Name())
-		return err
+		return "", err
 	}
 	f.Close()
-	return os.Rename(f.Name(), file)
+	return f.Name(), nil
+}
+
+func writeKeyFile(file string, content []byte) error {
+	name, err := writeTemporaryKeyFile(file, content)
+	if err != nil {
+		return err
+	}
+	return os.Rename(name, file)
 }
 
 // keyFileName implements the naming convention for keyfiles:
@@ -216,5 +227,6 @@ func toISO8601(t time.Time) string {
 	} else {
 		tz = fmt.Sprintf("%03d00", offset/3600)
 	}
-	return fmt.Sprintf("%04d-%02d-%02dT%02d-%02d-%02d.%09d%s", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), tz)
+	return fmt.Sprintf("%04d-%02d-%02dT%02d-%02d-%02d.%09d%s",
+		t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), tz)
 }

accounts/keystore/keystore.go

@@ -50,7 +50,7 @@ var (
 var KeyStoreType = reflect.TypeOf(&KeyStore{})
 
 // KeyStoreScheme is the protocol scheme prefixing account and wallet URLs.
-var KeyStoreScheme = "keystore"
+const KeyStoreScheme = "keystore"
 
 // Maximum time between wallet refreshes (if filesystem notifications don't work).
 const walletRefreshCycle = 3 * time.Second
@@ -78,7 +78,7 @@ type unlocked struct {
 // NewKeyStore creates a keystore for the given directory.
 func NewKeyStore(keydir string, scryptN, scryptP int) *KeyStore {
 	keydir, _ = filepath.Abs(keydir)
-	ks := &KeyStore{storage: &keyStorePassphrase{keydir, scryptN, scryptP}}
+	ks := &KeyStore{storage: &keyStorePassphrase{keydir, scryptN, scryptP, false}}
 	ks.init(keydir)
 	return ks
 }

accounts/keystore/passphrase.go

@@ -28,18 +28,19 @@ package keystore
 import (
 	"bytes"
 	"crypto/aes"
-	crand "crypto/rand"
+	"crypto/rand"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/crypto/randentropy"
 	"github.com/pborman/uuid"
 	"golang.org/x/crypto/pbkdf2"
 	"golang.org/x/crypto/scrypt"
@@ -72,6 +73,10 @@ type keyStorePassphrase struct {
 	keysDirPath string
 	scryptN     int
 	scryptP     int
+	// skipKeyFileVerification disables the security-feature which does
+	// reads and decrypts any newly created keyfiles. This should be 'false' in all
+	// cases except tests -- setting this to 'true' is not recommended.
+	skipKeyFileVerification bool
 }
 
 func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string) (*Key, error) {
@@ -93,7 +98,7 @@ func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string)
 
 // StoreKey generates a key, encrypts with 'auth' and stores in the given directory
 func StoreKey(dir, auth string, scryptN, scryptP int) (common.Address, error) {
-	_, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP}, crand.Reader, auth)
+	_, a, err := storeNewKey(&keyStorePassphrase{dir, scryptN, scryptP, false}, rand.Reader, auth)
 	return a.Address, err
 }
 
@@ -102,7 +107,25 @@ func (ks keyStorePassphrase) StoreKey(filename string, key *Key, auth string) er
 	if err != nil {
 		return err
 	}
-	return writeKeyFile(filename, keyjson)
+	// Write into temporary file
+	tmpName, err := writeTemporaryKeyFile(filename, keyjson)
+	if err != nil {
+		return err
+	}
+	if !ks.skipKeyFileVerification {
+		// Verify that we can decrypt the file with the given password.
+		_, err = ks.GetKey(key.Address, tmpName, auth)
+		if err != nil {
+			msg := "An error was encountered when saving and verifying the keystore file. \n" +
+				"This indicates that the keystore is corrupted. \n" +
+				"The corrupted file is stored at \n%v\n" +
+				"Please file a ticket at:\n\n" +
+				"https://github.com/ethereum/go-ethereum/issues." +
+				"The error was : %s"
+			return fmt.Errorf(msg, tmpName, err)
+		}
+	}
+	return os.Rename(tmpName, filename)
 }
 
 func (ks keyStorePassphrase) JoinPath(filename string) string {
@@ -112,22 +135,26 @@ func (ks keyStorePassphrase) JoinPath(filename string) string {
 	return filepath.Join(ks.keysDirPath, filename)
 }
 
-// EncryptKey encrypts a key using the specified scrypt parameters into a json
+// Encryptdata encrypts the data given as 'data' with the password 'auth'.
-// blob that can be decrypted later on.
+func EncryptDataV3(data, auth []byte, scryptN, scryptP int) (CryptoJSON, error) {
-func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
+
-	authArray := []byte(auth)
+	salt := make([]byte, 32)
-	salt := randentropy.GetEntropyCSPRNG(32)
+	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
-	derivedKey, err := scrypt.Key(authArray, salt, scryptN, scryptR, scryptP, scryptDKLen)
+		panic("reading from crypto/rand failed: " + err.Error())
+	}
+	derivedKey, err := scrypt.Key(auth, salt, scryptN, scryptR, scryptP, scryptDKLen)
 	if err != nil {
-		return nil, err
+		return CryptoJSON{}, err
 	}
 	encryptKey := derivedKey[:16]
-	keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
 
-	iv := randentropy.GetEntropyCSPRNG(aes.BlockSize) // 16
+	iv := make([]byte, aes.BlockSize) // 16
-	cipherText, err := aesCTRXOR(encryptKey, keyBytes, iv)
+	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
+		panic("reading from crypto/rand failed: " + err.Error())
+	}
+	cipherText, err := aesCTRXOR(encryptKey, data, iv)
 	if err != nil {
-		return nil, err
+		return CryptoJSON{}, err
 	}
 	mac := crypto.Keccak256(derivedKey[16:32], cipherText)
 
@@ -137,12 +164,11 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
 	scryptParamsJSON["p"] = scryptP
 	scryptParamsJSON["dklen"] = scryptDKLen
 	scryptParamsJSON["salt"] = hex.EncodeToString(salt)
-
 	cipherParamsJSON := cipherparamsJSON{
 		IV: hex.EncodeToString(iv),
 	}
 
-	cryptoStruct := cryptoJSON{
+	cryptoStruct := CryptoJSON{
 		Cipher:       "aes-128-ctr",
 		CipherText:   hex.EncodeToString(cipherText),
 		CipherParams: cipherParamsJSON,
@@ -150,6 +176,17 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
 		KDFParams:    scryptParamsJSON,
 		MAC:          hex.EncodeToString(mac),
 	}
+	return cryptoStruct, nil
+}
+
+// EncryptKey encrypts a key using the specified scrypt parameters into a json
+// blob that can be decrypted later on.
+func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
+	keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
+	cryptoStruct, err := EncryptDataV3(keyBytes, []byte(auth), scryptN, scryptP)
+	if err != nil {
+		return nil, err
+	}
 	encryptedKeyJSONV3 := encryptedKeyJSONV3{
 		hex.EncodeToString(key.Address[:]),
 		cryptoStruct,
@@ -197,42 +234,48 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) {
 	}, nil
 }
 
-func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byte, keyId []byte, err error) {
+func DecryptDataV3(cryptoJson CryptoJSON, auth string) ([]byte, error) {
-	if keyProtected.Version != version {
+	if cryptoJson.Cipher != "aes-128-ctr" {
-		return nil, nil, fmt.Errorf("Version not supported: %v", keyProtected.Version)
+		return nil, fmt.Errorf("Cipher not supported: %v", cryptoJson.Cipher)
 	}
-
+	mac, err := hex.DecodeString(cryptoJson.MAC)
-	if keyProtected.Crypto.Cipher != "aes-128-ctr" {
-		return nil, nil, fmt.Errorf("Cipher not supported: %v", keyProtected.Crypto.Cipher)
-	}
-
-	keyId = uuid.Parse(keyProtected.Id)
-	mac, err := hex.DecodeString(keyProtected.Crypto.MAC)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	iv, err := hex.DecodeString(keyProtected.Crypto.CipherParams.IV)
+	iv, err := hex.DecodeString(cryptoJson.CipherParams.IV)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	cipherText, err := hex.DecodeString(keyProtected.Crypto.CipherText)
+	cipherText, err := hex.DecodeString(cryptoJson.CipherText)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	derivedKey, err := getKDFKey(keyProtected.Crypto, auth)
+	derivedKey, err := getKDFKey(cryptoJson, auth)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	calculatedMAC := crypto.Keccak256(derivedKey[16:32], cipherText)
 	if !bytes.Equal(calculatedMAC, mac) {
-		return nil, nil, ErrDecrypt
+		return nil, ErrDecrypt
 	}
 
 	plainText, err := aesCTRXOR(derivedKey[:16], cipherText, iv)
+	if err != nil {
+		return nil, err
+	}
+	return plainText, err
+}
+
+func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byte, keyId []byte, err error) {
+	if keyProtected.Version != version {
+		return nil, nil, fmt.Errorf("Version not supported: %v", keyProtected.Version)
+	}
+	keyId = uuid.Parse(keyProtected.Id)
+	plainText, err := DecryptDataV3(keyProtected.Crypto, auth)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -273,7 +316,7 @@ func decryptKeyV1(keyProtected *encryptedKeyJSONV1, auth string) (keyBytes []byt
 	return plainText, keyId, err
 }
 
-func getKDFKey(cryptoJSON cryptoJSON, auth string) ([]byte, error) {
+func getKDFKey(cryptoJSON CryptoJSON, auth string) ([]byte, error) {
 	authArray := []byte(auth)
 	salt, err := hex.DecodeString(cryptoJSON.KDFParams["salt"].(string))
 	if err != nil {

accounts/keystore/plain_test.go

@@ -37,7 +37,7 @@ func tmpKeyStoreIface(t *testing.T, encrypted bool) (dir string, ks keyStore) {
 		t.Fatal(err)
 	}
 	if encrypted {
-		ks = &keyStorePassphrase{d, veryLightScryptN, veryLightScryptP}
+		ks = &keyStorePassphrase{d, veryLightScryptN, veryLightScryptP, true}
 	} else {
 		ks = &keyStorePlain{d}
 	}
@@ -191,7 +191,7 @@ func TestV1_1(t *testing.T) {
 
 func TestV1_2(t *testing.T) {
 	t.Parallel()
-	ks := &keyStorePassphrase{"testdata/v1", LightScryptN, LightScryptP}
+	ks := &keyStorePassphrase{"testdata/v1", LightScryptN, LightScryptP, true}
 	addr := common.HexToAddress("cb61d5a9c4896fb9658090b597ef0e7be6f7b67e")
 	file := "testdata/v1/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e/cb61d5a9c4896fb9658090b597ef0e7be6f7b67e"
 	k, err := ks.GetKey(addr, file, "g")

accounts/keystore/presale.go

@@ -38,7 +38,13 @@ func importPreSaleKey(keyStore keyStore, keyJSON []byte, password string) (accou
 		return accounts.Account{}, nil, err
 	}
 	key.Id = uuid.NewRandom()
-	a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: keyStore.JoinPath(keyFileName(key.Address))}}
+	a := accounts.Account{
+		Address: key.Address,
+		URL: accounts.URL{
+			Scheme: KeyStoreScheme,
+			Path:   keyStore.JoinPath(keyFileName(key.Address)),
+		},
+	}
 	err = keyStore.StoreKey(a.URL.Path, key, password)
 	return a, key, err
 }

accounts/url.go

@@ -76,12 +76,12 @@ func (u URL) MarshalJSON() ([]byte, error) {
 
 // UnmarshalJSON parses url.
 func (u *URL) UnmarshalJSON(input []byte) error {
-	var textUrl string
+	var textURL string
-	err := json.Unmarshal(input, &textUrl)
+	err := json.Unmarshal(input, &textURL)
 	if err != nil {
 		return err
 	}
-	url, err := parseURL(textUrl)
+	url, err := parseURL(textURL)
 	if err != nil {
 		return err
 	}

accounts/url_test.go

@@ -0,0 +1,96 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package accounts
+
+import (
+	"testing"
+)
+
+func TestURLParsing(t *testing.T) {
+	url, err := parseURL("https://ethereum.org")
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if url.Scheme != "https" {
+		t.Errorf("expected: %v, got: %v", "https", url.Scheme)
+	}
+	if url.Path != "ethereum.org" {
+		t.Errorf("expected: %v, got: %v", "ethereum.org", url.Path)
+	}
+
+	_, err = parseURL("ethereum.org")
+	if err == nil {
+		t.Error("expected err, got: nil")
+	}
+}
+
+func TestURLString(t *testing.T) {
+	url := URL{Scheme: "https", Path: "ethereum.org"}
+	if url.String() != "https://ethereum.org" {
+		t.Errorf("expected: %v, got: %v", "https://ethereum.org", url.String())
+	}
+
+	url = URL{Scheme: "", Path: "ethereum.org"}
+	if url.String() != "ethereum.org" {
+		t.Errorf("expected: %v, got: %v", "ethereum.org", url.String())
+	}
+}
+
+func TestURLMarshalJSON(t *testing.T) {
+	url := URL{Scheme: "https", Path: "ethereum.org"}
+	json, err := url.MarshalJSON()
+	if err != nil {
+		t.Errorf("unexpcted error: %v", err)
+	}
+	if string(json) != "\"https://ethereum.org\"" {
+		t.Errorf("expected: %v, got: %v", "\"https://ethereum.org\"", string(json))
+	}
+}
+
+func TestURLUnmarshalJSON(t *testing.T) {
+	url := &URL{}
+	err := url.UnmarshalJSON([]byte("\"https://ethereum.org\""))
+	if err != nil {
+		t.Errorf("unexpcted error: %v", err)
+	}
+	if url.Scheme != "https" {
+		t.Errorf("expected: %v, got: %v", "https", url.Scheme)
+	}
+	if url.Path != "ethereum.org" {
+		t.Errorf("expected: %v, got: %v", "https", url.Path)
+	}
+}
+
+func TestURLComparison(t *testing.T) {
+	tests := []struct {
+		urlA   URL
+		urlB   URL
+		expect int
+	}{
+		{URL{"https", "ethereum.org"}, URL{"https", "ethereum.org"}, 0},
+		{URL{"http", "ethereum.org"}, URL{"https", "ethereum.org"}, -1},
+		{URL{"https", "ethereum.org/a"}, URL{"https", "ethereum.org"}, 1},
+		{URL{"https", "abc.org"}, URL{"https", "ethereum.org"}, -1},
+	}
+
+	for i, tt := range tests {
+		result := tt.urlA.Cmp(tt.urlB)
+		if result != tt.expect {
+			t.Errorf("test %d: cmp mismatch: expected: %d, got: %d", i, tt.expect, result)
+		}
+	}
+}

accounts/usbwallet/internal/trezor/trezor.go

@@ -36,7 +36,7 @@ func Type(msg proto.Message) uint16 {
 }
 
 // Name returns the friendly message type name of a specific protocol buffer
-// type numbers.
+// type number.
 func Name(kind uint16) string {
 	name := MessageType_name[int32(kind)]
 	if len(name) < 12 {

accounts/usbwallet/ledger.go

@@ -302,7 +302,7 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction
 	for i, component := range derivationPath {
 		binary.BigEndian.PutUint32(path[1+4*i:], component)
 	}
-	// Create the transaction RLP based on whether legacy or EIP155 signing was requeste
+	// Create the transaction RLP based on whether legacy or EIP155 signing was requested
 	var (
 		txrlp []byte
 		err   error
@@ -350,7 +350,7 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction
 		signer = new(types.HomesteadSigner)
 	} else {
 		signer = types.NewEIP155Signer(chainID)
-		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
+		signature[64] -= byte(chainID.Uint64()*2 + 35)
 	}
 	signed, err := tx.WithSignature(signer, signature)
 	if err != nil {

accounts/usbwallet/trezor.go

@@ -221,7 +221,7 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction
 		signer = new(types.HomesteadSigner)
 	} else {
 		signer = types.NewEIP155Signer(chainID)
-		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
+		signature[64] -= byte(chainID.Uint64()*2 + 35)
 	}
 	// Inject the final signature into the transaction and sanity check the sender
 	signed, err := tx.WithSignature(signer, signature)

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.2.windows-%GETH_ARCH%.zip
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.2.windows-%GETH_ARCH%.zip
-  - 7z x go1.10.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - 7z x go1.11.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

bmt/bmt.go

@@ -1,560 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Package bmt provides a binary merkle tree implementation
-package bmt
-
-import (
-	"fmt"
-	"hash"
-	"io"
-	"strings"
-	"sync"
-	"sync/atomic"
-)
-
-/*
-Binary Merkle Tree Hash is a hash function over arbitrary datachunks of limited size
-It is defined as the root hash of the binary merkle tree built over fixed size segments
-of the underlying chunk using any base hash function (e.g keccak 256 SHA3)
-
-It is used as the chunk hash function in swarm which in turn is the basis for the
-128 branching swarm hash http://swarm-guide.readthedocs.io/en/latest/architecture.html#swarm-hash
-
-The BMT is optimal for providing compact inclusion proofs, i.e. prove that a
-segment is a substring of a chunk starting at a particular offset
-The size of the underlying segments is fixed at 32 bytes (called the resolution
-of the BMT hash), the EVM word size to optimize for on-chain BMT verification
-as well as the hash size optimal for inclusion proofs in the merkle tree of the swarm hash.
-
-Two implementations are provided:
-
-* RefHasher is optimized for code simplicity and meant as a reference implementation
-* Hasher is optimized for speed taking advantage of concurrency with minimalistic
-  control structure to coordinate the concurrent routines
-  It implements the ChunkHash interface as well as the go standard hash.Hash interface
-
-*/
-
-const (
-	// DefaultSegmentCount is the maximum number of segments of the underlying chunk
-	DefaultSegmentCount = 128 // Should be equal to storage.DefaultBranches
-	// DefaultPoolSize is the maximum number of bmt trees used by the hashers, i.e,
-	// the maximum number of concurrent BMT hashing operations performed by the same hasher
-	DefaultPoolSize = 8
-)
-
-// BaseHasher is a hash.Hash constructor function used for the base hash of the  BMT.
-type BaseHasher func() hash.Hash
-
-// Hasher a reusable hasher for fixed maximum size chunks representing a BMT
-// implements the hash.Hash interface
-// reuse pool of Tree-s for amortised memory allocation and resource control
-// supports order-agnostic concurrent segment writes
-// as well as sequential read and write
-// can not be called concurrently on more than one chunk
-// can be further appended after Sum
-// Reset gives back the Tree to the pool and guaranteed to leave
-// the tree and itself in a state reusable for hashing a new chunk
-type Hasher struct {
-	pool        *TreePool   // BMT resource pool
-	bmt         *Tree       // prebuilt BMT resource for flowcontrol and proofs
-	blocksize   int         // segment size (size of hash) also for hash.Hash
-	count       int         // segment count
-	size        int         // for hash.Hash same as hashsize
-	cur         int         // cursor position for rightmost currently open chunk
-	segment     []byte      // the rightmost open segment (not complete)
-	depth       int         // index of last level
-	result      chan []byte // result channel
-	hash        []byte      // to record the result
-	max         int32       // max segments for SegmentWriter interface
-	blockLength []byte      // The block length that needes to be added in Sum
-}
-
-// New creates a reusable Hasher
-// implements the hash.Hash interface
-// pulls a new Tree from a resource pool for hashing each chunk
-func New(p *TreePool) *Hasher {
-	return &Hasher{
-		pool:      p,
-		depth:     depth(p.SegmentCount),
-		size:      p.SegmentSize,
-		blocksize: p.SegmentSize,
-		count:     p.SegmentCount,
-		result:    make(chan []byte),
-	}
-}
-
-// Node is a reuseable segment hasher representing a node in a BMT
-// it allows for continued writes after a Sum
-// and is left in completely reusable state after Reset
-type Node struct {
-	level, index int   // position of node for information/logging only
-	initial      bool  // first and last node
-	root         bool  // whether the node is root to a smaller BMT
-	isLeft       bool  // whether it is left side of the parent double segment
-	unbalanced   bool  // indicates if a node has only the left segment
-	parent       *Node // BMT connections
-	state        int32 // atomic increment impl concurrent boolean toggle
-	left, right  []byte
-}
-
-// NewNode constructor for segment hasher nodes in the BMT
-func NewNode(level, index int, parent *Node) *Node {
-	return &Node{
-		parent:  parent,
-		level:   level,
-		index:   index,
-		initial: index == 0,
-		isLeft:  index%2 == 0,
-	}
-}
-
-// TreePool provides a pool of Trees used as resources by Hasher
-// a Tree popped from the pool is guaranteed to have clean state
-// for hashing a new chunk
-// Hasher Reset releases the Tree to the pool
-type TreePool struct {
-	lock         sync.Mutex
-	c            chan *Tree
-	hasher       BaseHasher
-	SegmentSize  int
-	SegmentCount int
-	Capacity     int
-	count        int
-}
-
-// NewTreePool creates a Tree pool with hasher, segment size, segment count and capacity
-// on GetTree it reuses free Trees or creates a new one if size is not reached
-func NewTreePool(hasher BaseHasher, segmentCount, capacity int) *TreePool {
-	return &TreePool{
-		c:            make(chan *Tree, capacity),
-		hasher:       hasher,
-		SegmentSize:  hasher().Size(),
-		SegmentCount: segmentCount,
-		Capacity:     capacity,
-	}
-}
-
-// Drain drains the pool until it has no more than n resources
-func (p *TreePool) Drain(n int) {
-	p.lock.Lock()
-	defer p.lock.Unlock()
-	for len(p.c) > n {
-		<-p.c
-		p.count--
-	}
-}
-
-// Reserve is blocking until it returns an available Tree
-// it reuses free Trees or creates a new one if size is not reached
-func (p *TreePool) Reserve() *Tree {
-	p.lock.Lock()
-	defer p.lock.Unlock()
-	var t *Tree
-	if p.count == p.Capacity {
-		return <-p.c
-	}
-	select {
-	case t = <-p.c:
-	default:
-		t = NewTree(p.hasher, p.SegmentSize, p.SegmentCount)
-		p.count++
-	}
-	return t
-}
-
-// Release gives back a Tree to the pool.
-// This Tree is guaranteed to be in reusable state
-// does not need locking
-func (p *TreePool) Release(t *Tree) {
-	p.c <- t // can never fail but...
-}
-
-// Tree is a reusable control structure representing a BMT
-// organised in a binary tree
-// Hasher uses a TreePool to pick one for each chunk hash
-// the Tree is 'locked' while not in the pool
-type Tree struct {
-	leaves []*Node
-}
-
-// Draw draws the BMT (badly)
-func (t *Tree) Draw(hash []byte, d int) string {
-	var left, right []string
-	var anc []*Node
-	for i, n := range t.leaves {
-		left = append(left, fmt.Sprintf("%v", hashstr(n.left)))
-		if i%2 == 0 {
-			anc = append(anc, n.parent)
-		}
-		right = append(right, fmt.Sprintf("%v", hashstr(n.right)))
-	}
-	anc = t.leaves
-	var hashes [][]string
-	for l := 0; len(anc) > 0; l++ {
-		var nodes []*Node
-		hash := []string{""}
-		for i, n := range anc {
-			hash = append(hash, fmt.Sprintf("%v|%v", hashstr(n.left), hashstr(n.right)))
-			if i%2 == 0 && n.parent != nil {
-				nodes = append(nodes, n.parent)
-			}
-		}
-		hash = append(hash, "")
-		hashes = append(hashes, hash)
-		anc = nodes
-	}
-	hashes = append(hashes, []string{"", fmt.Sprintf("%v", hashstr(hash)), ""})
-	total := 60
-	del := "                             "
-	var rows []string
-	for i := len(hashes) - 1; i >= 0; i-- {
-		var textlen int
-		hash := hashes[i]
-		for _, s := range hash {
-			textlen += len(s)
-		}
-		if total < textlen {
-			total = textlen + len(hash)
-		}
-		delsize := (total - textlen) / (len(hash) - 1)
-		if delsize > len(del) {
-			delsize = len(del)
-		}
-		row := fmt.Sprintf("%v: %v", len(hashes)-i-1, strings.Join(hash, del[:delsize]))
-		rows = append(rows, row)
-
-	}
-	rows = append(rows, strings.Join(left, "  "))
-	rows = append(rows, strings.Join(right, "  "))
-	return strings.Join(rows, "\n") + "\n"
-}
-
-// NewTree initialises the Tree by building up the nodes of a BMT
-// segment size is stipulated to be the size of the hash
-// segmentCount needs to be positive integer and does not need to be
-// a power of two and can even be an odd number
-// segmentSize * segmentCount determines the maximum chunk size
-// hashed using the tree
-func NewTree(hasher BaseHasher, segmentSize, segmentCount int) *Tree {
-	n := NewNode(0, 0, nil)
-	n.root = true
-	prevlevel := []*Node{n}
-	// iterate over levels and creates 2^level nodes
-	level := 1
-	count := 2
-	for d := 1; d <= depth(segmentCount); d++ {
-		nodes := make([]*Node, count)
-		for i := 0; i < len(nodes); i++ {
-			parent := prevlevel[i/2]
-			t := NewNode(level, i, parent)
-			nodes[i] = t
-		}
-		prevlevel = nodes
-		level++
-		count *= 2
-	}
-	// the datanode level is the nodes on the last level where
-	return &Tree{
-		leaves: prevlevel,
-	}
-}
-
-// methods needed by hash.Hash
-
-// Size returns the size
-func (h *Hasher) Size() int {
-	return h.size
-}
-
-// BlockSize returns the block size
-func (h *Hasher) BlockSize() int {
-	return h.blocksize
-}
-
-// Sum returns the hash of the buffer
-// hash.Hash interface Sum method appends the byte slice to the underlying
-// data before it calculates and returns the hash of the chunk
-func (h *Hasher) Sum(b []byte) (r []byte) {
-	t := h.bmt
-	i := h.cur
-	n := t.leaves[i]
-	j := i
-	// must run strictly before all nodes calculate
-	// datanodes are guaranteed to have a parent
-	if len(h.segment) > h.size && i > 0 && n.parent != nil {
-		n = n.parent
-	} else {
-		i *= 2
-	}
-	d := h.finalise(n, i)
-	h.writeSegment(j, h.segment, d)
-	c := <-h.result
-	h.releaseTree()
-
-	// sha3(length + BMT(pure_chunk))
-	if h.blockLength == nil {
-		return c
-	}
-	res := h.pool.hasher()
-	res.Reset()
-	res.Write(h.blockLength)
-	res.Write(c)
-	return res.Sum(nil)
-}
-
-// Hasher implements the SwarmHash interface
-
-// Hash waits for the hasher result and returns it
-// caller must call this on a BMT Hasher being written to
-func (h *Hasher) Hash() []byte {
-	return <-h.result
-}
-
-// Hasher implements the io.Writer interface
-
-// Write fills the buffer to hash
-// with every full segment complete launches a hasher go routine
-// that shoots up the BMT
-func (h *Hasher) Write(b []byte) (int, error) {
-	l := len(b)
-	if l <= 0 {
-		return 0, nil
-	}
-	s := h.segment
-	i := h.cur
-	count := (h.count + 1) / 2
-	need := h.count*h.size - h.cur*2*h.size
-	size := h.size
-	if need > size {
-		size *= 2
-	}
-	if l < need {
-		need = l
-	}
-	// calculate missing bit to complete current open segment
-	rest := size - len(s)
-	if need < rest {
-		rest = need
-	}
-	s = append(s, b[:rest]...)
-	need -= rest
-	// read full segments and the last possibly partial segment
-	for need > 0 && i < count-1 {
-		// push all finished chunks we read
-		h.writeSegment(i, s, h.depth)
-		need -= size
-		if need < 0 {
-			size += need
-		}
-		s = b[rest : rest+size]
-		rest += size
-		i++
-	}
-	h.segment = s
-	h.cur = i
-	// otherwise, we can assume len(s) == 0, so all buffer is read and chunk is not yet full
-	return l, nil
-}
-
-// Hasher implements the io.ReaderFrom interface
-
-// ReadFrom reads from io.Reader and appends to the data to hash using Write
-// it reads so that chunk to hash is maximum length or reader reaches EOF
-// caller must Reset the hasher prior to call
-func (h *Hasher) ReadFrom(r io.Reader) (m int64, err error) {
-	bufsize := h.size*h.count - h.size*h.cur - len(h.segment)
-	buf := make([]byte, bufsize)
-	var read int
-	for {
-		var n int
-		n, err = r.Read(buf)
-		read += n
-		if err == io.EOF || read == len(buf) {
-			hash := h.Sum(buf[:n])
-			if read == len(buf) {
-				err = NewEOC(hash)
-			}
-			break
-		}
-		if err != nil {
-			break
-		}
-		n, err = h.Write(buf[:n])
-		if err != nil {
-			break
-		}
-	}
-	return int64(read), err
-}
-
-// Reset needs to be called before writing to the hasher
-func (h *Hasher) Reset() {
-	h.getTree()
-	h.blockLength = nil
-}
-
-// Hasher implements the SwarmHash interface
-
-// ResetWithLength needs to be called before writing to the hasher
-// the argument is supposed to be the byte slice binary representation of
-// the length of the data subsumed under the hash
-func (h *Hasher) ResetWithLength(l []byte) {
-	h.Reset()
-	h.blockLength = l
-}
-
-// Release gives back the Tree to the pool whereby it unlocks
-// it resets tree, segment and index
-func (h *Hasher) releaseTree() {
-	if h.bmt != nil {
-		n := h.bmt.leaves[h.cur]
-		for ; n != nil; n = n.parent {
-			n.unbalanced = false
-			if n.parent != nil {
-				n.root = false
-			}
-		}
-		h.pool.Release(h.bmt)
-		h.bmt = nil
-
-	}
-	h.cur = 0
-	h.segment = nil
-}
-
-func (h *Hasher) writeSegment(i int, s []byte, d int) {
-	hash := h.pool.hasher()
-	n := h.bmt.leaves[i]
-
-	if len(s) > h.size && n.parent != nil {
-		go func() {
-			hash.Reset()
-			hash.Write(s)
-			s = hash.Sum(nil)
-
-			if n.root {
-				h.result <- s
-				return
-			}
-			h.run(n.parent, hash, d, n.index, s)
-		}()
-		return
-	}
-	go h.run(n, hash, d, i*2, s)
-}
-
-func (h *Hasher) run(n *Node, hash hash.Hash, d int, i int, s []byte) {
-	isLeft := i%2 == 0
-	for {
-		if isLeft {
-			n.left = s
-		} else {
-			n.right = s
-		}
-		if !n.unbalanced && n.toggle() {
-			return
-		}
-		if !n.unbalanced || !isLeft || i == 0 && d == 0 {
-			hash.Reset()
-			hash.Write(n.left)
-			hash.Write(n.right)
-			s = hash.Sum(nil)
-
-		} else {
-			s = append(n.left, n.right...)
-		}
-
-		h.hash = s
-		if n.root {
-			h.result <- s
-			return
-		}
-
-		isLeft = n.isLeft
-		n = n.parent
-		i++
-	}
-}
-
-// getTree obtains a BMT resource by reserving one from the pool
-func (h *Hasher) getTree() *Tree {
-	if h.bmt != nil {
-		return h.bmt
-	}
-	t := h.pool.Reserve()
-	h.bmt = t
-	return t
-}
-
-// atomic bool toggle implementing a concurrent reusable 2-state object
-// atomic addint with %2 implements atomic bool toggle
-// it returns true if the toggler just put it in the active/waiting state
-func (n *Node) toggle() bool {
-	return atomic.AddInt32(&n.state, 1)%2 == 1
-}
-
-func hashstr(b []byte) string {
-	end := len(b)
-	if end > 4 {
-		end = 4
-	}
-	return fmt.Sprintf("%x", b[:end])
-}
-
-func depth(n int) (d int) {
-	for l := (n - 1) / 2; l > 0; l /= 2 {
-		d++
-	}
-	return d
-}
-
-// finalise is following the zigzags on the tree belonging
-// to the final datasegment
-func (h *Hasher) finalise(n *Node, i int) (d int) {
-	isLeft := i%2 == 0
-	for {
-		// when the final segment's path is going via left segments
-		// the incoming data is pushed to the parent upon pulling the left
-		// we do not need toggle the state since this condition is
-		// detectable
-		n.unbalanced = isLeft
-		n.right = nil
-		if n.initial {
-			n.root = true
-			return d
-		}
-		isLeft = n.isLeft
-		n = n.parent
-		d++
-	}
-}
-
-// EOC (end of chunk) implements the error interface
-type EOC struct {
-	Hash []byte // read the hash of the chunk off the error
-}
-
-// Error returns the error string
-func (e *EOC) Error() string {
-	return fmt.Sprintf("hasher limit reached, chunk hash: %x", e.Hash)
-}
-
-// NewEOC creates new end of chunk error with the hash
-func NewEOC(hash []byte) *EOC {
-	return &EOC{hash}
-}

bmt/bmt_r.go

@@ -1,85 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// simple nonconcurrent reference implementation for hashsize segment based
-// Binary Merkle tree hash on arbitrary but fixed maximum chunksize
-//
-// This implementation does not take advantage of any paralellisms and uses
-// far more memory than necessary, but it is easy to see that it is correct.
-// It can be used for generating test cases for optimized implementations.
-// see testBMTHasherCorrectness function in bmt_test.go
-package bmt
-
-import (
-	"hash"
-)
-
-// RefHasher is the non-optimized easy to read reference implementation of BMT
-type RefHasher struct {
-	span    int
-	section int
-	cap     int
-	h       hash.Hash
-}
-
-// NewRefHasher returns a new RefHasher
-func NewRefHasher(hasher BaseHasher, count int) *RefHasher {
-	h := hasher()
-	hashsize := h.Size()
-	maxsize := hashsize * count
-	c := 2
-	for ; c < count; c *= 2 {
-	}
-	if c > 2 {
-		c /= 2
-	}
-	return &RefHasher{
-		section: 2 * hashsize,
-		span:    c * hashsize,
-		cap:     maxsize,
-		h:       h,
-	}
-}
-
-// Hash returns the BMT hash of the byte slice
-// implements the SwarmHash interface
-func (rh *RefHasher) Hash(d []byte) []byte {
-	if len(d) > rh.cap {
-		d = d[:rh.cap]
-	}
-
-	return rh.hash(d, rh.span)
-}
-
-func (rh *RefHasher) hash(d []byte, s int) []byte {
-	l := len(d)
-	left := d
-	var right []byte
-	if l > rh.section {
-		for ; s >= l; s /= 2 {
-		}
-		left = rh.hash(d[:s], s)
-		right = d[s:]
-		if l-s > rh.section/2 {
-			right = rh.hash(right, s)
-		}
-	}
-	defer rh.h.Reset()
-	rh.h.Write(left)
-	rh.h.Write(right)
-	h := rh.h.Sum(nil)
-	return h
-}

bmt/bmt_test.go

@@ -1,481 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package bmt
-
-import (
-	"bytes"
-	crand "crypto/rand"
-	"fmt"
-	"hash"
-	"io"
-	"math/rand"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/ethereum/go-ethereum/crypto/sha3"
-)
-
-const (
-	maxproccnt = 8
-)
-
-// TestRefHasher tests that the RefHasher computes the expected BMT hash for
-// all data lengths between 0 and 256 bytes
-func TestRefHasher(t *testing.T) {
-	hashFunc := sha3.NewKeccak256
-
-	sha3 := func(data ...[]byte) []byte {
-		h := hashFunc()
-		for _, v := range data {
-			h.Write(v)
-		}
-		return h.Sum(nil)
-	}
-
-	// the test struct is used to specify the expected BMT hash for data
-	// lengths between "from" and "to"
-	type test struct {
-		from     int64
-		to       int64
-		expected func([]byte) []byte
-	}
-
-	var tests []*test
-
-	// all lengths in [0,64] should be:
-	//
-	//   sha3(data)
-	//
-	tests = append(tests, &test{
-		from: 0,
-		to:   64,
-		expected: func(data []byte) []byte {
-			return sha3(data)
-		},
-	})
-
-	// all lengths in [65,96] should be:
-	//
-	//   sha3(
-	//     sha3(data[:64])
-	//     data[64:]
-	//   )
-	//
-	tests = append(tests, &test{
-		from: 65,
-		to:   96,
-		expected: func(data []byte) []byte {
-			return sha3(sha3(data[:64]), data[64:])
-		},
-	})
-
-	// all lengths in [97,128] should be:
-	//
-	//   sha3(
-	//     sha3(data[:64])
-	//     sha3(data[64:])
-	//   )
-	//
-	tests = append(tests, &test{
-		from: 97,
-		to:   128,
-		expected: func(data []byte) []byte {
-			return sha3(sha3(data[:64]), sha3(data[64:]))
-		},
-	})
-
-	// all lengths in [129,160] should be:
-	//
-	//   sha3(
-	//     sha3(
-	//       sha3(data[:64])
-	//       sha3(data[64:128])
-	//     )
-	//     data[128:]
-	//   )
-	//
-	tests = append(tests, &test{
-		from: 129,
-		to:   160,
-		expected: func(data []byte) []byte {
-			return sha3(sha3(sha3(data[:64]), sha3(data[64:128])), data[128:])
-		},
-	})
-
-	// all lengths in [161,192] should be:
-	//
-	//   sha3(
-	//     sha3(
-	//       sha3(data[:64])
-	//       sha3(data[64:128])
-	//     )
-	//     sha3(data[128:])
-	//   )
-	//
-	tests = append(tests, &test{
-		from: 161,
-		to:   192,
-		expected: func(data []byte) []byte {
-			return sha3(sha3(sha3(data[:64]), sha3(data[64:128])), sha3(data[128:]))
-		},
-	})
-
-	// all lengths in [193,224] should be:
-	//
-	//   sha3(
-	//     sha3(
-	//       sha3(data[:64])
-	//       sha3(data[64:128])
-	//     )
-	//     sha3(
-	//       sha3(data[128:192])
-	//       data[192:]
-	//     )
-	//   )
-	//
-	tests = append(tests, &test{
-		from: 193,
-		to:   224,
-		expected: func(data []byte) []byte {
-			return sha3(sha3(sha3(data[:64]), sha3(data[64:128])), sha3(sha3(data[128:192]), data[192:]))
-		},
-	})
-
-	// all lengths in [225,256] should be:
-	//
-	//   sha3(
-	//     sha3(
-	//       sha3(data[:64])
-	//       sha3(data[64:128])
-	//     )
-	//     sha3(
-	//       sha3(data[128:192])
-	//       sha3(data[192:])
-	//     )
-	//   )
-	//
-	tests = append(tests, &test{
-		from: 225,
-		to:   256,
-		expected: func(data []byte) []byte {
-			return sha3(sha3(sha3(data[:64]), sha3(data[64:128])), sha3(sha3(data[128:192]), sha3(data[192:])))
-		},
-	})
-
-	// run the tests
-	for _, x := range tests {
-		for length := x.from; length <= x.to; length++ {
-			t.Run(fmt.Sprintf("%d_bytes", length), func(t *testing.T) {
-				data := make([]byte, length)
-				if _, err := io.ReadFull(crand.Reader, data); err != nil && err != io.EOF {
-					t.Fatal(err)
-				}
-				expected := x.expected(data)
-				actual := NewRefHasher(hashFunc, 128).Hash(data)
-				if !bytes.Equal(actual, expected) {
-					t.Fatalf("expected %x, got %x", expected, actual)
-				}
-			})
-		}
-	}
-}
-
-func testDataReader(l int) (r io.Reader) {
-	return io.LimitReader(crand.Reader, int64(l))
-}
-
-func TestHasherCorrectness(t *testing.T) {
-	err := testHasher(testBaseHasher)
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func testHasher(f func(BaseHasher, []byte, int, int) error) error {
-	tdata := testDataReader(4128)
-	data := make([]byte, 4128)
-	tdata.Read(data)
-	hasher := sha3.NewKeccak256
-	size := hasher().Size()
-	counts := []int{1, 2, 3, 4, 5, 8, 16, 32, 64, 128}
-
-	var err error
-	for _, count := range counts {
-		max := count * size
-		incr := 1
-		for n := 0; n <= max+incr; n += incr {
-			err = f(hasher, data, n, count)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func TestHasherReuseWithoutRelease(t *testing.T) {
-	testHasherReuse(1, t)
-}
-
-func TestHasherReuseWithRelease(t *testing.T) {
-	testHasherReuse(maxproccnt, t)
-}
-
-func testHasherReuse(i int, t *testing.T) {
-	hasher := sha3.NewKeccak256
-	pool := NewTreePool(hasher, 128, i)
-	defer pool.Drain(0)
-	bmt := New(pool)
-
-	for i := 0; i < 500; i++ {
-		n := rand.Intn(4096)
-		tdata := testDataReader(n)
-		data := make([]byte, n)
-		tdata.Read(data)
-
-		err := testHasherCorrectness(bmt, hasher, data, n, 128)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
-func TestHasherConcurrency(t *testing.T) {
-	hasher := sha3.NewKeccak256
-	pool := NewTreePool(hasher, 128, maxproccnt)
-	defer pool.Drain(0)
-	wg := sync.WaitGroup{}
-	cycles := 100
-	wg.Add(maxproccnt * cycles)
-	errc := make(chan error)
-
-	for p := 0; p < maxproccnt; p++ {
-		for i := 0; i < cycles; i++ {
-			go func() {
-				bmt := New(pool)
-				n := rand.Intn(4096)
-				tdata := testDataReader(n)
-				data := make([]byte, n)
-				tdata.Read(data)
-				err := testHasherCorrectness(bmt, hasher, data, n, 128)
-				wg.Done()
-				if err != nil {
-					errc <- err
-				}
-			}()
-		}
-	}
-	go func() {
-		wg.Wait()
-		close(errc)
-	}()
-	var err error
-	select {
-	case <-time.NewTimer(5 * time.Second).C:
-		err = fmt.Errorf("timed out")
-	case err = <-errc:
-	}
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func testBaseHasher(hasher BaseHasher, d []byte, n, count int) error {
-	pool := NewTreePool(hasher, count, 1)
-	defer pool.Drain(0)
-	bmt := New(pool)
-	return testHasherCorrectness(bmt, hasher, d, n, count)
-}
-
-func testHasherCorrectness(bmt hash.Hash, hasher BaseHasher, d []byte, n, count int) (err error) {
-	data := d[:n]
-	rbmt := NewRefHasher(hasher, count)
-	exp := rbmt.Hash(data)
-	timeout := time.NewTimer(time.Second)
-	c := make(chan error)
-
-	go func() {
-		bmt.Reset()
-		bmt.Write(data)
-		got := bmt.Sum(nil)
-		if !bytes.Equal(got, exp) {
-			c <- fmt.Errorf("wrong hash: expected %x, got %x", exp, got)
-		}
-		close(c)
-	}()
-	select {
-	case <-timeout.C:
-		err = fmt.Errorf("BMT hash calculation timed out")
-	case err = <-c:
-	}
-	return err
-}
-
-func BenchmarkSHA3_4k(t *testing.B)   { benchmarkSHA3(4096, t) }
-func BenchmarkSHA3_2k(t *testing.B)   { benchmarkSHA3(4096/2, t) }
-func BenchmarkSHA3_1k(t *testing.B)   { benchmarkSHA3(4096/4, t) }
-func BenchmarkSHA3_512b(t *testing.B) { benchmarkSHA3(4096/8, t) }
-func BenchmarkSHA3_256b(t *testing.B) { benchmarkSHA3(4096/16, t) }
-func BenchmarkSHA3_128b(t *testing.B) { benchmarkSHA3(4096/32, t) }
-
-func BenchmarkBMTBaseline_4k(t *testing.B)   { benchmarkBMTBaseline(4096, t) }
-func BenchmarkBMTBaseline_2k(t *testing.B)   { benchmarkBMTBaseline(4096/2, t) }
-func BenchmarkBMTBaseline_1k(t *testing.B)   { benchmarkBMTBaseline(4096/4, t) }
-func BenchmarkBMTBaseline_512b(t *testing.B) { benchmarkBMTBaseline(4096/8, t) }
-func BenchmarkBMTBaseline_256b(t *testing.B) { benchmarkBMTBaseline(4096/16, t) }
-func BenchmarkBMTBaseline_128b(t *testing.B) { benchmarkBMTBaseline(4096/32, t) }
-
-func BenchmarkRefHasher_4k(t *testing.B)   { benchmarkRefHasher(4096, t) }
-func BenchmarkRefHasher_2k(t *testing.B)   { benchmarkRefHasher(4096/2, t) }
-func BenchmarkRefHasher_1k(t *testing.B)   { benchmarkRefHasher(4096/4, t) }
-func BenchmarkRefHasher_512b(t *testing.B) { benchmarkRefHasher(4096/8, t) }
-func BenchmarkRefHasher_256b(t *testing.B) { benchmarkRefHasher(4096/16, t) }
-func BenchmarkRefHasher_128b(t *testing.B) { benchmarkRefHasher(4096/32, t) }
-
-func BenchmarkHasher_4k(t *testing.B)   { benchmarkHasher(4096, t) }
-func BenchmarkHasher_2k(t *testing.B)   { benchmarkHasher(4096/2, t) }
-func BenchmarkHasher_1k(t *testing.B)   { benchmarkHasher(4096/4, t) }
-func BenchmarkHasher_512b(t *testing.B) { benchmarkHasher(4096/8, t) }
-func BenchmarkHasher_256b(t *testing.B) { benchmarkHasher(4096/16, t) }
-func BenchmarkHasher_128b(t *testing.B) { benchmarkHasher(4096/32, t) }
-
-func BenchmarkHasherNoReuse_4k(t *testing.B)   { benchmarkHasherReuse(1, 4096, t) }
-func BenchmarkHasherNoReuse_2k(t *testing.B)   { benchmarkHasherReuse(1, 4096/2, t) }
-func BenchmarkHasherNoReuse_1k(t *testing.B)   { benchmarkHasherReuse(1, 4096/4, t) }
-func BenchmarkHasherNoReuse_512b(t *testing.B) { benchmarkHasherReuse(1, 4096/8, t) }
-func BenchmarkHasherNoReuse_256b(t *testing.B) { benchmarkHasherReuse(1, 4096/16, t) }
-func BenchmarkHasherNoReuse_128b(t *testing.B) { benchmarkHasherReuse(1, 4096/32, t) }
-
-func BenchmarkHasherReuse_4k(t *testing.B)   { benchmarkHasherReuse(16, 4096, t) }
-func BenchmarkHasherReuse_2k(t *testing.B)   { benchmarkHasherReuse(16, 4096/2, t) }
-func BenchmarkHasherReuse_1k(t *testing.B)   { benchmarkHasherReuse(16, 4096/4, t) }
-func BenchmarkHasherReuse_512b(t *testing.B) { benchmarkHasherReuse(16, 4096/8, t) }
-func BenchmarkHasherReuse_256b(t *testing.B) { benchmarkHasherReuse(16, 4096/16, t) }
-func BenchmarkHasherReuse_128b(t *testing.B) { benchmarkHasherReuse(16, 4096/32, t) }
-
-// benchmarks the minimum hashing time for a balanced (for simplicity) BMT
-// by doing count/segmentsize parallel hashings of 2*segmentsize bytes
-// doing it on n maxproccnt each reusing the base hasher
-// the premise is that this is the minimum computation needed for a BMT
-// therefore this serves as a theoretical optimum for concurrent implementations
-func benchmarkBMTBaseline(n int, t *testing.B) {
-	tdata := testDataReader(64)
-	data := make([]byte, 64)
-	tdata.Read(data)
-	hasher := sha3.NewKeccak256
-
-	t.ReportAllocs()
-	t.ResetTimer()
-	for i := 0; i < t.N; i++ {
-		count := int32((n-1)/hasher().Size() + 1)
-		wg := sync.WaitGroup{}
-		wg.Add(maxproccnt)
-		var i int32
-		for j := 0; j < maxproccnt; j++ {
-			go func() {
-				defer wg.Done()
-				h := hasher()
-				for atomic.AddInt32(&i, 1) < count {
-					h.Reset()
-					h.Write(data)
-					h.Sum(nil)
-				}
-			}()
-		}
-		wg.Wait()
-	}
-}
-
-func benchmarkHasher(n int, t *testing.B) {
-	tdata := testDataReader(n)
-	data := make([]byte, n)
-	tdata.Read(data)
-
-	size := 1
-	hasher := sha3.NewKeccak256
-	segmentCount := 128
-	pool := NewTreePool(hasher, segmentCount, size)
-	bmt := New(pool)
-
-	t.ReportAllocs()
-	t.ResetTimer()
-	for i := 0; i < t.N; i++ {
-		bmt.Reset()
-		bmt.Write(data)
-		bmt.Sum(nil)
-	}
-}
-
-func benchmarkHasherReuse(poolsize, n int, t *testing.B) {
-	tdata := testDataReader(n)
-	data := make([]byte, n)
-	tdata.Read(data)
-
-	hasher := sha3.NewKeccak256
-	segmentCount := 128
-	pool := NewTreePool(hasher, segmentCount, poolsize)
-	cycles := 200
-
-	t.ReportAllocs()
-	t.ResetTimer()
-	for i := 0; i < t.N; i++ {
-		wg := sync.WaitGroup{}
-		wg.Add(cycles)
-		for j := 0; j < cycles; j++ {
-			bmt := New(pool)
-			go func() {
-				defer wg.Done()
-				bmt.Reset()
-				bmt.Write(data)
-				bmt.Sum(nil)
-			}()
-		}
-		wg.Wait()
-	}
-}
-
-func benchmarkSHA3(n int, t *testing.B) {
-	data := make([]byte, n)
-	tdata := testDataReader(n)
-	tdata.Read(data)
-	hasher := sha3.NewKeccak256
-	h := hasher()
-
-	t.ReportAllocs()
-	t.ResetTimer()
-	for i := 0; i < t.N; i++ {
-		h.Reset()
-		h.Write(data)
-		h.Sum(nil)
-	}
-}
-
-func benchmarkRefHasher(n int, t *testing.B) {
-	data := make([]byte, n)
-	tdata := testDataReader(n)
-	tdata.Read(data)
-	hasher := sha3.NewKeccak256
-	rbmt := NewRefHasher(hasher, 128)
-
-	t.ReportAllocs()
-	t.ResetTimer()
-	for i := 0; i < t.N; i++ {
-		rbmt.Hash(data)
-	}
-}

build/ci.go

@@ -26,7 +26,7 @@ Available commands are:
    install    [ -arch architecture ] [ -cc compiler ] [ packages... ]                          -- builds packages and executables
    test       [ -coverage ] [ packages... ]                                                    -- runs the tests
    lint                                                                                        -- runs certain pre-selected linters
-   archive    [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artefacts
+   archive    [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artifacts
    importkeys                                                                                  -- imports signing keys from env
    debsrc     [ -signer key-id ] [ -upload dest ]                                              -- creates a debian source package
    nsis                                                                                        -- creates a Windows NSIS installer
@@ -59,6 +59,8 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/internal/build"
+	"github.com/ethereum/go-ethereum/params"
+	sv "github.com/ethereum/go-ethereum/swarm/version"
 )
 
 var (
@@ -77,52 +79,84 @@ var (
 		executablePath("geth"),
 		executablePath("puppeth"),
 		executablePath("rlpdump"),
-		executablePath("swarm"),
 		executablePath("wnode"),
 	}
 
+	// Files that end up in the swarm*.zip archive.
+	swarmArchiveFiles = []string{
+		"COPYING",
+		executablePath("swarm"),
+	}
+
 	// A debian package is created for all executables listed here.
 	debExecutables = []debExecutable{
 		{
-			Name:        "abigen",
+			BinaryName:  "abigen",
 			Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.",
 		},
 		{
-			Name:        "bootnode",
+			BinaryName:  "bootnode",
 			Description: "Ethereum bootnode.",
 		},
 		{
-			Name:        "evm",
+			BinaryName:  "evm",
 			Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.",
 		},
 		{
-			Name:        "geth",
+			BinaryName:  "geth",
 			Description: "Ethereum CLI client.",
 		},
 		{
-			Name:        "puppeth",
+			BinaryName:  "puppeth",
 			Description: "Ethereum private network manager.",
 		},
 		{
-			Name:        "rlpdump",
+			BinaryName:  "rlpdump",
 			Description: "Developer utility tool that prints RLP structures.",
 		},
 		{
-			Name:        "swarm",
+			BinaryName:  "wnode",
-			Description: "Ethereum Swarm daemon and tools",
-		},
-		{
-			Name:        "wnode",
 			Description: "Ethereum Whisper diagnostic tool",
 		},
 	}
 
+	// A debian package is created for all executables listed here.
+	debSwarmExecutables = []debExecutable{
+		{
+			BinaryName:  "swarm",
+			PackageName: "ethereum-swarm",
+			Description: "Ethereum Swarm daemon and tools",
+		},
+	}
+
+	debEthereum = debPackage{
+		Name:        "ethereum",
+		Version:     params.Version,
+		Executables: debExecutables,
+	}
+
+	debSwarm = debPackage{
+		Name:        "ethereum-swarm",
+		Version:     sv.Version,
+		Executables: debSwarmExecutables,
+	}
+
+	// Debian meta packages to build and push to Ubuntu PPA
+	debPackages = []debPackage{
+		debSwarm,
+		debEthereum,
+	}
+
+	// Packages to be cross-compiled by the xgo command
+	allCrossCompiledArchiveFiles = append(allToolsArchiveFiles, swarmArchiveFiles...)
+
 	// Distros for which packages are created.
 	// Note: vivid is unsupported because there is no golang-1.6 package for it.
 	// Note: wily is unsupported because it was officially deprecated on lanchpad.
 	// Note: yakkety is unsupported because it was officially deprecated on lanchpad.
 	// Note: zesty is unsupported because it was officially deprecated on lanchpad.
-	debDistros = []string{"trusty", "xenial", "artful", "bionic"}
+	// Note: artful is unsupported because it was officially deprecated on lanchpad.
+	debDistros = []string{"trusty", "xenial", "bionic", "cosmic"}
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -286,9 +320,7 @@ func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd
 // "tests" also includes static analysis tools such as vet.
 
 func doTest(cmdline []string) {
-	var (
+	coverage := flag.Bool("coverage", false, "Whether to record code coverage")
-		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
-	)
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
 
@@ -298,14 +330,11 @@ func doTest(cmdline []string) {
 	}
 	packages = build.ExpandPackagesNoVendor(packages)
 
-	// Run analysis tools before the tests.
-	build.MustRun(goTool("vet", packages...))
-
 	// Run the actual tests.
-	gotest := goTool("test", buildFlags(env)...)
 	// Test a single package at a time. CI builders are slow
 	// and some tests run into timeouts under load.
-	gotest.Args = append(gotest.Args, "-p", "1")
+	gotest := goTool("test", buildFlags(env)...)
+	gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m")
 	if *coverage {
 		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
 	}
@@ -330,6 +359,7 @@ func doLint(cmdline []string) {
 	configs := []string{
 		"--vendor",
 		"--tests",
+		"--deadline=2m",
 		"--disable-all",
 		"--enable=goimports",
 		"--enable=varcheck",
@@ -349,7 +379,6 @@ func doLint(cmdline []string) {
 }
 
 // Release Packaging
-
 func doArchive(cmdline []string) {
 	var (
 		arch   = flag.String("arch", runtime.GOARCH, "Architecture cross packaging")
@@ -369,10 +398,14 @@ func doArchive(cmdline []string) {
 	}
 
 	var (
-		env      = build.Env()
+		env = build.Env()
-		base     = archiveBasename(*arch, env)
+
-		geth     = "geth-" + base + ext
+		basegeth = archiveBasename(*arch, params.ArchiveVersion(env.Commit))
-		alltools = "geth-alltools-" + base + ext
+		geth     = "geth-" + basegeth + ext
+		alltools = "geth-alltools-" + basegeth + ext
+
+		baseswarm = archiveBasename(*arch, sv.ArchiveVersion(env.Commit))
+		swarm     = "swarm-" + baseswarm + ext
 	)
 	maybeSkipArchive(env)
 	if err := build.WriteArchive(geth, gethArchiveFiles); err != nil {
@@ -381,14 +414,17 @@ func doArchive(cmdline []string) {
 	if err := build.WriteArchive(alltools, allToolsArchiveFiles); err != nil {
 		log.Fatal(err)
 	}
-	for _, archive := range []string{geth, alltools} {
+	if err := build.WriteArchive(swarm, swarmArchiveFiles); err != nil {
+		log.Fatal(err)
+	}
+	for _, archive := range []string{geth, alltools, swarm} {
 		if err := archiveUpload(archive, *upload, *signer); err != nil {
 			log.Fatal(err)
 		}
 	}
 }
 
-func archiveBasename(arch string, env build.Environment) string {
+func archiveBasename(arch string, archiveVersion string) string {
 	platform := runtime.GOOS + "-" + arch
 	if arch == "arm" {
 		platform += os.Getenv("GOARM")
@@ -399,18 +435,7 @@ func archiveBasename(arch string, env build.Environment) string {
 	if arch == "ios" {
 		platform = "ios-all"
 	}
-	return platform + "-" + archiveVersion(env)
+	return platform + "-" + archiveVersion
-}
-
-func archiveVersion(env build.Environment) string {
-	version := build.VERSION()
-	if isUnstableBuild(env) {
-		version += "-unstable"
-	}
-	if env.Commit != "" {
-		version += "-" + env.Commit[:8]
-	}
-	return version
 }
 
 func archiveUpload(archive string, blobstore string, signer string) error {
@@ -460,7 +485,6 @@ func maybeSkipArchive(env build.Environment) {
 }
 
 // Debian Packaging
-
 func doDebianSource(cmdline []string) {
 	var (
 		signer  = flag.String("signer", "", `Signing key name, also used as package author`)
@@ -484,21 +508,23 @@ func doDebianSource(cmdline []string) {
 		build.MustRun(gpg)
 	}
 
-	// Create the packages.
+	// Create Debian packages and upload them
-	for _, distro := range debDistros {
+	for _, pkg := range debPackages {
-		meta := newDebMetadata(distro, *signer, env, now)
+		for _, distro := range debDistros {
-		pkgdir := stageDebianSource(*workdir, meta)
+			meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
-		debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc")
+			pkgdir := stageDebianSource(*workdir, meta)
-		debuild.Dir = pkgdir
+			debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc")
-		build.MustRun(debuild)
+			debuild.Dir = pkgdir
+			build.MustRun(debuild)
 
-		changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString())
+			changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString())
-		changes = filepath.Join(*workdir, changes)
+			changes = filepath.Join(*workdir, changes)
-		if *signer != "" {
+			if *signer != "" {
-			build.MustRunCommand("debsign", changes)
+				build.MustRunCommand("debsign", changes)
-		}
+			}
-		if *upload != "" {
+			if *upload != "" {
-			build.MustRunCommand("dput", *upload, changes)
+				build.MustRunCommand("dput", *upload, changes)
+			}
 		}
 	}
 }
@@ -523,9 +549,17 @@ func isUnstableBuild(env build.Environment) bool {
 	return true
 }
 
+type debPackage struct {
+	Name        string          // the name of the Debian package to produce, e.g. "ethereum", or "ethereum-swarm"
+	Version     string          // the clean version of the debPackage, e.g. 1.8.12 or 0.3.0, without any metadata
+	Executables []debExecutable // executables to be included in the package
+}
+
 type debMetadata struct {
 	Env build.Environment
 
+	PackageName string
+
 	// go-ethereum version being built. Note that this
 	// is not the debian package version. The package version
 	// is constructed by VersionString.
@@ -537,21 +571,33 @@ type debMetadata struct {
 }
 
 type debExecutable struct {
-	Name, Description string
+	PackageName string
+	BinaryName  string
+	Description string
 }
 
-func newDebMetadata(distro, author string, env build.Environment, t time.Time) debMetadata {
+// Package returns the name of the package if present, or
+// fallbacks to BinaryName
+func (d debExecutable) Package() string {
+	if d.PackageName != "" {
+		return d.PackageName
+	}
+	return d.BinaryName
+}
+
+func newDebMetadata(distro, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata {
 	if author == "" {
 		// No signing key, use default author.
 		author = "Ethereum Builds <fjl@ethereum.org>"
 	}
 	return debMetadata{
+		PackageName: name,
 		Env:         env,
 		Author:      author,
 		Distro:      distro,
-		Version:     build.VERSION(),
+		Version:     version,
 		Time:        t.Format(time.RFC1123Z),
-		Executables: debExecutables,
+		Executables: exes,
 	}
 }
 
@@ -559,9 +605,9 @@ func newDebMetadata(distro, author string, env build.Environment, t time.Time) d
 // on all executable packages.
 func (meta debMetadata) Name() string {
 	if isUnstableBuild(meta.Env) {
-		return "ethereum-unstable"
+		return meta.PackageName + "-unstable"
 	}
-	return "ethereum"
+	return meta.PackageName
 }
 
 // VersionString returns the debian version of the packages.
@@ -588,9 +634,9 @@ func (meta debMetadata) ExeList() string {
 // ExeName returns the package name of an executable package.
 func (meta debMetadata) ExeName(exe debExecutable) string {
 	if isUnstableBuild(meta.Env) {
-		return exe.Name + "-unstable"
+		return exe.Package() + "-unstable"
 	}
-	return exe.Name
+	return exe.Package()
 }
 
 // ExeConflicts returns the content of the Conflicts field
@@ -605,7 +651,7 @@ func (meta debMetadata) ExeConflicts(exe debExecutable) string {
 		// be preferred and the conflicting files should be handled via
 		// alternates. We might do this eventually but using a conflict is
 		// easier now.
-		return "ethereum, " + exe.Name
+		return "ethereum, " + exe.Package()
 	}
 	return ""
 }
@@ -622,24 +668,23 @@ func stageDebianSource(tmpdir string, meta debMetadata) (pkgdir string) {
 
 	// Put the debian build files in place.
 	debian := filepath.Join(pkgdir, "debian")
-	build.Render("build/deb.rules", filepath.Join(debian, "rules"), 0755, meta)
+	build.Render("build/deb/"+meta.PackageName+"/deb.rules", filepath.Join(debian, "rules"), 0755, meta)
-	build.Render("build/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta)
+	build.Render("build/deb/"+meta.PackageName+"/deb.changelog", filepath.Join(debian, "changelog"), 0644, meta)
-	build.Render("build/deb.control", filepath.Join(debian, "control"), 0644, meta)
+	build.Render("build/deb/"+meta.PackageName+"/deb.control", filepath.Join(debian, "control"), 0644, meta)
-	build.Render("build/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta)
+	build.Render("build/deb/"+meta.PackageName+"/deb.copyright", filepath.Join(debian, "copyright"), 0644, meta)
 	build.RenderString("8\n", filepath.Join(debian, "compat"), 0644, meta)
 	build.RenderString("3.0 (native)\n", filepath.Join(debian, "source/format"), 0644, meta)
 	for _, exe := range meta.Executables {
 		install := filepath.Join(debian, meta.ExeName(exe)+".install")
 		docs := filepath.Join(debian, meta.ExeName(exe)+".docs")
-		build.Render("build/deb.install", install, 0644, exe)
+		build.Render("build/deb/"+meta.PackageName+"/deb.install", install, 0644, exe)
-		build.Render("build/deb.docs", docs, 0644, exe)
+		build.Render("build/deb/"+meta.PackageName+"/deb.docs", docs, 0644, exe)
 	}
 
 	return pkgdir
 }
 
 // Windows installer
-
 func doWindowsInstaller(cmdline []string) {
 	// Parse the flags and make skip installer generation on PRs
 	var (
@@ -689,11 +734,11 @@ func doWindowsInstaller(cmdline []string) {
 	// Build the installer. This assumes that all the needed files have been previously
 	// built (don't mix building and packaging to keep cross compilation complexity to a
 	// minimum).
-	version := strings.Split(build.VERSION(), ".")
+	version := strings.Split(params.Version, ".")
 	if env.Commit != "" {
 		version[2] += "-" + env.Commit[:8]
 	}
-	installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, env) + ".exe")
+	installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe")
 	build.MustRunCommand("makensis.exe",
 		"/DOUTPUTFILE="+installer,
 		"/DMAJORVERSION="+version[0],
@@ -745,7 +790,7 @@ func doAndroidArchive(cmdline []string) {
 	maybeSkipArchive(env)
 
 	// Sign and upload the archive to Azure
-	archive := "geth-" + archiveBasename("android", env) + ".aar"
+	archive := "geth-" + archiveBasename("android", params.ArchiveVersion(env.Commit)) + ".aar"
 	os.Rename("geth.aar", archive)
 
 	if err := archiveUpload(archive, *upload, *signer); err != nil {
@@ -830,7 +875,7 @@ func newMavenMetadata(env build.Environment) mavenMetadata {
 		}
 	}
 	// Render the version and package strings
-	version := build.VERSION()
+	version := params.Version
 	if isUnstableBuild(env) {
 		version += "-SNAPSHOT"
 	}
@@ -865,7 +910,7 @@ func doXCodeFramework(cmdline []string) {
 		build.MustRun(bind)
 		return
 	}
-	archive := "geth-" + archiveBasename("ios", env)
+	archive := "geth-" + archiveBasename("ios", params.ArchiveVersion(env.Commit))
 	if err := os.Mkdir(archive, os.ModePerm); err != nil {
 		log.Fatal(err)
 	}
@@ -921,7 +966,7 @@ func newPodMetadata(env build.Environment, archive string) podMetadata {
 			}
 		}
 	}
-	version := build.VERSION()
+	version := params.Version
 	if isUnstableBuild(env) {
 		version += "-unstable." + env.Buildnum
 	}
@@ -951,7 +996,7 @@ func doXgo(cmdline []string) {
 
 	if *alltools {
 		args = append(args, []string{"--dest", GOBIN}...)
-		for _, res := range allToolsArchiveFiles {
+		for _, res := range allCrossCompiledArchiveFiles {
 			if strings.HasPrefix(res, GOBIN) {
 				// Binary tool found, cross build it explicitly
 				args = append(args, "./"+filepath.Join("cmd", filepath.Base(res)))
@@ -990,7 +1035,7 @@ func xgoTool(args []string) *exec.Cmd {
 func doPurge(cmdline []string) {
 	var (
 		store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`)
-		limit = flag.Int("days", 30, `Age threshold above which to delete unstalbe archives`)
+		limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`)
 	)
 	flag.CommandLine.Parse(cmdline)
 
@@ -1017,23 +1062,14 @@ func doPurge(cmdline []string) {
 	}
 	for i := 0; i < len(blobs); i++ {
 		for j := i + 1; j < len(blobs); j++ {
-			iTime, err := time.Parse(time.RFC1123, blobs[i].Properties.LastModified)
+			if blobs[i].Properties.LastModified.After(blobs[j].Properties.LastModified) {
-			if err != nil {
-				log.Fatal(err)
-			}
-			jTime, err := time.Parse(time.RFC1123, blobs[j].Properties.LastModified)
-			if err != nil {
-				log.Fatal(err)
-			}
-			if iTime.After(jTime) {
 				blobs[i], blobs[j] = blobs[j], blobs[i]
 			}
 		}
 	}
 	// Filter out all archives more recent that the given threshold
 	for i, blob := range blobs {
-		timestamp, _ := time.Parse(time.RFC1123, blob.Properties.LastModified)
+		if time.Since(blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour {
-		if time.Since(timestamp) < time.Duration(*limit)*24*time.Hour {
 			blobs = blobs[:i]
 			break
 		}

build/clean_go_build_cache.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Cleaning the Go cache only makes sense if we actually have Go installed... or
+# if Go is actually callable. This does not hold true during deb packaging, so
+# we need an explicit check to avoid build failures.
+if ! command -v go > /dev/null; then
+  exit
+fi
+
+version_gt() {
+  test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"
+}
+
+golang_version=$(go version |cut -d' ' -f3 |sed 's/go//')
+
+# Clean go build cache when go version is greater than or equal to 1.10
+if !(version_gt 1.10 $golang_version); then
+    go clean -cache
+fi

build/deb.install

@@ -1 +0,0 @@
-build/bin/{{.Name}} usr/bin

build/deb/ethereum-swarm/deb.control

@@ -0,0 +1,19 @@
+Source: {{.Name}}
+Section: science
+Priority: extra
+Maintainer: {{.Author}}
+Build-Depends: debhelper (>= 8.0.0), golang-1.10
+Standards-Version: 3.9.5
+Homepage: https://ethereum.org
+Vcs-Git: git://github.com/ethereum/go-ethereum.git
+Vcs-Browser: https://github.com/ethereum/go-ethereum
+
+{{range .Executables}}
+Package: {{$.ExeName .}}
+Conflicts: {{$.ExeConflicts .}}
+Architecture: any
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Built-Using: ${misc:Built-Using}
+Description: {{.Description}}
+ {{.Description}}
+{{end}}

build/deb/ethereum-swarm/deb.copyright

@@ -1,4 +1,4 @@
-Copyright 2016 The go-ethereum Authors
+Copyright 2018 The go-ethereum Authors
 
 go-ethereum is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

build/deb/ethereum-swarm/deb.install

@@ -0,0 +1 @@
+build/bin/{{.BinaryName}} usr/bin

build/deb/ethereum/deb.changelog

@@ -0,0 +1,5 @@
+{{.Name}} ({{.VersionString}}) {{.Distro}}; urgency=low
+
+  * git build of {{.Env.Commit}}
+
+ -- {{.Author}}  {{.Time}}

build/deb/ethereum/deb.control

@@ -11,8 +11,8 @@ Vcs-Browser: https://github.com/ethereum/go-ethereum
 Package: {{.Name}}
 Architecture: any
 Depends: ${misc:Depends}, {{.ExeList}}
-Description: Meta-package to install geth and other tools
+Description: Meta-package to install geth, swarm, and other tools
- Meta-package to install geth and other tools
+ Meta-package to install geth, swarm and other tools
 
 {{range .Executables}}
 Package: {{$.ExeName .}}

build/deb/ethereum/deb.copyright

@@ -0,0 +1,14 @@
+Copyright 2018 The go-ethereum Authors
+
+go-ethereum is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+go-ethereum is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

build/deb/ethereum/deb.docs

@@ -0,0 +1 @@
+AUTHORS

build/deb/ethereum/deb.install

@@ -0,0 +1 @@
+build/bin/{{.BinaryName}} usr/bin

build/deb/ethereum/deb.rules

@@ -0,0 +1,13 @@
+#!/usr/bin/make -f
+# -*- makefile -*-
+
+# Uncomment this to turn on verbose mode.
+#export DH_VERBOSE=1
+
+override_dh_auto_build:
+	build/env.sh /usr/lib/go-1.10/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
+
+override_dh_auto_test:
+
+%:
+	dh $@

build/goimports.sh

@@ -1,18 +1,18 @@
-#!/usr/bin/env bash
+#!/bin/sh
 
 find_files() {
-  find . -not \( \
+  find . ! \( \
       \( \
-        -wholename '.github' \
+        -path '.github' \
-        -o -wholename './build/_workspace' \
+        -o -path './build/_workspace' \
-        -o -wholename './build/bin' \
+        -o -path './build/bin' \
-        -o -wholename './crypto/bn256' \
+        -o -path './crypto/bn256' \
-        -o -wholename '*/vendor/*' \
+        -o -path '*/vendor/*' \
       \) -prune \
     \) -name '*.go'
 }
 
-GOFMT="gofmt -s -w";
+GOFMT="gofmt -s -w"
-GOIMPORTS="goimports -w";
+GOIMPORTS="goimports -w"
-find_files | xargs $GOFMT;
+find_files | xargs $GOFMT
-find_files | xargs $GOIMPORTS;
+find_files | xargs $GOIMPORTS

cmd/abigen/main.go

@@ -75,7 +75,7 @@ func main() {
 		bins  []string
 		types []string
 	)
-	if *solFlag != "" || *abiFlag == "-" {
+	if *solFlag != "" || (*abiFlag == "-" && *pkgFlag == "") {
 		// Generate the list of types to exclude from binding
 		exclude := make(map[string]bool)
 		for _, kind := range strings.Split(*excFlag, ",") {
@@ -111,7 +111,13 @@ func main() {
 		}
 	} else {
 		// Otherwise load up the ABI, optional bytecode and type name from the parameters
-		abi, err := ioutil.ReadFile(*abiFlag)
+		var abi []byte
+		var err error
+		if *abiFlag == "-" {
+			abi, err = ioutil.ReadAll(os.Stdin)
+		} else {
+			abi, err = ioutil.ReadFile(*abiFlag)
+		}
 		if err != nil {
 			fmt.Printf("Failed to read input ABI: %v\n", err)
 			os.Exit(-1)
@@ -155,6 +161,5 @@ func contractsFromStdin() (map[string]*compiler.Contract, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	return compiler.ParseCombinedJSON(bytes, "", "", "", "")
 }

cmd/bootnode/main.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/discv5"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 )
@@ -37,7 +38,7 @@ func main() {
 	var (
 		listenAddr  = flag.String("addr", ":30301", "listen address")
 		genKey      = flag.String("genkey", "", "generate a node key")
-		writeAddr   = flag.Bool("writeaddress", false, "write out the node's pubkey hash and quit")
+		writeAddr   = flag.Bool("writeaddress", false, "write out the node's public key and quit")
 		nodeKeyFile = flag.String("nodekey", "", "private key filename")
 		nodeKeyHex  = flag.String("nodekeyhex", "", "private key as hex (for testing)")
 		natdesc     = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|extip:<IP>)")
@@ -85,7 +86,7 @@ func main() {
 	}
 
 	if *writeAddr {
-		fmt.Printf("%v\n", discover.PubkeyID(&nodeKey.PublicKey))
+		fmt.Printf("%x\n", crypto.FromECDSAPub(&nodeKey.PublicKey)[1:])
 		os.Exit(0)
 	}
 
@@ -118,16 +119,17 @@ func main() {
 	}
 
 	if *runv5 {
-		if _, err := discv5.ListenUDP(nodeKey, conn, realaddr, "", restrictList); err != nil {
+		if _, err := discv5.ListenUDP(nodeKey, conn, "", restrictList); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	} else {
+		db, _ := enode.OpenDB("")
+		ln := enode.NewLocalNode(db, nodeKey)
 		cfg := discover.Config{
-			PrivateKey:   nodeKey,
+			PrivateKey:  nodeKey,
-			AnnounceAddr: realaddr,
+			NetRestrict: restrictList,
-			NetRestrict:  restrictList,
 		}
-		if _, err := discover.ListenUDP(conn, cfg); err != nil {
+		if _, err := discover.ListenUDP(conn, ln, cfg); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	}

cmd/clef/README.md

@@ -91,7 +91,7 @@ invoking methods with the following info:
   * [x] Version info about the signer
   * [x] Address of API (http/ipc)
   * [ ] List of known accounts
-* [ ] Have a default timeout on signing operations, so that if the user has not answered withing e.g. 60 seconds, the request is rejected.
+* [ ] Have a default timeout on signing operations, so that if the user has not answered within e.g. 60 seconds, the request is rejected.
 * [ ] `account_signRawTransaction`
 * [ ] `account_bulkSignTransactions([] transactions)` should
    * only exist if enabled via config/flag
@@ -129,7 +129,7 @@ The signer listens to HTTP requests on `rpcaddr`:`rpcport`, with the same JSONRP
 expected to be JSON [jsonrpc 2.0 standard](http://www.jsonrpc.org/specification).
 
 Some of these call can require user interaction. Clients must be aware that responses
-may be delayed significanlty or may never be received if a users decides to ignore the confirmation request.
+may be delayed significantly or may never be received if a users decides to ignore the confirmation request.
 
 The External API is **untrusted** : it does not accept credentials over this api, nor does it expect
 that requests have any authority.
@@ -862,7 +862,7 @@ A UI should conform to the following rules.
 * A UI SHOULD inform the user about the `SHA256` or `MD5` hash of the binary being executed
 * A UI SHOULD NOT maintain a secondary storage of data, e.g. list of accounts
   * The signer provides accounts
-* A UI SHOULD, to the best extent possible, use static linking / bundling, so that requried libraries are bundled
+* A UI SHOULD, to the best extent possible, use static linking / bundling, so that required libraries are bundled
 along with the UI.
 
 
@@ -875,3 +875,4 @@ There are a couple of implementation for a UI. We'll try to keep this list up to
 | QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: |  :+1: (partially)|
 | GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: |  :x: |
 | Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: |  :x: |
+| Clef UI| https://github.com/kyokan/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: |  :+1: (approve tx only)|

cmd/clef/extapi_changelog.md

@@ -1,6 +1,13 @@
 ### Changelog for external API
 
+#### 4.0.0
 
+* The external `account_Ecrecover`-method was removed. 
+* The external `account_Import`-method was removed.
+
+#### 3.0.0
+
+* The external `account_List`-method was changed to not expose `url`, which contained info about the local filesystem. It now returns only a list of addresses. 
 
 #### 2.0.0
 

cmd/clef/intapi_changelog.md

@@ -1,5 +1,24 @@
 ### Changelog for internal API (ui-api)
 
+### 3.0.0
+
+* Make use of `OnInputRequired(info UserInputRequest)` for obtaining master password during startup
+
+### 2.1.0
+
+* Add `OnInputRequired(info UserInputRequest)` to internal API. This method is used when Clef needs user input, e.g. passwords.
+
+The following structures are used:
+```golang
+       UserInputRequest struct {
+               Prompt     string `json:"prompt"`
+               Title      string `json:"title"`
+               IsPassword bool   `json:"isPassword"`
+       }
+       UserInputResponse struct {
+               Text string `json:"text"`
+       }
+
 ### 2.0.0
 
 * Modify how `call_info` on a transaction is conveyed. New format:

cmd/clef/main.go

@@ -35,8 +35,10 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
@@ -48,10 +50,10 @@ import (
 )
 
 // ExternalAPIVersion -- see extapi_changelog.md
-const ExternalAPIVersion = "2.0.0"
+const ExternalAPIVersion = "4.0.0"
 
 // InternalAPIVersion -- see intapi_changelog.md
-const InternalAPIVersion = "2.0.0"
+const InternalAPIVersion = "3.0.0"
 
 const legalWarning = `
 WARNING! 
@@ -70,6 +72,10 @@ var (
 		Value: 4,
 		Usage: "log level to emit to the screen",
 	}
+	advancedMode = cli.BoolFlag{
+		Name:  "advanced",
+		Usage: "If enabled, issues warnings instead of rejections for suspicious requests. Default off",
+	}
 	keystoreFlag = cli.StringFlag{
 		Name:  "keystore",
 		Value: filepath.Join(node.DefaultDataDir(), "keystore"),
@@ -87,7 +93,7 @@ var (
 	}
 	signerSecretFlag = cli.StringFlag{
 		Name:  "signersecret",
-		Usage: "A file containing the password used to encrypt Clef credentials, e.g. keystore credentials and ruleset hash",
+		Usage: "A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash",
 	}
 	dBFlag = cli.StringFlag{
 		Name:  "4bytedb",
@@ -151,18 +157,18 @@ Whenever you make an edit to the rule file, you need to use attestation to tell
 Clef that the file is 'safe' to execute.`,
 	}
 
-	addCredentialCommand = cli.Command{
+	setCredentialCommand = cli.Command{
-		Action:    utils.MigrateFlags(addCredential),
+		Action:    utils.MigrateFlags(setCredential),
-		Name:      "addpw",
+		Name:      "setpw",
 		Usage:     "Store a credential for a keystore file",
-		ArgsUsage: "<address> <password>",
+		ArgsUsage: "<address>",
 		Flags: []cli.Flag{
 			logLevelFlag,
 			configdirFlag,
 			signerSecretFlag,
 		},
 		Description: `
-The addpw command stores a password for a given address (keyfile). If you invoke it with only one parameter, it will 
+		The setpw command stores a password for a given address (keyfile). If you enter a blank passphrase, it will 
 remove any stored credential for that address (keyfile)
 `,
 	}
@@ -191,9 +197,10 @@ func init() {
 		ruleFlag,
 		stdiouiFlag,
 		testFlag,
+		advancedMode,
 	}
 	app.Action = signer
-	app.Commands = []cli.Command{initCommand, attestCommand, addCredentialCommand}
+	app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand}
 
 }
 func main() {
@@ -207,25 +214,45 @@ func initializeSecrets(c *cli.Context) error {
 	if err := initialize(c); err != nil {
 		return err
 	}
-	configDir := c.String(configdirFlag.Name)
+	configDir := c.GlobalString(configdirFlag.Name)
 
 	masterSeed := make([]byte, 256)
-	n, err := io.ReadFull(rand.Reader, masterSeed)
+	num, err := io.ReadFull(rand.Reader, masterSeed)
 	if err != nil {
 		return err
 	}
-	if n != len(masterSeed) {
+	if num != len(masterSeed) {
 		return fmt.Errorf("failed to read enough random")
 	}
+
+	n, p := keystore.StandardScryptN, keystore.StandardScryptP
+	if c.GlobalBool(utils.LightKDFFlag.Name) {
+		n, p = keystore.LightScryptN, keystore.LightScryptP
+	}
+	text := "The master seed of clef is locked with a password. Please give a password. Do not forget this password."
+	var password string
+	for {
+		password = getPassPhrase(text, true)
+		if err := core.ValidatePasswordFormat(password); err != nil {
+			fmt.Printf("invalid password: %v\n", err)
+		} else {
+			break
+		}
+	}
+	cipherSeed, err := encryptSeed(masterSeed, []byte(password), n, p)
+	if err != nil {
+		return fmt.Errorf("failed to encrypt master seed: %v", err)
+	}
+
 	err = os.Mkdir(configDir, 0700)
 	if err != nil && !os.IsExist(err) {
 		return err
 	}
-	location := filepath.Join(configDir, "secrets.dat")
+	location := filepath.Join(configDir, "masterseed.json")
 	if _, err := os.Stat(location); err == nil {
 		return fmt.Errorf("file %v already exists, will not overwrite", location)
 	}
-	err = ioutil.WriteFile(location, masterSeed, 0700)
+	err = ioutil.WriteFile(location, cipherSeed, 0400)
 	if err != nil {
 		return err
 	}
@@ -250,11 +277,11 @@ func attestFile(ctx *cli.Context) error {
 		return err
 	}
 
-	stretchedKey, err := readMasterKey(ctx)
+	stretchedKey, err := readMasterKey(ctx, nil)
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	configDir := ctx.String(configdirFlag.Name)
+	configDir := ctx.GlobalString(configdirFlag.Name)
 	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 	confKey := crypto.Keccak256([]byte("config"), stretchedKey)
 
@@ -266,38 +293,36 @@ func attestFile(ctx *cli.Context) error {
 	return nil
 }
 
-func addCredential(ctx *cli.Context) error {
+func setCredential(ctx *cli.Context) error {
 	if len(ctx.Args()) < 1 {
-		utils.Fatalf("This command requires at leaste one argument.")
+		utils.Fatalf("This command requires an address to be passed as an argument.")
 	}
 	if err := initialize(ctx); err != nil {
 		return err
 	}
 
-	stretchedKey, err := readMasterKey(ctx)
+	address := ctx.Args().First()
+	password := getPassPhrase("Enter a passphrase to store with this address.", true)
+
+	stretchedKey, err := readMasterKey(ctx, nil)
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	configDir := ctx.String(configdirFlag.Name)
+	configDir := ctx.GlobalString(configdirFlag.Name)
 	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 	pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
 
 	// Initialize the encrypted storages
 	pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
-	key := ctx.Args().First()
+	pwStorage.Put(address, password)
-	value := ""
+	log.Info("Credential store updated", "key", address)
-	if len(ctx.Args()) > 1 {
-		value = ctx.Args().Get(1)
-	}
-	pwStorage.Put(key, value)
-	log.Info("Credential store updated", "key", key)
 	return nil
 }
 
 func initialize(c *cli.Context) error {
 	// Set up the logger to print everything
 	logOutput := os.Stdout
-	if c.Bool(stdiouiFlag.Name) {
+	if c.GlobalBool(stdiouiFlag.Name) {
 		logOutput = os.Stderr
 		// If using the stdioui, we can't do the 'confirm'-flow
 		fmt.Fprintf(logOutput, legalWarning)
@@ -318,26 +343,28 @@ func signer(c *cli.Context) error {
 	var (
 		ui core.SignerUI
 	)
-	if c.Bool(stdiouiFlag.Name) {
+	if c.GlobalBool(stdiouiFlag.Name) {
 		log.Info("Using stdin/stdout as UI-channel")
 		ui = core.NewStdIOUI()
 	} else {
 		log.Info("Using CLI as UI-channel")
 		ui = core.NewCommandlineUI()
 	}
-	db, err := core.NewAbiDBFromFiles(c.String(dBFlag.Name), c.String(customDBFlag.Name))
+	fourByteDb := c.GlobalString(dBFlag.Name)
+	fourByteLocal := c.GlobalString(customDBFlag.Name)
+	db, err := core.NewAbiDBFromFiles(fourByteDb, fourByteLocal)
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	log.Info("Loaded 4byte db", "signatures", db.Size(), "file", c.String("4bytedb"))
+	log.Info("Loaded 4byte db", "signatures", db.Size(), "file", fourByteDb, "local", fourByteLocal)
 
 	var (
 		api core.ExternalAPI
 	)
 
-	configDir := c.String(configdirFlag.Name)
+	configDir := c.GlobalString(configdirFlag.Name)
-	if stretchedKey, err := readMasterKey(c); err != nil {
+	if stretchedKey, err := readMasterKey(c, ui); err != nil {
-		log.Info("No master seed provided, rules disabled")
+		log.Info("No master seed provided, rules disabled", "error", err)
 	} else {
 
 		if err != nil {
@@ -356,7 +383,7 @@ func signer(c *cli.Context) error {
 		configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey)
 
 		//Do we have a rule-file?
-		ruleJS, err := ioutil.ReadFile(c.String(ruleFlag.Name))
+		ruleJS, err := ioutil.ReadFile(c.GlobalString(ruleFlag.Name))
 		if err != nil {
 			log.Info("Could not load rulefile, rules not enabled", "file", "rulefile")
 		} else {
@@ -380,16 +407,15 @@ func signer(c *cli.Context) error {
 	}
 
 	apiImpl := core.NewSignerAPI(
-		c.Int64(utils.NetworkIdFlag.Name),
+		c.GlobalInt64(utils.NetworkIdFlag.Name),
-		c.String(keystoreFlag.Name),
+		c.GlobalString(keystoreFlag.Name),
-		c.Bool(utils.NoUSBFlag.Name),
+		c.GlobalBool(utils.NoUSBFlag.Name),
 		ui, db,
-		c.Bool(utils.LightKDFFlag.Name))
+		c.GlobalBool(utils.LightKDFFlag.Name),
-
+		c.GlobalBool(advancedMode.Name))
 	api = apiImpl
-
 	// Audit logging
-	if logfile := c.String(auditLogFlag.Name); logfile != "" {
+	if logfile := c.GlobalString(auditLogFlag.Name); logfile != "" {
 		api, err = core.NewAuditLogger(logfile, api)
 		if err != nil {
 			utils.Fatalf(err.Error())
@@ -408,14 +434,14 @@ func signer(c *cli.Context) error {
 			Service:   api,
 			Version:   "1.0"},
 	}
-	if c.Bool(utils.RPCEnabledFlag.Name) {
+	if c.GlobalBool(utils.RPCEnabledFlag.Name) {
 
 		vhosts := splitAndTrim(c.GlobalString(utils.RPCVirtualHostsFlag.Name))
 		cors := splitAndTrim(c.GlobalString(utils.RPCCORSDomainFlag.Name))
 
 		// start http server
-		httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
+		httpEndpoint := fmt.Sprintf("%s:%d", c.GlobalString(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
-		listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts)
+		listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts, rpc.DefaultHTTPTimeouts)
 		if err != nil {
 			utils.Fatalf("Could not start RPC api: %v", err)
 		}
@@ -428,9 +454,9 @@ func signer(c *cli.Context) error {
 		}()
 
 	}
-	if !c.Bool(utils.IPCDisabledFlag.Name) {
+	if !c.GlobalBool(utils.IPCDisabledFlag.Name) {
 		if c.IsSet(utils.IPCPathFlag.Name) {
-			ipcapiURL = c.String(utils.IPCPathFlag.Name)
+			ipcapiURL = c.GlobalString(utils.IPCPathFlag.Name)
 		} else {
 			ipcapiURL = filepath.Join(configDir, "clef.ipc")
 		}
@@ -447,7 +473,7 @@ func signer(c *cli.Context) error {
 
 	}
 
-	if c.Bool(testFlag.Name) {
+	if c.GlobalBool(testFlag.Name) {
 		log.Info("Performing UI test")
 		go testExternalUI(apiImpl)
 	}
@@ -506,48 +532,64 @@ func homeDir() string {
 	}
 	return ""
 }
-func readMasterKey(ctx *cli.Context) ([]byte, error) {
+func readMasterKey(ctx *cli.Context, ui core.SignerUI) ([]byte, error) {
 	var (
 		file      string
-		configDir = ctx.String(configdirFlag.Name)
+		configDir = ctx.GlobalString(configdirFlag.Name)
 	)
-	if ctx.IsSet(signerSecretFlag.Name) {
+	if ctx.GlobalIsSet(signerSecretFlag.Name) {
-		file = ctx.String(signerSecretFlag.Name)
+		file = ctx.GlobalString(signerSecretFlag.Name)
 	} else {
-		file = filepath.Join(configDir, "secrets.dat")
+		file = filepath.Join(configDir, "masterseed.json")
 	}
 	if err := checkFile(file); err != nil {
 		return nil, err
 	}
-	masterKey, err := ioutil.ReadFile(file)
+	cipherKey, err := ioutil.ReadFile(file)
 	if err != nil {
 		return nil, err
 	}
-	if len(masterKey) < 256 {
+	var password string
-		return nil, fmt.Errorf("master key of insufficient length, expected >255 bytes, got %d", len(masterKey))
+	// If ui is not nil, get the password from ui.
+	if ui != nil {
+		resp, err := ui.OnInputRequired(core.UserInputRequest{
+			Title:      "Master Password",
+			Prompt:     "Please enter the password to decrypt the master seed",
+			IsPassword: true})
+		if err != nil {
+			return nil, err
+		}
+		password = resp.Text
+	} else {
+		password = getPassPhrase("Decrypt master seed of clef", false)
 	}
+	masterSeed, err := decryptSeed(cipherKey, password)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decrypt the master seed of clef")
+	}
+	if len(masterSeed) < 256 {
+		return nil, fmt.Errorf("master seed of insufficient length, expected >255 bytes, got %d", len(masterSeed))
+	}
+
 	// Create vault location
-	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterKey)[:10]))
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterSeed)[:10]))
 	err = os.Mkdir(vaultLocation, 0700)
 	if err != nil && !os.IsExist(err) {
 		return nil, err
 	}
-	//!TODO, use KDF to stretch the master key
+	return masterSeed, nil
-	//			stretched_key := stretch_key(master_key)
-
-	return masterKey, nil
 }
 
 // checkFile is a convenience function to check if a file
 // * exists
-// * is mode 0600
+// * is mode 0400
 func checkFile(filename string) error {
 	info, err := os.Stat(filename)
 	if err != nil {
 		return fmt.Errorf("failed stat on %s: %v", filename, err)
 	}
 	// Check the unix permission bits
-	if info.Mode().Perm()&077 != 0 {
+	if info.Mode().Perm()&0377 != 0 {
 		return fmt.Errorf("file (%v) has insecure file permissions (%v)", filename, info.Mode().String())
 	}
 	return nil
@@ -613,6 +655,59 @@ func testExternalUI(api *core.SignerAPI) {
 
 }
 
+// getPassPhrase retrieves the password associated with clef, either fetched
+// from a list of preloaded passphrases, or requested interactively from the user.
+// TODO: there are many `getPassPhrase` functions, it will be better to abstract them into one.
+func getPassPhrase(prompt string, confirmation bool) string {
+	fmt.Println(prompt)
+	password, err := console.Stdin.PromptPassword("Passphrase: ")
+	if err != nil {
+		utils.Fatalf("Failed to read passphrase: %v", err)
+	}
+	if confirmation {
+		confirm, err := console.Stdin.PromptPassword("Repeat passphrase: ")
+		if err != nil {
+			utils.Fatalf("Failed to read passphrase confirmation: %v", err)
+		}
+		if password != confirm {
+			utils.Fatalf("Passphrases do not match")
+		}
+	}
+	return password
+}
+
+type encryptedSeedStorage struct {
+	Description string              `json:"description"`
+	Version     int                 `json:"version"`
+	Params      keystore.CryptoJSON `json:"params"`
+}
+
+// encryptSeed uses a similar scheme as the keystore uses, but with a different wrapping,
+// to encrypt the master seed
+func encryptSeed(seed []byte, auth []byte, scryptN, scryptP int) ([]byte, error) {
+	cryptoStruct, err := keystore.EncryptDataV3(seed, auth, scryptN, scryptP)
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(&encryptedSeedStorage{"Clef seed", 1, cryptoStruct})
+}
+
+// decryptSeed decrypts the master seed
+func decryptSeed(keyjson []byte, auth string) ([]byte, error) {
+	var encSeed encryptedSeedStorage
+	if err := json.Unmarshal(keyjson, &encSeed); err != nil {
+		return nil, err
+	}
+	if encSeed.Version != 1 {
+		log.Warn(fmt.Sprintf("unsupported encryption format of seed: %d, operation will likely fail", encSeed.Version))
+	}
+	seed, err := keystore.DecryptDataV3(encSeed.Params, auth)
+	if err != nil {
+		return nil, err
+	}
+	return seed, err
+}
+
 /**
 //Create Account
 

cmd/clef/tutorial.md

@@ -31,43 +31,51 @@ NOTE: This file does not contain your accounts. Those need to be backed up separ
 
 ## Creating rules
 
-Now, you can create a rule-file.
+Now, you can create a rule-file. Note that it is not mandatory to use predefined rules, but it's really handy.
 
 ```javascript
 function ApproveListing(){
     return "Approve"
 }
 ```
-Get the `sha256` hash....
+
+Get the `sha256` hash. If you have openssl, you can do `openssl sha256 rules.js`...
 ```text
 #sha256sum rules.js
 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72  rules.js
 ```
-...And then `attest` the file:
+...now `attest` the file...
 ```text
 #./signer attest 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
 
 INFO [02-21|12:14:38] Ruleset attestation updated              sha256=6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
 ```
-At this point, we then start the signer with the rule-file:
 
+...and (this is required only for non-production versions) load a mock-up `4byte.json` by copying the file from the source to your current working directory:
 ```text
-#./signer --rules rules.json
+#cp $GOPATH/src/github.com/ethereum/go-ethereum/cmd/clef/4byte.json $PWD
+```
 
-INFO [02-21|12:15:18] Using CLI as UI-channel
+At this point, we can start the signer with the rule-file:
-INFO [02-21|12:15:18] Loaded 4byte db                          signatures=5509 file=./4byte.json
+```text
-INFO [02-21|12:15:18] Could not load rulefile, rules not enabled file=rulefile
+#./signer --rules rules.js --rpc
-DEBUG[02-21|12:15:18] FS scan times                            list=35.335µs set=5.536µs diff=5.073µs
+
-DEBUG[02-21|12:15:18] Ledger support enabled
+INFO [09-25|20:28:11.866] Using CLI as UI-channel 
-DEBUG[02-21|12:15:18] Trezor support enabled
+INFO [09-25|20:28:11.876] Loaded 4byte db                          signatures=5509 file=./4byte.json
-INFO [02-21|12:15:18] Audit logs configured                    file=audit.log
+INFO [09-25|20:28:11.877] Rule engine configured                   file=./rules.js
-INFO [02-21|12:15:18] HTTP endpoint opened                     url=http://localhost:8550
+DEBUG[09-25|20:28:11.877] FS scan times                            list=100.781µs set=13.253µs diff=5.761µs
+DEBUG[09-25|20:28:11.884] Ledger support enabled 
+DEBUG[09-25|20:28:11.888] Trezor support enabled 
+INFO [09-25|20:28:11.888] Audit logs configured                    file=audit.log
+DEBUG[09-25|20:28:11.888] HTTP registered                          namespace=account
+INFO [09-25|20:28:11.890] HTTP endpoint opened                     url=http://localhost:8550
+DEBUG[09-25|20:28:11.890] IPC registered                           namespace=account
+INFO [09-25|20:28:11.890] IPC endpoint opened                      url=<nil>
 ------- Signer info -------
+* extapi_version : 2.0.0
+* intapi_version : 2.0.0
 * extapi_http : http://localhost:8550
 * extapi_ipc : <nil>
-* extapi_version : 2.0.0
-* intapi_version : 1.2.0
-
 ```
 
 Any list-requests will now be auto-approved by our rule-file.
@@ -107,16 +115,16 @@ The `master_seed` was then used to derive a few other things:
 
 ## Adding credentials
 
-In order to make more useful rules; sign transactions, the signer needs access to the passwords needed to unlock keystores.
+In order to make more useful rules like signing transactions, the signer needs access to the passwords needed to unlock keystores.
 
 ```text
-#./signer addpw 0x694267f14675d7e1b9494fd8d72fefe1755710fa test
+#./signer addpw "0x694267f14675d7e1b9494fd8d72fefe1755710fa" "test_password"
 
 INFO [02-21|13:43:21] Credential store updated                 key=0x694267f14675d7e1b9494fd8d72fefe1755710fa
 ```
 ## More advanced rules
 
-Now let's update the rules to make use of credentials
+Now let's update the rules to make use of credentials:
 
 ```javascript
 function ApproveListing(){
@@ -134,13 +142,15 @@ function ApproveSignData(r){
 }
 
 ```
-In this example,
+In this example:
-* any requests to sign data with the account `0x694...` will be
+* Any requests to sign data with the account `0x694...` will be
-    * auto-approved if the message contains with `bazonk`,
+    * auto-approved if the message contains with `bazonk`
-    * and auto-rejected if it does not.
+    * auto-rejected if it does not.
-    * Any other signing-requests will be passed along for manual approve/reject.
+* Any other signing-requests will be passed along for manual approve/reject.
 
-..attest the new file
+_Note: make sure that `0x694...` is an account you have access to. You can create it either via the clef or the traditional account cli tool. If the latter was chosen, make sure both clef and geth use the same keystore by specifing `--keystore path/to/your/keystore` when running clef._
+
+Attest the new file...
 ```text
 #sha256sum rules.js
 2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f  rules.js
@@ -153,23 +163,26 @@ INFO [02-21|14:36:30] Ruleset attestation updated              sha256=2a0cb661da
 And start the signer:
 
 ```
-#./signer --rules rules.js
+#./signer --rules rules.js --rpc
 
-INFO [02-21|14:41:56] Using CLI as UI-channel
+INFO [09-25|21:02:16.450] Using CLI as UI-channel 
-INFO [02-21|14:41:56] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [09-25|21:02:16.466] Loaded 4byte db                          signatures=5509 file=./4byte.json
-INFO [02-21|14:41:56] Rule engine configured                   file=rules.js
+INFO [09-25|21:02:16.467] Rule engine configured                   file=./rules.js
-DEBUG[02-21|14:41:56] FS scan times                            list=34.607µs set=4.509µs diff=4.87µs
+DEBUG[09-25|21:02:16.468] FS scan times                            list=1.45262ms set=21.926µs diff=6.944µs
-DEBUG[02-21|14:41:56] Ledger support enabled
+DEBUG[09-25|21:02:16.473] Ledger support enabled 
-DEBUG[02-21|14:41:56] Trezor support enabled
+DEBUG[09-25|21:02:16.475] Trezor support enabled 
-INFO [02-21|14:41:56] Audit logs configured                    file=audit.log
+INFO [09-25|21:02:16.476] Audit logs configured                    file=audit.log
-INFO [02-21|14:41:56] HTTP endpoint opened                     url=http://localhost:8550
+DEBUG[09-25|21:02:16.476] HTTP registered                          namespace=account
+INFO [09-25|21:02:16.478] HTTP endpoint opened                     url=http://localhost:8550
+DEBUG[09-25|21:02:16.478] IPC registered                           namespace=account
+INFO [09-25|21:02:16.478] IPC endpoint opened                      url=<nil>
 ------- Signer info -------
 * extapi_version : 2.0.0
-* intapi_version : 1.2.0
+* intapi_version : 2.0.0
 * extapi_http : http://localhost:8550
 * extapi_ipc : <nil>
-INFO [02-21|14:41:56] error occurred during execution          error="ReferenceError: 'OnSignerStartup' is not defined"
 ```
+
 And then test signing, once with `bazonk` and once without:
 
 ```
@@ -190,9 +203,9 @@ INFO [02-21|14:42:56] Op rejected
 The signer also stores all traffic over the external API in a log file. The last 4 lines shows the two requests and their responses:
 
 ```text
-#tail audit.log -n 4
+#tail -n 4 audit.log
 t=2018-02-21T14:42:41+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49706\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=202062617a6f6e6b2062617a2067617a0a
 t=2018-02-21T14:42:42+0100 lvl=info msg=Sign       api=signer type=response data=93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c error=nil
 t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49708\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=2020626f6e6b2062617a2067617a0a
 t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=response data=                                                                                                                                   error="Request denied"
-```
+```

cmd/ethkey/README.md

@@ -21,21 +21,33 @@ Private key information can be printed by using the `--private` flag;
 make sure to use this feature with great caution!
 
 
-### `ethkey sign <keyfile> <message/file>`
+### `ethkey signmessage <keyfile> <message/file>`
 
 Sign the message with a keyfile.
 It is possible to refer to a file containing the message.
+To sign a message contained in a file, use the `--msgfile` flag.
 
 
-### `ethkey verify <address> <signature> <message/file>`
+### `ethkey verifymessage <address> <signature> <message/file>`
 
 Verify the signature of the message.
 It is possible to refer to a file containing the message.
+To sign a message contained in a file, use the --msgfile flag.
+
+
+### `ethkey changepassphrase <keyfile>`
+
+Change the passphrase of a keyfile.
+use the `--newpasswordfile` to point to the new password file.
 
 
 ## Passphrases
 
 For every command that uses a keyfile, you will be prompted to provide the 
 passphrase for decrypting the keyfile.  To avoid this message, it is possible
-to pass the passphrase by using the `--passphrase` flag pointing to a file that
+to pass the passphrase by using the `--passwordfile` flag pointing to a file that
 contains the passphrase.
+
+## JSON
+
+In case you need to output the result in a JSON format, you shall by using the `--json` flag.

cmd/evm/disasm.go

@@ -44,7 +44,7 @@ func disasmCmd(ctx *cli.Context) error {
 		return err
 	}
 
-	code := strings.TrimSpace(string(in[:]))
+	code := strings.TrimSpace(string(in))
 	fmt.Printf("%v\n", code)
 	return asm.PrintDisassembled(code)
 }

cmd/evm/runner.go

@@ -80,16 +80,16 @@ func runCmd(ctx *cli.Context) error {
 	}
 
 	var (
-		tracer      vm.Tracer
+		tracer        vm.Tracer
-		debugLogger *vm.StructLogger
+		debugLogger   *vm.StructLogger
-		statedb     *state.StateDB
+		statedb       *state.StateDB
-		chainConfig *params.ChainConfig
+		chainConfig   *params.ChainConfig
-		sender      = common.BytesToAddress([]byte("sender"))
+		sender        = common.BytesToAddress([]byte("sender"))
-		receiver    = common.BytesToAddress([]byte("receiver"))
+		receiver      = common.BytesToAddress([]byte("receiver"))
-		blockNumber uint64
+		genesisConfig *core.Genesis
 	)
 	if ctx.GlobalBool(MachineFlag.Name) {
-		tracer = NewJSONLogger(logconfig, os.Stdout)
+		tracer = vm.NewJSONLogger(logconfig, os.Stdout)
 	} else if ctx.GlobalBool(DebugFlag.Name) {
 		debugLogger = vm.NewStructLogger(logconfig)
 		tracer = debugLogger
@@ -98,13 +98,14 @@ func runCmd(ctx *cli.Context) error {
 	}
 	if ctx.GlobalString(GenesisFlag.Name) != "" {
 		gen := readGenesis(ctx.GlobalString(GenesisFlag.Name))
+		genesisConfig = gen
 		db := ethdb.NewMemDatabase()
 		genesis := gen.ToBlock(db)
 		statedb, _ = state.New(genesis.Root(), state.NewDatabase(db))
 		chainConfig = gen.Config
-		blockNumber = gen.Number
 	} else {
 		statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+		genesisConfig = new(core.Genesis)
 	}
 	if ctx.GlobalString(SenderFlag.Name) != "" {
 		sender = common.HexToAddress(ctx.GlobalString(SenderFlag.Name))
@@ -156,13 +157,19 @@ func runCmd(ctx *cli.Context) error {
 	}
 
 	initialGas := ctx.GlobalUint64(GasFlag.Name)
+	if genesisConfig.GasLimit != 0 {
+		initialGas = genesisConfig.GasLimit
+	}
 	runtimeConfig := runtime.Config{
 		Origin:      sender,
 		State:       statedb,
 		GasLimit:    initialGas,
 		GasPrice:    utils.GlobalBig(ctx, PriceFlag.Name),
 		Value:       utils.GlobalBig(ctx, ValueFlag.Name),
-		BlockNumber: new(big.Int).SetUint64(blockNumber),
+		Difficulty:  genesisConfig.Difficulty,
+		Time:        new(big.Int).SetUint64(genesisConfig.Timestamp),
+		Coinbase:    genesisConfig.Coinbase,
+		BlockNumber: new(big.Int).SetUint64(genesisConfig.Number),
 		EVMConfig: vm.Config{
 			Tracer: tracer,
 			Debug:  ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name),
@@ -199,6 +206,7 @@ func runCmd(ctx *cli.Context) error {
 	execTime := time.Since(tstart)
 
 	if ctx.GlobalBool(DumpFlag.Name) {
+		statedb.Commit(true)
 		statedb.IntermediateRoot(true)
 		fmt.Println(string(statedb.Dump()))
 	}

cmd/evm/staterunner.go

@@ -68,7 +68,7 @@ func stateTestCmd(ctx *cli.Context) error {
 	)
 	switch {
 	case ctx.GlobalBool(MachineFlag.Name):
-		tracer = NewJSONLogger(config, os.Stderr)
+		tracer = vm.NewJSONLogger(config, os.Stderr)
 
 	case ctx.GlobalBool(DebugFlag.Name):
 		debugger = vm.NewStructLogger(config)
@@ -97,6 +97,10 @@ func stateTestCmd(ctx *cli.Context) error {
 			// Run the test and aggregate the result
 			result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
 			state, err := test.Run(st, cfg)
+			// print state root for evmlab tracing
+			if ctx.GlobalBool(MachineFlag.Name) && state != nil {
+				fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))
+			}
 			if err != nil {
 				// Test failed, mark as so and dump any state to aid debugging
 				result.Pass, result.Error = false, err.Error()
@@ -105,10 +109,6 @@ func stateTestCmd(ctx *cli.Context) error {
 					result.State = &dump
 				}
 			}
-			// print state root for evmlab tracing (already committed above, so no need to delete objects again
-			if ctx.GlobalBool(MachineFlag.Name) && state != nil {
-				fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))
-			}
 
 			results = append(results, *result)
 

cmd/faucet/faucet.go

@@ -54,8 +54,8 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/discv5"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/params"
 	"golang.org/x/net/websocket"
@@ -77,9 +77,6 @@ var (
 	accJSONFlag = flag.String("account.json", "", "Key json file to fund user requests with")
 	accPassFlag = flag.String("account.pass", "", "Decryption password to access faucet funds")
 
-	githubUser  = flag.String("github.user", "", "GitHub user to authenticate with for Gist access")
-	githubToken = flag.String("github.token", "", "GitHub personal token to access Gists with")
-
 	captchaToken  = flag.String("captcha.token", "", "Recaptcha site key to authenticate client side")
 	captchaSecret = flag.String("captcha.secret", "", "Recaptcha secret key to authenticate server side")
 
@@ -160,7 +157,8 @@ func main() {
 	if blob, err = ioutil.ReadFile(*accPassFlag); err != nil {
 		log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err)
 	}
-	pass := string(blob)
+	// Delete trailing newline in password
+	pass := strings.TrimSuffix(string(blob), "\n")
 
 	ks := keystore.NewKeyStore(filepath.Join(os.Getenv("HOME"), ".faucet", "keys"), keystore.StandardScryptN, keystore.StandardScryptP)
 	if blob, err = ioutil.ReadFile(*accJSONFlag); err != nil {
@@ -201,6 +199,8 @@ type faucet struct {
 
 	keystore *keystore.KeyStore // Keystore containing the single signer
 	account  accounts.Account   // Account funding user faucet requests
+	head     *types.Header      // Current head header of the faucet
+	balance  *big.Int           // Current balance of the faucet
 	nonce    uint64             // Current pending nonce of the faucet
 	price    *big.Int           // Current gas price to issue funds with
 
@@ -216,7 +216,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u
 	// Assemble the raw devp2p protocol stack
 	stack, err := node.New(&node.Config{
 		Name:    "geth",
-		Version: params.Version,
+		Version: params.VersionWithMeta,
 		DataDir: filepath.Join(os.Getenv("HOME"), ".faucet"),
 		P2P: p2p.Config{
 			NAT:              nat.Any(),
@@ -255,8 +255,10 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u
 		return nil, err
 	}
 	for _, boot := range enodes {
-		old, _ := discover.ParseNode(boot.String())
+		old, err := enode.ParseV4(boot.String())
-		stack.Server().AddPeer(old)
+		if err == nil {
+			stack.Server().AddPeer(old)
+		}
 	}
 	// Attach to the client and retrieve and interesting metadatas
 	api, err := stack.Attach()
@@ -326,33 +328,30 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 		nonce   uint64
 		err     error
 	)
-	for {
+	for head == nil || balance == nil {
-		// Attempt to retrieve the stats, may error on no faucet connectivity
+		// Retrieve the current stats cached by the faucet
-		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+		f.lock.RLock()
-		head, err = f.client.HeaderByNumber(ctx, nil)
+		if f.head != nil {
-		if err == nil {
+			head = types.CopyHeader(f.head)
-			balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number)
-			if err == nil {
-				nonce, err = f.client.NonceAt(ctx, f.account.Address, nil)
-			}
 		}
-		cancel()
+		if f.balance != nil {
+			balance = new(big.Int).Set(f.balance)
+		}
+		nonce = f.nonce
+		f.lock.RUnlock()
 
-		// If stats retrieval failed, wait a bit and retry
+		if head == nil || balance == nil {
-		if err != nil {
+			// Report the faucet offline until initial stats are ready
-			if err = sendError(conn, errors.New("Faucet offline: "+err.Error())); err != nil {
+			if err = sendError(conn, errors.New("Faucet offline")); err != nil {
 				log.Warn("Failed to send faucet error to client", "err", err)
 				return
 			}
 			time.Sleep(3 * time.Second)
-			continue
 		}
-		// Initial stats reported successfully, proceed with user interaction
-		break
 	}
 	// Send over the initial stats and the latest header
 	if err = send(conn, map[string]interface{}{
-		"funds":    balance.Div(balance, ether),
+		"funds":    new(big.Int).Div(balance, ether),
 		"funded":   nonce,
 		"peers":    f.stack.Server().PeerCount(),
 		"requests": f.reqs,
@@ -522,6 +521,47 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 	}
 }
 
+// refresh attempts to retrieve the latest header from the chain and extract the
+// associated faucet balance and nonce for connectivity caching.
+func (f *faucet) refresh(head *types.Header) error {
+	// Ensure a state update does not run for too long
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	// If no header was specified, use the current chain head
+	var err error
+	if head == nil {
+		if head, err = f.client.HeaderByNumber(ctx, nil); err != nil {
+			return err
+		}
+	}
+	// Retrieve the balance, nonce and gas price from the current head
+	var (
+		balance *big.Int
+		nonce   uint64
+		price   *big.Int
+	)
+	if balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number); err != nil {
+		return err
+	}
+	if nonce, err = f.client.NonceAt(ctx, f.account.Address, head.Number); err != nil {
+		return err
+	}
+	if price, err = f.client.SuggestGasPrice(ctx); err != nil {
+		return err
+	}
+	// Everything succeeded, update the cached stats and eject old requests
+	f.lock.Lock()
+	f.head, f.balance = head, balance
+	f.price, f.nonce = price, nonce
+	for len(f.reqs) > 0 && f.reqs[0].Tx.Nonce() < f.nonce {
+		f.reqs = f.reqs[1:]
+	}
+	f.lock.Unlock()
+
+	return nil
+}
+
 // loop keeps waiting for interesting events and pushes them out to connected
 // websockets.
 func (f *faucet) loop() {
@@ -539,45 +579,27 @@ func (f *faucet) loop() {
 	go func() {
 		for head := range update {
 			// New chain head arrived, query the current stats and stream to clients
-			var (
+			timestamp := time.Unix(head.Time.Int64(), 0)
-				balance *big.Int
+			if time.Since(timestamp) > time.Hour {
-				nonce   uint64
+				log.Warn("Skipping faucet refresh, head too old", "number", head.Number, "hash", head.Hash(), "age", common.PrettyAge(timestamp))
-				price   *big.Int
+				continue
-				err     error
-			)
-			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-			balance, err = f.client.BalanceAt(ctx, f.account.Address, head.Number)
-			if err == nil {
-				nonce, err = f.client.NonceAt(ctx, f.account.Address, nil)
-				if err == nil {
-					price, err = f.client.SuggestGasPrice(ctx)
-				}
 			}
-			cancel()
+			if err := f.refresh(head); err != nil {
-
-			// If querying the data failed, try for the next block
-			if err != nil {
 				log.Warn("Failed to update faucet state", "block", head.Number, "hash", head.Hash(), "err", err)
 				continue
-			} else {
-				log.Info("Updated faucet state", "block", head.Number, "hash", head.Hash(), "balance", balance, "nonce", nonce, "price", price)
 			}
 			// Faucet state retrieved, update locally and send to clients
-			balance = new(big.Int).Div(balance, ether)
-
-			f.lock.Lock()
-			f.price, f.nonce = price, nonce
-			for len(f.reqs) > 0 && f.reqs[0].Tx.Nonce() < f.nonce {
-				f.reqs = f.reqs[1:]
-			}
-			f.lock.Unlock()
-
 			f.lock.RLock()
+			log.Info("Updated faucet state", "number", head.Number, "hash", head.Hash(), "age", common.PrettyAge(timestamp), "balance", f.balance, "nonce", f.nonce, "price", f.price)
+
+			balance := new(big.Int).Div(f.balance, ether)
+			peers := f.stack.Server().PeerCount()
+
 			for _, conn := range f.conns {
 				if err := send(conn, map[string]interface{}{
 					"funds":    balance,
 					"funded":   f.nonce,
-					"peers":    f.stack.Server().PeerCount(),
+					"peers":    peers,
 					"requests": f.reqs,
 				}, time.Second); err != nil {
 					log.Warn("Failed to send stats to client", "err", err)
@@ -638,59 +660,6 @@ func sendSuccess(conn *websocket.Conn, msg string) error {
 	return send(conn, map[string]string{"success": msg}, time.Second)
 }
 
-// authGitHub tries to authenticate a faucet request using GitHub gists, returning
-// the username, avatar URL and Ethereum address to fund on success.
-func authGitHub(url string) (string, string, common.Address, error) {
-	// Retrieve the gist from the GitHub Gist APIs
-	parts := strings.Split(url, "/")
-	req, _ := http.NewRequest("GET", "https://api.github.com/gists/"+parts[len(parts)-1], nil)
-	if *githubUser != "" {
-		req.SetBasicAuth(*githubUser, *githubToken)
-	}
-	res, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", "", common.Address{}, err
-	}
-	var gist struct {
-		Owner struct {
-			Login string `json:"login"`
-		} `json:"owner"`
-		Files map[string]struct {
-			Content string `json:"content"`
-		} `json:"files"`
-	}
-	err = json.NewDecoder(res.Body).Decode(&gist)
-	res.Body.Close()
-	if err != nil {
-		return "", "", common.Address{}, err
-	}
-	if gist.Owner.Login == "" {
-		return "", "", common.Address{}, errors.New("Anonymous Gists not allowed")
-	}
-	// Iterate over all the files and look for Ethereum addresses
-	var address common.Address
-	for _, file := range gist.Files {
-		content := strings.TrimSpace(file.Content)
-		if len(content) == 2+common.AddressLength*2 {
-			address = common.HexToAddress(content)
-		}
-	}
-	if address == (common.Address{}) {
-		return "", "", common.Address{}, errors.New("No Ethereum address found to fund")
-	}
-	// Validate the user's existence since the API is unhelpful here
-	if res, err = http.Head("https://github.com/" + gist.Owner.Login); err != nil {
-		return "", "", common.Address{}, err
-	}
-	res.Body.Close()
-
-	if res.StatusCode != 200 {
-		return "", "", common.Address{}, errors.New("Invalid user... boom!")
-	}
-	// Everything passed validation, return the gathered infos
-	return gist.Owner.Login + "@github", fmt.Sprintf("https://github.com/%s.png?size=64", gist.Owner.Login), address, nil
-}
-
 // authTwitter tries to authenticate a faucet request using Twitter posts, returning
 // the username, avatar URL and Ethereum address to fund on success.
 func authTwitter(url string) (string, string, common.Address, error) {

cmd/geth/bugcmd.go

@@ -51,7 +51,7 @@ func reportBug(ctx *cli.Context) error {
 
 	fmt.Fprintln(&buff, "#### System information")
 	fmt.Fprintln(&buff)
-	fmt.Fprintln(&buff, "Version:", params.Version)
+	fmt.Fprintln(&buff, "Version:", params.VersionWithMeta)
 	fmt.Fprintln(&buff, "Go Version:", runtime.Version())
 	fmt.Fprintln(&buff, "OS:", runtime.GOOS)
 	printOSDetails(&buff)

cmd/geth/chaincmd.go

@@ -48,7 +48,6 @@ var (
 		ArgsUsage: "<genesisPath>",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
-			utils.LightModeFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
@@ -66,7 +65,7 @@ It expects the genesis file as argument.`,
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
 			utils.CacheFlag,
-			utils.LightModeFlag,
+			utils.SyncModeFlag,
 			utils.GCModeFlag,
 			utils.CacheDatabaseFlag,
 			utils.CacheGCFlag,
@@ -87,14 +86,15 @@ processing will proceed even if an individual RLP-file import failure occurs.`,
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
 			utils.CacheFlag,
-			utils.LightModeFlag,
+			utils.SyncModeFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
 Requires a first argument of the file to write to.
 Optional second and third arguments control the first and
 last block to write. In this mode, the file will be appended
-if already existing.`,
+if already existing. If the file ends with .gz, the output will
+be gzipped.`,
 	}
 	importPreimagesCommand = cli.Command{
 		Action:    utils.MigrateFlags(importPreimages),
@@ -104,7 +104,7 @@ if already existing.`,
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
 			utils.CacheFlag,
-			utils.LightModeFlag,
+			utils.SyncModeFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
@@ -118,7 +118,7 @@ if already existing.`,
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
 			utils.CacheFlag,
-			utils.LightModeFlag,
+			utils.SyncModeFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
@@ -148,7 +148,6 @@ The first argument must be the directory containing the blockchain to download f
 		ArgsUsage: " ",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
-			utils.LightModeFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
@@ -162,7 +161,7 @@ Remove blockchain and state databases`,
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
 			utils.CacheFlag,
-			utils.LightModeFlag,
+			utils.SyncModeFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
@@ -341,9 +340,9 @@ func importPreimages(ctx *cli.Context) error {
 
 	start := time.Now()
 	if err := utils.ImportPreimages(diskdb, ctx.Args().First()); err != nil {
-		utils.Fatalf("Export error: %v\n", err)
+		utils.Fatalf("Import error: %v\n", err)
 	}
-	fmt.Printf("Export done in %v\n", time.Since(start))
+	fmt.Printf("Import done in %v\n", time.Since(start))
 	return nil
 }
 

cmd/geth/config.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math/big"
 	"os"
 	"reflect"
 	"unicode"
@@ -152,7 +153,9 @@ func enableWhisper(ctx *cli.Context) bool {
 
 func makeFullNode(ctx *cli.Context) *node.Node {
 	stack, cfg := makeConfigNode(ctx)
-
+	if ctx.GlobalIsSet(utils.ConstantinopleOverrideFlag.Name) {
+		cfg.Eth.ConstantinopleOverride = new(big.Int).SetUint64(ctx.GlobalUint64(utils.ConstantinopleOverrideFlag.Name))
+	}
 	utils.RegisterEthService(stack, &cfg.Eth)
 
 	if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) {
@@ -168,6 +171,9 @@ func makeFullNode(ctx *cli.Context) *node.Node {
 		if ctx.GlobalIsSet(utils.WhisperMinPOWFlag.Name) {
 			cfg.Shh.MinimumAcceptedPOW = ctx.Float64(utils.WhisperMinPOWFlag.Name)
 		}
+		if ctx.GlobalIsSet(utils.WhisperRestrictConnectionBetweenLightClientsFlag.Name) {
+			cfg.Shh.RestrictConnectionBetweenLightClients = true
+		}
 		utils.RegisterShhService(stack, &cfg.Shh)
 	}
 

cmd/geth/consolecmd_test.go

@@ -31,7 +31,7 @@ import (
 )
 
 const (
-	ipcAPIs  = "admin:1.0 debug:1.0 eth:1.0 miner:1.0 net:1.0 personal:1.0 rpc:1.0 shh:1.0 txpool:1.0 web3:1.0"
+	ipcAPIs  = "admin:1.0 debug:1.0 eth:1.0 ethash:1.0 miner:1.0 net:1.0 personal:1.0 rpc:1.0 shh:1.0 txpool:1.0 web3:1.0"
 	httpAPIs = "eth:1.0 net:1.0 rpc:1.0 web3:1.0"
 )
 
@@ -50,7 +50,7 @@ func TestConsoleWelcome(t *testing.T) {
 	geth.SetTemplateFunc("goos", func() string { return runtime.GOOS })
 	geth.SetTemplateFunc("goarch", func() string { return runtime.GOARCH })
 	geth.SetTemplateFunc("gover", runtime.Version)
-	geth.SetTemplateFunc("gethver", func() string { return params.Version })
+	geth.SetTemplateFunc("gethver", func() string { return params.VersionWithMeta })
 	geth.SetTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
 	geth.SetTemplateFunc("apis", func() string { return ipcAPIs })
 
@@ -133,7 +133,7 @@ func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) {
 	attach.SetTemplateFunc("goos", func() string { return runtime.GOOS })
 	attach.SetTemplateFunc("goarch", func() string { return runtime.GOARCH })
 	attach.SetTemplateFunc("gover", runtime.Version)
-	attach.SetTemplateFunc("gethver", func() string { return params.Version })
+	attach.SetTemplateFunc("gethver", func() string { return params.VersionWithMeta })
 	attach.SetTemplateFunc("etherbase", func() string { return geth.Etherbase })
 	attach.SetTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) })
 	attach.SetTemplateFunc("ipc", func() bool { return strings.HasPrefix(endpoint, "ipc") })

cmd/geth/genesis_test.go

@@ -77,7 +77,7 @@ var customGenesisTests = []struct {
 				"homesteadBlock" : 314,
 				"daoForkBlock"   : 141,
 				"daoForkSupport" : true
-			},
+			}
 		}`,
 		query:  "eth.getBlock(0).nonce",
 		result: "0x0000000000000042",

cmd/geth/main.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"math"
 	"os"
-	"runtime"
 	godebug "runtime/debug"
 	"sort"
 	"strconv"
@@ -72,6 +71,7 @@ var (
 		utils.EthashDatasetDirFlag,
 		utils.EthashDatasetsInMemoryFlag,
 		utils.EthashDatasetsOnDiskFlag,
+		utils.TxPoolLocalsFlag,
 		utils.TxPoolNoLocalsFlag,
 		utils.TxPoolJournalFlag,
 		utils.TxPoolRejournalFlag,
@@ -82,25 +82,35 @@ var (
 		utils.TxPoolAccountQueueFlag,
 		utils.TxPoolGlobalQueueFlag,
 		utils.TxPoolLifetimeFlag,
-		utils.FastSyncFlag,
-		utils.LightModeFlag,
 		utils.SyncModeFlag,
 		utils.GCModeFlag,
 		utils.LightServFlag,
 		utils.LightPeersFlag,
 		utils.LightKDFFlag,
+		utils.WhitelistFlag,
 		utils.CacheFlag,
 		utils.CacheDatabaseFlag,
+		utils.CacheTrieFlag,
 		utils.CacheGCFlag,
 		utils.TrieCacheGenFlag,
 		utils.ListenPortFlag,
 		utils.MaxPeersFlag,
 		utils.MaxPendingPeersFlag,
-		utils.EtherbaseFlag,
-		utils.GasPriceFlag,
-		utils.MinerThreadsFlag,
 		utils.MiningEnabledFlag,
-		utils.TargetGasLimitFlag,
+		utils.MinerThreadsFlag,
+		utils.MinerLegacyThreadsFlag,
+		utils.MinerNotifyFlag,
+		utils.MinerGasTargetFlag,
+		utils.MinerLegacyGasTargetFlag,
+		utils.MinerGasLimitFlag,
+		utils.MinerGasPriceFlag,
+		utils.MinerLegacyGasPriceFlag,
+		utils.MinerEtherbaseFlag,
+		utils.MinerLegacyEtherbaseFlag,
+		utils.MinerExtraDataFlag,
+		utils.MinerLegacyExtraDataFlag,
+		utils.MinerRecommitIntervalFlag,
+		utils.MinerNoVerfiyFlag,
 		utils.NATFlag,
 		utils.NoDiscoverFlag,
 		utils.DiscoveryV5Flag,
@@ -113,6 +123,7 @@ var (
 		utils.RinkebyFlag,
 		utils.VMEnableDebugFlag,
 		utils.NetworkIdFlag,
+		utils.ConstantinopleOverrideFlag,
 		utils.RPCCORSDomainFlag,
 		utils.RPCVirtualHostsFlag,
 		utils.EthStatsURLFlag,
@@ -121,7 +132,8 @@ var (
 		utils.NoCompactionFlag,
 		utils.GpoBlocksFlag,
 		utils.GpoPercentileFlag,
-		utils.ExtraDataFlag,
+		utils.EWASMInterpreterFlag,
+		utils.EVMInterpreterFlag,
 		configFileFlag,
 	}
 
@@ -143,6 +155,16 @@ var (
 		utils.WhisperEnabledFlag,
 		utils.WhisperMaxMessageSizeFlag,
 		utils.WhisperMinPOWFlag,
+		utils.WhisperRestrictConnectionBetweenLightClientsFlag,
+	}
+
+	metricsFlags = []cli.Flag{
+		utils.MetricsEnableInfluxDBFlag,
+		utils.MetricsInfluxDBEndpointFlag,
+		utils.MetricsInfluxDBDatabaseFlag,
+		utils.MetricsInfluxDBUsernameFlag,
+		utils.MetricsInfluxDBPasswordFlag,
+		utils.MetricsInfluxDBHostTagFlag,
 	}
 )
 
@@ -186,13 +208,17 @@ func init() {
 	app.Flags = append(app.Flags, consoleFlags...)
 	app.Flags = append(app.Flags, debug.Flags...)
 	app.Flags = append(app.Flags, whisperFlags...)
+	app.Flags = append(app.Flags, metricsFlags...)
 
 	app.Before = func(ctx *cli.Context) error {
-		runtime.GOMAXPROCS(runtime.NumCPU())
+		logdir := ""
-		if err := debug.Setup(ctx); err != nil {
+		if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) {
+			logdir = (&node.Config{DataDir: utils.MakeDataDir(ctx)}).ResolvePath("logs")
+		}
+		if err := debug.Setup(ctx, logdir); err != nil {
 			return err
 		}
-		// Cap the cache allowance and tune the garbage colelctor
+		// Cap the cache allowance and tune the garbage collector
 		var mem gosigar.Mem
 		if err := mem.Get(); err == nil {
 			allowance := int(mem.Total / 1024 / 1024 / 3)
@@ -208,10 +234,12 @@ func init() {
 		log.Debug("Sanitizing Go's GC trigger", "percent", int(gogc))
 		godebug.SetGCPercent(int(gogc))
 
+		// Start metrics export if enabled
+		utils.SetupMetrics(ctx)
+
 		// Start system runtime metrics collection
 		go metrics.CollectProcessMetrics(3 * time.Second)
 
-		utils.SetupNetwork(ctx)
 		return nil
 	}
 
@@ -233,6 +261,9 @@ func main() {
 // It creates a default node based on the command line arguments and runs it in
 // blocking mode, waiting for it to be shut down.
 func geth(ctx *cli.Context) error {
+	if args := ctx.Args(); len(args) > 0 {
+		return fmt.Errorf("invalid command: %q", args[0])
+	}
 	node := makeFullNode(ctx)
 	startNode(ctx, node)
 	node.Wait()
@@ -287,11 +318,11 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 				status, _ := event.Wallet.Status()
 				log.Info("New wallet appeared", "url", event.Wallet.URL(), "status", status)
 
+				derivationPath := accounts.DefaultBaseDerivationPath
 				if event.Wallet.URL().Scheme == "ledger" {
-					event.Wallet.SelfDerive(accounts.DefaultLedgerBaseDerivationPath, stateReader)
+					derivationPath = accounts.DefaultLedgerBaseDerivationPath
-				} else {
-					event.Wallet.SelfDerive(accounts.DefaultBaseDerivationPath, stateReader)
 				}
+				event.Wallet.SelfDerive(derivationPath, stateReader)
 
 			case accounts.WalletDropped:
 				log.Info("Old wallet dropped", "url", event.Wallet.URL())
@@ -302,25 +333,25 @@ func startNode(ctx *cli.Context, stack *node.Node) {
 	// Start auxiliary services if enabled
 	if ctx.GlobalBool(utils.MiningEnabledFlag.Name) || ctx.GlobalBool(utils.DeveloperFlag.Name) {
 		// Mining only makes sense if a full Ethereum node is running
-		if ctx.GlobalBool(utils.LightModeFlag.Name) || ctx.GlobalString(utils.SyncModeFlag.Name) == "light" {
+		if ctx.GlobalString(utils.SyncModeFlag.Name) == "light" {
 			utils.Fatalf("Light clients do not support mining")
 		}
 		var ethereum *eth.Ethereum
 		if err := stack.Service(&ethereum); err != nil {
 			utils.Fatalf("Ethereum service not running: %v", err)
 		}
-		// Use a reduced number of threads if requested
-		if threads := ctx.GlobalInt(utils.MinerThreadsFlag.Name); threads > 0 {
-			type threaded interface {
-				SetThreads(threads int)
-			}
-			if th, ok := ethereum.Engine().(threaded); ok {
-				th.SetThreads(threads)
-			}
-		}
 		// Set the gas price to the limits from the CLI and start mining
-		ethereum.TxPool().SetGasPrice(utils.GlobalBig(ctx, utils.GasPriceFlag.Name))
+		gasprice := utils.GlobalBig(ctx, utils.MinerLegacyGasPriceFlag.Name)
-		if err := ethereum.StartMining(true); err != nil {
+		if ctx.IsSet(utils.MinerGasPriceFlag.Name) {
+			gasprice = utils.GlobalBig(ctx, utils.MinerGasPriceFlag.Name)
+		}
+		ethereum.TxPool().SetGasPrice(gasprice)
+
+		threads := ctx.GlobalInt(utils.MinerLegacyThreadsFlag.Name)
+		if ctx.GlobalIsSet(utils.MinerThreadsFlag.Name) {
+			threads = ctx.GlobalInt(utils.MinerThreadsFlag.Name)
+		}
+		if err := ethereum.StartMining(threads); err != nil {
 			utils.Fatalf("Failed to start mining: %v", err)
 		}
 	}

cmd/geth/misccmd.go

@@ -108,7 +108,7 @@ func makedag(ctx *cli.Context) error {
 
 func version(ctx *cli.Context) error {
 	fmt.Println(strings.Title(clientIdentifier))
-	fmt.Println("Version:", params.Version)
+	fmt.Println("Version:", params.VersionWithMeta)
 	if gitCommit != "" {
 		fmt.Println("Git Commit:", gitCommit)
 	}

cmd/geth/monitorcmd.go

@@ -185,12 +185,12 @@ func resolveMetric(metrics map[string]interface{}, pattern string, path string)
 	parts := strings.SplitN(pattern, "/", 2)
 	if len(parts) > 1 {
 		for _, variation := range strings.Split(parts[0], ",") {
-			if submetrics, ok := metrics[variation].(map[string]interface{}); !ok {
+			submetrics, ok := metrics[variation].(map[string]interface{})
+			if !ok {
 				utils.Fatalf("Failed to retrieve system metrics: %s", path+variation)
 				return nil
-			} else {
-				results = append(results, resolveMetric(submetrics, parts[1], path+variation+"/")...)
 			}
+			results = append(results, resolveMetric(submetrics, parts[1], path+variation+"/")...)
 		}
 		return results
 	}

cmd/geth/usage.go

@@ -81,9 +81,11 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.LightServFlag,
 			utils.LightPeersFlag,
 			utils.LightKDFFlag,
+			utils.WhitelistFlag,
 		},
 	},
-	{Name: "DEVELOPER CHAIN",
+	{
+		Name: "DEVELOPER CHAIN",
 		Flags: []cli.Flag{
 			utils.DeveloperFlag,
 			utils.DeveloperPeriodFlag,
@@ -113,6 +115,7 @@ var AppHelpFlagGroups = []flagGroup{
 	{
 		Name: "TRANSACTION POOL",
 		Flags: []cli.Flag{
+			utils.TxPoolLocalsFlag,
 			utils.TxPoolNoLocalsFlag,
 			utils.TxPoolJournalFlag,
 			utils.TxPoolRejournalFlag,
@@ -130,6 +133,7 @@ var AppHelpFlagGroups = []flagGroup{
 		Flags: []cli.Flag{
 			utils.CacheFlag,
 			utils.CacheDatabaseFlag,
+			utils.CacheTrieFlag,
 			utils.CacheGCFlag,
 			utils.TrieCacheGenFlag,
 		},
@@ -184,10 +188,14 @@ var AppHelpFlagGroups = []flagGroup{
 		Flags: []cli.Flag{
 			utils.MiningEnabledFlag,
 			utils.MinerThreadsFlag,
-			utils.EtherbaseFlag,
+			utils.MinerNotifyFlag,
-			utils.TargetGasLimitFlag,
+			utils.MinerGasPriceFlag,
-			utils.GasPriceFlag,
+			utils.MinerGasTargetFlag,
-			utils.ExtraDataFlag,
+			utils.MinerGasLimitFlag,
+			utils.MinerEtherbaseFlag,
+			utils.MinerExtraDataFlag,
+			utils.MinerRecommitIntervalFlag,
+			utils.MinerNoVerfiyFlag,
 		},
 	},
 	{
@@ -201,16 +209,29 @@ var AppHelpFlagGroups = []flagGroup{
 		Name: "VIRTUAL MACHINE",
 		Flags: []cli.Flag{
 			utils.VMEnableDebugFlag,
+			utils.EVMInterpreterFlag,
+			utils.EWASMInterpreterFlag,
 		},
 	},
 	{
 		Name: "LOGGING AND DEBUGGING",
 		Flags: append([]cli.Flag{
-			utils.MetricsEnabledFlag,
 			utils.FakePoWFlag,
 			utils.NoCompactionFlag,
 		}, debug.Flags...),
 	},
+	{
+		Name: "METRICS AND STATS",
+		Flags: []cli.Flag{
+			utils.MetricsEnabledFlag,
+			utils.MetricsEnableInfluxDBFlag,
+			utils.MetricsInfluxDBEndpointFlag,
+			utils.MetricsInfluxDBDatabaseFlag,
+			utils.MetricsInfluxDBUsernameFlag,
+			utils.MetricsInfluxDBPasswordFlag,
+			utils.MetricsInfluxDBHostTagFlag,
+		},
+	},
 	{
 		Name:  "WHISPER (EXPERIMENTAL)",
 		Flags: whisperFlags,
@@ -218,8 +239,11 @@ var AppHelpFlagGroups = []flagGroup{
 	{
 		Name: "DEPRECATED",
 		Flags: []cli.Flag{
-			utils.FastSyncFlag,
+			utils.MinerLegacyThreadsFlag,
-			utils.LightModeFlag,
+			utils.MinerLegacyGasTargetFlag,
+			utils.MinerLegacyGasPriceFlag,
+			utils.MinerLegacyEtherbaseFlag,
+			utils.MinerLegacyExtraDataFlag,
 		},
 	},
 	{

cmd/p2psim/main.go

@@ -47,7 +47,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/simulations"
 	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -180,7 +180,10 @@ func main() {
 			},
 		},
 	}
-	app.Run(os.Args)
+	if err := app.Run(os.Args); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
 }
 
 func showNetwork(ctx *cli.Context) error {
@@ -275,15 +278,14 @@ func createNode(ctx *cli.Context) error {
 	if len(ctx.Args()) != 0 {
 		return cli.ShowCommandHelp(ctx, ctx.Command.Name)
 	}
-	config := &adapters.NodeConfig{
+	config := adapters.RandomNodeConfig()
-		Name: ctx.String("name"),
+	config.Name = ctx.String("name")
-	}
 	if key := ctx.String("key"); key != "" {
 		privKey, err := crypto.HexToECDSA(key)
 		if err != nil {
 			return err
 		}
-		config.ID = discover.PubkeyID(&privKey.PublicKey)
+		config.ID = enode.PubkeyToIDV4(&privKey.PublicKey)
 		config.PrivateKey = privKey
 	}
 	if services := ctx.String("services"); services != "" {

cmd/puppeth/genesis.go

@@ -20,35 +20,41 @@ import (
 	"encoding/binary"
 	"errors"
 	"math"
+	"math/big"
+	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	math2 "github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/params"
 )
 
-// cppEthereumGenesisSpec represents the genesis specification format used by the
+// alethGenesisSpec represents the genesis specification format used by the
 // C++ Ethereum implementation.
-type cppEthereumGenesisSpec struct {
+type alethGenesisSpec struct {
 	SealEngine string `json:"sealEngine"`
 	Params     struct {
-		AccountStartNonce       hexutil.Uint64 `json:"accountStartNonce"`
+		AccountStartNonce       math2.HexOrDecimal64   `json:"accountStartNonce"`
-		HomesteadForkBlock      hexutil.Uint64 `json:"homesteadForkBlock"`
+		MaximumExtraDataSize    hexutil.Uint64         `json:"maximumExtraDataSize"`
-		EIP150ForkBlock         hexutil.Uint64 `json:"EIP150ForkBlock"`
+		HomesteadForkBlock      hexutil.Uint64         `json:"homesteadForkBlock"`
-		EIP158ForkBlock         hexutil.Uint64 `json:"EIP158ForkBlock"`
+		DaoHardforkBlock        math2.HexOrDecimal64   `json:"daoHardforkBlock"`
-		ByzantiumForkBlock      hexutil.Uint64 `json:"byzantiumForkBlock"`
+		EIP150ForkBlock         hexutil.Uint64         `json:"EIP150ForkBlock"`
-		ConstantinopleForkBlock hexutil.Uint64 `json:"constantinopleForkBlock"`
+		EIP158ForkBlock         hexutil.Uint64         `json:"EIP158ForkBlock"`
-		NetworkID               hexutil.Uint64 `json:"networkID"`
+		ByzantiumForkBlock      hexutil.Uint64         `json:"byzantiumForkBlock"`
-		ChainID                 hexutil.Uint64 `json:"chainID"`
+		ConstantinopleForkBlock hexutil.Uint64         `json:"constantinopleForkBlock"`
-		MaximumExtraDataSize    hexutil.Uint64 `json:"maximumExtraDataSize"`
+		MinGasLimit             hexutil.Uint64         `json:"minGasLimit"`
-		MinGasLimit             hexutil.Uint64 `json:"minGasLimit"`
+		MaxGasLimit             hexutil.Uint64         `json:"maxGasLimit"`
-		MaxGasLimit             hexutil.Uint64 `json:"maxGasLimit"`
+		TieBreakingGas          bool                   `json:"tieBreakingGas"`
-		GasLimitBoundDivisor    hexutil.Uint64 `json:"gasLimitBoundDivisor"`
+		GasLimitBoundDivisor    math2.HexOrDecimal64   `json:"gasLimitBoundDivisor"`
-		MinimumDifficulty       *hexutil.Big   `json:"minimumDifficulty"`
+		MinimumDifficulty       *hexutil.Big           `json:"minimumDifficulty"`
-		DifficultyBoundDivisor  *hexutil.Big   `json:"difficultyBoundDivisor"`
+		DifficultyBoundDivisor  *math2.HexOrDecimal256 `json:"difficultyBoundDivisor"`
-		DurationLimit           *hexutil.Big   `json:"durationLimit"`
+		DurationLimit           *math2.HexOrDecimal256 `json:"durationLimit"`
-		BlockReward             *hexutil.Big   `json:"blockReward"`
+		BlockReward             *hexutil.Big           `json:"blockReward"`
+		NetworkID               hexutil.Uint64         `json:"networkID"`
+		ChainID                 hexutil.Uint64         `json:"chainID"`
+		AllowFutureBlocks       bool                   `json:"allowFutureBlocks"`
 	} `json:"params"`
 
 	Genesis struct {
@@ -62,57 +68,68 @@ type cppEthereumGenesisSpec struct {
 		GasLimit   hexutil.Uint64 `json:"gasLimit"`
 	} `json:"genesis"`
 
-	Accounts map[common.Address]*cppEthereumGenesisSpecAccount `json:"accounts"`
+	Accounts map[common.UnprefixedAddress]*alethGenesisSpecAccount `json:"accounts"`
 }
 
-// cppEthereumGenesisSpecAccount is the prefunded genesis account and/or precompiled
+// alethGenesisSpecAccount is the prefunded genesis account and/or precompiled
 // contract definition.
-type cppEthereumGenesisSpecAccount struct {
+type alethGenesisSpecAccount struct {
-	Balance     *hexutil.Big                   `json:"balance"`
+	Balance     *math2.HexOrDecimal256   `json:"balance"`
-	Nonce       uint64                         `json:"nonce,omitempty"`
+	Nonce       uint64                   `json:"nonce,omitempty"`
-	Precompiled *cppEthereumGenesisSpecBuiltin `json:"precompiled,omitempty"`
+	Precompiled *alethGenesisSpecBuiltin `json:"precompiled,omitempty"`
 }
 
-// cppEthereumGenesisSpecBuiltin is the precompiled contract definition.
+// alethGenesisSpecBuiltin is the precompiled contract definition.
-type cppEthereumGenesisSpecBuiltin struct {
+type alethGenesisSpecBuiltin struct {
-	Name          string                               `json:"name,omitempty"`
+	Name          string                         `json:"name,omitempty"`
-	StartingBlock hexutil.Uint64                       `json:"startingBlock,omitempty"`
+	StartingBlock hexutil.Uint64                 `json:"startingBlock,omitempty"`
-	Linear        *cppEthereumGenesisSpecLinearPricing `json:"linear,omitempty"`
+	Linear        *alethGenesisSpecLinearPricing `json:"linear,omitempty"`
 }
 
-type cppEthereumGenesisSpecLinearPricing struct {
+type alethGenesisSpecLinearPricing struct {
 	Base uint64 `json:"base"`
 	Word uint64 `json:"word"`
 }
 
-// newCppEthereumGenesisSpec converts a go-ethereum genesis block into a Parity specific
+// newAlethGenesisSpec converts a go-ethereum genesis block into a Aleth-specific
 // chain specification format.
-func newCppEthereumGenesisSpec(network string, genesis *core.Genesis) (*cppEthereumGenesisSpec, error) {
+func newAlethGenesisSpec(network string, genesis *core.Genesis) (*alethGenesisSpec, error) {
-	// Only ethash is currently supported between go-ethereum and cpp-ethereum
+	// Only ethash is currently supported between go-ethereum and aleth
 	if genesis.Config.Ethash == nil {
 		return nil, errors.New("unsupported consensus engine")
 	}
-	// Reconstruct the chain spec in Parity's format
+	// Reconstruct the chain spec in Aleth format
-	spec := &cppEthereumGenesisSpec{
+	spec := &alethGenesisSpec{
 		SealEngine: "Ethash",
 	}
+	// Some defaults
 	spec.Params.AccountStartNonce = 0
+	spec.Params.TieBreakingGas = false
+	spec.Params.AllowFutureBlocks = false
+	spec.Params.DaoHardforkBlock = 0
+
 	spec.Params.HomesteadForkBlock = (hexutil.Uint64)(genesis.Config.HomesteadBlock.Uint64())
 	spec.Params.EIP150ForkBlock = (hexutil.Uint64)(genesis.Config.EIP150Block.Uint64())
 	spec.Params.EIP158ForkBlock = (hexutil.Uint64)(genesis.Config.EIP158Block.Uint64())
-	spec.Params.ByzantiumForkBlock = (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64())
+
-	spec.Params.ConstantinopleForkBlock = (hexutil.Uint64)(math.MaxUint64)
+	// Byzantium
+	if num := genesis.Config.ByzantiumBlock; num != nil {
+		spec.setByzantium(num)
+	}
+	// Constantinople
+	if num := genesis.Config.ConstantinopleBlock; num != nil {
+		spec.setConstantinople(num)
+	}
 
 	spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64())
 	spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64())
-
 	spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize)
 	spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit)
-	spec.Params.MaxGasLimit = (hexutil.Uint64)(math.MaxUint64)
+	spec.Params.MaxGasLimit = (hexutil.Uint64)(math.MaxInt64)
 	spec.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty)
-	spec.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor)
+	spec.Params.DifficultyBoundDivisor = (*math2.HexOrDecimal256)(params.DifficultyBoundDivisor)
-	spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor)
+	spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor)
-	spec.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit)
+	spec.Params.DurationLimit = (*math2.HexOrDecimal256)(params.DurationLimit)
 	spec.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward)
 
 	spec.Genesis.Nonce = (hexutil.Bytes)(make([]byte, 8))
@@ -126,77 +143,104 @@ func newCppEthereumGenesisSpec(network string, genesis *core.Genesis) (*cppEther
 	spec.Genesis.ExtraData = (hexutil.Bytes)(genesis.ExtraData)
 	spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit)
 
-	spec.Accounts = make(map[common.Address]*cppEthereumGenesisSpecAccount)
 	for address, account := range genesis.Alloc {
-		spec.Accounts[address] = &cppEthereumGenesisSpecAccount{
+		spec.setAccount(address, account)
-			Balance: (*hexutil.Big)(account.Balance),
-			Nonce:   account.Nonce,
-		}
-	}
-	spec.Accounts[common.BytesToAddress([]byte{1})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-		Name: "ecrecover", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 3000},
-	}
-	spec.Accounts[common.BytesToAddress([]byte{2})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-		Name: "sha256", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 60, Word: 12},
-	}
-	spec.Accounts[common.BytesToAddress([]byte{3})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-		Name: "ripemd160", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 600, Word: 120},
-	}
-	spec.Accounts[common.BytesToAddress([]byte{4})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-		Name: "identity", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 15, Word: 3},
 	}
+
+	spec.setPrecompile(1, &alethGenesisSpecBuiltin{Name: "ecrecover",
+		Linear: &alethGenesisSpecLinearPricing{Base: 3000}})
+	spec.setPrecompile(2, &alethGenesisSpecBuiltin{Name: "sha256",
+		Linear: &alethGenesisSpecLinearPricing{Base: 60, Word: 12}})
+	spec.setPrecompile(3, &alethGenesisSpecBuiltin{Name: "ripemd160",
+		Linear: &alethGenesisSpecLinearPricing{Base: 600, Word: 120}})
+	spec.setPrecompile(4, &alethGenesisSpecBuiltin{Name: "identity",
+		Linear: &alethGenesisSpecLinearPricing{Base: 15, Word: 3}})
 	if genesis.Config.ByzantiumBlock != nil {
-		spec.Accounts[common.BytesToAddress([]byte{5})].Precompiled = &cppEthereumGenesisSpecBuiltin{
+		spec.setPrecompile(5, &alethGenesisSpecBuiltin{Name: "modexp",
-			Name: "modexp", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()),
+			StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64())})
-		}
+		spec.setPrecompile(6, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_add",
-		spec.Accounts[common.BytesToAddress([]byte{6})].Precompiled = &cppEthereumGenesisSpecBuiltin{
+			StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()),
-			Name: "alt_bn128_G1_add", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), Linear: &cppEthereumGenesisSpecLinearPricing{Base: 500},
+			Linear:        &alethGenesisSpecLinearPricing{Base: 500}})
-		}
+		spec.setPrecompile(7, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_mul",
-		spec.Accounts[common.BytesToAddress([]byte{7})].Precompiled = &cppEthereumGenesisSpecBuiltin{
+			StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()),
-			Name: "alt_bn128_G1_mul", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), Linear: &cppEthereumGenesisSpecLinearPricing{Base: 40000},
+			Linear:        &alethGenesisSpecLinearPricing{Base: 40000}})
-		}
+		spec.setPrecompile(8, &alethGenesisSpecBuiltin{Name: "alt_bn128_pairing_product",
-		spec.Accounts[common.BytesToAddress([]byte{8})].Precompiled = &cppEthereumGenesisSpecBuiltin{
+			StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64())})
-			Name: "alt_bn128_pairing_product", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()),
-		}
 	}
 	return spec, nil
 }
 
+func (spec *alethGenesisSpec) setPrecompile(address byte, data *alethGenesisSpecBuiltin) {
+	if spec.Accounts == nil {
+		spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount)
+	}
+	spec.Accounts[common.UnprefixedAddress(common.BytesToAddress([]byte{address}))].Precompiled = data
+}
+
+func (spec *alethGenesisSpec) setAccount(address common.Address, account core.GenesisAccount) {
+	if spec.Accounts == nil {
+		spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount)
+	}
+
+	a, exist := spec.Accounts[common.UnprefixedAddress(address)]
+	if !exist {
+		a = &alethGenesisSpecAccount{}
+		spec.Accounts[common.UnprefixedAddress(address)] = a
+	}
+	a.Balance = (*math2.HexOrDecimal256)(account.Balance)
+	a.Nonce = account.Nonce
+
+}
+
+func (spec *alethGenesisSpec) setByzantium(num *big.Int) {
+	spec.Params.ByzantiumForkBlock = hexutil.Uint64(num.Uint64())
+}
+
+func (spec *alethGenesisSpec) setConstantinople(num *big.Int) {
+	spec.Params.ConstantinopleForkBlock = hexutil.Uint64(num.Uint64())
+}
+
 // parityChainSpec is the chain specification format used by Parity.
 type parityChainSpec struct {
-	Name   string `json:"name"`
+	Name    string `json:"name"`
-	Engine struct {
+	Datadir string `json:"dataDir"`
+	Engine  struct {
 		Ethash struct {
 			Params struct {
-				MinimumDifficulty      *hexutil.Big `json:"minimumDifficulty"`
+				MinimumDifficulty      *hexutil.Big      `json:"minimumDifficulty"`
-				DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"`
+				DifficultyBoundDivisor *hexutil.Big      `json:"difficultyBoundDivisor"`
-				DurationLimit          *hexutil.Big `json:"durationLimit"`
+				DurationLimit          *hexutil.Big      `json:"durationLimit"`
-				BlockReward            *hexutil.Big `json:"blockReward"`
+				BlockReward            map[string]string `json:"blockReward"`
-				HomesteadTransition    uint64       `json:"homesteadTransition"`
+				DifficultyBombDelays   map[string]string `json:"difficultyBombDelays"`
-				EIP150Transition       uint64       `json:"eip150Transition"`
+				HomesteadTransition    hexutil.Uint64    `json:"homesteadTransition"`
-				EIP160Transition       uint64       `json:"eip160Transition"`
+				EIP100bTransition      hexutil.Uint64    `json:"eip100bTransition"`
-				EIP161abcTransition    uint64       `json:"eip161abcTransition"`
-				EIP161dTransition      uint64       `json:"eip161dTransition"`
-				EIP649Reward           *hexutil.Big `json:"eip649Reward"`
-				EIP100bTransition      uint64       `json:"eip100bTransition"`
-				EIP649Transition       uint64       `json:"eip649Transition"`
 			} `json:"params"`
 		} `json:"Ethash"`
 	} `json:"engine"`
 
 	Params struct {
-		MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"`
+		AccountStartNonce     hexutil.Uint64       `json:"accountStartNonce"`
-		MinGasLimit          hexutil.Uint64 `json:"minGasLimit"`
+		MaximumExtraDataSize  hexutil.Uint64       `json:"maximumExtraDataSize"`
-		GasLimitBoundDivisor hexutil.Uint64 `json:"gasLimitBoundDivisor"`
+		MinGasLimit           hexutil.Uint64       `json:"minGasLimit"`
-		NetworkID            hexutil.Uint64 `json:"networkID"`
+		GasLimitBoundDivisor  math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"`
-		MaxCodeSize          uint64         `json:"maxCodeSize"`
+		NetworkID             hexutil.Uint64       `json:"networkID"`
-		EIP155Transition     uint64         `json:"eip155Transition"`
+		ChainID               hexutil.Uint64       `json:"chainID"`
-		EIP98Transition      uint64         `json:"eip98Transition"`
+		MaxCodeSize           hexutil.Uint64       `json:"maxCodeSize"`
-		EIP86Transition      uint64         `json:"eip86Transition"`
+		MaxCodeSizeTransition hexutil.Uint64       `json:"maxCodeSizeTransition"`
-		EIP140Transition     uint64         `json:"eip140Transition"`
+		EIP98Transition       hexutil.Uint64       `json:"eip98Transition"`
-		EIP211Transition     uint64         `json:"eip211Transition"`
+		EIP150Transition      hexutil.Uint64       `json:"eip150Transition"`
-		EIP214Transition     uint64         `json:"eip214Transition"`
+		EIP160Transition      hexutil.Uint64       `json:"eip160Transition"`
-		EIP658Transition     uint64         `json:"eip658Transition"`
+		EIP161abcTransition   hexutil.Uint64       `json:"eip161abcTransition"`
+		EIP161dTransition     hexutil.Uint64       `json:"eip161dTransition"`
+		EIP155Transition      hexutil.Uint64       `json:"eip155Transition"`
+		EIP140Transition      hexutil.Uint64       `json:"eip140Transition"`
+		EIP211Transition      hexutil.Uint64       `json:"eip211Transition"`
+		EIP214Transition      hexutil.Uint64       `json:"eip214Transition"`
+		EIP658Transition      hexutil.Uint64       `json:"eip658Transition"`
+		EIP145Transition      hexutil.Uint64       `json:"eip145Transition"`
+		EIP1014Transition     hexutil.Uint64       `json:"eip1014Transition"`
+		EIP1052Transition     hexutil.Uint64       `json:"eip1052Transition"`
+		EIP1283Transition     hexutil.Uint64       `json:"eip1283Transition"`
 	} `json:"params"`
 
 	Genesis struct {
@@ -215,22 +259,22 @@ type parityChainSpec struct {
 		GasLimit   hexutil.Uint64 `json:"gasLimit"`
 	} `json:"genesis"`
 
-	Nodes    []string                                   `json:"nodes"`
+	Nodes    []string                                             `json:"nodes"`
-	Accounts map[common.Address]*parityChainSpecAccount `json:"accounts"`
+	Accounts map[common.UnprefixedAddress]*parityChainSpecAccount `json:"accounts"`
 }
 
 // parityChainSpecAccount is the prefunded genesis account and/or precompiled
 // contract definition.
 type parityChainSpecAccount struct {
-	Balance *hexutil.Big            `json:"balance"`
+	Balance math2.HexOrDecimal256   `json:"balance"`
-	Nonce   uint64                  `json:"nonce,omitempty"`
+	Nonce   math2.HexOrDecimal64    `json:"nonce,omitempty"`
 	Builtin *parityChainSpecBuiltin `json:"builtin,omitempty"`
 }
 
 // parityChainSpecBuiltin is the precompiled contract definition.
 type parityChainSpecBuiltin struct {
 	Name       string                  `json:"name,omitempty"`
-	ActivateAt uint64                  `json:"activate_at,omitempty"`
+	ActivateAt math2.HexOrDecimal64    `json:"activate_at,omitempty"`
 	Pricing    *parityChainSpecPricing `json:"pricing,omitempty"`
 }
 
@@ -265,34 +309,51 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin
 	}
 	// Reconstruct the chain spec in Parity's format
 	spec := &parityChainSpec{
-		Name:  network,
+		Name:    network,
-		Nodes: bootnodes,
+		Nodes:   bootnodes,
+		Datadir: strings.ToLower(network),
 	}
+	spec.Engine.Ethash.Params.BlockReward = make(map[string]string)
+	spec.Engine.Ethash.Params.DifficultyBombDelays = make(map[string]string)
+	// Frontier
 	spec.Engine.Ethash.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty)
 	spec.Engine.Ethash.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor)
 	spec.Engine.Ethash.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit)
-	spec.Engine.Ethash.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward)
+	spec.Engine.Ethash.Params.BlockReward["0x0"] = hexutil.EncodeBig(ethash.FrontierBlockReward)
-	spec.Engine.Ethash.Params.HomesteadTransition = genesis.Config.HomesteadBlock.Uint64()
-	spec.Engine.Ethash.Params.EIP150Transition = genesis.Config.EIP150Block.Uint64()
-	spec.Engine.Ethash.Params.EIP160Transition = genesis.Config.EIP155Block.Uint64()
-	spec.Engine.Ethash.Params.EIP161abcTransition = genesis.Config.EIP158Block.Uint64()
-	spec.Engine.Ethash.Params.EIP161dTransition = genesis.Config.EIP158Block.Uint64()
-	spec.Engine.Ethash.Params.EIP649Reward = (*hexutil.Big)(ethash.ByzantiumBlockReward)
-	spec.Engine.Ethash.Params.EIP100bTransition = genesis.Config.ByzantiumBlock.Uint64()
-	spec.Engine.Ethash.Params.EIP649Transition = genesis.Config.ByzantiumBlock.Uint64()
 
+	// Homestead
+	spec.Engine.Ethash.Params.HomesteadTransition = hexutil.Uint64(genesis.Config.HomesteadBlock.Uint64())
+
+	// Tangerine Whistle : 150
+	// https://github.com/ethereum/EIPs/blob/master/EIPS/eip-608.md
+	spec.Params.EIP150Transition = hexutil.Uint64(genesis.Config.EIP150Block.Uint64())
+
+	// Spurious Dragon: 155, 160, 161, 170
+	// https://github.com/ethereum/EIPs/blob/master/EIPS/eip-607.md
+	spec.Params.EIP155Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64())
+	spec.Params.EIP160Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64())
+	spec.Params.EIP161abcTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64())
+	spec.Params.EIP161dTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64())
+
+	// Byzantium
+	if num := genesis.Config.ByzantiumBlock; num != nil {
+		spec.setByzantium(num)
+	}
+	// Constantinople
+	if num := genesis.Config.ConstantinopleBlock; num != nil {
+		spec.setConstantinople(num)
+	}
 	spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize)
 	spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit)
-	spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor)
+	spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor)
 	spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64())
+	spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64())
 	spec.Params.MaxCodeSize = params.MaxCodeSize
-	spec.Params.EIP155Transition = genesis.Config.EIP155Block.Uint64()
+	// geth has it set from zero
-	spec.Params.EIP98Transition = math.MaxUint64
+	spec.Params.MaxCodeSizeTransition = 0
-	spec.Params.EIP86Transition = math.MaxUint64
+
-	spec.Params.EIP140Transition = genesis.Config.ByzantiumBlock.Uint64()
+	// Disable this one
-	spec.Params.EIP211Transition = genesis.Config.ByzantiumBlock.Uint64()
+	spec.Params.EIP98Transition = math.MaxInt64
-	spec.Params.EIP214Transition = genesis.Config.ByzantiumBlock.Uint64()
-	spec.Params.EIP658Transition = genesis.Config.ByzantiumBlock.Uint64()
 
 	spec.Genesis.Seal.Ethereum.Nonce = (hexutil.Bytes)(make([]byte, 8))
 	binary.LittleEndian.PutUint64(spec.Genesis.Seal.Ethereum.Nonce[:], genesis.Nonce)
@@ -305,42 +366,77 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin
 	spec.Genesis.ExtraData = (hexutil.Bytes)(genesis.ExtraData)
 	spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit)
 
-	spec.Accounts = make(map[common.Address]*parityChainSpecAccount)
+	spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount)
 	for address, account := range genesis.Alloc {
-		spec.Accounts[address] = &parityChainSpecAccount{
+		bal := math2.HexOrDecimal256(*account.Balance)
-			Balance: (*hexutil.Big)(account.Balance),
+
-			Nonce:   account.Nonce,
+		spec.Accounts[common.UnprefixedAddress(address)] = &parityChainSpecAccount{
+			Balance: bal,
+			Nonce:   math2.HexOrDecimal64(account.Nonce),
 		}
 	}
-	spec.Accounts[common.BytesToAddress([]byte{1})].Builtin = &parityChainSpecBuiltin{
+	spec.setPrecompile(1, &parityChainSpecBuiltin{Name: "ecrecover",
-		Name: "ecrecover", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 3000}},
+		Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 3000}}})
-	}
+
-	spec.Accounts[common.BytesToAddress([]byte{2})].Builtin = &parityChainSpecBuiltin{
+	spec.setPrecompile(2, &parityChainSpecBuiltin{
 		Name: "sha256", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 60, Word: 12}},
-	}
+	})
-	spec.Accounts[common.BytesToAddress([]byte{3})].Builtin = &parityChainSpecBuiltin{
+	spec.setPrecompile(3, &parityChainSpecBuiltin{
 		Name: "ripemd160", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 600, Word: 120}},
-	}
+	})
-	spec.Accounts[common.BytesToAddress([]byte{4})].Builtin = &parityChainSpecBuiltin{
+	spec.setPrecompile(4, &parityChainSpecBuiltin{
 		Name: "identity", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 15, Word: 3}},
-	}
+	})
 	if genesis.Config.ByzantiumBlock != nil {
-		spec.Accounts[common.BytesToAddress([]byte{5})].Builtin = &parityChainSpecBuiltin{
+		blnum := math2.HexOrDecimal64(genesis.Config.ByzantiumBlock.Uint64())
-			Name: "modexp", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{ModExp: &parityChainSpecModExpPricing{Divisor: 20}},
+		spec.setPrecompile(5, &parityChainSpecBuiltin{
-		}
+			Name: "modexp", ActivateAt: blnum, Pricing: &parityChainSpecPricing{ModExp: &parityChainSpecModExpPricing{Divisor: 20}},
-		spec.Accounts[common.BytesToAddress([]byte{6})].Builtin = &parityChainSpecBuiltin{
+		})
-			Name: "alt_bn128_add", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 500}},
+		spec.setPrecompile(6, &parityChainSpecBuiltin{
-		}
+			Name: "alt_bn128_add", ActivateAt: blnum, Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 500}},
-		spec.Accounts[common.BytesToAddress([]byte{7})].Builtin = &parityChainSpecBuiltin{
+		})
-			Name: "alt_bn128_mul", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 40000}},
+		spec.setPrecompile(7, &parityChainSpecBuiltin{
-		}
+			Name: "alt_bn128_mul", ActivateAt: blnum, Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 40000}},
-		spec.Accounts[common.BytesToAddress([]byte{8})].Builtin = &parityChainSpecBuiltin{
+		})
-			Name: "alt_bn128_pairing", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{AltBnPairing: &parityChainSpecAltBnPairingPricing{Base: 100000, Pair: 80000}},
+		spec.setPrecompile(8, &parityChainSpecBuiltin{
-		}
+			Name: "alt_bn128_pairing", ActivateAt: blnum, Pricing: &parityChainSpecPricing{AltBnPairing: &parityChainSpecAltBnPairingPricing{Base: 100000, Pair: 80000}},
+		})
 	}
 	return spec, nil
 }
 
+func (spec *parityChainSpec) setPrecompile(address byte, data *parityChainSpecBuiltin) {
+	if spec.Accounts == nil {
+		spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount)
+	}
+	a := common.UnprefixedAddress(common.BytesToAddress([]byte{address}))
+	if _, exist := spec.Accounts[a]; !exist {
+		spec.Accounts[a] = &parityChainSpecAccount{}
+	}
+	spec.Accounts[a].Builtin = data
+}
+
+func (spec *parityChainSpec) setByzantium(num *big.Int) {
+	spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ByzantiumBlockReward)
+	spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(3000000)
+	n := hexutil.Uint64(num.Uint64())
+	spec.Engine.Ethash.Params.EIP100bTransition = n
+	spec.Params.EIP140Transition = n
+	spec.Params.EIP211Transition = n
+	spec.Params.EIP214Transition = n
+	spec.Params.EIP658Transition = n
+}
+
+func (spec *parityChainSpec) setConstantinople(num *big.Int) {
+	spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ConstantinopleBlockReward)
+	spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(2000000)
+	n := hexutil.Uint64(num.Uint64())
+	spec.Params.EIP145Transition = n
+	spec.Params.EIP1014Transition = n
+	spec.Params.EIP1052Transition = n
+	spec.Params.EIP1283Transition = n
+}
+
 // pyEthereumGenesisSpec represents the genesis specification format used by the
 // Python Ethereum implementation.
 type pyEthereumGenesisSpec struct {

cmd/puppeth/genesis_test.go

@@ -0,0 +1,109 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/ethereum/go-ethereum/core"
+)
+
+// Tests the go-ethereum to Aleth chainspec conversion for the Stureby testnet.
+func TestAlethSturebyConverter(t *testing.T) {
+	blob, err := ioutil.ReadFile("testdata/stureby_geth.json")
+	if err != nil {
+		t.Fatalf("could not read file: %v", err)
+	}
+	var genesis core.Genesis
+	if err := json.Unmarshal(blob, &genesis); err != nil {
+		t.Fatalf("failed parsing genesis: %v", err)
+	}
+	spec, err := newAlethGenesisSpec("stureby", &genesis)
+	if err != nil {
+		t.Fatalf("failed creating chainspec: %v", err)
+	}
+
+	expBlob, err := ioutil.ReadFile("testdata/stureby_aleth.json")
+	if err != nil {
+		t.Fatalf("could not read file: %v", err)
+	}
+	expspec := &alethGenesisSpec{}
+	if err := json.Unmarshal(expBlob, expspec); err != nil {
+		t.Fatalf("failed parsing genesis: %v", err)
+	}
+	if !reflect.DeepEqual(expspec, spec) {
+		t.Errorf("chainspec mismatch")
+		c := spew.ConfigState{
+			DisablePointerAddresses: true,
+			SortKeys:                true,
+		}
+		exp := strings.Split(c.Sdump(expspec), "\n")
+		got := strings.Split(c.Sdump(spec), "\n")
+		for i := 0; i < len(exp) && i < len(got); i++ {
+			if exp[i] != got[i] {
+				fmt.Printf("got: %v\nexp: %v\n", exp[i], got[i])
+			}
+		}
+	}
+}
+
+// Tests the go-ethereum to Parity chainspec conversion for the Stureby testnet.
+func TestParitySturebyConverter(t *testing.T) {
+	blob, err := ioutil.ReadFile("testdata/stureby_geth.json")
+	if err != nil {
+		t.Fatalf("could not read file: %v", err)
+	}
+	var genesis core.Genesis
+	if err := json.Unmarshal(blob, &genesis); err != nil {
+		t.Fatalf("failed parsing genesis: %v", err)
+	}
+	spec, err := newParityChainSpec("Stureby", &genesis, []string{})
+	if err != nil {
+		t.Fatalf("failed creating chainspec: %v", err)
+	}
+
+	expBlob, err := ioutil.ReadFile("testdata/stureby_parity.json")
+	if err != nil {
+		t.Fatalf("could not read file: %v", err)
+	}
+	expspec := &parityChainSpec{}
+	if err := json.Unmarshal(expBlob, expspec); err != nil {
+		t.Fatalf("failed parsing genesis: %v", err)
+	}
+	expspec.Nodes = []string{}
+
+	if !reflect.DeepEqual(expspec, spec) {
+		t.Errorf("chainspec mismatch")
+		c := spew.ConfigState{
+			DisablePointerAddresses: true,
+			SortKeys:                true,
+		}
+		exp := strings.Split(c.Sdump(expspec), "\n")
+		got := strings.Split(c.Sdump(spec), "\n")
+		for i := 0; i < len(exp) && i < len(got); i++ {
+			if exp[i] != got[i] {
+				fmt.Printf("got: %v\nexp: %v\n", exp[i], got[i])
+			}
+		}
+	}
+}

cmd/puppeth/module_dashboard.go

@@ -640,7 +640,7 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da
 	files[filepath.Join(workdir, network+".json")] = genesis
 
 	if conf.Genesis.Config.Ethash != nil {
-		cppSpec, err := newCppEthereumGenesisSpec(network, conf.Genesis)
+		cppSpec, err := newAlethGenesisSpec(network, conf.Genesis)
 		if err != nil {
 			return nil, err
 		}
@@ -678,9 +678,9 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da
 
 	// Build and deploy the dashboard service
 	if nocache {
-		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
+		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network))
 	}
-	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network))
 }
 
 // dashboardInfos is returned from a dashboard status check to allow reporting

cmd/puppeth/module_ethstats.go

@@ -43,7 +43,8 @@ version: '2'
 services:
   ethstats:
     build: .
-    image: {{.Network}}/ethstats{{if not .VHost}}
+    image: {{.Network}}/ethstats
+    container_name: {{.Network}}_ethstats_1{{if not .VHost}}
     ports:
       - "{{.Port}}:3000"{{end}}
     environment:
@@ -100,9 +101,9 @@ func deployEthstats(client *sshClient, network string, port int, secret string,
 
 	// Build and deploy the ethstats service
 	if nocache {
-		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
+		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network))
 	}
-	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network))
 }
 
 // ethstatsInfos is returned from an ethstats status check to allow reporting
@@ -122,7 +123,7 @@ func (info *ethstatsInfos) Report() map[string]string {
 		"Website address":       info.host,
 		"Website listener port": strconv.Itoa(info.port),
 		"Login secret":          info.secret,
-		"Banned addresses":      fmt.Sprintf("%v", info.banned),
+		"Banned addresses":      strings.Join(info.banned, "\n"),
 	}
 }
 

cmd/puppeth/module_explorer.go

@@ -38,7 +38,7 @@ ADD chain.json /chain.json
 RUN \
   echo '(cd ../eth-net-intelligence-api && pm2 start /ethstats.json)' >  explorer.sh && \
 	echo '(cd ../etherchain-light && npm start &)'                      >> explorer.sh && \
-	echo '/parity/parity --chain=/chain.json --port={{.NodePort}} --tracing=on --fat-db=on --pruning=archive' >> explorer.sh
+	echo 'exec /parity/parity --chain=/chain.json --port={{.NodePort}} --tracing=on --fat-db=on --pruning=archive' >> explorer.sh
 
 ENTRYPOINT ["/bin/sh", "explorer.sh"]
 `
@@ -77,6 +77,7 @@ services:
   explorer:
     build: .
     image: {{.Network}}/explorer
+    container_name: {{.Network}}_explorer_1
     ports:
       - "{{.NodePort}}:{{.NodePort}}"
       - "{{.NodePort}}:{{.NodePort}}/udp"{{if not .VHost}}
@@ -140,9 +141,9 @@ func deployExplorer(client *sshClient, network string, chainspec []byte, config
 
 	// Build and deploy the boot or seal node service
 	if nocache {
-		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
+		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network))
 	}
-	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network))
 }
 
 // explorerInfos is returned from a block explorer status check to allow reporting

cmd/puppeth/module_faucet.go

@@ -56,8 +56,10 @@ services:
   faucet:
     build: .
     image: {{.Network}}/faucet
+    container_name: {{.Network}}_faucet_1
     ports:
-      - "{{.EthPort}}:{{.EthPort}}"{{if not .VHost}}
+      - "{{.EthPort}}:{{.EthPort}}"
+      - "{{.EthPort}}:{{.EthPort}}/udp"{{if not .VHost}}
       - "{{.ApiPort}}:8080"{{end}}
     volumes:
       - {{.Datadir}}:/root/.faucet
@@ -133,9 +135,9 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config
 
 	// Build and deploy the faucet service
 	if nocache {
-		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
+		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network))
 	}
-	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network))
 }
 
 // faucetInfos is returned from a faucet status check to allow reporting various

cmd/puppeth/module_nginx.go

@@ -40,6 +40,7 @@ services:
   nginx:
     build: .
     image: {{.Network}}/nginx
+    container_name: {{.Network}}_nginx_1
     ports:
       - "{{.Port}}:80"
     volumes:
@@ -81,9 +82,9 @@ func deployNginx(client *sshClient, network string, port int, nocache bool) ([]b
 
 	// Build and deploy the reverse-proxy service
 	if nocache {
-		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
+		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network))
 	}
-	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network))
 }
 
 // nginxInfos is returned from an nginx reverse-proxy status check to allow

cmd/puppeth/module_node.go

@@ -42,7 +42,7 @@ ADD genesis.json /genesis.json
 RUN \
   echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}}
 	echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
-	echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh
+	echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --nat extip:{{.IP}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh
 
 ENTRYPOINT ["/bin/sh", "geth.sh"]
 `
@@ -55,6 +55,7 @@ services:
   {{.Type}}:
     build: .
     image: {{.Network}}/{{.Type}}
+    container_name: {{.Network}}_{{.Type}}_1
     ports:
       - "{{.Port}}:{{.Port}}"
       - "{{.Port}}:{{.Port}}/udp"
@@ -68,6 +69,7 @@ services:
       - STATS_NAME={{.Ethstats}}
       - MINER_NAME={{.Etherbase}}
       - GAS_TARGET={{.GasTarget}}
+      - GAS_LIMIT={{.GasLimit}}
       - GAS_PRICE={{.GasPrice}}
     logging:
       driver: "json-file"
@@ -98,12 +100,14 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n
 	template.Must(template.New("").Parse(nodeDockerfile)).Execute(dockerfile, map[string]interface{}{
 		"NetworkID": config.network,
 		"Port":      config.port,
+		"IP":        client.address,
 		"Peers":     config.peersTotal,
 		"LightFlag": lightFlag,
 		"Bootnodes": strings.Join(bootnodes, ","),
 		"Ethstats":  config.ethstats,
 		"Etherbase": config.etherbase,
 		"GasTarget": uint64(1000000 * config.gasTarget),
+		"GasLimit":  uint64(1000000 * config.gasLimit),
 		"GasPrice":  uint64(1000000000 * config.gasPrice),
 		"Unlock":    config.keyJSON != "",
 	})
@@ -122,6 +126,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n
 		"Ethstats":   config.ethstats[:strings.Index(config.ethstats, ":")],
 		"Etherbase":  config.etherbase,
 		"GasTarget":  config.gasTarget,
+		"GasLimit":   config.gasLimit,
 		"GasPrice":   config.gasPrice,
 	})
 	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
@@ -139,9 +144,9 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n
 
 	// Build and deploy the boot or seal node service
 	if nocache {
-		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
+		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network))
 	}
-	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network))
 }
 
 // nodeInfos is returned from a boot or seal node status check to allow reporting
@@ -160,6 +165,7 @@ type nodeInfos struct {
 	keyJSON    string
 	keyPass    string
 	gasTarget  float64
+	gasLimit   float64
 	gasPrice   float64
 }
 
@@ -175,8 +181,9 @@ func (info *nodeInfos) Report() map[string]string {
 	}
 	if info.gasTarget > 0 {
 		// Miner or signer node
-		report["Gas limit (baseline target)"] = fmt.Sprintf("%0.3f MGas", info.gasTarget)
 		report["Gas price (minimum accepted)"] = fmt.Sprintf("%0.3f GWei", info.gasPrice)
+		report["Gas floor (baseline target)"] = fmt.Sprintf("%0.3f MGas", info.gasTarget)
+		report["Gas ceil  (target maximum)"] = fmt.Sprintf("%0.3f MGas", info.gasLimit)
 
 		if info.etherbase != "" {
 			// Ethash proof-of-work miner
@@ -217,14 +224,15 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
 	totalPeers, _ := strconv.Atoi(infos.envvars["TOTAL_PEERS"])
 	lightPeers, _ := strconv.Atoi(infos.envvars["LIGHT_PEERS"])
 	gasTarget, _ := strconv.ParseFloat(infos.envvars["GAS_TARGET"], 64)
+	gasLimit, _ := strconv.ParseFloat(infos.envvars["GAS_LIMIT"], 64)
 	gasPrice, _ := strconv.ParseFloat(infos.envvars["GAS_PRICE"], 64)
 
 	// Container available, retrieve its node ID and its genesis json
 	var out []byte
-	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.id attach", network, kind)); err != nil {
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.enode --cache=16 attach", network, kind)); err != nil {
 		return nil, ErrServiceUnreachable
 	}
-	id := bytes.Trim(bytes.TrimSpace(out), "\"")
+	enode := bytes.Trim(bytes.TrimSpace(out), "\"")
 
 	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /genesis.json", network, kind)); err != nil {
 		return nil, ErrServiceUnreachable
@@ -256,9 +264,10 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
 		keyJSON:    keyJSON,
 		keyPass:    keyPass,
 		gasTarget:  gasTarget,
+		gasLimit:   gasLimit,
 		gasPrice:   gasPrice,
 	}
-	stats.enode = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.port)
+	stats.enode = string(enode)
 
 	return stats, nil
 }

cmd/puppeth/module_wallet.go

@@ -37,7 +37,7 @@ ADD genesis.json /genesis.json
 RUN \
   echo 'node server.js &'                     > wallet.sh && \
 	echo 'geth --cache 512 init /genesis.json' >> wallet.sh && \
-	echo $'geth --networkid {{.NetworkID}} --port {{.NodePort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --rpc --rpcaddr=0.0.0.0 --rpccorsdomain "*" --rpcvhosts "*"' >> wallet.sh
+	echo $'exec geth --networkid {{.NetworkID}} --port {{.NodePort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --rpc --rpcaddr=0.0.0.0 --rpccorsdomain "*" --rpcvhosts "*"' >> wallet.sh
 
 RUN \
 	sed -i 's/PuppethNetworkID/{{.NetworkID}}/g' dist/js/etherwallet-master.js && \
@@ -57,6 +57,7 @@ services:
   wallet:
     build: .
     image: {{.Network}}/wallet
+    container_name: {{.Network}}_wallet_1
     ports:
       - "{{.NodePort}}:{{.NodePort}}"
       - "{{.NodePort}}:{{.NodePort}}/udp"
@@ -120,9 +121,9 @@ func deployWallet(client *sshClient, network string, bootnodes []string, config
 
 	// Build and deploy the boot or seal node service
 	if nocache {
-		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate", workdir, network, network))
+		return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s build --pull --no-cache && docker-compose -p %s up -d --force-recreate --timeout 60", workdir, network, network))
 	}
-	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate", workdir, network))
+	return nil, client.Stream(fmt.Sprintf("cd %s && docker-compose -p %s up -d --build --force-recreate --timeout 60", workdir, network))
 }
 
 // walletInfos is returned from a web wallet status check to allow reporting

cmd/puppeth/puppeth.go

@@ -43,18 +43,23 @@ func main() {
 			Usage: "log level to emit to the screen",
 		},
 	}
-	app.Action = func(c *cli.Context) error {
+	app.Before = func(c *cli.Context) error {
 		// Set up the logger to print everything and the random generator
 		log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int("loglevel")), log.StreamHandler(os.Stdout, log.TerminalFormat(true))))
 		rand.Seed(time.Now().UnixNano())
 
-		network := c.String("network")
-		if strings.Contains(network, " ") || strings.Contains(network, "-") {
-			log.Crit("No spaces or hyphens allowed in network name")
-		}
-		// Start the wizard and relinquish control
-		makeWizard(c.String("network")).run()
 		return nil
 	}
+	app.Action = runWizard
 	app.Run(os.Args)
 }
+
+// runWizard start the wizard and relinquish control to it.
+func runWizard(c *cli.Context) error {
+	network := c.String("network")
+	if strings.Contains(network, " ") || strings.Contains(network, "-") || strings.ToLower(network) != network {
+		log.Crit("No spaces, hyphens or capital letters allowed in network name")
+	}
+	makeWizard(c.String("network")).run()
+	return nil
+}