params, swarm: release Geth v1.8.20 and Swarm v0.3.8

* geth/core/eth: implement constantinople override flag

* les: implemnent constantinople override flag for les clients

* cmd/geth, eth, les: fix typo, move flag to experimentals

.github/CODEOWNERS

@@ -9,23 +9,26 @@ les/                            @zsfelfoldi
 light/                          @zsfelfoldi
 mobile/                         @karalabe
 p2p/                            @fjl @zsfelfoldi
+p2p/simulations                 @lmars
+p2p/protocols                   @zelig
+swarm/api/http                  @justelad
 swarm/bmt                       @zelig
 swarm/dev                       @lmars
 swarm/fuse                      @jmozah @holisticode
 swarm/grafana_dashboards        @nonsense
 swarm/metrics                   @nonsense @holisticode
 swarm/multihash                 @nolash
-swarm/network/bitvector         @zelig @janos @gbalint
-swarm/network/priorityqueue     @zelig @janos @gbalint
-swarm/network/simulations       @zelig
-swarm/network/stream            @janos @zelig @gbalint @holisticode @justelad
+swarm/network/bitvector         @zelig @janos 
+swarm/network/priorityqueue     @zelig @janos 
+swarm/network/simulations       @zelig @janos
+swarm/network/stream            @janos @zelig @holisticode @justelad
 swarm/network/stream/intervals  @janos
 swarm/network/stream/testing    @zelig
 swarm/pot                       @zelig
 swarm/pss                       @nolash @zelig @nonsense
 swarm/services                  @zelig
 swarm/state                     @justelad
-swarm/storage/encryption        @gbalint @zelig @nagydani
+swarm/storage/encryption        @zelig @nagydani
 swarm/storage/mock              @janos
 swarm/storage/feed              @nolash @jpeletier
 swarm/testutil                  @lmars

.travis.yml

@@ -29,6 +29,14 @@ matrix:
     - os: osx
       go: 1.11.x
       script:
+        - echo "Increase the maximum number of open file descriptors on macOS"
+        - NOFILE=20480
+        - sudo sysctl -w kern.maxfiles=$NOFILE
+        - sudo sysctl -w kern.maxfilesperproc=$NOFILE
+        - sudo launchctl limit maxfiles $NOFILE $NOFILE
+        - sudo launchctl limit maxfiles
+        - ulimit -S -n $NOFILE
+        - ulimit -n
         - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703
         - go run build/ci.go install
         - go run build/ci.go test -coverage $TEST_PACKAGES
@@ -148,7 +156,7 @@ matrix:
       git:
         submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.11.1.linux-amd64.tar.gz | tar -xz
+        - curl https://storage.googleapis.com/golang/go1.11.2.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go

README.md

@@ -18,7 +18,7 @@ For prerequisites and detailed build instructions please read the
 [Installation Instructions](https://github.com/ethereum/go-ethereum/wiki/Building-Ethereum)
 on the wiki.
 
-Building geth requires both a Go (version 1.7 or later) and a C compiler.
+Building geth requires both a Go (version 1.9 or later) and a C compiler.
 You can install them using your favourite package manager.
 Once the dependencies are installed, run
 
@@ -168,7 +168,7 @@ HTTP based JSON-RPC API options:
   * `--ipcpath` Filename for IPC socket/pipe within the datadir (explicit paths escape it)
 
 You'll need to use your own programming environments' capabilities (libraries, tools, etc) to connect
-via HTTP, WS or IPC to a Geth node configured with the above flags and you'll need to speak [JSON-RPC](http://www.jsonrpc.org/specification)
+via HTTP, WS or IPC to a Geth node configured with the above flags and you'll need to speak [JSON-RPC](https://www.jsonrpc.org/specification)
 on all transports. You can reuse the same connection for multiple requests!
 
 **Note: Please understand the security implications of opening up an HTTP/WS based transport before

accounts/abi/argument.go

@@ -243,11 +243,7 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) {
 	// input offset is the bytes offset for packed output
 	inputOffset := 0
 	for _, abiArg := range abiArgs {
-		if abiArg.Type.T == ArrayTy {
-			inputOffset += 32 * abiArg.Type.Size
-		} else {
-			inputOffset += 32
-		}
+		inputOffset += getDynamicTypeOffset(abiArg.Type)
 	}
 	var ret []byte
 	for i, a := range args {
@@ -257,14 +253,13 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) {
 		if err != nil {
 			return nil, err
 		}
-		// check for a slice type (string, bytes, slice)
-		if input.Type.requiresLengthPrefix() {
-			// calculate the offset
-			offset := inputOffset + len(variableInput)
+		// check for dynamic types
+		if isDynamicType(input.Type) {
 			// set the offset
-			ret = append(ret, packNum(reflect.ValueOf(offset))...)
-			// Append the packed output to the variable input. The variable input
-			// will be appended at the end of the input.
+			ret = append(ret, packNum(reflect.ValueOf(inputOffset))...)
+			// calculate next offset
+			inputOffset += len(packed)
+			// append to variable input
 			variableInput = append(variableInput, packed...)
 		} else {
 			// append the packed value to the input

accounts/abi/pack_test.go

@@ -324,6 +324,66 @@ func TestPack(t *testing.T) {
 			"foobar",
 			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000006666f6f6261720000000000000000000000000000000000000000000000000000"),
 		},
+		{
+			"string[]",
+			[]string{"hello", "foobar"},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // len(array) = 2
+				"0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0
+				"0000000000000000000000000000000000000000000000000000000000000080" + // offset 128 to i = 1
+				"0000000000000000000000000000000000000000000000000000000000000005" + // len(str[0]) = 5
+				"68656c6c6f000000000000000000000000000000000000000000000000000000" + // str[0]
+				"0000000000000000000000000000000000000000000000000000000000000006" + // len(str[1]) = 6
+				"666f6f6261720000000000000000000000000000000000000000000000000000"), // str[1]
+		},
+		{
+			"string[2]",
+			[]string{"hello", "foobar"},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // offset to i = 0
+				"0000000000000000000000000000000000000000000000000000000000000080" + // offset to i = 1
+				"0000000000000000000000000000000000000000000000000000000000000005" + // len(str[0]) = 5
+				"68656c6c6f000000000000000000000000000000000000000000000000000000" + // str[0]
+				"0000000000000000000000000000000000000000000000000000000000000006" + // len(str[1]) = 6
+				"666f6f6261720000000000000000000000000000000000000000000000000000"), // str[1]
+		},
+		{
+			"bytes32[][]",
+			[][]common.Hash{{{1}, {2}}, {{3}, {4}, {5}}},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // len(array) = 2
+				"0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0
+				"00000000000000000000000000000000000000000000000000000000000000a0" + // offset 160 to i = 1
+				"0000000000000000000000000000000000000000000000000000000000000002" + // len(array[0]) = 2
+				"0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0]
+				"0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1]
+				"0000000000000000000000000000000000000000000000000000000000000003" + // len(array[1]) = 3
+				"0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0]
+				"0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1]
+				"0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2]
+		},
+
+		{
+			"bytes32[][2]",
+			[][]common.Hash{{{1}, {2}}, {{3}, {4}, {5}}},
+			common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0
+				"00000000000000000000000000000000000000000000000000000000000000a0" + // offset 160 to i = 1
+				"0000000000000000000000000000000000000000000000000000000000000002" + // len(array[0]) = 2
+				"0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0]
+				"0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1]
+				"0000000000000000000000000000000000000000000000000000000000000003" + // len(array[1]) = 3
+				"0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0]
+				"0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1]
+				"0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2]
+		},
+
+		{
+			"bytes32[3][2]",
+			[][]common.Hash{{{1}, {2}, {3}}, {{3}, {4}, {5}}},
+			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0]
+				"0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1]
+				"0300000000000000000000000000000000000000000000000000000000000000" + // array[0][2]
+				"0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0]
+				"0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1]
+				"0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2]
+		},
 	} {
 		typ, err := NewType(test.typ)
 		if err != nil {
@@ -336,7 +396,7 @@ func TestPack(t *testing.T) {
 		}
 
 		if !bytes.Equal(output, test.output) {
-			t.Errorf("%d failed. Expected bytes: '%x' Got: '%x'", i, test.output, output)
+			t.Errorf("input %d for typ: %v failed. Expected bytes: '%x' Got: '%x'", i, typ.String(), test.output, output)
 		}
 	}
 }

accounts/abi/type.go

@@ -183,23 +183,39 @@ func (t Type) pack(v reflect.Value) ([]byte, error) {
 		return nil, err
 	}
 
-	if t.T == SliceTy || t.T == ArrayTy {
-		var packed []byte
+	switch t.T {
+	case SliceTy, ArrayTy:
+		var ret []byte
 
+		if t.requiresLengthPrefix() {
+			// append length
+			ret = append(ret, packNum(reflect.ValueOf(v.Len()))...)
+		}
+
+		// calculate offset if any
+		offset := 0
+		offsetReq := isDynamicType(*t.Elem)
+		if offsetReq {
+			offset = getDynamicTypeOffset(*t.Elem) * v.Len()
+		}
+		var tail []byte
 		for i := 0; i < v.Len(); i++ {
 			val, err := t.Elem.pack(v.Index(i))
 			if err != nil {
 				return nil, err
 			}
-			packed = append(packed, val...)
-		}
-		if t.T == SliceTy {
-			return packBytesSlice(packed, v.Len()), nil
-		} else if t.T == ArrayTy {
-			return packed, nil
+			if !offsetReq {
+				ret = append(ret, val...)
+				continue
+			}
+			ret = append(ret, packNum(reflect.ValueOf(offset))...)
+			offset += len(val)
+			tail = append(tail, val...)
 		}
+		return append(ret, tail...), nil
+	default:
+		return packElement(t, v), nil
 	}
-	return packElement(t, v), nil
 }
 
 // requireLengthPrefix returns whether the type requires any sort of length
@@ -207,3 +223,27 @@ func (t Type) pack(v reflect.Value) ([]byte, error) {
 func (t Type) requiresLengthPrefix() bool {
 	return t.T == StringTy || t.T == BytesTy || t.T == SliceTy
 }
+
+// isDynamicType returns true if the type is dynamic.
+// StringTy, BytesTy, and SliceTy(irrespective of slice element type) are dynamic types
+// ArrayTy is considered dynamic if and only if the Array element is a dynamic type.
+// This function recursively checks the type for slice and array elements.
+func isDynamicType(t Type) bool {
+	// dynamic types
+	// array is also a dynamic type if the array type is dynamic
+	return t.T == StringTy || t.T == BytesTy || t.T == SliceTy || (t.T == ArrayTy && isDynamicType(*t.Elem))
+}
+
+// getDynamicTypeOffset returns the offset for the type.
+// See `isDynamicType` to know which types are considered dynamic.
+// If the type t is an array and element type is not a dynamic type, then we consider it a static type and
+// return 32 * size of array since length prefix is not required.
+// If t is a dynamic type or element type(for slices and arrays) is dynamic, then we simply return 32 as offset.
+func getDynamicTypeOffset(t Type) int {
+	// if it is an array and there are no dynamic types
+	// then the array is static type
+	if t.T == ArrayTy && !isDynamicType(*t.Elem) {
+		return 32 * t.Size
+	}
+	return 32
+}

accounts/hd.go

@@ -30,8 +30,8 @@ import (
 var DefaultRootDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}
 
 // DefaultBaseDerivationPath is the base path from which custom derivation endpoints
-// are incremented. As such, the first account will be at m/44'/60'/0'/0, the second
-// at m/44'/60'/0'/1, etc.
+// are incremented. As such, the first account will be at m/44'/60'/0'/0/0, the second
+// at m/44'/60'/0'/0/1, etc.
 var DefaultBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0}
 
 // DefaultLedgerBaseDerivationPath is the base path from which custom derivation endpoints

accounts/keystore/account_cache.go

@@ -265,7 +265,10 @@ func (ac *accountCache) scanAccounts() error {
 		case (addr == common.Address{}):
 			log.Debug("Failed to decode keystore key", "path", path, "err", "missing or zero address")
 		default:
-			return &accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}}
+			return &accounts.Account{
+				Address: addr,
+				URL:     accounts.URL{Scheme: KeyStoreScheme, Path: path},
+			}
 		}
 		return nil
 	}

accounts/keystore/key.go

@@ -66,19 +66,19 @@ type plainKeyJSON struct {
 
 type encryptedKeyJSONV3 struct {
 	Address string     `json:"address"`
-	Crypto  cryptoJSON `json:"crypto"`
+	Crypto  CryptoJSON `json:"crypto"`
 	Id      string     `json:"id"`
 	Version int        `json:"version"`
 }
 
 type encryptedKeyJSONV1 struct {
 	Address string     `json:"address"`
-	Crypto  cryptoJSON `json:"crypto"`
+	Crypto  CryptoJSON `json:"crypto"`
 	Id      string     `json:"id"`
 	Version string     `json:"version"`
 }
 
-type cryptoJSON struct {
+type CryptoJSON struct {
 	Cipher       string                 `json:"cipher"`
 	CipherText   string                 `json:"ciphertext"`
 	CipherParams cipherparamsJSON       `json:"cipherparams"`
@@ -171,7 +171,10 @@ func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, accounts.Accou
 	if err != nil {
 		return nil, accounts.Account{}, err
 	}
-	a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))}}
+	a := accounts.Account{
+		Address: key.Address,
+		URL:     accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))},
+	}
 	if err := ks.StoreKey(a.URL.Path, key, auth); err != nil {
 		zeroKey(key.PrivateKey)
 		return nil, a, err
@@ -224,5 +227,6 @@ func toISO8601(t time.Time) string {
 	} else {
 		tz = fmt.Sprintf("%03d00", offset/3600)
 	}
-	return fmt.Sprintf("%04d-%02d-%02dT%02d-%02d-%02d.%09d%s", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), tz)
+	return fmt.Sprintf("%04d-%02d-%02dT%02d-%02d-%02d.%09d%s",
+		t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), tz)
 }

accounts/keystore/passphrase.go

@@ -135,29 +135,26 @@ func (ks keyStorePassphrase) JoinPath(filename string) string {
 	return filepath.Join(ks.keysDirPath, filename)
 }
 
-// EncryptKey encrypts a key using the specified scrypt parameters into a json
-// blob that can be decrypted later on.
-func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
-	authArray := []byte(auth)
+// Encryptdata encrypts the data given as 'data' with the password 'auth'.
+func EncryptDataV3(data, auth []byte, scryptN, scryptP int) (CryptoJSON, error) {
 
 	salt := make([]byte, 32)
 	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
 		panic("reading from crypto/rand failed: " + err.Error())
 	}
-	derivedKey, err := scrypt.Key(authArray, salt, scryptN, scryptR, scryptP, scryptDKLen)
+	derivedKey, err := scrypt.Key(auth, salt, scryptN, scryptR, scryptP, scryptDKLen)
 	if err != nil {
-		return nil, err
+		return CryptoJSON{}, err
 	}
 	encryptKey := derivedKey[:16]
-	keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
 
 	iv := make([]byte, aes.BlockSize) // 16
 	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
 		panic("reading from crypto/rand failed: " + err.Error())
 	}
-	cipherText, err := aesCTRXOR(encryptKey, keyBytes, iv)
+	cipherText, err := aesCTRXOR(encryptKey, data, iv)
 	if err != nil {
-		return nil, err
+		return CryptoJSON{}, err
 	}
 	mac := crypto.Keccak256(derivedKey[16:32], cipherText)
 
@@ -167,12 +164,11 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
 	scryptParamsJSON["p"] = scryptP
 	scryptParamsJSON["dklen"] = scryptDKLen
 	scryptParamsJSON["salt"] = hex.EncodeToString(salt)
-
 	cipherParamsJSON := cipherparamsJSON{
 		IV: hex.EncodeToString(iv),
 	}
 
-	cryptoStruct := cryptoJSON{
+	cryptoStruct := CryptoJSON{
 		Cipher:       "aes-128-ctr",
 		CipherText:   hex.EncodeToString(cipherText),
 		CipherParams: cipherParamsJSON,
@@ -180,6 +176,17 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
 		KDFParams:    scryptParamsJSON,
 		MAC:          hex.EncodeToString(mac),
 	}
+	return cryptoStruct, nil
+}
+
+// EncryptKey encrypts a key using the specified scrypt parameters into a json
+// blob that can be decrypted later on.
+func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
+	keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
+	cryptoStruct, err := EncryptDataV3(keyBytes, []byte(auth), scryptN, scryptP)
+	if err != nil {
+		return nil, err
+	}
 	encryptedKeyJSONV3 := encryptedKeyJSONV3{
 		hex.EncodeToString(key.Address[:]),
 		cryptoStruct,
@@ -227,42 +234,48 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) {
 	}, nil
 }
 
-func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byte, keyId []byte, err error) {
-	if keyProtected.Version != version {
-		return nil, nil, fmt.Errorf("Version not supported: %v", keyProtected.Version)
+func DecryptDataV3(cryptoJson CryptoJSON, auth string) ([]byte, error) {
+	if cryptoJson.Cipher != "aes-128-ctr" {
+		return nil, fmt.Errorf("Cipher not supported: %v", cryptoJson.Cipher)
 	}
-
-	if keyProtected.Crypto.Cipher != "aes-128-ctr" {
-		return nil, nil, fmt.Errorf("Cipher not supported: %v", keyProtected.Crypto.Cipher)
-	}
-
-	keyId = uuid.Parse(keyProtected.Id)
-	mac, err := hex.DecodeString(keyProtected.Crypto.MAC)
+	mac, err := hex.DecodeString(cryptoJson.MAC)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	iv, err := hex.DecodeString(keyProtected.Crypto.CipherParams.IV)
+	iv, err := hex.DecodeString(cryptoJson.CipherParams.IV)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	cipherText, err := hex.DecodeString(keyProtected.Crypto.CipherText)
+	cipherText, err := hex.DecodeString(cryptoJson.CipherText)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	derivedKey, err := getKDFKey(keyProtected.Crypto, auth)
+	derivedKey, err := getKDFKey(cryptoJson, auth)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	calculatedMAC := crypto.Keccak256(derivedKey[16:32], cipherText)
 	if !bytes.Equal(calculatedMAC, mac) {
-		return nil, nil, ErrDecrypt
+		return nil, ErrDecrypt
 	}
 
 	plainText, err := aesCTRXOR(derivedKey[:16], cipherText, iv)
+	if err != nil {
+		return nil, err
+	}
+	return plainText, err
+}
+
+func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byte, keyId []byte, err error) {
+	if keyProtected.Version != version {
+		return nil, nil, fmt.Errorf("Version not supported: %v", keyProtected.Version)
+	}
+	keyId = uuid.Parse(keyProtected.Id)
+	plainText, err := DecryptDataV3(keyProtected.Crypto, auth)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -303,7 +316,7 @@ func decryptKeyV1(keyProtected *encryptedKeyJSONV1, auth string) (keyBytes []byt
 	return plainText, keyId, err
 }
 
-func getKDFKey(cryptoJSON cryptoJSON, auth string) ([]byte, error) {
+func getKDFKey(cryptoJSON CryptoJSON, auth string) ([]byte, error) {
 	authArray := []byte(auth)
 	salt, err := hex.DecodeString(cryptoJSON.KDFParams["salt"].(string))
 	if err != nil {

accounts/keystore/presale.go

@@ -38,7 +38,13 @@ func importPreSaleKey(keyStore keyStore, keyJSON []byte, password string) (accou
 		return accounts.Account{}, nil, err
 	}
 	key.Id = uuid.NewRandom()
-	a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: keyStore.JoinPath(keyFileName(key.Address))}}
+	a := accounts.Account{
+		Address: key.Address,
+		URL: accounts.URL{
+			Scheme: KeyStoreScheme,
+			Path:   keyStore.JoinPath(keyFileName(key.Address)),
+		},
+	}
 	err = keyStore.StoreKey(a.URL.Path, key, password)
 	return a, key, err
 }

accounts/usbwallet/ledger.go

@@ -350,7 +350,7 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction
 		signer = new(types.HomesteadSigner)
 	} else {
 		signer = types.NewEIP155Signer(chainID)
-		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
+		signature[64] -= byte(chainID.Uint64()*2 + 35)
 	}
 	signed, err := tx.WithSignature(signer, signature)
 	if err != nil {

accounts/usbwallet/trezor.go

@@ -221,7 +221,7 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction
 		signer = new(types.HomesteadSigner)
 	} else {
 		signer = types.NewEIP155Signer(chainID)
-		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
+		signature[64] -= byte(chainID.Uint64()*2 + 35)
 	}
 	// Inject the final signature into the transaction and sanity check the sender
 	signed, err := tx.WithSignature(signer, signature)

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.1.windows-%GETH_ARCH%.zip
-  - 7z x go1.11.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.2.windows-%GETH_ARCH%.zip
+  - 7z x go1.11.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

cmd/bootnode/main.go

@@ -38,7 +38,7 @@ func main() {
 	var (
 		listenAddr  = flag.String("addr", ":30301", "listen address")
 		genKey      = flag.String("genkey", "", "generate a node key")
-		writeAddr   = flag.Bool("writeaddress", false, "write out the node's pubkey hash and quit")
+		writeAddr   = flag.Bool("writeaddress", false, "write out the node's public key and quit")
 		nodeKeyFile = flag.String("nodekey", "", "private key filename")
 		nodeKeyHex  = flag.String("nodekeyhex", "", "private key as hex (for testing)")
 		natdesc     = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|extip:<IP>)")
@@ -86,7 +86,7 @@ func main() {
 	}
 
 	if *writeAddr {
-		fmt.Printf("%v\n", enode.PubkeyToIDV4(&nodeKey.PublicKey))
+		fmt.Printf("%x\n", crypto.FromECDSAPub(&nodeKey.PublicKey)[1:])
 		os.Exit(0)
 	}
 
@@ -119,16 +119,17 @@ func main() {
 	}
 
 	if *runv5 {
-		if _, err := discv5.ListenUDP(nodeKey, conn, realaddr, "", restrictList); err != nil {
+		if _, err := discv5.ListenUDP(nodeKey, conn, "", restrictList); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	} else {
+		db, _ := enode.OpenDB("")
+		ln := enode.NewLocalNode(db, nodeKey)
 		cfg := discover.Config{
-			PrivateKey:   nodeKey,
-			AnnounceAddr: realaddr,
-			NetRestrict:  restrictList,
+			PrivateKey:  nodeKey,
+			NetRestrict: restrictList,
 		}
-		if _, err := discover.ListenUDP(conn, cfg); err != nil {
+		if _, err := discover.ListenUDP(conn, ln, cfg); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	}

cmd/clef/README.md

@@ -91,7 +91,7 @@ invoking methods with the following info:
   * [x] Version info about the signer
   * [x] Address of API (http/ipc)
   * [ ] List of known accounts
-* [ ] Have a default timeout on signing operations, so that if the user has not answered withing e.g. 60 seconds, the request is rejected.
+* [ ] Have a default timeout on signing operations, so that if the user has not answered within e.g. 60 seconds, the request is rejected.
 * [ ] `account_signRawTransaction`
 * [ ] `account_bulkSignTransactions([] transactions)` should
    * only exist if enabled via config/flag
@@ -129,7 +129,7 @@ The signer listens to HTTP requests on `rpcaddr`:`rpcport`, with the same JSONRP
 expected to be JSON [jsonrpc 2.0 standard](http://www.jsonrpc.org/specification).
 
 Some of these call can require user interaction. Clients must be aware that responses
-may be delayed significanlty or may never be received if a users decides to ignore the confirmation request.
+may be delayed significantly or may never be received if a users decides to ignore the confirmation request.
 
 The External API is **untrusted** : it does not accept credentials over this api, nor does it expect
 that requests have any authority.
@@ -862,7 +862,7 @@ A UI should conform to the following rules.
 * A UI SHOULD inform the user about the `SHA256` or `MD5` hash of the binary being executed
 * A UI SHOULD NOT maintain a secondary storage of data, e.g. list of accounts
   * The signer provides accounts
-* A UI SHOULD, to the best extent possible, use static linking / bundling, so that requried libraries are bundled
+* A UI SHOULD, to the best extent possible, use static linking / bundling, so that required libraries are bundled
 along with the UI.
 
 

cmd/clef/intapi_changelog.md

@@ -1,5 +1,9 @@
 ### Changelog for internal API (ui-api)
 
+### 3.0.0
+
+* Make use of `OnInputRequired(info UserInputRequest)` for obtaining master password during startup
+
 ### 2.1.0
 
 * Add `OnInputRequired(info UserInputRequest)` to internal API. This method is used when Clef needs user input, e.g. passwords.
@@ -14,7 +18,6 @@ The following structures are used:
        UserInputResponse struct {
                Text string `json:"text"`
        }
-```
 
 ### 2.0.0
 

cmd/clef/main.go

@@ -35,8 +35,10 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
@@ -48,10 +50,10 @@ import (
 )
 
 // ExternalAPIVersion -- see extapi_changelog.md
-const ExternalAPIVersion = "3.0.0"
+const ExternalAPIVersion = "4.0.0"
 
 // InternalAPIVersion -- see intapi_changelog.md
-const InternalAPIVersion = "2.0.0"
+const InternalAPIVersion = "3.0.0"
 
 const legalWarning = `
 WARNING! 
@@ -91,7 +93,7 @@ var (
 	}
 	signerSecretFlag = cli.StringFlag{
 		Name:  "signersecret",
-		Usage: "A file containing the password used to encrypt Clef credentials, e.g. keystore credentials and ruleset hash",
+		Usage: "A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash",
 	}
 	dBFlag = cli.StringFlag{
 		Name:  "4bytedb",
@@ -155,18 +157,18 @@ Whenever you make an edit to the rule file, you need to use attestation to tell
 Clef that the file is 'safe' to execute.`,
 	}
 
-	addCredentialCommand = cli.Command{
-		Action:    utils.MigrateFlags(addCredential),
-		Name:      "addpw",
+	setCredentialCommand = cli.Command{
+		Action:    utils.MigrateFlags(setCredential),
+		Name:      "setpw",
 		Usage:     "Store a credential for a keystore file",
-		ArgsUsage: "<address> <password>",
+		ArgsUsage: "<address>",
 		Flags: []cli.Flag{
 			logLevelFlag,
 			configdirFlag,
 			signerSecretFlag,
 		},
 		Description: `
-The addpw command stores a password for a given address (keyfile). If you invoke it with only one parameter, it will 
+		The setpw command stores a password for a given address (keyfile). If you enter a blank passphrase, it will 
 remove any stored credential for that address (keyfile)
 `,
 	}
@@ -198,7 +200,7 @@ func init() {
 		advancedMode,
 	}
 	app.Action = signer
-	app.Commands = []cli.Command{initCommand, attestCommand, addCredentialCommand}
+	app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand}
 
 }
 func main() {
@@ -212,25 +214,45 @@ func initializeSecrets(c *cli.Context) error {
 	if err := initialize(c); err != nil {
 		return err
 	}
-	configDir := c.String(configdirFlag.Name)
+	configDir := c.GlobalString(configdirFlag.Name)
 
 	masterSeed := make([]byte, 256)
-	n, err := io.ReadFull(rand.Reader, masterSeed)
+	num, err := io.ReadFull(rand.Reader, masterSeed)
 	if err != nil {
 		return err
 	}
-	if n != len(masterSeed) {
+	if num != len(masterSeed) {
 		return fmt.Errorf("failed to read enough random")
 	}
+
+	n, p := keystore.StandardScryptN, keystore.StandardScryptP
+	if c.GlobalBool(utils.LightKDFFlag.Name) {
+		n, p = keystore.LightScryptN, keystore.LightScryptP
+	}
+	text := "The master seed of clef is locked with a password. Please give a password. Do not forget this password."
+	var password string
+	for {
+		password = getPassPhrase(text, true)
+		if err := core.ValidatePasswordFormat(password); err != nil {
+			fmt.Printf("invalid password: %v\n", err)
+		} else {
+			break
+		}
+	}
+	cipherSeed, err := encryptSeed(masterSeed, []byte(password), n, p)
+	if err != nil {
+		return fmt.Errorf("failed to encrypt master seed: %v", err)
+	}
+
 	err = os.Mkdir(configDir, 0700)
 	if err != nil && !os.IsExist(err) {
 		return err
 	}
-	location := filepath.Join(configDir, "secrets.dat")
+	location := filepath.Join(configDir, "masterseed.json")
 	if _, err := os.Stat(location); err == nil {
 		return fmt.Errorf("file %v already exists, will not overwrite", location)
 	}
-	err = ioutil.WriteFile(location, masterSeed, 0400)
+	err = ioutil.WriteFile(location, cipherSeed, 0400)
 	if err != nil {
 		return err
 	}
@@ -255,11 +277,11 @@ func attestFile(ctx *cli.Context) error {
 		return err
 	}
 
-	stretchedKey, err := readMasterKey(ctx)
+	stretchedKey, err := readMasterKey(ctx, nil)
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	configDir := ctx.String(configdirFlag.Name)
+	configDir := ctx.GlobalString(configdirFlag.Name)
 	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 	confKey := crypto.Keccak256([]byte("config"), stretchedKey)
 
@@ -271,38 +293,36 @@ func attestFile(ctx *cli.Context) error {
 	return nil
 }
 
-func addCredential(ctx *cli.Context) error {
+func setCredential(ctx *cli.Context) error {
 	if len(ctx.Args()) < 1 {
-		utils.Fatalf("This command requires at leaste one argument.")
+		utils.Fatalf("This command requires an address to be passed as an argument.")
 	}
 	if err := initialize(ctx); err != nil {
 		return err
 	}
 
-	stretchedKey, err := readMasterKey(ctx)
+	address := ctx.Args().First()
+	password := getPassPhrase("Enter a passphrase to store with this address.", true)
+
+	stretchedKey, err := readMasterKey(ctx, nil)
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	configDir := ctx.String(configdirFlag.Name)
+	configDir := ctx.GlobalString(configdirFlag.Name)
 	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 	pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
 
 	// Initialize the encrypted storages
 	pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
-	key := ctx.Args().First()
-	value := ""
-	if len(ctx.Args()) > 1 {
-		value = ctx.Args().Get(1)
-	}
-	pwStorage.Put(key, value)
-	log.Info("Credential store updated", "key", key)
+	pwStorage.Put(address, password)
+	log.Info("Credential store updated", "key", address)
 	return nil
 }
 
 func initialize(c *cli.Context) error {
 	// Set up the logger to print everything
 	logOutput := os.Stdout
-	if c.Bool(stdiouiFlag.Name) {
+	if c.GlobalBool(stdiouiFlag.Name) {
 		logOutput = os.Stderr
 		// If using the stdioui, we can't do the 'confirm'-flow
 		fmt.Fprintf(logOutput, legalWarning)
@@ -323,26 +343,28 @@ func signer(c *cli.Context) error {
 	var (
 		ui core.SignerUI
 	)
-	if c.Bool(stdiouiFlag.Name) {
+	if c.GlobalBool(stdiouiFlag.Name) {
 		log.Info("Using stdin/stdout as UI-channel")
 		ui = core.NewStdIOUI()
 	} else {
 		log.Info("Using CLI as UI-channel")
 		ui = core.NewCommandlineUI()
 	}
-	db, err := core.NewAbiDBFromFiles(c.String(dBFlag.Name), c.String(customDBFlag.Name))
+	fourByteDb := c.GlobalString(dBFlag.Name)
+	fourByteLocal := c.GlobalString(customDBFlag.Name)
+	db, err := core.NewAbiDBFromFiles(fourByteDb, fourByteLocal)
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	log.Info("Loaded 4byte db", "signatures", db.Size(), "file", c.String("4bytedb"))
+	log.Info("Loaded 4byte db", "signatures", db.Size(), "file", fourByteDb, "local", fourByteLocal)
 
 	var (
 		api core.ExternalAPI
 	)
 
-	configDir := c.String(configdirFlag.Name)
-	if stretchedKey, err := readMasterKey(c); err != nil {
-		log.Info("No master seed provided, rules disabled")
+	configDir := c.GlobalString(configdirFlag.Name)
+	if stretchedKey, err := readMasterKey(c, ui); err != nil {
+		log.Info("No master seed provided, rules disabled", "error", err)
 	} else {
 
 		if err != nil {
@@ -361,7 +383,7 @@ func signer(c *cli.Context) error {
 		configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey)
 
 		//Do we have a rule-file?
-		ruleJS, err := ioutil.ReadFile(c.String(ruleFlag.Name))
+		ruleJS, err := ioutil.ReadFile(c.GlobalString(ruleFlag.Name))
 		if err != nil {
 			log.Info("Could not load rulefile, rules not enabled", "file", "rulefile")
 		} else {
@@ -385,17 +407,15 @@ func signer(c *cli.Context) error {
 	}
 
 	apiImpl := core.NewSignerAPI(
-		c.Int64(utils.NetworkIdFlag.Name),
-		c.String(keystoreFlag.Name),
-		c.Bool(utils.NoUSBFlag.Name),
+		c.GlobalInt64(utils.NetworkIdFlag.Name),
+		c.GlobalString(keystoreFlag.Name),
+		c.GlobalBool(utils.NoUSBFlag.Name),
 		ui, db,
-		c.Bool(utils.LightKDFFlag.Name),
-		c.Bool(advancedMode.Name))
-
+		c.GlobalBool(utils.LightKDFFlag.Name),
+		c.GlobalBool(advancedMode.Name))
 	api = apiImpl
-
 	// Audit logging
-	if logfile := c.String(auditLogFlag.Name); logfile != "" {
+	if logfile := c.GlobalString(auditLogFlag.Name); logfile != "" {
 		api, err = core.NewAuditLogger(logfile, api)
 		if err != nil {
 			utils.Fatalf(err.Error())
@@ -414,13 +434,13 @@ func signer(c *cli.Context) error {
 			Service:   api,
 			Version:   "1.0"},
 	}
-	if c.Bool(utils.RPCEnabledFlag.Name) {
+	if c.GlobalBool(utils.RPCEnabledFlag.Name) {
 
 		vhosts := splitAndTrim(c.GlobalString(utils.RPCVirtualHostsFlag.Name))
 		cors := splitAndTrim(c.GlobalString(utils.RPCCORSDomainFlag.Name))
 
 		// start http server
-		httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
+		httpEndpoint := fmt.Sprintf("%s:%d", c.GlobalString(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
 		listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts, rpc.DefaultHTTPTimeouts)
 		if err != nil {
 			utils.Fatalf("Could not start RPC api: %v", err)
@@ -434,9 +454,9 @@ func signer(c *cli.Context) error {
 		}()
 
 	}
-	if !c.Bool(utils.IPCDisabledFlag.Name) {
+	if !c.GlobalBool(utils.IPCDisabledFlag.Name) {
 		if c.IsSet(utils.IPCPathFlag.Name) {
-			ipcapiURL = c.String(utils.IPCPathFlag.Name)
+			ipcapiURL = c.GlobalString(utils.IPCPathFlag.Name)
 		} else {
 			ipcapiURL = filepath.Join(configDir, "clef.ipc")
 		}
@@ -453,7 +473,7 @@ func signer(c *cli.Context) error {
 
 	}
 
-	if c.Bool(testFlag.Name) {
+	if c.GlobalBool(testFlag.Name) {
 		log.Info("Performing UI test")
 		go testExternalUI(apiImpl)
 	}
@@ -512,36 +532,52 @@ func homeDir() string {
 	}
 	return ""
 }
-func readMasterKey(ctx *cli.Context) ([]byte, error) {
+func readMasterKey(ctx *cli.Context, ui core.SignerUI) ([]byte, error) {
 	var (
 		file      string
-		configDir = ctx.String(configdirFlag.Name)
+		configDir = ctx.GlobalString(configdirFlag.Name)
 	)
-	if ctx.IsSet(signerSecretFlag.Name) {
-		file = ctx.String(signerSecretFlag.Name)
+	if ctx.GlobalIsSet(signerSecretFlag.Name) {
+		file = ctx.GlobalString(signerSecretFlag.Name)
 	} else {
-		file = filepath.Join(configDir, "secrets.dat")
+		file = filepath.Join(configDir, "masterseed.json")
 	}
 	if err := checkFile(file); err != nil {
 		return nil, err
 	}
-	masterKey, err := ioutil.ReadFile(file)
+	cipherKey, err := ioutil.ReadFile(file)
 	if err != nil {
 		return nil, err
 	}
-	if len(masterKey) < 256 {
-		return nil, fmt.Errorf("master key of insufficient length, expected >255 bytes, got %d", len(masterKey))
+	var password string
+	// If ui is not nil, get the password from ui.
+	if ui != nil {
+		resp, err := ui.OnInputRequired(core.UserInputRequest{
+			Title:      "Master Password",
+			Prompt:     "Please enter the password to decrypt the master seed",
+			IsPassword: true})
+		if err != nil {
+			return nil, err
+		}
+		password = resp.Text
+	} else {
+		password = getPassPhrase("Decrypt master seed of clef", false)
 	}
+	masterSeed, err := decryptSeed(cipherKey, password)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decrypt the master seed of clef")
+	}
+	if len(masterSeed) < 256 {
+		return nil, fmt.Errorf("master seed of insufficient length, expected >255 bytes, got %d", len(masterSeed))
+	}
+
 	// Create vault location
-	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterKey)[:10]))
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterSeed)[:10]))
 	err = os.Mkdir(vaultLocation, 0700)
 	if err != nil && !os.IsExist(err) {
 		return nil, err
 	}
-	//!TODO, use KDF to stretch the master key
-	//			stretched_key := stretch_key(master_key)
-
-	return masterKey, nil
+	return masterSeed, nil
 }
 
 // checkFile is a convenience function to check if a file
@@ -619,6 +655,59 @@ func testExternalUI(api *core.SignerAPI) {
 
 }
 
+// getPassPhrase retrieves the password associated with clef, either fetched
+// from a list of preloaded passphrases, or requested interactively from the user.
+// TODO: there are many `getPassPhrase` functions, it will be better to abstract them into one.
+func getPassPhrase(prompt string, confirmation bool) string {
+	fmt.Println(prompt)
+	password, err := console.Stdin.PromptPassword("Passphrase: ")
+	if err != nil {
+		utils.Fatalf("Failed to read passphrase: %v", err)
+	}
+	if confirmation {
+		confirm, err := console.Stdin.PromptPassword("Repeat passphrase: ")
+		if err != nil {
+			utils.Fatalf("Failed to read passphrase confirmation: %v", err)
+		}
+		if password != confirm {
+			utils.Fatalf("Passphrases do not match")
+		}
+	}
+	return password
+}
+
+type encryptedSeedStorage struct {
+	Description string              `json:"description"`
+	Version     int                 `json:"version"`
+	Params      keystore.CryptoJSON `json:"params"`
+}
+
+// encryptSeed uses a similar scheme as the keystore uses, but with a different wrapping,
+// to encrypt the master seed
+func encryptSeed(seed []byte, auth []byte, scryptN, scryptP int) ([]byte, error) {
+	cryptoStruct, err := keystore.EncryptDataV3(seed, auth, scryptN, scryptP)
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(&encryptedSeedStorage{"Clef seed", 1, cryptoStruct})
+}
+
+// decryptSeed decrypts the master seed
+func decryptSeed(keyjson []byte, auth string) ([]byte, error) {
+	var encSeed encryptedSeedStorage
+	if err := json.Unmarshal(keyjson, &encSeed); err != nil {
+		return nil, err
+	}
+	if encSeed.Version != 1 {
+		log.Warn(fmt.Sprintf("unsupported encryption format of seed: %d, operation will likely fail", encSeed.Version))
+	}
+	seed, err := keystore.DecryptDataV3(encSeed.Params, auth)
+	if err != nil {
+		return nil, err
+	}
+	return seed, err
+}
+
 /**
 //Create Account
 

cmd/evm/runner.go

@@ -89,7 +89,7 @@ func runCmd(ctx *cli.Context) error {
 		genesisConfig *core.Genesis
 	)
 	if ctx.GlobalBool(MachineFlag.Name) {
-		tracer = NewJSONLogger(logconfig, os.Stdout)
+		tracer = vm.NewJSONLogger(logconfig, os.Stdout)
 	} else if ctx.GlobalBool(DebugFlag.Name) {
 		debugLogger = vm.NewStructLogger(logconfig)
 		tracer = debugLogger
@@ -206,6 +206,7 @@ func runCmd(ctx *cli.Context) error {
 	execTime := time.Since(tstart)
 
 	if ctx.GlobalBool(DumpFlag.Name) {
+		statedb.Commit(true)
 		statedb.IntermediateRoot(true)
 		fmt.Println(string(statedb.Dump()))
 	}

cmd/evm/staterunner.go

@@ -68,7 +68,7 @@ func stateTestCmd(ctx *cli.Context) error {
 	)
 	switch {
 	case ctx.GlobalBool(MachineFlag.Name):
-		tracer = NewJSONLogger(config, os.Stderr)
+		tracer = vm.NewJSONLogger(config, os.Stderr)
 
 	case ctx.GlobalBool(DebugFlag.Name):
 		debugger = vm.NewStructLogger(config)

cmd/faucet/faucet.go

@@ -256,7 +256,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u
 	}
 	for _, boot := range enodes {
 		old, err := enode.ParseV4(boot.String())
-		if err != nil {
+		if err == nil {
 			stack.Server().AddPeer(old)
 		}
 	}

cmd/geth/config.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math/big"
 	"os"
 	"reflect"
 	"unicode"
@@ -152,7 +153,9 @@ func enableWhisper(ctx *cli.Context) bool {
 
 func makeFullNode(ctx *cli.Context) *node.Node {
 	stack, cfg := makeConfigNode(ctx)
-
+	if ctx.GlobalIsSet(utils.ConstantinopleOverrideFlag.Name) {
+		cfg.Eth.ConstantinopleOverride = new(big.Int).SetUint64(ctx.GlobalUint64(utils.ConstantinopleOverrideFlag.Name))
+	}
 	utils.RegisterEthService(stack, &cfg.Eth)
 
 	if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) {

cmd/geth/main.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"math"
 	"os"
-	"runtime"
 	godebug "runtime/debug"
 	"sort"
 	"strconv"
@@ -88,8 +87,10 @@ var (
 		utils.LightServFlag,
 		utils.LightPeersFlag,
 		utils.LightKDFFlag,
+		utils.WhitelistFlag,
 		utils.CacheFlag,
 		utils.CacheDatabaseFlag,
+		utils.CacheTrieFlag,
 		utils.CacheGCFlag,
 		utils.TrieCacheGenFlag,
 		utils.ListenPortFlag,
@@ -122,6 +123,7 @@ var (
 		utils.RinkebyFlag,
 		utils.VMEnableDebugFlag,
 		utils.NetworkIdFlag,
+		utils.ConstantinopleOverrideFlag,
 		utils.RPCCORSDomainFlag,
 		utils.RPCVirtualHostsFlag,
 		utils.EthStatsURLFlag,
@@ -209,8 +211,6 @@ func init() {
 	app.Flags = append(app.Flags, metricsFlags...)
 
 	app.Before = func(ctx *cli.Context) error {
-		runtime.GOMAXPROCS(runtime.NumCPU())
-
 		logdir := ""
 		if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) {
 			logdir = (&node.Config{DataDir: utils.MakeDataDir(ctx)}).ResolvePath("logs")

cmd/geth/usage.go

@@ -81,6 +81,7 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.LightServFlag,
 			utils.LightPeersFlag,
 			utils.LightKDFFlag,
+			utils.WhitelistFlag,
 		},
 	},
 	{
@@ -132,6 +133,7 @@ var AppHelpFlagGroups = []flagGroup{
 		Flags: []cli.Flag{
 			utils.CacheFlag,
 			utils.CacheDatabaseFlag,
+			utils.CacheTrieFlag,
 			utils.CacheGCFlag,
 			utils.TrieCacheGenFlag,
 		},

cmd/puppeth/genesis.go

@@ -20,35 +20,41 @@ import (
 	"encoding/binary"
 	"errors"
 	"math"
+	"math/big"
+	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	math2 "github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/params"
 )
 
-// cppEthereumGenesisSpec represents the genesis specification format used by the
+// alethGenesisSpec represents the genesis specification format used by the
 // C++ Ethereum implementation.
-type cppEthereumGenesisSpec struct {
+type alethGenesisSpec struct {
 	SealEngine string `json:"sealEngine"`
 	Params     struct {
-		AccountStartNonce       hexutil.Uint64 `json:"accountStartNonce"`
-		HomesteadForkBlock      hexutil.Uint64 `json:"homesteadForkBlock"`
-		EIP150ForkBlock         hexutil.Uint64 `json:"EIP150ForkBlock"`
-		EIP158ForkBlock         hexutil.Uint64 `json:"EIP158ForkBlock"`
-		ByzantiumForkBlock      hexutil.Uint64 `json:"byzantiumForkBlock"`
-		ConstantinopleForkBlock hexutil.Uint64 `json:"constantinopleForkBlock"`
-		NetworkID               hexutil.Uint64 `json:"networkID"`
-		ChainID                 hexutil.Uint64 `json:"chainID"`
-		MaximumExtraDataSize    hexutil.Uint64 `json:"maximumExtraDataSize"`
-		MinGasLimit             hexutil.Uint64 `json:"minGasLimit"`
-		MaxGasLimit             hexutil.Uint64 `json:"maxGasLimit"`
-		GasLimitBoundDivisor    hexutil.Uint64 `json:"gasLimitBoundDivisor"`
-		MinimumDifficulty       *hexutil.Big   `json:"minimumDifficulty"`
-		DifficultyBoundDivisor  *hexutil.Big   `json:"difficultyBoundDivisor"`
-		DurationLimit           *hexutil.Big   `json:"durationLimit"`
-		BlockReward             *hexutil.Big   `json:"blockReward"`
+		AccountStartNonce       math2.HexOrDecimal64   `json:"accountStartNonce"`
+		MaximumExtraDataSize    hexutil.Uint64         `json:"maximumExtraDataSize"`
+		HomesteadForkBlock      hexutil.Uint64         `json:"homesteadForkBlock"`
+		DaoHardforkBlock        math2.HexOrDecimal64   `json:"daoHardforkBlock"`
+		EIP150ForkBlock         hexutil.Uint64         `json:"EIP150ForkBlock"`
+		EIP158ForkBlock         hexutil.Uint64         `json:"EIP158ForkBlock"`
+		ByzantiumForkBlock      hexutil.Uint64         `json:"byzantiumForkBlock"`
+		ConstantinopleForkBlock hexutil.Uint64         `json:"constantinopleForkBlock"`
+		MinGasLimit             hexutil.Uint64         `json:"minGasLimit"`
+		MaxGasLimit             hexutil.Uint64         `json:"maxGasLimit"`
+		TieBreakingGas          bool                   `json:"tieBreakingGas"`
+		GasLimitBoundDivisor    math2.HexOrDecimal64   `json:"gasLimitBoundDivisor"`
+		MinimumDifficulty       *hexutil.Big           `json:"minimumDifficulty"`
+		DifficultyBoundDivisor  *math2.HexOrDecimal256 `json:"difficultyBoundDivisor"`
+		DurationLimit           *math2.HexOrDecimal256 `json:"durationLimit"`
+		BlockReward             *hexutil.Big           `json:"blockReward"`
+		NetworkID               hexutil.Uint64         `json:"networkID"`
+		ChainID                 hexutil.Uint64         `json:"chainID"`
+		AllowFutureBlocks       bool                   `json:"allowFutureBlocks"`
 	} `json:"params"`
 
 	Genesis struct {
@@ -62,57 +68,68 @@ type cppEthereumGenesisSpec struct {
 		GasLimit   hexutil.Uint64 `json:"gasLimit"`
 	} `json:"genesis"`
 
-	Accounts map[common.Address]*cppEthereumGenesisSpecAccount `json:"accounts"`
+	Accounts map[common.UnprefixedAddress]*alethGenesisSpecAccount `json:"accounts"`
 }
 
-// cppEthereumGenesisSpecAccount is the prefunded genesis account and/or precompiled
+// alethGenesisSpecAccount is the prefunded genesis account and/or precompiled
 // contract definition.
-type cppEthereumGenesisSpecAccount struct {
-	Balance     *hexutil.Big                   `json:"balance"`
-	Nonce       uint64                         `json:"nonce,omitempty"`
-	Precompiled *cppEthereumGenesisSpecBuiltin `json:"precompiled,omitempty"`
+type alethGenesisSpecAccount struct {
+	Balance     *math2.HexOrDecimal256   `json:"balance"`
+	Nonce       uint64                   `json:"nonce,omitempty"`
+	Precompiled *alethGenesisSpecBuiltin `json:"precompiled,omitempty"`
 }
 
-// cppEthereumGenesisSpecBuiltin is the precompiled contract definition.
-type cppEthereumGenesisSpecBuiltin struct {
-	Name          string                               `json:"name,omitempty"`
-	StartingBlock hexutil.Uint64                       `json:"startingBlock,omitempty"`
-	Linear        *cppEthereumGenesisSpecLinearPricing `json:"linear,omitempty"`
+// alethGenesisSpecBuiltin is the precompiled contract definition.
+type alethGenesisSpecBuiltin struct {
+	Name          string                         `json:"name,omitempty"`
+	StartingBlock hexutil.Uint64                 `json:"startingBlock,omitempty"`
+	Linear        *alethGenesisSpecLinearPricing `json:"linear,omitempty"`
 }
 
-type cppEthereumGenesisSpecLinearPricing struct {
+type alethGenesisSpecLinearPricing struct {
 	Base uint64 `json:"base"`
 	Word uint64 `json:"word"`
 }
 
-// newCppEthereumGenesisSpec converts a go-ethereum genesis block into a Parity specific
+// newAlethGenesisSpec converts a go-ethereum genesis block into a Aleth-specific
 // chain specification format.
-func newCppEthereumGenesisSpec(network string, genesis *core.Genesis) (*cppEthereumGenesisSpec, error) {
-	// Only ethash is currently supported between go-ethereum and cpp-ethereum
+func newAlethGenesisSpec(network string, genesis *core.Genesis) (*alethGenesisSpec, error) {
+	// Only ethash is currently supported between go-ethereum and aleth
 	if genesis.Config.Ethash == nil {
 		return nil, errors.New("unsupported consensus engine")
 	}
-	// Reconstruct the chain spec in Parity's format
-	spec := &cppEthereumGenesisSpec{
+	// Reconstruct the chain spec in Aleth format
+	spec := &alethGenesisSpec{
 		SealEngine: "Ethash",
 	}
+	// Some defaults
 	spec.Params.AccountStartNonce = 0
+	spec.Params.TieBreakingGas = false
+	spec.Params.AllowFutureBlocks = false
+	spec.Params.DaoHardforkBlock = 0
+
 	spec.Params.HomesteadForkBlock = (hexutil.Uint64)(genesis.Config.HomesteadBlock.Uint64())
 	spec.Params.EIP150ForkBlock = (hexutil.Uint64)(genesis.Config.EIP150Block.Uint64())
 	spec.Params.EIP158ForkBlock = (hexutil.Uint64)(genesis.Config.EIP158Block.Uint64())
-	spec.Params.ByzantiumForkBlock = (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64())
-	spec.Params.ConstantinopleForkBlock = (hexutil.Uint64)(math.MaxUint64)
+
+	// Byzantium
+	if num := genesis.Config.ByzantiumBlock; num != nil {
+		spec.setByzantium(num)
+	}
+	// Constantinople
+	if num := genesis.Config.ConstantinopleBlock; num != nil {
+		spec.setConstantinople(num)
+	}
 
 	spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64())
 	spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64())
-
 	spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize)
 	spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit)
-	spec.Params.MaxGasLimit = (hexutil.Uint64)(math.MaxUint64)
+	spec.Params.MaxGasLimit = (hexutil.Uint64)(math.MaxInt64)
 	spec.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty)
-	spec.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor)
-	spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor)
-	spec.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit)
+	spec.Params.DifficultyBoundDivisor = (*math2.HexOrDecimal256)(params.DifficultyBoundDivisor)
+	spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor)
+	spec.Params.DurationLimit = (*math2.HexOrDecimal256)(params.DurationLimit)
 	spec.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward)
 
 	spec.Genesis.Nonce = (hexutil.Bytes)(make([]byte, 8))
@@ -126,77 +143,104 @@ func newCppEthereumGenesisSpec(network string, genesis *core.Genesis) (*cppEther
 	spec.Genesis.ExtraData = (hexutil.Bytes)(genesis.ExtraData)
 	spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit)
 
-	spec.Accounts = make(map[common.Address]*cppEthereumGenesisSpecAccount)
 	for address, account := range genesis.Alloc {
-		spec.Accounts[address] = &cppEthereumGenesisSpecAccount{
-			Balance: (*hexutil.Big)(account.Balance),
-			Nonce:   account.Nonce,
-		}
-	}
-	spec.Accounts[common.BytesToAddress([]byte{1})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-		Name: "ecrecover", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 3000},
-	}
-	spec.Accounts[common.BytesToAddress([]byte{2})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-		Name: "sha256", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 60, Word: 12},
-	}
-	spec.Accounts[common.BytesToAddress([]byte{3})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-		Name: "ripemd160", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 600, Word: 120},
-	}
-	spec.Accounts[common.BytesToAddress([]byte{4})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-		Name: "identity", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 15, Word: 3},
+		spec.setAccount(address, account)
 	}
+
+	spec.setPrecompile(1, &alethGenesisSpecBuiltin{Name: "ecrecover",
+		Linear: &alethGenesisSpecLinearPricing{Base: 3000}})
+	spec.setPrecompile(2, &alethGenesisSpecBuiltin{Name: "sha256",
+		Linear: &alethGenesisSpecLinearPricing{Base: 60, Word: 12}})
+	spec.setPrecompile(3, &alethGenesisSpecBuiltin{Name: "ripemd160",
+		Linear: &alethGenesisSpecLinearPricing{Base: 600, Word: 120}})
+	spec.setPrecompile(4, &alethGenesisSpecBuiltin{Name: "identity",
+		Linear: &alethGenesisSpecLinearPricing{Base: 15, Word: 3}})
 	if genesis.Config.ByzantiumBlock != nil {
-		spec.Accounts[common.BytesToAddress([]byte{5})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-			Name: "modexp", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()),
-		}
-		spec.Accounts[common.BytesToAddress([]byte{6})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-			Name: "alt_bn128_G1_add", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), Linear: &cppEthereumGenesisSpecLinearPricing{Base: 500},
-		}
-		spec.Accounts[common.BytesToAddress([]byte{7})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-			Name: "alt_bn128_G1_mul", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), Linear: &cppEthereumGenesisSpecLinearPricing{Base: 40000},
-		}
-		spec.Accounts[common.BytesToAddress([]byte{8})].Precompiled = &cppEthereumGenesisSpecBuiltin{
-			Name: "alt_bn128_pairing_product", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()),
-		}
+		spec.setPrecompile(5, &alethGenesisSpecBuiltin{Name: "modexp",
+			StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64())})
+		spec.setPrecompile(6, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_add",
+			StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()),
+			Linear:        &alethGenesisSpecLinearPricing{Base: 500}})
+		spec.setPrecompile(7, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_mul",
+			StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()),
+			Linear:        &alethGenesisSpecLinearPricing{Base: 40000}})
+		spec.setPrecompile(8, &alethGenesisSpecBuiltin{Name: "alt_bn128_pairing_product",
+			StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64())})
 	}
 	return spec, nil
 }
 
+func (spec *alethGenesisSpec) setPrecompile(address byte, data *alethGenesisSpecBuiltin) {
+	if spec.Accounts == nil {
+		spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount)
+	}
+	spec.Accounts[common.UnprefixedAddress(common.BytesToAddress([]byte{address}))].Precompiled = data
+}
+
+func (spec *alethGenesisSpec) setAccount(address common.Address, account core.GenesisAccount) {
+	if spec.Accounts == nil {
+		spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount)
+	}
+
+	a, exist := spec.Accounts[common.UnprefixedAddress(address)]
+	if !exist {
+		a = &alethGenesisSpecAccount{}
+		spec.Accounts[common.UnprefixedAddress(address)] = a
+	}
+	a.Balance = (*math2.HexOrDecimal256)(account.Balance)
+	a.Nonce = account.Nonce
+
+}
+
+func (spec *alethGenesisSpec) setByzantium(num *big.Int) {
+	spec.Params.ByzantiumForkBlock = hexutil.Uint64(num.Uint64())
+}
+
+func (spec *alethGenesisSpec) setConstantinople(num *big.Int) {
+	spec.Params.ConstantinopleForkBlock = hexutil.Uint64(num.Uint64())
+}
+
 // parityChainSpec is the chain specification format used by Parity.
 type parityChainSpec struct {
-	Name   string `json:"name"`
-	Engine struct {
+	Name    string `json:"name"`
+	Datadir string `json:"dataDir"`
+	Engine  struct {
 		Ethash struct {
 			Params struct {
-				MinimumDifficulty      *hexutil.Big `json:"minimumDifficulty"`
-				DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"`
-				DurationLimit          *hexutil.Big `json:"durationLimit"`
-				BlockReward            *hexutil.Big `json:"blockReward"`
-				HomesteadTransition    uint64       `json:"homesteadTransition"`
-				EIP150Transition       uint64       `json:"eip150Transition"`
-				EIP160Transition       uint64       `json:"eip160Transition"`
-				EIP161abcTransition    uint64       `json:"eip161abcTransition"`
-				EIP161dTransition      uint64       `json:"eip161dTransition"`
-				EIP649Reward           *hexutil.Big `json:"eip649Reward"`
-				EIP100bTransition      uint64       `json:"eip100bTransition"`
-				EIP649Transition       uint64       `json:"eip649Transition"`
+				MinimumDifficulty      *hexutil.Big      `json:"minimumDifficulty"`
+				DifficultyBoundDivisor *hexutil.Big      `json:"difficultyBoundDivisor"`
+				DurationLimit          *hexutil.Big      `json:"durationLimit"`
+				BlockReward            map[string]string `json:"blockReward"`
+				DifficultyBombDelays   map[string]string `json:"difficultyBombDelays"`
+				HomesteadTransition    hexutil.Uint64    `json:"homesteadTransition"`
+				EIP100bTransition      hexutil.Uint64    `json:"eip100bTransition"`
 			} `json:"params"`
 		} `json:"Ethash"`
 	} `json:"engine"`
 
 	Params struct {
-		MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"`
-		MinGasLimit          hexutil.Uint64 `json:"minGasLimit"`
-		GasLimitBoundDivisor hexutil.Uint64 `json:"gasLimitBoundDivisor"`
-		NetworkID            hexutil.Uint64 `json:"networkID"`
-		MaxCodeSize          uint64         `json:"maxCodeSize"`
-		EIP155Transition     uint64         `json:"eip155Transition"`
-		EIP98Transition      uint64         `json:"eip98Transition"`
-		EIP86Transition      uint64         `json:"eip86Transition"`
-		EIP140Transition     uint64         `json:"eip140Transition"`
-		EIP211Transition     uint64         `json:"eip211Transition"`
-		EIP214Transition     uint64         `json:"eip214Transition"`
-		EIP658Transition     uint64         `json:"eip658Transition"`
+		AccountStartNonce     hexutil.Uint64       `json:"accountStartNonce"`
+		MaximumExtraDataSize  hexutil.Uint64       `json:"maximumExtraDataSize"`
+		MinGasLimit           hexutil.Uint64       `json:"minGasLimit"`
+		GasLimitBoundDivisor  math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"`
+		NetworkID             hexutil.Uint64       `json:"networkID"`
+		ChainID               hexutil.Uint64       `json:"chainID"`
+		MaxCodeSize           hexutil.Uint64       `json:"maxCodeSize"`
+		MaxCodeSizeTransition hexutil.Uint64       `json:"maxCodeSizeTransition"`
+		EIP98Transition       hexutil.Uint64       `json:"eip98Transition"`
+		EIP150Transition      hexutil.Uint64       `json:"eip150Transition"`
+		EIP160Transition      hexutil.Uint64       `json:"eip160Transition"`
+		EIP161abcTransition   hexutil.Uint64       `json:"eip161abcTransition"`
+		EIP161dTransition     hexutil.Uint64       `json:"eip161dTransition"`
+		EIP155Transition      hexutil.Uint64       `json:"eip155Transition"`
+		EIP140Transition      hexutil.Uint64       `json:"eip140Transition"`
+		EIP211Transition      hexutil.Uint64       `json:"eip211Transition"`
+		EIP214Transition      hexutil.Uint64       `json:"eip214Transition"`
+		EIP658Transition      hexutil.Uint64       `json:"eip658Transition"`
+		EIP145Transition      hexutil.Uint64       `json:"eip145Transition"`
+		EIP1014Transition     hexutil.Uint64       `json:"eip1014Transition"`
+		EIP1052Transition     hexutil.Uint64       `json:"eip1052Transition"`
+		EIP1283Transition     hexutil.Uint64       `json:"eip1283Transition"`
 	} `json:"params"`
 
 	Genesis struct {
@@ -215,22 +259,22 @@ type parityChainSpec struct {
 		GasLimit   hexutil.Uint64 `json:"gasLimit"`
 	} `json:"genesis"`
 
-	Nodes    []string                                   `json:"nodes"`
-	Accounts map[common.Address]*parityChainSpecAccount `json:"accounts"`
+	Nodes    []string                                             `json:"nodes"`
+	Accounts map[common.UnprefixedAddress]*parityChainSpecAccount `json:"accounts"`
 }
 
 // parityChainSpecAccount is the prefunded genesis account and/or precompiled
 // contract definition.
 type parityChainSpecAccount struct {
-	Balance *hexutil.Big            `json:"balance"`
-	Nonce   uint64                  `json:"nonce,omitempty"`
+	Balance math2.HexOrDecimal256   `json:"balance"`
+	Nonce   math2.HexOrDecimal64    `json:"nonce,omitempty"`
 	Builtin *parityChainSpecBuiltin `json:"builtin,omitempty"`
 }
 
 // parityChainSpecBuiltin is the precompiled contract definition.
 type parityChainSpecBuiltin struct {
 	Name       string                  `json:"name,omitempty"`
-	ActivateAt uint64                  `json:"activate_at,omitempty"`
+	ActivateAt math2.HexOrDecimal64    `json:"activate_at,omitempty"`
 	Pricing    *parityChainSpecPricing `json:"pricing,omitempty"`
 }
 
@@ -265,34 +309,51 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin
 	}
 	// Reconstruct the chain spec in Parity's format
 	spec := &parityChainSpec{
-		Name:  network,
-		Nodes: bootnodes,
+		Name:    network,
+		Nodes:   bootnodes,
+		Datadir: strings.ToLower(network),
 	}
+	spec.Engine.Ethash.Params.BlockReward = make(map[string]string)
+	spec.Engine.Ethash.Params.DifficultyBombDelays = make(map[string]string)
+	// Frontier
 	spec.Engine.Ethash.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty)
 	spec.Engine.Ethash.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor)
 	spec.Engine.Ethash.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit)
-	spec.Engine.Ethash.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward)
-	spec.Engine.Ethash.Params.HomesteadTransition = genesis.Config.HomesteadBlock.Uint64()
-	spec.Engine.Ethash.Params.EIP150Transition = genesis.Config.EIP150Block.Uint64()
-	spec.Engine.Ethash.Params.EIP160Transition = genesis.Config.EIP155Block.Uint64()
-	spec.Engine.Ethash.Params.EIP161abcTransition = genesis.Config.EIP158Block.Uint64()
-	spec.Engine.Ethash.Params.EIP161dTransition = genesis.Config.EIP158Block.Uint64()
-	spec.Engine.Ethash.Params.EIP649Reward = (*hexutil.Big)(ethash.ByzantiumBlockReward)
-	spec.Engine.Ethash.Params.EIP100bTransition = genesis.Config.ByzantiumBlock.Uint64()
-	spec.Engine.Ethash.Params.EIP649Transition = genesis.Config.ByzantiumBlock.Uint64()
+	spec.Engine.Ethash.Params.BlockReward["0x0"] = hexutil.EncodeBig(ethash.FrontierBlockReward)
 
+	// Homestead
+	spec.Engine.Ethash.Params.HomesteadTransition = hexutil.Uint64(genesis.Config.HomesteadBlock.Uint64())
+
+	// Tangerine Whistle : 150
+	// https://github.com/ethereum/EIPs/blob/master/EIPS/eip-608.md
+	spec.Params.EIP150Transition = hexutil.Uint64(genesis.Config.EIP150Block.Uint64())
+
+	// Spurious Dragon: 155, 160, 161, 170
+	// https://github.com/ethereum/EIPs/blob/master/EIPS/eip-607.md
+	spec.Params.EIP155Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64())
+	spec.Params.EIP160Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64())
+	spec.Params.EIP161abcTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64())
+	spec.Params.EIP161dTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64())
+
+	// Byzantium
+	if num := genesis.Config.ByzantiumBlock; num != nil {
+		spec.setByzantium(num)
+	}
+	// Constantinople
+	if num := genesis.Config.ConstantinopleBlock; num != nil {
+		spec.setConstantinople(num)
+	}
 	spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize)
 	spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit)
-	spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor)
+	spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor)
 	spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64())
+	spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64())
 	spec.Params.MaxCodeSize = params.MaxCodeSize
-	spec.Params.EIP155Transition = genesis.Config.EIP155Block.Uint64()
-	spec.Params.EIP98Transition = math.MaxUint64
-	spec.Params.EIP86Transition = math.MaxUint64
-	spec.Params.EIP140Transition = genesis.Config.ByzantiumBlock.Uint64()
-	spec.Params.EIP211Transition = genesis.Config.ByzantiumBlock.Uint64()
-	spec.Params.EIP214Transition = genesis.Config.ByzantiumBlock.Uint64()
-	spec.Params.EIP658Transition = genesis.Config.ByzantiumBlock.Uint64()
+	// geth has it set from zero
+	spec.Params.MaxCodeSizeTransition = 0
+
+	// Disable this one
+	spec.Params.EIP98Transition = math.MaxInt64
 
 	spec.Genesis.Seal.Ethereum.Nonce = (hexutil.Bytes)(make([]byte, 8))
 	binary.LittleEndian.PutUint64(spec.Genesis.Seal.Ethereum.Nonce[:], genesis.Nonce)
@@ -305,42 +366,77 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin
 	spec.Genesis.ExtraData = (hexutil.Bytes)(genesis.ExtraData)
 	spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit)
 
-	spec.Accounts = make(map[common.Address]*parityChainSpecAccount)
+	spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount)
 	for address, account := range genesis.Alloc {
-		spec.Accounts[address] = &parityChainSpecAccount{
-			Balance: (*hexutil.Big)(account.Balance),
-			Nonce:   account.Nonce,
+		bal := math2.HexOrDecimal256(*account.Balance)
+
+		spec.Accounts[common.UnprefixedAddress(address)] = &parityChainSpecAccount{
+			Balance: bal,
+			Nonce:   math2.HexOrDecimal64(account.Nonce),
 		}
 	}
-	spec.Accounts[common.BytesToAddress([]byte{1})].Builtin = &parityChainSpecBuiltin{
-		Name: "ecrecover", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 3000}},
-	}
-	spec.Accounts[common.BytesToAddress([]byte{2})].Builtin = &parityChainSpecBuiltin{
+	spec.setPrecompile(1, &parityChainSpecBuiltin{Name: "ecrecover",
+		Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 3000}}})
+
+	spec.setPrecompile(2, &parityChainSpecBuiltin{
 		Name: "sha256", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 60, Word: 12}},
-	}
-	spec.Accounts[common.BytesToAddress([]byte{3})].Builtin = &parityChainSpecBuiltin{
+	})
+	spec.setPrecompile(3, &parityChainSpecBuiltin{
 		Name: "ripemd160", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 600, Word: 120}},
-	}
-	spec.Accounts[common.BytesToAddress([]byte{4})].Builtin = &parityChainSpecBuiltin{
+	})
+	spec.setPrecompile(4, &parityChainSpecBuiltin{
 		Name: "identity", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 15, Word: 3}},
-	}
+	})
 	if genesis.Config.ByzantiumBlock != nil {
-		spec.Accounts[common.BytesToAddress([]byte{5})].Builtin = &parityChainSpecBuiltin{
-			Name: "modexp", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{ModExp: &parityChainSpecModExpPricing{Divisor: 20}},
-		}
-		spec.Accounts[common.BytesToAddress([]byte{6})].Builtin = &parityChainSpecBuiltin{
-			Name: "alt_bn128_add", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 500}},
-		}
-		spec.Accounts[common.BytesToAddress([]byte{7})].Builtin = &parityChainSpecBuiltin{
-			Name: "alt_bn128_mul", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 40000}},
-		}
-		spec.Accounts[common.BytesToAddress([]byte{8})].Builtin = &parityChainSpecBuiltin{
-			Name: "alt_bn128_pairing", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{AltBnPairing: &parityChainSpecAltBnPairingPricing{Base: 100000, Pair: 80000}},
-		}
+		blnum := math2.HexOrDecimal64(genesis.Config.ByzantiumBlock.Uint64())
+		spec.setPrecompile(5, &parityChainSpecBuiltin{
+			Name: "modexp", ActivateAt: blnum, Pricing: &parityChainSpecPricing{ModExp: &parityChainSpecModExpPricing{Divisor: 20}},
+		})
+		spec.setPrecompile(6, &parityChainSpecBuiltin{
+			Name: "alt_bn128_add", ActivateAt: blnum, Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 500}},
+		})
+		spec.setPrecompile(7, &parityChainSpecBuiltin{
+			Name: "alt_bn128_mul", ActivateAt: blnum, Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 40000}},
+		})
+		spec.setPrecompile(8, &parityChainSpecBuiltin{
+			Name: "alt_bn128_pairing", ActivateAt: blnum, Pricing: &parityChainSpecPricing{AltBnPairing: &parityChainSpecAltBnPairingPricing{Base: 100000, Pair: 80000}},
+		})
 	}
 	return spec, nil
 }
 
+func (spec *parityChainSpec) setPrecompile(address byte, data *parityChainSpecBuiltin) {
+	if spec.Accounts == nil {
+		spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount)
+	}
+	a := common.UnprefixedAddress(common.BytesToAddress([]byte{address}))
+	if _, exist := spec.Accounts[a]; !exist {
+		spec.Accounts[a] = &parityChainSpecAccount{}
+	}
+	spec.Accounts[a].Builtin = data
+}
+
+func (spec *parityChainSpec) setByzantium(num *big.Int) {
+	spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ByzantiumBlockReward)
+	spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(3000000)
+	n := hexutil.Uint64(num.Uint64())
+	spec.Engine.Ethash.Params.EIP100bTransition = n
+	spec.Params.EIP140Transition = n
+	spec.Params.EIP211Transition = n
+	spec.Params.EIP214Transition = n
+	spec.Params.EIP658Transition = n
+}
+
+func (spec *parityChainSpec) setConstantinople(num *big.Int) {
+	spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ConstantinopleBlockReward)
+	spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(2000000)
+	n := hexutil.Uint64(num.Uint64())
+	spec.Params.EIP145Transition = n
+	spec.Params.EIP1014Transition = n
+	spec.Params.EIP1052Transition = n
+	spec.Params.EIP1283Transition = n
+}
+
 // pyEthereumGenesisSpec represents the genesis specification format used by the
 // Python Ethereum implementation.
 type pyEthereumGenesisSpec struct {

cmd/puppeth/genesis_test.go

@@ -0,0 +1,109 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/ethereum/go-ethereum/core"
+)
+
+// Tests the go-ethereum to Aleth chainspec conversion for the Stureby testnet.
+func TestAlethSturebyConverter(t *testing.T) {
+	blob, err := ioutil.ReadFile("testdata/stureby_geth.json")
+	if err != nil {
+		t.Fatalf("could not read file: %v", err)
+	}
+	var genesis core.Genesis
+	if err := json.Unmarshal(blob, &genesis); err != nil {
+		t.Fatalf("failed parsing genesis: %v", err)
+	}
+	spec, err := newAlethGenesisSpec("stureby", &genesis)
+	if err != nil {
+		t.Fatalf("failed creating chainspec: %v", err)
+	}
+
+	expBlob, err := ioutil.ReadFile("testdata/stureby_aleth.json")
+	if err != nil {
+		t.Fatalf("could not read file: %v", err)
+	}
+	expspec := &alethGenesisSpec{}
+	if err := json.Unmarshal(expBlob, expspec); err != nil {
+		t.Fatalf("failed parsing genesis: %v", err)
+	}
+	if !reflect.DeepEqual(expspec, spec) {
+		t.Errorf("chainspec mismatch")
+		c := spew.ConfigState{
+			DisablePointerAddresses: true,
+			SortKeys:                true,
+		}
+		exp := strings.Split(c.Sdump(expspec), "\n")
+		got := strings.Split(c.Sdump(spec), "\n")
+		for i := 0; i < len(exp) && i < len(got); i++ {
+			if exp[i] != got[i] {
+				fmt.Printf("got: %v\nexp: %v\n", exp[i], got[i])
+			}
+		}
+	}
+}
+
+// Tests the go-ethereum to Parity chainspec conversion for the Stureby testnet.
+func TestParitySturebyConverter(t *testing.T) {
+	blob, err := ioutil.ReadFile("testdata/stureby_geth.json")
+	if err != nil {
+		t.Fatalf("could not read file: %v", err)
+	}
+	var genesis core.Genesis
+	if err := json.Unmarshal(blob, &genesis); err != nil {
+		t.Fatalf("failed parsing genesis: %v", err)
+	}
+	spec, err := newParityChainSpec("Stureby", &genesis, []string{})
+	if err != nil {
+		t.Fatalf("failed creating chainspec: %v", err)
+	}
+
+	expBlob, err := ioutil.ReadFile("testdata/stureby_parity.json")
+	if err != nil {
+		t.Fatalf("could not read file: %v", err)
+	}
+	expspec := &parityChainSpec{}
+	if err := json.Unmarshal(expBlob, expspec); err != nil {
+		t.Fatalf("failed parsing genesis: %v", err)
+	}
+	expspec.Nodes = []string{}
+
+	if !reflect.DeepEqual(expspec, spec) {
+		t.Errorf("chainspec mismatch")
+		c := spew.ConfigState{
+			DisablePointerAddresses: true,
+			SortKeys:                true,
+		}
+		exp := strings.Split(c.Sdump(expspec), "\n")
+		got := strings.Split(c.Sdump(spec), "\n")
+		for i := 0; i < len(exp) && i < len(got); i++ {
+			if exp[i] != got[i] {
+				fmt.Printf("got: %v\nexp: %v\n", exp[i], got[i])
+			}
+		}
+	}
+}

cmd/puppeth/module_dashboard.go

@@ -640,7 +640,7 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da
 	files[filepath.Join(workdir, network+".json")] = genesis
 
 	if conf.Genesis.Config.Ethash != nil {
-		cppSpec, err := newCppEthereumGenesisSpec(network, conf.Genesis)
+		cppSpec, err := newAlethGenesisSpec(network, conf.Genesis)
 		if err != nil {
 			return nil, err
 		}

cmd/puppeth/module_ethstats.go

@@ -43,7 +43,8 @@ version: '2'
 services:
   ethstats:
     build: .
-    image: {{.Network}}/ethstats{{if not .VHost}}
+    image: {{.Network}}/ethstats
+    container_name: {{.Network}}_ethstats_1{{if not .VHost}}
     ports:
       - "{{.Port}}:3000"{{end}}
     environment:

cmd/puppeth/module_explorer.go

@@ -77,6 +77,7 @@ services:
   explorer:
     build: .
     image: {{.Network}}/explorer
+    container_name: {{.Network}}_explorer_1
     ports:
       - "{{.NodePort}}:{{.NodePort}}"
       - "{{.NodePort}}:{{.NodePort}}/udp"{{if not .VHost}}

cmd/puppeth/module_faucet.go

@@ -56,8 +56,10 @@ services:
   faucet:
     build: .
     image: {{.Network}}/faucet
+    container_name: {{.Network}}_faucet_1
     ports:
-      - "{{.EthPort}}:{{.EthPort}}"{{if not .VHost}}
+      - "{{.EthPort}}:{{.EthPort}}"
+      - "{{.EthPort}}:{{.EthPort}}/udp"{{if not .VHost}}
       - "{{.ApiPort}}:8080"{{end}}
     volumes:
       - {{.Datadir}}:/root/.faucet

cmd/puppeth/module_nginx.go

@@ -40,6 +40,7 @@ services:
   nginx:
     build: .
     image: {{.Network}}/nginx
+    container_name: {{.Network}}_nginx_1
     ports:
       - "{{.Port}}:80"
     volumes:

cmd/puppeth/module_node.go

@@ -55,6 +55,7 @@ services:
   {{.Type}}:
     build: .
     image: {{.Network}}/{{.Type}}
+    container_name: {{.Network}}_{{.Type}}_1
     ports:
       - "{{.Port}}:{{.Port}}"
       - "{{.Port}}:{{.Port}}/udp"

cmd/puppeth/module_wallet.go

@@ -57,6 +57,7 @@ services:
   wallet:
     build: .
     image: {{.Network}}/wallet
+    container_name: {{.Network}}_wallet_1
     ports:
       - "{{.NodePort}}:{{.NodePort}}"
       - "{{.NodePort}}:{{.NodePort}}/udp"

cmd/puppeth/puppeth.go

@@ -43,18 +43,23 @@ func main() {
 			Usage: "log level to emit to the screen",
 		},
 	}
-	app.Action = func(c *cli.Context) error {
+	app.Before = func(c *cli.Context) error {
 		// Set up the logger to print everything and the random generator
 		log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int("loglevel")), log.StreamHandler(os.Stdout, log.TerminalFormat(true))))
 		rand.Seed(time.Now().UnixNano())
 
-		network := c.String("network")
-		if strings.Contains(network, " ") || strings.Contains(network, "-") {
-			log.Crit("No spaces or hyphens allowed in network name")
-		}
-		// Start the wizard and relinquish control
-		makeWizard(c.String("network")).run()
 		return nil
 	}
+	app.Action = runWizard
 	app.Run(os.Args)
 }
+
+// runWizard start the wizard and relinquish control to it.
+func runWizard(c *cli.Context) error {
+	network := c.String("network")
+	if strings.Contains(network, " ") || strings.Contains(network, "-") || strings.ToLower(network) != network {
+		log.Crit("No spaces, hyphens or capital letters allowed in network name")
+	}
+	makeWizard(c.String("network")).run()
+	return nil
+}

cmd/puppeth/testdata/stureby_aleth.json

@@ -0,0 +1,112 @@
+{
+  "sealEngine":"Ethash",
+  "params":{
+    "accountStartNonce":"0x00",
+    "maximumExtraDataSize":"0x20",
+    "homesteadForkBlock":"0x2710",
+    "daoHardforkBlock":"0x00",
+    "EIP150ForkBlock":"0x3a98",
+    "EIP158ForkBlock":"0x59d8",
+    "byzantiumForkBlock":"0x7530",
+    "constantinopleForkBlock":"0x9c40",
+    "minGasLimit":"0x1388",
+    "maxGasLimit":"0x7fffffffffffffff",
+    "tieBreakingGas":false,
+    "gasLimitBoundDivisor":"0x0400",
+    "minimumDifficulty":"0x20000",
+    "difficultyBoundDivisor":"0x0800",
+    "durationLimit":"0x0d",
+    "blockReward":"0x4563918244F40000",
+    "networkID":"0x4cb2e",
+    "chainID":"0x4cb2e",
+    "allowFutureBlocks":false
+  },
+  "genesis":{
+    "nonce":"0x0000000000000000",
+    "difficulty":"0x20000",
+    "mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000",
+    "author":"0x0000000000000000000000000000000000000000",
+    "timestamp":"0x59a4e76d",
+    "parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000",
+    "extraData":"0x0000000000000000000000000000000000000000000000000000000b4dc0ffee",
+    "gasLimit":"0x47b760"
+  },
+  "accounts":{
+    "0000000000000000000000000000000000000001":{
+      "balance":"1",
+      "precompiled":{
+        "name":"ecrecover",
+        "linear":{
+          "base":3000,
+          "word":0
+        }
+      }
+    },
+    "0000000000000000000000000000000000000002":{
+      "balance":"1",
+      "precompiled":{
+        "name":"sha256",
+        "linear":{
+          "base":60,
+          "word":12
+        }
+      }
+    },
+    "0000000000000000000000000000000000000003":{
+      "balance":"1",
+      "precompiled":{
+        "name":"ripemd160",
+        "linear":{
+          "base":600,
+          "word":120
+        }
+      }
+    },
+    "0000000000000000000000000000000000000004":{
+      "balance":"1",
+      "precompiled":{
+        "name":"identity",
+        "linear":{
+          "base":15,
+          "word":3
+        }
+      }
+    },
+    "0000000000000000000000000000000000000005":{
+      "balance":"1",
+      "precompiled":{
+        "name":"modexp",
+        "startingBlock":"0x7530"
+      }
+    },
+    "0000000000000000000000000000000000000006":{
+      "balance":"1",
+      "precompiled":{
+        "name":"alt_bn128_G1_add",
+        "startingBlock":"0x7530",
+        "linear":{
+          "base":500,
+          "word":0
+        }
+      }
+    },
+    "0000000000000000000000000000000000000007":{
+      "balance":"1",
+      "precompiled":{
+        "name":"alt_bn128_G1_mul",
+        "startingBlock":"0x7530",
+        "linear":{
+          "base":40000,
+          "word":0
+        }
+      }
+    },
+    "0000000000000000000000000000000000000008":{
+      "balance":"1",
+      "precompiled":{
+        "name":"alt_bn128_pairing_product",
+        "startingBlock":"0x7530"
+      }
+    }
+  }
+}

cmd/puppeth/testdata/stureby_geth.json

@@ -0,0 +1,47 @@
+{
+  "config": {
+    "ethash":{},
+    "chainId": 314158,
+    "homesteadBlock": 10000,
+    "eip150Block": 15000,
+    "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+    "eip155Block": 23000,
+    "eip158Block": 23000,
+    "byzantiumBlock": 30000,
+    "constantinopleBlock": 40000
+  },
+  "nonce": "0x0",
+  "timestamp": "0x59a4e76d",
+  "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "extraData": "0x0000000000000000000000000000000000000000000000000000000b4dc0ffee",
+  "gasLimit": "0x47b760",
+  "difficulty": "0x20000",
+  "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "coinbase": "0x0000000000000000000000000000000000000000",
+  "alloc": {
+    "0000000000000000000000000000000000000001": {
+      "balance": "0x01"
+    },
+    "0000000000000000000000000000000000000002": {
+      "balance": "0x01"
+    },
+    "0000000000000000000000000000000000000003": {
+      "balance": "0x01"
+    },
+    "0000000000000000000000000000000000000004": {
+      "balance": "0x01"
+    },
+    "0000000000000000000000000000000000000005": {
+      "balance": "0x01"
+    },
+    "0000000000000000000000000000000000000006": {
+      "balance": "0x01"
+    },
+    "0000000000000000000000000000000000000007": {
+      "balance": "0x01"
+    },
+    "0000000000000000000000000000000000000008": {
+      "balance": "0x01"
+    }
+  }
+}

cmd/puppeth/testdata/stureby_parity.json

@@ -0,0 +1,181 @@
+{
+  "name":"Stureby",
+  "dataDir":"stureby",
+  "engine":{
+    "Ethash":{
+      "params":{
+        "minimumDifficulty":"0x20000",
+        "difficultyBoundDivisor":"0x800",
+        "durationLimit":"0xd",
+        "blockReward":{
+          "0x0":"0x4563918244f40000",
+          "0x7530":"0x29a2241af62c0000",
+          "0x9c40":"0x1bc16d674ec80000"
+        },
+        "homesteadTransition":"0x2710",
+        "eip100bTransition":"0x7530",
+        "difficultyBombDelays":{
+          "0x7530":"0x2dc6c0",
+          "0x9c40":"0x1e8480"
+        }
+      }
+    }
+  },
+  "params":{
+    "accountStartNonce":"0x0",
+    "maximumExtraDataSize":"0x20",
+    "gasLimitBoundDivisor":"0x400",
+    "minGasLimit":"0x1388",
+    "networkID":"0x4cb2e",
+    "chainID":"0x4cb2e",
+    "maxCodeSize":"0x6000",
+    "maxCodeSizeTransition":"0x0",
+    "eip98Transition": "0x7fffffffffffffff",
+    "eip150Transition":"0x3a98",
+    "eip160Transition":"0x59d8",
+    "eip161abcTransition":"0x59d8",
+    "eip161dTransition":"0x59d8",
+    "eip155Transition":"0x59d8",
+    "eip140Transition":"0x7530",
+    "eip211Transition":"0x7530",
+    "eip214Transition":"0x7530",
+    "eip658Transition":"0x7530",
+    "eip145Transition":"0x9c40",
+    "eip1014Transition":"0x9c40",
+    "eip1052Transition":"0x9c40",
+    "eip1283Transition":"0x9c40"
+  },
+  "genesis":{
+    "seal":{
+      "ethereum":{
+        "nonce":"0x0000000000000000",
+        "mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000"
+      }
+    },
+    "difficulty":"0x20000",
+    "author":"0x0000000000000000000000000000000000000000",
+    "timestamp":"0x59a4e76d",
+    "parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000",
+    "extraData":"0x0000000000000000000000000000000000000000000000000000000b4dc0ffee",
+    "gasLimit":"0x47b760"
+  },
+  "nodes":[
+    "enode://dfa7aca3f5b635fbfe7d0b20575f25e40d9e27b4bfbb3cf74364a42023ad9f25c1a4383bcc8cced86ee511a7d03415345a4df05be37f1dff040e4c780699f1c0@168.61.153.255:31303",
+    "enode://ef441b20dd70aeabf0eac35c3b8a2854e5ce04db0e30be9152ea9fd129359dcbb3f803993303ff5781c755dfd7223f3fe43505f583cccb740949407677412ba9@40.74.91.252:31303",
+    "enode://953b5ea1c8987cf46008232a0160324fd00d41320ecf00e23af86ec8f5396b19eb57ddab37c78141be56f62e9077de4f4dfa0747fa768ed8c8531bbfb1046237@40.70.214.166:31303",
+    "enode://276e613dd4b277a66591e565711e6c8bb107f0905248a9f8f8228c1a87992e156e5114bb9937c02824a9d9d25f76340442cf86e2028bf5293cae19904fb2b98e@35.178.251.52:30303",
+    "enode://064c820d41e52ed7d426ac64b60506c2998235bedc7e67cb497c6faf7bb4fc54fe56fc82d0add3180b747c0c4f40a1108a6f84d7d0629ed606d504528e61cc57@3.8.5.3:30303",
+    "enode://90069fdabcc5e684fa5d59430bebbb12755d9362dfe5006a1485b13d71a78a3812d36e74dd7d88e50b51add01e097ea80f16263aeaa4f0230db6c79e2a97e7ca@217.29.191.142:30303",
+    "enode://0aac74b7fd28726275e466acb5e03bc88a95927e9951eb66b5efb239b2f798ada0690853b2f2823fe4efa408f0f3d4dd258430bc952a5ff70677b8625b3e3b14@40.115.33.57:40404",
+    "enode://0b96415a10f835106d83e090a0528eed5e7887e5c802a6d084e9f1993a9d0fc713781e6e4101f6365e9b91259712f291acc0a9e6e667e22023050d602c36fbe2@40.115.33.57:40414"
+  ],
+  "accounts":{
+    "0000000000000000000000000000000000000001":{
+      "balance":"1",
+      "nonce":"0",
+      "builtin":{
+        "name":"ecrecover",
+        "pricing":{
+          "linear":{
+            "base":3000,
+            "word":0
+          }
+        }
+      }
+    },
+    "0000000000000000000000000000000000000002":{
+      "balance":"1",
+      "nonce":"0",
+      "builtin":{
+        "name":"sha256",
+        "pricing":{
+          "linear":{
+            "base":60,
+            "word":12
+          }
+        }
+      }
+    },
+    "0000000000000000000000000000000000000003":{
+      "balance":"1",
+      "nonce":"0",
+      "builtin":{
+        "name":"ripemd160",
+        "pricing":{
+          "linear":{
+            "base":600,
+            "word":120
+          }
+        }
+      }
+    },
+    "0000000000000000000000000000000000000004":{
+      "balance":"1",
+      "nonce":"0",
+      "builtin":{
+        "name":"identity",
+        "pricing":{
+          "linear":{
+            "base":15,
+            "word":3
+          }
+        }
+      }
+    },
+    "0000000000000000000000000000000000000005":{
+      "balance":"1",
+      "nonce":"0",
+      "builtin":{
+        "name":"modexp",
+        "activate_at":"0x7530",
+        "pricing":{
+          "modexp":{
+            "divisor":20
+          }
+        }
+      }
+    },
+    "0000000000000000000000000000000000000006":{
+      "balance":"1",
+      "nonce":"0",
+      "builtin":{
+        "name":"alt_bn128_add",
+        "activate_at":"0x7530",
+        "pricing":{
+          "linear":{
+            "base":500,
+            "word":0
+          }
+        }
+      }
+    },
+    "0000000000000000000000000000000000000007":{
+      "balance":"1",
+      "nonce":"0",
+      "builtin":{
+        "name":"alt_bn128_mul",
+        "activate_at":"0x7530",
+        "pricing":{
+          "linear":{
+            "base":40000,
+            "word":0
+          }
+        }
+      }
+    },
+    "0000000000000000000000000000000000000008":{
+      "balance":"1",
+      "nonce":"0",
+      "builtin":{
+        "name":"alt_bn128_pairing",
+        "activate_at":"0x7530",
+        "pricing":{
+          "alt_bn128_pairing":{
+            "base":100000,
+            "pair":80000
+          }
+        }
+      }
+    }
+  }
+}

cmd/puppeth/wizard.go

@@ -23,6 +23,7 @@ import (
 	"io/ioutil"
 	"math/big"
 	"net"
+	"net/url"
 	"os"
 	"path/filepath"
 	"sort"
@@ -118,6 +119,47 @@ func (w *wizard) readDefaultString(def string) string {
 	return def
 }
 
+// readDefaultYesNo reads a single line from stdin, trimming if from spaces and
+// interpreting it as a 'yes' or a 'no'. If an empty line is entered, the default
+// value is returned.
+func (w *wizard) readDefaultYesNo(def bool) bool {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		if text = strings.ToLower(strings.TrimSpace(text)); text == "" {
+			return def
+		}
+		if text == "y" || text == "yes" {
+			return true
+		}
+		if text == "n" || text == "no" {
+			return false
+		}
+		log.Error("Invalid input, expected 'y', 'yes', 'n', 'no' or empty")
+	}
+}
+
+// readURL reads a single line from stdin, trimming if from spaces and trying to
+// interpret it as a URL (http, https or file).
+func (w *wizard) readURL() *url.URL {
+	for {
+		fmt.Printf("> ")
+		text, err := w.in.ReadString('\n')
+		if err != nil {
+			log.Crit("Failed to read user input", "err", err)
+		}
+		uri, err := url.Parse(strings.TrimSpace(text))
+		if err != nil {
+			log.Error("Invalid input, expected URL", "err", err)
+			continue
+		}
+		return uri
+	}
+}
+
 // readInt reads a single line from stdin, trimming if from spaces, enforcing it
 // to parse into an integer.
 func (w *wizard) readInt() int {

cmd/puppeth/wizard_dashboard.go

@@ -137,14 +137,14 @@ func (w *wizard) deployDashboard() {
 	if w.conf.ethstats != "" {
 		fmt.Println()
 		fmt.Println("Include ethstats secret on dashboard (y/n)? (default = yes)")
-		infos.trusted = w.readDefaultString("y") == "y"
+		infos.trusted = w.readDefaultYesNo(true)
 	}
 	// Try to deploy the dashboard container on the host
 	nocache := false
 	if existed {
 		fmt.Println()
 		fmt.Printf("Should the dashboard be built from scratch (y/n)? (default = no)\n")
-		nocache = w.readDefaultString("n") != "n"
+		nocache = w.readDefaultYesNo(false)
 	}
 	if out, err := deployDashboard(client, w.network, &w.conf, infos, nocache); err != nil {
 		log.Error("Failed to deploy dashboard container", "err", err)

cmd/puppeth/wizard_ethstats.go

@@ -67,11 +67,11 @@ func (w *wizard) deployEthstats() {
 	if existed {
 		fmt.Println()
 		fmt.Printf("Keep existing IP %v blacklist (y/n)? (default = yes)\n", infos.banned)
-		if w.readDefaultString("y") != "y" {
+		if !w.readDefaultYesNo(true) {
 			// The user might want to clear the entire list, although generally probably not
 			fmt.Println()
 			fmt.Printf("Clear out blacklist and start over (y/n)? (default = no)\n")
-			if w.readDefaultString("n") != "n" {
+			if w.readDefaultYesNo(false) {
 				infos.banned = nil
 			}
 			// Offer the user to explicitly add/remove certain IP addresses
@@ -106,7 +106,7 @@ func (w *wizard) deployEthstats() {
 	if existed {
 		fmt.Println()
 		fmt.Printf("Should the ethstats be built from scratch (y/n)? (default = no)\n")
-		nocache = w.readDefaultString("n") != "n"
+		nocache = w.readDefaultYesNo(false)
 	}
 	trusted := make([]string, 0, len(w.servers))
 	for _, client := range w.servers {

cmd/puppeth/wizard_explorer.go

@@ -100,7 +100,7 @@ func (w *wizard) deployExplorer() {
 	if existed {
 		fmt.Println()
 		fmt.Printf("Should the explorer be built from scratch (y/n)? (default = no)\n")
-		nocache = w.readDefaultString("n") != "n"
+		nocache = w.readDefaultYesNo(false)
 	}
 	if out, err := deployExplorer(client, w.network, chain, infos, nocache); err != nil {
 		log.Error("Failed to deploy explorer container", "err", err)

cmd/puppeth/wizard_faucet.go

@@ -81,7 +81,7 @@ func (w *wizard) deployFaucet() {
 	if infos.captchaToken != "" {
 		fmt.Println()
 		fmt.Println("Reuse previous reCaptcha API authorization (y/n)? (default = yes)")
-		if w.readDefaultString("y") != "y" {
+		if !w.readDefaultYesNo(true) {
 			infos.captchaToken, infos.captchaSecret = "", ""
 		}
 	}
@@ -89,7 +89,7 @@ func (w *wizard) deployFaucet() {
 		// No previous authorization (or old one discarded)
 		fmt.Println()
 		fmt.Println("Enable reCaptcha protection against robots (y/n)? (default = no)")
-		if w.readDefaultString("n") == "n" {
+		if !w.readDefaultYesNo(false) {
 			log.Warn("Users will be able to requests funds via automated scripts")
 		} else {
 			// Captcha protection explicitly requested, read the site and secret keys
@@ -132,7 +132,7 @@ func (w *wizard) deployFaucet() {
 		} else {
 			fmt.Println()
 			fmt.Printf("Reuse previous (%s) funding account (y/n)? (default = yes)\n", key.Address.Hex())
-			if w.readDefaultString("y") != "y" {
+			if !w.readDefaultYesNo(true) {
 				infos.node.keyJSON, infos.node.keyPass = "", ""
 			}
 		}
@@ -166,7 +166,7 @@ func (w *wizard) deployFaucet() {
 	if existed {
 		fmt.Println()
 		fmt.Printf("Should the faucet be built from scratch (y/n)? (default = no)\n")
-		nocache = w.readDefaultString("n") != "n"
+		nocache = w.readDefaultYesNo(false)
 	}
 	if out, err := deployFaucet(client, w.network, w.conf.bootnodes, infos, nocache); err != nil {
 		log.Error("Failed to deploy faucet container", "err", err)

cmd/puppeth/wizard_genesis.go

@@ -20,9 +20,13 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"math/big"
 	"math/rand"
+	"net/http"
+	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -40,11 +44,12 @@ func (w *wizard) makeGenesis() {
 		Difficulty: big.NewInt(524288),
 		Alloc:      make(core.GenesisAlloc),
 		Config: &params.ChainConfig{
-			HomesteadBlock: big.NewInt(1),
-			EIP150Block:    big.NewInt(2),
-			EIP155Block:    big.NewInt(3),
-			EIP158Block:    big.NewInt(3),
-			ByzantiumBlock: big.NewInt(4),
+			HomesteadBlock:      big.NewInt(1),
+			EIP150Block:         big.NewInt(2),
+			EIP155Block:         big.NewInt(3),
+			EIP158Block:         big.NewInt(3),
+			ByzantiumBlock:      big.NewInt(4),
+			ConstantinopleBlock: big.NewInt(5),
 		},
 	}
 	// Figure out which consensus engine to choose
@@ -114,9 +119,13 @@ func (w *wizard) makeGenesis() {
 		}
 		break
 	}
-	// Add a batch of precompile balances to avoid them getting deleted
-	for i := int64(0); i < 256; i++ {
-		genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(1)}
+	fmt.Println()
+	fmt.Println("Should the precompile-addresses (0x1 .. 0xff) be pre-funded with 1 wei? (advisable yes)")
+	if w.readDefaultYesNo(true) {
+		// Add a batch of precompile balances to avoid them getting deleted
+		for i := int64(0); i < 256; i++ {
+			genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(1)}
+		}
 	}
 	// Query the user for some custom extras
 	fmt.Println()
@@ -130,53 +139,130 @@ func (w *wizard) makeGenesis() {
 	w.conf.flush()
 }
 
+// importGenesis imports a Geth genesis spec into puppeth.
+func (w *wizard) importGenesis() {
+	// Request the genesis JSON spec URL from the user
+	fmt.Println()
+	fmt.Println("Where's the genesis file? (local file or http/https url)")
+	url := w.readURL()
+
+	// Convert the various allowed URLs to a reader stream
+	var reader io.Reader
+
+	switch url.Scheme {
+	case "http", "https":
+		// Remote web URL, retrieve it via an HTTP client
+		res, err := http.Get(url.String())
+		if err != nil {
+			log.Error("Failed to retrieve remote genesis", "err", err)
+			return
+		}
+		defer res.Body.Close()
+		reader = res.Body
+
+	case "":
+		// Schemaless URL, interpret as a local file
+		file, err := os.Open(url.String())
+		if err != nil {
+			log.Error("Failed to open local genesis", "err", err)
+			return
+		}
+		defer file.Close()
+		reader = file
+
+	default:
+		log.Error("Unsupported genesis URL scheme", "scheme", url.Scheme)
+		return
+	}
+	// Parse the genesis file and inject it successful
+	var genesis core.Genesis
+	if err := json.NewDecoder(reader).Decode(&genesis); err != nil {
+		log.Error("Invalid genesis spec: %v", err)
+		return
+	}
+	log.Info("Imported genesis block")
+
+	w.conf.Genesis = &genesis
+	w.conf.flush()
+}
+
 // manageGenesis permits the modification of chain configuration parameters in
 // a genesis config and the export of the entire genesis spec.
 func (w *wizard) manageGenesis() {
 	// Figure out whether to modify or export the genesis
 	fmt.Println()
 	fmt.Println(" 1. Modify existing fork rules")
-	fmt.Println(" 2. Export genesis configuration")
+	fmt.Println(" 2. Export genesis configurations")
 	fmt.Println(" 3. Remove genesis configuration")
 
 	choice := w.read()
-	switch {
-	case choice == "1":
+	switch choice {
+	case "1":
 		// Fork rule updating requested, iterate over each fork
 		fmt.Println()
 		fmt.Printf("Which block should Homestead come into effect? (default = %v)\n", w.conf.Genesis.Config.HomesteadBlock)
 		w.conf.Genesis.Config.HomesteadBlock = w.readDefaultBigInt(w.conf.Genesis.Config.HomesteadBlock)
 
 		fmt.Println()
-		fmt.Printf("Which block should EIP150 come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP150Block)
+		fmt.Printf("Which block should EIP150 (Tangerine Whistle) come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP150Block)
 		w.conf.Genesis.Config.EIP150Block = w.readDefaultBigInt(w.conf.Genesis.Config.EIP150Block)
 
 		fmt.Println()
-		fmt.Printf("Which block should EIP155 come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP155Block)
+		fmt.Printf("Which block should EIP155 (Spurious Dragon) come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP155Block)
 		w.conf.Genesis.Config.EIP155Block = w.readDefaultBigInt(w.conf.Genesis.Config.EIP155Block)
 
 		fmt.Println()
-		fmt.Printf("Which block should EIP158 come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP158Block)
+		fmt.Printf("Which block should EIP158/161 (also Spurious Dragon) come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP158Block)
 		w.conf.Genesis.Config.EIP158Block = w.readDefaultBigInt(w.conf.Genesis.Config.EIP158Block)
 
 		fmt.Println()
 		fmt.Printf("Which block should Byzantium come into effect? (default = %v)\n", w.conf.Genesis.Config.ByzantiumBlock)
 		w.conf.Genesis.Config.ByzantiumBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ByzantiumBlock)
 
+		fmt.Println()
+		fmt.Printf("Which block should Constantinople come into effect? (default = %v)\n", w.conf.Genesis.Config.ConstantinopleBlock)
+		w.conf.Genesis.Config.ConstantinopleBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock)
+
 		out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", "  ")
 		fmt.Printf("Chain configuration updated:\n\n%s\n", out)
 
-	case choice == "2":
+	case "2":
 		// Save whatever genesis configuration we currently have
 		fmt.Println()
-		fmt.Printf("Which file to save the genesis into? (default = %s.json)\n", w.network)
-		out, _ := json.MarshalIndent(w.conf.Genesis, "", "  ")
-		if err := ioutil.WriteFile(w.readDefaultString(fmt.Sprintf("%s.json", w.network)), out, 0644); err != nil {
-			log.Error("Failed to save genesis file", "err", err)
-		}
-		log.Info("Exported existing genesis block")
+		fmt.Printf("Which folder to save the genesis specs into? (default = current)\n")
+		fmt.Printf("  Will create %s.json, %s-aleth.json, %s-harmony.json, %s-parity.json\n", w.network, w.network, w.network, w.network)
 
-	case choice == "3":
+		folder := w.readDefaultString(".")
+		if err := os.MkdirAll(folder, 0755); err != nil {
+			log.Error("Failed to create spec folder", "folder", folder, "err", err)
+			return
+		}
+		out, _ := json.MarshalIndent(w.conf.Genesis, "", "  ")
+
+		// Export the native genesis spec used by puppeth and Geth
+		gethJson := filepath.Join(folder, fmt.Sprintf("%s.json", w.network))
+		if err := ioutil.WriteFile((gethJson), out, 0644); err != nil {
+			log.Error("Failed to save genesis file", "err", err)
+			return
+		}
+		log.Info("Saved native genesis chain spec", "path", gethJson)
+
+		// Export the genesis spec used by Aleth (formerly C++ Ethereum)
+		if spec, err := newAlethGenesisSpec(w.network, w.conf.Genesis); err != nil {
+			log.Error("Failed to create Aleth chain spec", "err", err)
+		} else {
+			saveGenesis(folder, w.network, "aleth", spec)
+		}
+		// Export the genesis spec used by Parity
+		if spec, err := newParityChainSpec(w.network, w.conf.Genesis, []string{}); err != nil {
+			log.Error("Failed to create Parity chain spec", "err", err)
+		} else {
+			saveGenesis(folder, w.network, "parity", spec)
+		}
+		// Export the genesis spec used by Harmony (formerly EthereumJ
+		saveGenesis(folder, w.network, "harmony", w.conf.Genesis)
+
+	case "3":
 		// Make sure we don't have any services running
 		if len(w.conf.servers()) > 0 {
 			log.Error("Genesis reset requires all services and servers torn down")
@@ -186,8 +272,20 @@ func (w *wizard) manageGenesis() {
 
 		w.conf.Genesis = nil
 		w.conf.flush()
-
 	default:
 		log.Error("That's not something I can do")
+		return
 	}
 }
+
+// saveGenesis JSON encodes an arbitrary genesis spec into a pre-defined file.
+func saveGenesis(folder, network, client string, spec interface{}) {
+	path := filepath.Join(folder, fmt.Sprintf("%s-%s.json", network, client))
+
+	out, _ := json.Marshal(spec)
+	if err := ioutil.WriteFile(path, out, 0644); err != nil {
+		log.Error("Failed to save genesis file", "client", client, "err", err)
+		return
+	}
+	log.Info("Saved genesis chain spec", "client", client, "path", path)
+}

cmd/puppeth/wizard_intro.go

@@ -61,14 +61,14 @@ func (w *wizard) run() {
 	// Make sure we have a good network name to work with	fmt.Println()
 	// Docker accepts hyphens in image names, but doesn't like it for container names
 	if w.network == "" {
-		fmt.Println("Please specify a network name to administer (no spaces or hyphens, please)")
+		fmt.Println("Please specify a network name to administer (no spaces, hyphens or capital letters please)")
 		for {
 			w.network = w.readString()
-			if !strings.Contains(w.network, " ") && !strings.Contains(w.network, "-") {
+			if !strings.Contains(w.network, " ") && !strings.Contains(w.network, "-") && strings.ToLower(w.network) == w.network {
 				fmt.Printf("\nSweet, you can set this via --network=%s next time!\n\n", w.network)
 				break
 			}
-			log.Error("I also like to live dangerously, still no spaces or hyphens")
+			log.Error("I also like to live dangerously, still no spaces, hyphens or capital letters")
 		}
 	}
 	log.Info("Administering Ethereum network", "name", w.network)
@@ -131,7 +131,20 @@ func (w *wizard) run() {
 
 		case choice == "2":
 			if w.conf.Genesis == nil {
-				w.makeGenesis()
+				fmt.Println()
+				fmt.Println("What would you like to do? (default = create)")
+				fmt.Println(" 1. Create new genesis from scratch")
+				fmt.Println(" 2. Import already existing genesis")
+
+				choice := w.read()
+				switch {
+				case choice == "" || choice == "1":
+					w.makeGenesis()
+				case choice == "2":
+					w.importGenesis()
+				default:
+					log.Error("That's not something I can do")
+				}
 			} else {
 				w.manageGenesis()
 			}
@@ -149,7 +162,6 @@ func (w *wizard) run() {
 			} else {
 				w.manageComponents()
 			}
-
 		default:
 			log.Error("That's not something I can do")
 		}

cmd/puppeth/wizard_nginx.go

@@ -41,12 +41,12 @@ func (w *wizard) ensureVirtualHost(client *sshClient, port int, def string) (str
 	// Reverse proxy is not running, offer to deploy a new one
 	fmt.Println()
 	fmt.Println("Allow sharing the port with other services (y/n)? (default = yes)")
-	if w.readDefaultString("y") == "y" {
+	if w.readDefaultYesNo(true) {
 		nocache := false
 		if proxy != nil {
 			fmt.Println()
 			fmt.Printf("Should the reverse-proxy be rebuilt from scratch (y/n)? (default = no)\n")
-			nocache = w.readDefaultString("n") != "n"
+			nocache = w.readDefaultYesNo(false)
 		}
 		if out, err := deployNginx(client, w.network, port, nocache); err != nil {
 			log.Error("Failed to deploy reverse-proxy", "err", err)

cmd/puppeth/wizard_node.go

@@ -126,7 +126,7 @@ func (w *wizard) deployNode(boot bool) {
 				} else {
 					fmt.Println()
 					fmt.Printf("Reuse previous (%s) signing account (y/n)? (default = yes)\n", key.Address.Hex())
-					if w.readDefaultString("y") != "y" {
+					if !w.readDefaultYesNo(true) {
 						infos.keyJSON, infos.keyPass = "", ""
 					}
 				}
@@ -165,7 +165,7 @@ func (w *wizard) deployNode(boot bool) {
 	if existed {
 		fmt.Println()
 		fmt.Printf("Should the node be built from scratch (y/n)? (default = no)\n")
-		nocache = w.readDefaultString("n") != "n"
+		nocache = w.readDefaultYesNo(false)
 	}
 	if out, err := deployNode(client, w.network, w.conf.bootnodes, infos, nocache); err != nil {
 		log.Error("Failed to deploy Ethereum node container", "err", err)

cmd/puppeth/wizard_wallet.go

@@ -96,7 +96,7 @@ func (w *wizard) deployWallet() {
 	if existed {
 		fmt.Println()
 		fmt.Printf("Should the wallet be built from scratch (y/n)? (default = no)\n")
-		nocache = w.readDefaultString("n") != "n"
+		nocache = w.readDefaultYesNo(false)
 	}
 	if out, err := deployWallet(client, w.network, w.conf.bootnodes, infos, nocache); err != nil {
 		log.Error("Failed to deploy wallet container", "err", err)

cmd/swarm/access.go

@@ -29,7 +29,65 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
-var salt = make([]byte, 32)
+var (
+	salt          = make([]byte, 32)
+	accessCommand = cli.Command{
+		CustomHelpTemplate: helpTemplate,
+		Name:               "access",
+		Usage:              "encrypts a reference and embeds it into a root manifest",
+		ArgsUsage:          "<ref>",
+		Description:        "encrypts a reference and embeds it into a root manifest",
+		Subcommands: []cli.Command{
+			{
+				CustomHelpTemplate: helpTemplate,
+				Name:               "new",
+				Usage:              "encrypts a reference and embeds it into a root manifest",
+				ArgsUsage:          "<ref>",
+				Description:        "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+				Subcommands: []cli.Command{
+					{
+						Action:             accessNewPass,
+						CustomHelpTemplate: helpTemplate,
+						Flags: []cli.Flag{
+							utils.PasswordFileFlag,
+							SwarmDryRunFlag,
+						},
+						Name:        "pass",
+						Usage:       "encrypts a reference with a password and embeds it into a root manifest",
+						ArgsUsage:   "<ref>",
+						Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+					},
+					{
+						Action:             accessNewPK,
+						CustomHelpTemplate: helpTemplate,
+						Flags: []cli.Flag{
+							utils.PasswordFileFlag,
+							SwarmDryRunFlag,
+							SwarmAccessGrantKeyFlag,
+						},
+						Name:        "pk",
+						Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
+						ArgsUsage:   "<ref>",
+						Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+					},
+					{
+						Action:             accessNewACT,
+						CustomHelpTemplate: helpTemplate,
+						Flags: []cli.Flag{
+							SwarmAccessGrantKeysFlag,
+							SwarmDryRunFlag,
+							utils.PasswordFileFlag,
+						},
+						Name:        "act",
+						Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
+						ArgsUsage:   "<ref>",
+						Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+					},
+				},
+			},
+		},
+	}
+)
 
 func init() {
 	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
@@ -56,6 +114,9 @@ func accessNewPass(ctx *cli.Context) {
 		utils.Fatalf("error getting session key: %v", err)
 	}
 	m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae)
+	if err != nil {
+		utils.Fatalf("had an error generating the manifest: %v", err)
+	}
 	if dryRun {
 		err = printManifests(m, nil)
 		if err != nil {
@@ -89,6 +150,9 @@ func accessNewPK(ctx *cli.Context) {
 		utils.Fatalf("error getting session key: %v", err)
 	}
 	m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae)
+	if err != nil {
+		utils.Fatalf("had an error generating the manifest: %v", err)
+	}
 	if dryRun {
 		err = printManifests(m, nil)
 		if err != nil {

cmd/swarm/access_test.go

@@ -14,8 +14,6 @@
 // You should have received a copy of the GNU General Public License
 // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
 
-// +build !windows
-
 package main
 
 import (
@@ -28,6 +26,7 @@ import (
 	gorand "math/rand"
 	"net/http"
 	"os"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -37,7 +36,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto/sha3"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/api"
-	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	swarmapi "github.com/ethereum/go-ethereum/swarm/api/client"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
@@ -48,23 +47,41 @@ const (
 
 var DefaultCurve = crypto.S256()
 
-// TestAccessPassword tests for the correct creation of an ACT manifest protected by a password.
+func TestACT(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
+	initCluster(t)
+
+	cases := []struct {
+		name string
+		f    func(t *testing.T)
+	}{
+		{"Password", testPassword},
+		{"PK", testPK},
+		{"ACTWithoutBogus", testACTWithoutBogus},
+		{"ACTWithBogus", testACTWithBogus},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, tc.f)
+	}
+}
+
+// testPassword tests for the correct creation of an ACT manifest protected by a password.
 // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
 // The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded
 // is then fetched through 2nd node. since the tested code is not key-aware - we can just
 // fetch from the 2nd node using HTTP BasicAuth
-func TestAccessPassword(t *testing.T) {
-	cluster := newTestCluster(t, 1)
-	defer cluster.Shutdown()
-	proxyNode := cluster.Nodes[0]
-
+func testPassword(t *testing.T) {
 	dataFilename := testutil.TempFileWithContent(t, data)
 	defer os.RemoveAll(dataFilename)
 
 	// upload the file with 'swarm up' and expect a hash
 	up := runSwarm(t,
 		"--bzzapi",
-		proxyNode.URL, //it doesn't matter through which node we upload content
+		cluster.Nodes[0].URL,
 		"up",
 		"--encrypt",
 		dataFilename)
@@ -138,16 +155,17 @@ func TestAccessPassword(t *testing.T) {
 	if a.Publisher != "" {
 		t.Fatal("should be empty")
 	}
-	client := swarm.NewClient(cluster.Nodes[0].URL)
+
+	client := swarmapi.NewClient(cluster.Nodes[0].URL)
 
 	hash, err := client.UploadManifest(&m, false)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	httpClient := &http.Client{}
-
 	url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash
+
+	httpClient := &http.Client{}
 	response, err := httpClient.Get(url)
 	if err != nil {
 		t.Fatal(err)
@@ -189,7 +207,7 @@ func TestAccessPassword(t *testing.T) {
 	//download file with 'swarm down' with wrong password
 	up = runSwarm(t,
 		"--bzzapi",
-		proxyNode.URL,
+		cluster.Nodes[0].URL,
 		"down",
 		"bzz:/"+hash,
 		tmp,
@@ -203,16 +221,12 @@ func TestAccessPassword(t *testing.T) {
 	up.ExpectExit()
 }
 
-// TestAccessPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee).
+// testPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee).
 // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
 // The parties participating - node (publisher), uploads to second node (which is also the grantee) then disappears.
 // Content which was uploaded is then fetched through the grantee's http proxy. Since the tested code is private-key aware,
 // the test will fail if the proxy's given private key is not granted on the ACT.
-func TestAccessPK(t *testing.T) {
-	// Setup Swarm and upload a test file to it
-	cluster := newTestCluster(t, 2)
-	defer cluster.Shutdown()
-
+func testPK(t *testing.T) {
 	dataFilename := testutil.TempFileWithContent(t, data)
 	defer os.RemoveAll(dataFilename)
 
@@ -318,7 +332,7 @@ func TestAccessPK(t *testing.T) {
 	if a.Publisher != pkComp {
 		t.Fatal("publisher key did not match")
 	}
-	client := swarm.NewClient(cluster.Nodes[0].URL)
+	client := swarmapi.NewClient(cluster.Nodes[0].URL)
 
 	hash, err := client.UploadManifest(&m, false)
 	if err != nil {
@@ -344,29 +358,24 @@ func TestAccessPK(t *testing.T) {
 	}
 }
 
-// TestAccessACT tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
-func TestAccessACT(t *testing.T) {
-	testAccessACT(t, 0)
+// testACTWithoutBogus tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
+func testACTWithoutBogus(t *testing.T) {
+	testACT(t, 0)
 }
 
-// TestAccessACTScale tests the creation of the ACT manifest end-to-end, with 1000 bogus entries (i.e. 1000 EC keys + default scenario = 3 nodes 1 unauthorized = 1003 keys in the ACT manifest)
-func TestAccessACTScale(t *testing.T) {
-	testAccessACT(t, 1000)
+// testACTWithBogus tests the creation of the ACT manifest end-to-end, with 100 bogus entries (i.e. 100 EC keys + default scenario = 3 nodes 1 unauthorized = 103 keys in the ACT manifest)
+func testACTWithBogus(t *testing.T) {
+	testACT(t, 100)
 }
 
-// TestAccessACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
+// testACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
 // the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data
 // set and also protects the ACT with a password. the third node should fail decoding the reference as it will not be granted access.
 // the third node then then tries to download using a correct password (and succeeds) then uses a wrong password and fails.
 // the publisher uploads through one of the nodes then disappears.
-func testAccessACT(t *testing.T, bogusEntries int) {
-	// Setup Swarm and upload a test file to it
-	const clusterSize = 3
-	cluster := newTestCluster(t, clusterSize)
-	defer cluster.Shutdown()
-
+func testACT(t *testing.T, bogusEntries int) {
 	var uploadThroughNode = cluster.Nodes[0]
-	client := swarm.NewClient(uploadThroughNode.URL)
+	client := swarmapi.NewClient(uploadThroughNode.URL)
 
 	r1 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
 	nodeToSkip := r1.Intn(clusterSize) // a number between 0 and 2 (node indices in `cluster`)

cmd/swarm/config.go

@@ -80,6 +80,7 @@ const (
 	SWARM_ENV_STORE_CAPACITY          = "SWARM_STORE_CAPACITY"
 	SWARM_ENV_STORE_CACHE_CAPACITY    = "SWARM_STORE_CACHE_CAPACITY"
 	SWARM_ACCESS_PASSWORD             = "SWARM_ACCESS_PASSWORD"
+	SWARM_AUTO_DEFAULTPATH            = "SWARM_AUTO_DEFAULTPATH"
 	GETH_ENV_DATADIR                  = "GETH_DATADIR"
 )
 

cmd/swarm/config_test.go

@@ -26,14 +26,14 @@ import (
 	"testing"
 	"time"
 
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm"
 	"github.com/ethereum/go-ethereum/swarm/api"
-
-	"github.com/docker/docker/pkg/reexec"
 )
 
-func TestDumpConfig(t *testing.T) {
+func TestConfigDump(t *testing.T) {
 	swarm := runSwarm(t, "dumpconfig")
 	defaultConf := api.NewConfig()
 	out, err := tomlSettings.Marshal(&defaultConf)
@@ -91,8 +91,8 @@ func TestConfigCmdLineOverrides(t *testing.T) {
 		fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
 		fmt.Sprintf("--%s", SwarmDeliverySkipCheckFlag.Name),
 		fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
-		"--datadir", dir,
-		"--ipcpath", conf.IPCPath,
+		fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
 	}
 	node.Cmd = runSwarm(t, flags...)
 	node.Cmd.InputLine(testPassphrase)
@@ -189,9 +189,9 @@ func TestConfigFileOverrides(t *testing.T) {
 	flags := []string{
 		fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(),
 		fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
-		"--ens-api", "",
-		"--ipcpath", conf.IPCPath,
-		"--datadir", dir,
+		fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
+		fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
 	}
 	node.Cmd = runSwarm(t, flags...)
 	node.Cmd.InputLine(testPassphrase)
@@ -407,9 +407,9 @@ func TestConfigCmdLineOverridesFile(t *testing.T) {
 		fmt.Sprintf("--%s", SwarmSyncDisabledFlag.Name),
 		fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(),
 		fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
-		"--ens-api", "",
-		"--datadir", dir,
-		"--ipcpath", conf.IPCPath,
+		fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
+		fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
 	}
 	node.Cmd = runSwarm(t, flags...)
 	node.Cmd.InputLine(testPassphrase)
@@ -466,7 +466,7 @@ func TestConfigCmdLineOverridesFile(t *testing.T) {
 	node.Shutdown()
 }
 
-func TestValidateConfig(t *testing.T) {
+func TestConfigValidate(t *testing.T) {
 	for _, c := range []struct {
 		cfg *api.Config
 		err string

cmd/swarm/db.go

@@ -29,6 +29,48 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var dbCommand = cli.Command{
+	Name:               "db",
+	CustomHelpTemplate: helpTemplate,
+	Usage:              "manage the local chunk database",
+	ArgsUsage:          "db COMMAND",
+	Description:        "Manage the local chunk database",
+	Subcommands: []cli.Command{
+		{
+			Action:             dbExport,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "export",
+			Usage:              "export a local chunk database as a tar archive (use - to send to stdout)",
+			ArgsUsage:          "<chunkdb> <file>",
+			Description: `
+Export a local chunk database as a tar archive (use - to send to stdout).
+
+    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
+
+The export may be quite large, consider piping the output through the Unix
+pv(1) tool to get a progress bar:
+
+    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks - | pv > chunks.tar
+`,
+		},
+		{
+			Action:             dbImport,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "import",
+			Usage:              "import chunks from a tar archive into a local chunk database (use - to read from stdin)",
+			ArgsUsage:          "<chunkdb> <file>",
+			Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin).
+
+    swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
+
+The import may be quite large, consider piping the input through the Unix
+pv(1) tool to get a progress bar:
+
+    pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`,
+		},
+	},
+}
+
 func dbExport(ctx *cli.Context) {
 	args := ctx.Args()
 	if len(args) != 3 {

cmd/swarm/download.go

@@ -28,6 +28,15 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var downloadCommand = cli.Command{
+	Action:      download,
+	Name:        "down",
+	Flags:       []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag},
+	Usage:       "downloads a swarm manifest or a file inside a manifest",
+	ArgsUsage:   " <uri> [<dir>]",
+	Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`,
+}
+
 func download(ctx *cli.Context) {
 	log.Debug("downloading content using swarm down")
 	args := ctx.Args()

cmd/swarm/export_test.go

@@ -19,9 +19,7 @@ package main
 import (
 	"bytes"
 	"crypto/md5"
-	"crypto/rand"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"runtime"
@@ -29,6 +27,7 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/swarm"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 // TestCLISwarmExportImport perform the following test:
@@ -44,12 +43,13 @@ func TestCLISwarmExportImport(t *testing.T) {
 	}
 	cluster := newTestCluster(t, 1)
 
-	// generate random 10mb file
-	f, cleanup := generateRandomFile(t, 10000000)
-	defer cleanup()
+	// generate random 1mb file
+	content := testutil.RandomBytes(1, 1000000)
+	fileName := testutil.TempFileWithContent(t, string(content))
+	defer os.Remove(fileName)
 
 	// upload the file with 'swarm up' and expect a hash
-	up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", f.Name())
+	up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", fileName)
 	_, matches := up.ExpectRegexp(`[a-f\d]{64}`)
 	up.ExpectExit()
 	hash := matches[0]
@@ -96,7 +96,7 @@ func TestCLISwarmExportImport(t *testing.T) {
 	}
 
 	// compare downloaded file with the generated random file
-	mustEqualFiles(t, f, res.Body)
+	mustEqualFiles(t, bytes.NewReader(content), res.Body)
 }
 
 func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
@@ -117,27 +117,3 @@ func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
 		t.Fatalf("downloaded imported file md5=%x (length %v) is not the same as the generated one mp5=%x (length %v)", downHash, downLen, upHash, upLen)
 	}
 }
-
-func generateRandomFile(t *testing.T, size int) (f *os.File, teardown func()) {
-	// create a tmp file
-	tmp, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// callback for tmp file cleanup
-	teardown = func() {
-		tmp.Close()
-		os.Remove(tmp.Name())
-	}
-
-	// write 10mb random data to file
-	buf := make([]byte, 10000000)
-	_, err = rand.Read(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ioutil.WriteFile(tmp.Name(), buf, 0755)
-
-	return tmp, teardown
-}

cmd/swarm/feeds.go

@@ -31,6 +31,68 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var feedCommand = cli.Command{
+	CustomHelpTemplate: helpTemplate,
+	Name:               "feed",
+	Usage:              "(Advanced) Create and update Swarm Feeds",
+	ArgsUsage:          "<create|update|info>",
+	Description:        "Works with Swarm Feeds",
+	Subcommands: []cli.Command{
+		{
+			Action:             feedCreateManifest,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "create",
+			Usage:              "creates and publishes a new feed manifest",
+			Description: `creates and publishes a new feed manifest pointing to a specified user's updates about a particular topic.
+					The feed topic can be built in the following ways:
+					* use --topic to set the topic to an arbitrary binary hex string.
+					* use --name to set the topic to a human-readable name.
+					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
+					* use both --topic and --name to create named subtopics. 
+						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
+						this feed tracks a discussion about that contract.
+					The --user flag allows to have this manifest refer to a user other than yourself. If not specified,
+					it will then default to your local account (--bzzaccount)`,
+			Flags: []cli.Flag{SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
+		},
+		{
+			Action:             feedUpdate,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "update",
+			Usage:              "updates the content of an existing Swarm Feed",
+			ArgsUsage:          "<0x Hex data>",
+			Description: `publishes a new update on the specified topic
+					The feed topic can be built in the following ways:
+					* use --topic to set the topic to an arbitrary binary hex string.
+					* use --name to set the topic to a human-readable name.
+					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
+					* use both --topic and --name to create named subtopics. 
+						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
+						this feed tracks a discussion about that contract.
+					
+					If you have a manifest, you can specify it with --manifest to refer to the feed,
+					instead of using --topic / --name
+					`,
+			Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag},
+		},
+		{
+			Action:             feedInfo,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "info",
+			Usage:              "obtains information about an existing Swarm feed",
+			Description: `obtains information about an existing Swarm feed
+					The topic can be specified directly with the --topic flag as an hex string
+					If no topic is specified, the default topic (zero) will be used
+					The --name flag can be used to specify subtopics with a specific name.
+					The --user flag allows to refer to a user other than yourself. If not specified,
+					it will then default to your local account (--bzzaccount)
+					If you have a manifest, you can specify it with --manifest instead of --topic / --name / ---user
+					to refer to the feed`,
+			Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
+		},
+	},
+}
+
 func NewGenericSigner(ctx *cli.Context) feed.Signer {
 	return feed.NewGenericSigner(getPrivKey(ctx))
 }
@@ -107,7 +169,6 @@ func feedUpdate(ctx *cli.Context) {
 		query = new(feed.Query)
 		query.User = signer.Address()
 		query.Topic = getTopic(ctx)
-
 	}
 
 	// Retrieve a feed update request
@@ -116,6 +177,11 @@ func feedUpdate(ctx *cli.Context) {
 		utils.Fatalf("Error retrieving feed status: %s", err.Error())
 	}
 
+	// Check that the provided signer matches the request to sign
+	if updateRequest.User != signer.Address() {
+		utils.Fatalf("Signer address does not match the update request")
+	}
+
 	// set the new data
 	updateRequest.SetData(data)
 

cmd/swarm/feeds_test.go

@@ -19,50 +19,35 @@ package main
 import (
 	"bytes"
 	"encoding/json"
-	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/swarm/api"
-	"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
-	"github.com/ethereum/go-ethereum/swarm/testutil"
-
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/swarm/storage/feed"
-
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 func TestCLIFeedUpdate(t *testing.T) {
-
-	srv := testutil.NewTestSwarmServer(t, func(api *api.API) testutil.TestServer {
+	srv := swarmhttp.NewTestSwarmServer(t, func(api *api.API) swarmhttp.TestServer {
 		return swarmhttp.NewServer(api, "")
 	}, nil)
 	log.Info("starting a test swarm server")
 	defer srv.Close()
 
 	// create a private key file for signing
-	pkfile, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer pkfile.Close()
-	defer os.Remove(pkfile.Name())
-
 	privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979"
 	privKey, _ := crypto.HexToECDSA(privkeyHex)
 	address := crypto.PubkeyToAddress(privKey.PublicKey)
 
-	// save the private key to a file
-	_, err = io.WriteString(pkfile, privkeyHex)
-	if err != nil {
-		t.Fatal(err)
-	}
+	pkFileName := testutil.TempFileWithContent(t, privkeyHex)
+	defer os.Remove(pkFileName)
 
 	// compose a topic. We'll be doing quotes about Miguel de Cervantes
 	var topic feed.Topic
@@ -76,26 +61,23 @@ func TestCLIFeedUpdate(t *testing.T) {
 
 	flags := []string{
 		"--bzzapi", srv.URL,
-		"--bzzaccount", pkfile.Name(),
+		"--bzzaccount", pkFileName,
 		"feed", "update",
 		"--topic", topic.Hex(),
 		"--name", name,
 		hexData}
 
 	// create an update and expect an exit without errors
-	log.Info(fmt.Sprintf("updating a feed with 'swarm feed update'"))
+	log.Info("updating a feed with 'swarm feed update'")
 	cmd := runSwarm(t, flags...)
 	cmd.ExpectExit()
 
 	// now try to get the update using the client
 	client := swarm.NewClient(srv.URL)
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	// build the same topic as before, this time
 	// we use NewTopic to create a topic automatically.
-	topic, err = feed.NewTopic(name, subject)
+	topic, err := feed.NewTopic(name, subject)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -133,7 +115,7 @@ func TestCLIFeedUpdate(t *testing.T) {
 		"--user", address.Hex(),
 	}
 
-	log.Info(fmt.Sprintf("getting feed info with 'swarm feed info'"))
+	log.Info("getting feed info with 'swarm feed info'")
 	cmd = runSwarm(t, flags...)
 	_, matches := cmd.ExpectRegexp(`.*`) // regex hack to extract stdout
 	cmd.ExpectExit()
@@ -153,14 +135,14 @@ func TestCLIFeedUpdate(t *testing.T) {
 	// test publishing a manifest
 	flags = []string{
 		"--bzzapi", srv.URL,
-		"--bzzaccount", pkfile.Name(),
+		"--bzzaccount", pkFileName,
 		"feed", "create",
 		"--topic", topic.Hex(),
 	}
 
-	log.Info(fmt.Sprintf("Publishing manifest with 'swarm feed create'"))
+	log.Info("Publishing manifest with 'swarm feed create'")
 	cmd = runSwarm(t, flags...)
-	_, matches = cmd.ExpectRegexp(`[a-f\d]{64}`) // regex hack to extract stdout
+	_, matches = cmd.ExpectRegexp(`[a-f\d]{64}`)
 	cmd.ExpectExit()
 
 	manifestAddress := matches[0] // read the received feed manifest
@@ -179,4 +161,36 @@ func TestCLIFeedUpdate(t *testing.T) {
 	if !bytes.Equal(data, retrieved) {
 		t.Fatalf("Received %s, expected %s", retrieved, data)
 	}
+
+	// test publishing a manifest for a different user
+	flags = []string{
+		"--bzzapi", srv.URL,
+		"feed", "create",
+		"--topic", topic.Hex(),
+		"--user", "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", // different user
+	}
+
+	log.Info("Publishing manifest with 'swarm feed create' for a different user")
+	cmd = runSwarm(t, flags...)
+	_, matches = cmd.ExpectRegexp(`[a-f\d]{64}`)
+	cmd.ExpectExit()
+
+	manifestAddress = matches[0] // read the received feed manifest
+
+	// now let's try to update that user's manifest which we don't have the private key for
+	flags = []string{
+		"--bzzapi", srv.URL,
+		"--bzzaccount", pkFileName,
+		"feed", "update",
+		"--manifest", manifestAddress,
+		hexData}
+
+	// create an update and expect an error given there is a user mismatch
+	log.Info("updating a feed with 'swarm feed update'")
+	cmd = runSwarm(t, flags...)
+	cmd.ExpectRegexp("Fatal:.*") // best way so far to detect a failure.
+	cmd.ExpectExit()
+	if cmd.ExitStatus() == 0 {
+		t.Fatal("Expected nonzero exit code when updating a manifest with the wrong user. Got 0.")
+	}
 }

cmd/swarm/flags.go

@@ -0,0 +1,179 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// Command feed allows the user to create and update signed Swarm feeds
+package main
+
+import cli "gopkg.in/urfave/cli.v1"
+
+var (
+	ChequebookAddrFlag = cli.StringFlag{
+		Name:   "chequebook",
+		Usage:  "chequebook contract address",
+		EnvVar: SWARM_ENV_CHEQUEBOOK_ADDR,
+	}
+	SwarmAccountFlag = cli.StringFlag{
+		Name:   "bzzaccount",
+		Usage:  "Swarm account key file",
+		EnvVar: SWARM_ENV_ACCOUNT,
+	}
+	SwarmListenAddrFlag = cli.StringFlag{
+		Name:   "httpaddr",
+		Usage:  "Swarm HTTP API listening interface",
+		EnvVar: SWARM_ENV_LISTEN_ADDR,
+	}
+	SwarmPortFlag = cli.StringFlag{
+		Name:   "bzzport",
+		Usage:  "Swarm local http api port",
+		EnvVar: SWARM_ENV_PORT,
+	}
+	SwarmNetworkIdFlag = cli.IntFlag{
+		Name:   "bzznetworkid",
+		Usage:  "Network identifier (integer, default 3=swarm testnet)",
+		EnvVar: SWARM_ENV_NETWORK_ID,
+	}
+	SwarmSwapEnabledFlag = cli.BoolFlag{
+		Name:   "swap",
+		Usage:  "Swarm SWAP enabled (default false)",
+		EnvVar: SWARM_ENV_SWAP_ENABLE,
+	}
+	SwarmSwapAPIFlag = cli.StringFlag{
+		Name:   "swap-api",
+		Usage:  "URL of the Ethereum API provider to use to settle SWAP payments",
+		EnvVar: SWARM_ENV_SWAP_API,
+	}
+	SwarmSyncDisabledFlag = cli.BoolTFlag{
+		Name:   "nosync",
+		Usage:  "Disable swarm syncing",
+		EnvVar: SWARM_ENV_SYNC_DISABLE,
+	}
+	SwarmSyncUpdateDelay = cli.DurationFlag{
+		Name:   "sync-update-delay",
+		Usage:  "Duration for sync subscriptions update after no new peers are added (default 15s)",
+		EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY,
+	}
+	SwarmMaxStreamPeerServersFlag = cli.IntFlag{
+		Name:   "max-stream-peer-servers",
+		Usage:  "Limit of Stream peer servers, 0 denotes unlimited",
+		EnvVar: SWARM_ENV_MAX_STREAM_PEER_SERVERS,
+		Value:  10000, // A very large default value is possible as stream servers have very small memory footprint
+	}
+	SwarmLightNodeEnabled = cli.BoolFlag{
+		Name:   "lightnode",
+		Usage:  "Enable Swarm LightNode (default false)",
+		EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE,
+	}
+	SwarmDeliverySkipCheckFlag = cli.BoolFlag{
+		Name:   "delivery-skip-check",
+		Usage:  "Skip chunk delivery check (default false)",
+		EnvVar: SWARM_ENV_DELIVERY_SKIP_CHECK,
+	}
+	EnsAPIFlag = cli.StringSliceFlag{
+		Name:   "ens-api",
+		Usage:  "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url",
+		EnvVar: SWARM_ENV_ENS_API,
+	}
+	SwarmApiFlag = cli.StringFlag{
+		Name:  "bzzapi",
+		Usage: "Specifies the Swarm HTTP endpoint to connect to",
+		Value: "http://127.0.0.1:8500",
+	}
+	SwarmRecursiveFlag = cli.BoolFlag{
+		Name:  "recursive",
+		Usage: "Upload directories recursively",
+	}
+	SwarmWantManifestFlag = cli.BoolTFlag{
+		Name:  "manifest",
+		Usage: "Automatic manifest upload (default true)",
+	}
+	SwarmUploadDefaultPath = cli.StringFlag{
+		Name:  "defaultpath",
+		Usage: "path to file served for empty url path (none)",
+	}
+	SwarmAccessGrantKeyFlag = cli.StringFlag{
+		Name:  "grant-key",
+		Usage: "grants a given public key access to an ACT",
+	}
+	SwarmAccessGrantKeysFlag = cli.StringFlag{
+		Name:  "grant-keys",
+		Usage: "grants a given list of public keys in the following file (separated by line breaks) access to an ACT",
+	}
+	SwarmUpFromStdinFlag = cli.BoolFlag{
+		Name:  "stdin",
+		Usage: "reads data to be uploaded from stdin",
+	}
+	SwarmUploadMimeType = cli.StringFlag{
+		Name:  "mime",
+		Usage: "Manually specify MIME type",
+	}
+	SwarmEncryptedFlag = cli.BoolFlag{
+		Name:  "encrypt",
+		Usage: "use encrypted upload",
+	}
+	SwarmAccessPasswordFlag = cli.StringFlag{
+		Name:   "password",
+		Usage:  "Password",
+		EnvVar: SWARM_ACCESS_PASSWORD,
+	}
+	SwarmDryRunFlag = cli.BoolFlag{
+		Name:  "dry-run",
+		Usage: "dry-run",
+	}
+	CorsStringFlag = cli.StringFlag{
+		Name:   "corsdomain",
+		Usage:  "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
+		EnvVar: SWARM_ENV_CORS,
+	}
+	SwarmStorePath = cli.StringFlag{
+		Name:   "store.path",
+		Usage:  "Path to leveldb chunk DB (default <$GETH_ENV_DIR>/swarm/bzz-<$BZZ_KEY>/chunks)",
+		EnvVar: SWARM_ENV_STORE_PATH,
+	}
+	SwarmStoreCapacity = cli.Uint64Flag{
+		Name:   "store.size",
+		Usage:  "Number of chunks (5M is roughly 20-25GB) (default 5000000)",
+		EnvVar: SWARM_ENV_STORE_CAPACITY,
+	}
+	SwarmStoreCacheCapacity = cli.UintFlag{
+		Name:   "store.cache.size",
+		Usage:  "Number of recent chunks cached in memory (default 5000)",
+		EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
+	}
+	SwarmCompressedFlag = cli.BoolFlag{
+		Name:  "compressed",
+		Usage: "Prints encryption keys in compressed form",
+	}
+	SwarmFeedNameFlag = cli.StringFlag{
+		Name:  "name",
+		Usage: "User-defined name for the new feed, limited to 32 characters. If combined with topic, it will refer to a subtopic with this name",
+	}
+	SwarmFeedTopicFlag = cli.StringFlag{
+		Name:  "topic",
+		Usage: "User-defined topic this feed is tracking, hex encoded. Limited to 64 hexadecimal characters",
+	}
+	SwarmFeedDataOnCreateFlag = cli.StringFlag{
+		Name:  "data",
+		Usage: "Initializes the feed with the given hex-encoded data. Data must be prefixed by 0x",
+	}
+	SwarmFeedManifestFlag = cli.StringFlag{
+		Name:  "manifest",
+		Usage: "Refers to the feed through a manifest",
+	}
+	SwarmFeedUserFlag = cli.StringFlag{
+		Name:  "user",
+		Usage: "Indicates the user who updates the feed",
+	}
+)

cmd/swarm/fs.go

@@ -24,16 +24,50 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm/fuse"
 	"gopkg.in/urfave/cli.v1"
 )
 
+var fsCommand = cli.Command{
+	Name:               "fs",
+	CustomHelpTemplate: helpTemplate,
+	Usage:              "perform FUSE operations",
+	ArgsUsage:          "fs COMMAND",
+	Description:        "Performs FUSE operations by mounting/unmounting/listing mount points. This assumes you already have a Swarm node running locally. For all operation you must reference the correct path to bzzd.ipc in order to communicate with the node",
+	Subcommands: []cli.Command{
+		{
+			Action:             mount,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "mount",
+			Usage:              "mount a swarm hash to a mount point",
+			ArgsUsage:          "swarm fs mount <manifest hash> <mount point>",
+			Description:        "Mounts a Swarm manifest hash to a given mount point. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
+		},
+		{
+			Action:             unmount,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "unmount",
+			Usage:              "unmount a swarmfs mount",
+			ArgsUsage:          "swarm fs unmount <mount point>",
+			Description:        "Unmounts a swarmfs mount residing at <mount point>. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
+		},
+		{
+			Action:             listMounts,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "list",
+			Usage:              "list swarmfs mounts",
+			ArgsUsage:          "swarm fs list",
+			Description:        "Lists all mounted swarmfs volumes. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
+		},
+	},
+}
+
 func mount(cliContext *cli.Context) {
 	args := cliContext.Args()
 	if len(args) < 2 {
-		utils.Fatalf("Usage: swarm fs mount --ipcpath <path to bzzd.ipc> <manifestHash> <file name>")
+		utils.Fatalf("Usage: swarm fs mount <manifestHash> <file name>")
 	}
 
 	client, err := dialRPC(cliContext)
@@ -60,7 +94,7 @@ func unmount(cliContext *cli.Context) {
 	args := cliContext.Args()
 
 	if len(args) < 1 {
-		utils.Fatalf("Usage: swarm fs unmount --ipcpath <path to bzzd.ipc> <mount path>")
+		utils.Fatalf("Usage: swarm fs unmount <mount path>")
 	}
 	client, err := dialRPC(cliContext)
 	if err != nil {
@@ -108,20 +142,21 @@ func listMounts(cliContext *cli.Context) {
 }
 
 func dialRPC(ctx *cli.Context) (*rpc.Client, error) {
-	var endpoint string
+	endpoint := getIPCEndpoint(ctx)
+	log.Info("IPC endpoint", "path", endpoint)
+	return rpc.Dial(endpoint)
+}
 
-	if ctx.IsSet(utils.IPCPathFlag.Name) {
-		endpoint = ctx.String(utils.IPCPathFlag.Name)
-	} else {
-		utils.Fatalf("swarm ipc endpoint not specified")
-	}
+func getIPCEndpoint(ctx *cli.Context) string {
+	cfg := defaultNodeConfig
+	utils.SetNodeConfig(ctx, &cfg)
 
-	if endpoint == "" {
-		endpoint = node.DefaultIPCEndpoint(clientIdentifier)
-	} else if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") {
+	endpoint := cfg.IPCEndpoint()
+
+	if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") {
 		// Backwards compatibility with geth < 1.5 which required
 		// these prefixes.
 		endpoint = endpoint[4:]
 	}
-	return rpc.Dial(endpoint)
+	return endpoint
 }

cmd/swarm/fs_test.go

@@ -20,6 +20,7 @@ package main
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -28,20 +29,35 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/log"
-	colorable "github.com/mattn/go-colorable"
 )
 
-func init() {
-	log.PrintOrigins(true)
-	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
-}
-
 type testFile struct {
 	filePath string
 	content  string
 }
 
+// TestCLISwarmFsDefaultIPCPath tests if the most basic fs command, i.e., list
+// can find and correctly connect to a running Swarm node on the default
+// IPCPath.
+func TestCLISwarmFsDefaultIPCPath(t *testing.T) {
+	cluster := newTestCluster(t, 1)
+	defer cluster.Shutdown()
+
+	handlingNode := cluster.Nodes[0]
+	list := runSwarm(t, []string{
+		"--datadir", handlingNode.Dir,
+		"fs",
+		"list",
+	}...)
+
+	list.WaitExit()
+	if list.Err != nil {
+		t.Fatal(list.Err)
+	}
+}
+
 // TestCLISwarmFs is a high-level test of swarmfs
 //
 // This test fails on travis for macOS as this executable exits with code 1
@@ -65,9 +81,9 @@ func TestCLISwarmFs(t *testing.T) {
 	log.Debug("swarmfs cli test: mounting first run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
 
 	mount := runSwarm(t, []string{
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		"fs",
 		"mount",
-		"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		mhash,
 		mountPoint,
 	}...)
@@ -80,6 +96,9 @@ func TestCLISwarmFs(t *testing.T) {
 		t.Fatal(err)
 	}
 	dirPath2, err := createDirInDir(dirPath, "AnotherTestSubDir")
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	dummyContent := "somerandomtestcontentthatshouldbeasserted"
 	dirs := []string{
@@ -104,9 +123,9 @@ func TestCLISwarmFs(t *testing.T) {
 	log.Debug("swarmfs cli test: unmounting first run...", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
 
 	unmount := runSwarm(t, []string{
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		"fs",
 		"unmount",
-		"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		mountPoint,
 	}...)
 	_, matches := unmount.ExpectRegexp(hashRegexp)
@@ -139,9 +158,9 @@ func TestCLISwarmFs(t *testing.T) {
 
 	//remount, check files
 	newMount := runSwarm(t, []string{
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		"fs",
 		"mount",
-		"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		hash, // the latest hash
 		secondMountPoint,
 	}...)
@@ -175,9 +194,9 @@ func TestCLISwarmFs(t *testing.T) {
 	log.Debug("swarmfs cli test: unmounting second run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
 
 	unmountSec := runSwarm(t, []string{
+		fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		"fs",
 		"unmount",
-		"--ipcpath", filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
 		secondMountPoint,
 	}...)
 

cmd/swarm/hash.go

@@ -27,6 +27,15 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var hashCommand = cli.Command{
+	Action:             hash,
+	CustomHelpTemplate: helpTemplate,
+	Name:               "hash",
+	Usage:              "print the swarm hash of a file or directory",
+	ArgsUsage:          "<file>",
+	Description:        "Prints the swarm hash of file or directory",
+}
+
 func hash(ctx *cli.Context) {
 	args := ctx.Args()
 	if len(args) < 1 {

cmd/swarm/list.go

@@ -27,6 +27,15 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var listCommand = cli.Command{
+	Action:             list,
+	CustomHelpTemplate: helpTemplate,
+	Name:               "ls",
+	Usage:              "list files and directories contained in a manifest",
+	ArgsUsage:          "<manifest> [<prefix>]",
+	Description:        "Lists files and directories contained in a manifest",
+}
+
 func list(ctx *cli.Context) {
 	args := ctx.Args()
 

cmd/swarm/main.go

@@ -70,165 +70,6 @@ var (
 	gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
 )
 
-var (
-	ChequebookAddrFlag = cli.StringFlag{
-		Name:   "chequebook",
-		Usage:  "chequebook contract address",
-		EnvVar: SWARM_ENV_CHEQUEBOOK_ADDR,
-	}
-	SwarmAccountFlag = cli.StringFlag{
-		Name:   "bzzaccount",
-		Usage:  "Swarm account key file",
-		EnvVar: SWARM_ENV_ACCOUNT,
-	}
-	SwarmListenAddrFlag = cli.StringFlag{
-		Name:   "httpaddr",
-		Usage:  "Swarm HTTP API listening interface",
-		EnvVar: SWARM_ENV_LISTEN_ADDR,
-	}
-	SwarmPortFlag = cli.StringFlag{
-		Name:   "bzzport",
-		Usage:  "Swarm local http api port",
-		EnvVar: SWARM_ENV_PORT,
-	}
-	SwarmNetworkIdFlag = cli.IntFlag{
-		Name:   "bzznetworkid",
-		Usage:  "Network identifier (integer, default 3=swarm testnet)",
-		EnvVar: SWARM_ENV_NETWORK_ID,
-	}
-	SwarmSwapEnabledFlag = cli.BoolFlag{
-		Name:   "swap",
-		Usage:  "Swarm SWAP enabled (default false)",
-		EnvVar: SWARM_ENV_SWAP_ENABLE,
-	}
-	SwarmSwapAPIFlag = cli.StringFlag{
-		Name:   "swap-api",
-		Usage:  "URL of the Ethereum API provider to use to settle SWAP payments",
-		EnvVar: SWARM_ENV_SWAP_API,
-	}
-	SwarmSyncDisabledFlag = cli.BoolTFlag{
-		Name:   "nosync",
-		Usage:  "Disable swarm syncing",
-		EnvVar: SWARM_ENV_SYNC_DISABLE,
-	}
-	SwarmSyncUpdateDelay = cli.DurationFlag{
-		Name:   "sync-update-delay",
-		Usage:  "Duration for sync subscriptions update after no new peers are added (default 15s)",
-		EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY,
-	}
-	SwarmMaxStreamPeerServersFlag = cli.IntFlag{
-		Name:   "max-stream-peer-servers",
-		Usage:  "Limit of Stream peer servers, 0 denotes unlimited",
-		EnvVar: SWARM_ENV_MAX_STREAM_PEER_SERVERS,
-		Value:  10000, // A very large default value is possible as stream servers have very small memory footprint
-	}
-	SwarmLightNodeEnabled = cli.BoolFlag{
-		Name:   "lightnode",
-		Usage:  "Enable Swarm LightNode (default false)",
-		EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE,
-	}
-	SwarmDeliverySkipCheckFlag = cli.BoolFlag{
-		Name:   "delivery-skip-check",
-		Usage:  "Skip chunk delivery check (default false)",
-		EnvVar: SWARM_ENV_DELIVERY_SKIP_CHECK,
-	}
-	EnsAPIFlag = cli.StringSliceFlag{
-		Name:   "ens-api",
-		Usage:  "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url",
-		EnvVar: SWARM_ENV_ENS_API,
-	}
-	SwarmApiFlag = cli.StringFlag{
-		Name:  "bzzapi",
-		Usage: "Swarm HTTP endpoint",
-		Value: "http://127.0.0.1:8500",
-	}
-	SwarmRecursiveFlag = cli.BoolFlag{
-		Name:  "recursive",
-		Usage: "Upload directories recursively",
-	}
-	SwarmWantManifestFlag = cli.BoolTFlag{
-		Name:  "manifest",
-		Usage: "Automatic manifest upload (default true)",
-	}
-	SwarmUploadDefaultPath = cli.StringFlag{
-		Name:  "defaultpath",
-		Usage: "path to file served for empty url path (none)",
-	}
-	SwarmAccessGrantKeyFlag = cli.StringFlag{
-		Name:  "grant-key",
-		Usage: "grants a given public key access to an ACT",
-	}
-	SwarmAccessGrantKeysFlag = cli.StringFlag{
-		Name:  "grant-keys",
-		Usage: "grants a given list of public keys in the following file (separated by line breaks) access to an ACT",
-	}
-	SwarmUpFromStdinFlag = cli.BoolFlag{
-		Name:  "stdin",
-		Usage: "reads data to be uploaded from stdin",
-	}
-	SwarmUploadMimeType = cli.StringFlag{
-		Name:  "mime",
-		Usage: "Manually specify MIME type",
-	}
-	SwarmEncryptedFlag = cli.BoolFlag{
-		Name:  "encrypt",
-		Usage: "use encrypted upload",
-	}
-	SwarmAccessPasswordFlag = cli.StringFlag{
-		Name:   "password",
-		Usage:  "Password",
-		EnvVar: SWARM_ACCESS_PASSWORD,
-	}
-	SwarmDryRunFlag = cli.BoolFlag{
-		Name:  "dry-run",
-		Usage: "dry-run",
-	}
-	CorsStringFlag = cli.StringFlag{
-		Name:   "corsdomain",
-		Usage:  "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
-		EnvVar: SWARM_ENV_CORS,
-	}
-	SwarmStorePath = cli.StringFlag{
-		Name:   "store.path",
-		Usage:  "Path to leveldb chunk DB (default <$GETH_ENV_DIR>/swarm/bzz-<$BZZ_KEY>/chunks)",
-		EnvVar: SWARM_ENV_STORE_PATH,
-	}
-	SwarmStoreCapacity = cli.Uint64Flag{
-		Name:   "store.size",
-		Usage:  "Number of chunks (5M is roughly 20-25GB) (default 5000000)",
-		EnvVar: SWARM_ENV_STORE_CAPACITY,
-	}
-	SwarmStoreCacheCapacity = cli.UintFlag{
-		Name:   "store.cache.size",
-		Usage:  "Number of recent chunks cached in memory (default 5000)",
-		EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
-	}
-	SwarmCompressedFlag = cli.BoolFlag{
-		Name:  "compressed",
-		Usage: "Prints encryption keys in compressed form",
-	}
-	SwarmFeedNameFlag = cli.StringFlag{
-		Name:  "name",
-		Usage: "User-defined name for the new feed, limited to 32 characters. If combined with topic, it will refer to a subtopic with this name",
-	}
-	SwarmFeedTopicFlag = cli.StringFlag{
-		Name:  "topic",
-		Usage: "User-defined topic this feed is tracking, hex encoded. Limited to 64 hexadecimal characters",
-	}
-	SwarmFeedDataOnCreateFlag = cli.StringFlag{
-		Name:  "data",
-		Usage: "Initializes the feed with the given hex-encoded data. Data must be prefixed by 0x",
-	}
-	SwarmFeedManifestFlag = cli.StringFlag{
-		Name:  "manifest",
-		Usage: "Refers to the feed through a manifest",
-	}
-	SwarmFeedUserFlag = cli.StringFlag{
-		Name:  "user",
-		Usage: "Indicates the user who updates the feed",
-	}
-)
-
 //declare a few constant error messages, useful for later error check comparisons in test
 var (
 	SWARM_ERR_NO_BZZACCOUNT   = "bzzaccount option is required but not set; check your config file, command line or environment variables"
@@ -279,267 +120,24 @@ func init() {
 			Usage:              "Print public key information",
 			Description:        "The output of this command is supposed to be machine-readable",
 		},
-		{
-			Action:             upload,
-			CustomHelpTemplate: helpTemplate,
-			Name:               "up",
-			Usage:              "uploads a file or directory to swarm using the HTTP API",
-			ArgsUsage:          "<file>",
-			Flags:              []cli.Flag{SwarmEncryptedFlag},
-			Description:        "uploads a file or directory to swarm using the HTTP API and prints the root hash",
-		},
-		{
-			CustomHelpTemplate: helpTemplate,
-			Name:               "access",
-			Usage:              "encrypts a reference and embeds it into a root manifest",
-			ArgsUsage:          "<ref>",
-			Description:        "encrypts a reference and embeds it into a root manifest",
-			Subcommands: []cli.Command{
-				{
-					CustomHelpTemplate: helpTemplate,
-					Name:               "new",
-					Usage:              "encrypts a reference and embeds it into a root manifest",
-					ArgsUsage:          "<ref>",
-					Description:        "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
-					Subcommands: []cli.Command{
-						{
-							Action:             accessNewPass,
-							CustomHelpTemplate: helpTemplate,
-							Flags: []cli.Flag{
-								utils.PasswordFileFlag,
-								SwarmDryRunFlag,
-							},
-							Name:        "pass",
-							Usage:       "encrypts a reference with a password and embeds it into a root manifest",
-							ArgsUsage:   "<ref>",
-							Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
-						},
-						{
-							Action:             accessNewPK,
-							CustomHelpTemplate: helpTemplate,
-							Flags: []cli.Flag{
-								utils.PasswordFileFlag,
-								SwarmDryRunFlag,
-								SwarmAccessGrantKeyFlag,
-							},
-							Name:        "pk",
-							Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
-							ArgsUsage:   "<ref>",
-							Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
-						},
-						{
-							Action:             accessNewACT,
-							CustomHelpTemplate: helpTemplate,
-							Flags: []cli.Flag{
-								SwarmAccessGrantKeysFlag,
-								SwarmDryRunFlag,
-								utils.PasswordFileFlag,
-							},
-							Name:        "act",
-							Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
-							ArgsUsage:   "<ref>",
-							Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
-						},
-					},
-				},
-			},
-		},
-		{
-			CustomHelpTemplate: helpTemplate,
-			Name:               "feed",
-			Usage:              "(Advanced) Create and update Swarm Feeds",
-			ArgsUsage:          "<create|update|info>",
-			Description:        "Works with Swarm Feeds",
-			Subcommands: []cli.Command{
-				{
-					Action:             feedCreateManifest,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "create",
-					Usage:              "creates and publishes a new feed manifest",
-					Description: `creates and publishes a new feed manifest pointing to a specified user's updates about a particular topic.
-					The feed topic can be built in the following ways:
-					* use --topic to set the topic to an arbitrary binary hex string.
-					* use --name to set the topic to a human-readable name.
-					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
-					* use both --topic and --name to create named subtopics. 
-						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
-						this feed tracks a discussion about that contract.
-					The --user flag allows to have this manifest refer to a user other than yourself. If not specified,
-					it will then default to your local account (--bzzaccount)`,
-					Flags: []cli.Flag{SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
-				},
-				{
-					Action:             feedUpdate,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "update",
-					Usage:              "updates the content of an existing Swarm Feed",
-					ArgsUsage:          "<0x Hex data>",
-					Description: `publishes a new update on the specified topic
-					The feed topic can be built in the following ways:
-					* use --topic to set the topic to an arbitrary binary hex string.
-					* use --name to set the topic to a human-readable name.
-					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
-					* use both --topic and --name to create named subtopics. 
-						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
-						this feed tracks a discussion about that contract.
-					
-					If you have a manifest, you can specify it with --manifest to refer to the feed,
-					instead of using --topic / --name
-					`,
-					Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag},
-				},
-				{
-					Action:             feedInfo,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "info",
-					Usage:              "obtains information about an existing Swarm feed",
-					Description: `obtains information about an existing Swarm feed
-					The topic can be specified directly with the --topic flag as an hex string
-					If no topic is specified, the default topic (zero) will be used
-					The --name flag can be used to specify subtopics with a specific name.
-					The --user flag allows to refer to a user other than yourself. If not specified,
-					it will then default to your local account (--bzzaccount)
-					If you have a manifest, you can specify it with --manifest instead of --topic / --name / ---user
-					to refer to the feed`,
-					Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
-				},
-			},
-		},
-		{
-			Action:             list,
-			CustomHelpTemplate: helpTemplate,
-			Name:               "ls",
-			Usage:              "list files and directories contained in a manifest",
-			ArgsUsage:          "<manifest> [<prefix>]",
-			Description:        "Lists files and directories contained in a manifest",
-		},
-		{
-			Action:             hash,
-			CustomHelpTemplate: helpTemplate,
-			Name:               "hash",
-			Usage:              "print the swarm hash of a file or directory",
-			ArgsUsage:          "<file>",
-			Description:        "Prints the swarm hash of file or directory",
-		},
-		{
-			Action:      download,
-			Name:        "down",
-			Flags:       []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag},
-			Usage:       "downloads a swarm manifest or a file inside a manifest",
-			ArgsUsage:   " <uri> [<dir>]",
-			Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`,
-		},
-		{
-			Name:               "manifest",
-			CustomHelpTemplate: helpTemplate,
-			Usage:              "perform operations on swarm manifests",
-			ArgsUsage:          "COMMAND",
-			Description:        "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove",
-			Subcommands: []cli.Command{
-				{
-					Action:             manifestAdd,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "add",
-					Usage:              "add a new path to the manifest",
-					ArgsUsage:          "<MANIFEST> <path> <hash>",
-					Description:        "Adds a new path to the manifest",
-				},
-				{
-					Action:             manifestUpdate,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "update",
-					Usage:              "update the hash for an already existing path in the manifest",
-					ArgsUsage:          "<MANIFEST> <path> <newhash>",
-					Description:        "Update the hash for an already existing path in the manifest",
-				},
-				{
-					Action:             manifestRemove,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "remove",
-					Usage:              "removes a path from the manifest",
-					ArgsUsage:          "<MANIFEST> <path>",
-					Description:        "Removes a path from the manifest",
-				},
-			},
-		},
-		{
-			Name:               "fs",
-			CustomHelpTemplate: helpTemplate,
-			Usage:              "perform FUSE operations",
-			ArgsUsage:          "fs COMMAND",
-			Description:        "Performs FUSE operations by mounting/unmounting/listing mount points. This assumes you already have a Swarm node running locally. For all operation you must reference the correct path to bzzd.ipc in order to communicate with the node",
-			Subcommands: []cli.Command{
-				{
-					Action:             mount,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "mount",
-					Flags:              []cli.Flag{utils.IPCPathFlag},
-					Usage:              "mount a swarm hash to a mount point",
-					ArgsUsage:          "swarm fs mount --ipcpath <path to bzzd.ipc> <manifest hash> <mount point>",
-					Description:        "Mounts a Swarm manifest hash to a given mount point. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
-				},
-				{
-					Action:             unmount,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "unmount",
-					Flags:              []cli.Flag{utils.IPCPathFlag},
-					Usage:              "unmount a swarmfs mount",
-					ArgsUsage:          "swarm fs unmount --ipcpath <path to bzzd.ipc> <mount point>",
-					Description:        "Unmounts a swarmfs mount residing at <mount point>. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
-				},
-				{
-					Action:             listMounts,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "list",
-					Flags:              []cli.Flag{utils.IPCPathFlag},
-					Usage:              "list swarmfs mounts",
-					ArgsUsage:          "swarm fs list --ipcpath <path to bzzd.ipc>",
-					Description:        "Lists all mounted swarmfs volumes. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
-				},
-			},
-		},
-		{
-			Name:               "db",
-			CustomHelpTemplate: helpTemplate,
-			Usage:              "manage the local chunk database",
-			ArgsUsage:          "db COMMAND",
-			Description:        "Manage the local chunk database",
-			Subcommands: []cli.Command{
-				{
-					Action:             dbExport,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "export",
-					Usage:              "export a local chunk database as a tar archive (use - to send to stdout)",
-					ArgsUsage:          "<chunkdb> <file>",
-					Description: `
-Export a local chunk database as a tar archive (use - to send to stdout).
-
-    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
-
-The export may be quite large, consider piping the output through the Unix
-pv(1) tool to get a progress bar:
-
-    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks - | pv > chunks.tar
-`,
-				},
-				{
-					Action:             dbImport,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "import",
-					Usage:              "import chunks from a tar archive into a local chunk database (use - to read from stdin)",
-					ArgsUsage:          "<chunkdb> <file>",
-					Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin).
-
-    swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
-
-The import may be quite large, consider piping the input through the Unix
-pv(1) tool to get a progress bar:
-
-    pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`,
-				},
-			},
-		},
-
+		// See upload.go
+		upCommand,
+		// See access.go
+		accessCommand,
+		// See feeds.go
+		feedCommand,
+		// See list.go
+		listCommand,
+		// See hash.go
+		hashCommand,
+		// See download.go
+		downloadCommand,
+		// See manifest.go
+		manifestCommand,
+		// See fs.go
+		fsCommand,
+		// See db.go
+		dbCommand,
 		// See config.go
 		DumpConfigCommand,
 	}

cmd/swarm/manifest.go

@@ -28,6 +28,40 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var manifestCommand = cli.Command{
+	Name:               "manifest",
+	CustomHelpTemplate: helpTemplate,
+	Usage:              "perform operations on swarm manifests",
+	ArgsUsage:          "COMMAND",
+	Description:        "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove",
+	Subcommands: []cli.Command{
+		{
+			Action:             manifestAdd,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "add",
+			Usage:              "add a new path to the manifest",
+			ArgsUsage:          "<MANIFEST> <path> <hash>",
+			Description:        "Adds a new path to the manifest",
+		},
+		{
+			Action:             manifestUpdate,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "update",
+			Usage:              "update the hash for an already existing path in the manifest",
+			ArgsUsage:          "<MANIFEST> <path> <newhash>",
+			Description:        "Update the hash for an already existing path in the manifest",
+		},
+		{
+			Action:             manifestRemove,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "remove",
+			Usage:              "removes a path from the manifest",
+			ArgsUsage:          "<MANIFEST> <path>",
+			Description:        "Removes a path from the manifest",
+		},
+	},
+}
+
 // manifestAdd adds a new entry to the manifest at the given path.
 // New entry hash, the last argument, must be the hash of a manifest
 // with only one entry, which meta-data will be added to the original manifest.

cmd/swarm/manifest_test.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
 )
 
 // TestManifestChange tests manifest add, update and remove
@@ -57,8 +58,8 @@ func TestManifestChangeEncrypted(t *testing.T) {
 // Argument encrypt controls whether to use encryption or not.
 func testManifestChange(t *testing.T, encrypt bool) {
 	t.Parallel()
-	cluster := newTestCluster(t, 1)
-	defer cluster.Shutdown()
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
+	defer srv.Close()
 
 	tmp, err := ioutil.TempDir("", "swarm-manifest-test")
 	if err != nil {
@@ -94,7 +95,7 @@ func testManifestChange(t *testing.T, encrypt bool) {
 
 	args := []string{
 		"--bzzapi",
-		cluster.Nodes[0].URL,
+		srv.URL,
 		"--recursive",
 		"--defaultpath",
 		indexDataFilename,
@@ -109,7 +110,7 @@ func testManifestChange(t *testing.T, encrypt bool) {
 
 	checkHashLength(t, origManifestHash, encrypt)
 
-	client := swarm.NewClient(cluster.Nodes[0].URL)
+	client := swarm.NewClient(srv.URL)
 
 	// upload a new file and use its manifest to add it the original manifest.
 	t.Run("add", func(t *testing.T) {
@@ -122,14 +123,14 @@ func testManifestChange(t *testing.T, encrypt bool) {
 
 		humansManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"up",
 			humansDataFilename,
 		)
 
 		newManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"manifest",
 			"add",
 			origManifestHash,
@@ -177,14 +178,14 @@ func testManifestChange(t *testing.T, encrypt bool) {
 
 		robotsManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"up",
 			robotsDataFilename,
 		)
 
 		newManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"manifest",
 			"add",
 			origManifestHash,
@@ -237,14 +238,14 @@ func testManifestChange(t *testing.T, encrypt bool) {
 
 		indexManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"up",
 			indexDataFilename,
 		)
 
 		newManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"manifest",
 			"update",
 			origManifestHash,
@@ -295,14 +296,14 @@ func testManifestChange(t *testing.T, encrypt bool) {
 
 		humansManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"up",
 			robotsDataFilename,
 		)
 
 		newManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"manifest",
 			"update",
 			origManifestHash,
@@ -348,7 +349,7 @@ func testManifestChange(t *testing.T, encrypt bool) {
 	t.Run("remove", func(t *testing.T) {
 		newManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"manifest",
 			"remove",
 			origManifestHash,
@@ -376,7 +377,7 @@ func testManifestChange(t *testing.T, encrypt bool) {
 	t.Run("remove nested", func(t *testing.T) {
 		newManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"manifest",
 			"remove",
 			origManifestHash,
@@ -429,8 +430,8 @@ func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) {
 
 func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
 	t.Parallel()
-	cluster := newTestCluster(t, 1)
-	defer cluster.Shutdown()
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
+	defer srv.Close()
 
 	tmp, err := ioutil.TempDir("", "swarm-manifest-test")
 	if err != nil {
@@ -458,7 +459,7 @@ func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
 
 	args := []string{
 		"--bzzapi",
-		cluster.Nodes[0].URL,
+		srv.URL,
 		"--recursive",
 		"--defaultpath",
 		indexDataFilename,
@@ -473,7 +474,7 @@ func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
 
 	checkHashLength(t, origManifestHash, encrypt)
 
-	client := swarm.NewClient(cluster.Nodes[0].URL)
+	client := swarm.NewClient(srv.URL)
 
 	newIndexData := []byte("<h1>Ethereum Swarm</h1>")
 	newIndexDataFilename := filepath.Join(tmp, "index.html")
@@ -484,14 +485,14 @@ func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
 
 	newIndexManifestHash := runSwarmExpectHash(t,
 		"--bzzapi",
-		cluster.Nodes[0].URL,
+		srv.URL,
 		"up",
 		newIndexDataFilename,
 	)
 
 	newManifestHash := runSwarmExpectHash(t,
 		"--bzzapi",
-		cluster.Nodes[0].URL,
+		srv.URL,
 		"manifest",
 		"update",
 		origManifestHash,

cmd/swarm/run_test.go

@@ -40,6 +40,8 @@ import (
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm"
+	"github.com/ethereum/go-ethereum/swarm/api"
+	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
 )
 
 var loglevel = flag.Int("loglevel", 3, "verbosity of logs")
@@ -55,6 +57,20 @@ func init() {
 	})
 }
 
+const clusterSize = 3
+
+var clusteronce sync.Once
+var cluster *testCluster
+
+func initCluster(t *testing.T) {
+	clusteronce.Do(func() {
+		cluster = newTestCluster(t, clusterSize)
+	})
+}
+
+func serverFunc(api *api.API) swarmhttp.TestServer {
+	return swarmhttp.NewServer(api, "")
+}
 func TestMain(m *testing.M) {
 	// check if we have been reexec'd
 	if reexec.Init() {

cmd/swarm/swarm-smoke/feed_upload_and_sync.go

@@ -0,0 +1,366 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptrace"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/swarm/api/client"
+	"github.com/ethereum/go-ethereum/swarm/spancontext"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
+	colorable "github.com/mattn/go-colorable"
+	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/pborman/uuid"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+const (
+	feedRandomDataLength = 8
+)
+
+func cliFeedUploadAndSync(c *cli.Context) error {
+	metrics.GetOrRegisterCounter("feed-and-sync", nil).Inc(1)
+	log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))))
+
+	errc := make(chan error)
+	go func() {
+		errc <- feedUploadAndSync(c)
+	}()
+
+	select {
+	case err := <-errc:
+		if err != nil {
+			metrics.GetOrRegisterCounter("feed-and-sync.fail", nil).Inc(1)
+		}
+		return err
+	case <-time.After(time.Duration(timeout) * time.Second):
+		metrics.GetOrRegisterCounter("feed-and-sync.timeout", nil).Inc(1)
+		return fmt.Errorf("timeout after %v sec", timeout)
+	}
+}
+
+// TODO: retrieve with manifest + extract repeating code
+func feedUploadAndSync(c *cli.Context) error {
+	defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now())
+
+	generateEndpoints(scheme, cluster, appName, from, to)
+
+	log.Info("generating and uploading feeds to " + endpoints[0] + " and syncing")
+
+	// create a random private key to sign updates with and derive the address
+	pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test")
+	if err != nil {
+		return err
+	}
+	defer pkFile.Close()
+	defer os.Remove(pkFile.Name())
+
+	privkeyHex := "0000000000000000000000000000000000000000000000000000000000001976"
+	privKey, err := crypto.HexToECDSA(privkeyHex)
+	if err != nil {
+		return err
+	}
+	user := crypto.PubkeyToAddress(privKey.PublicKey)
+	userHex := hexutil.Encode(user.Bytes())
+
+	// save the private key to a file
+	_, err = io.WriteString(pkFile, privkeyHex)
+	if err != nil {
+		return err
+	}
+
+	// keep hex strings for topic and subtopic
+	var topicHex string
+	var subTopicHex string
+
+	// and create combination hex topics for bzz-feed retrieval
+	// xor'ed with topic (zero-value topic if no topic)
+	var subTopicOnlyHex string
+	var mergedSubTopicHex string
+
+	// generate random topic and subtopic and put a hex on them
+	topicBytes, err := generateRandomData(feed.TopicLength)
+	topicHex = hexutil.Encode(topicBytes)
+	subTopicBytes, err := generateRandomData(8)
+	subTopicHex = hexutil.Encode(subTopicBytes)
+	if err != nil {
+		return err
+	}
+	mergedSubTopic, err := feed.NewTopic(subTopicHex, topicBytes)
+	if err != nil {
+		return err
+	}
+	mergedSubTopicHex = hexutil.Encode(mergedSubTopic[:])
+	subTopicOnlyBytes, err := feed.NewTopic(subTopicHex, nil)
+	if err != nil {
+		return err
+	}
+	subTopicOnlyHex = hexutil.Encode(subTopicOnlyBytes[:])
+
+	// create feed manifest, topic only
+	var out bytes.Buffer
+	cmd := exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--user", userHex)
+	cmd.Stdout = &out
+	log.Debug("create feed manifest topic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	manifestWithTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
+	if len(manifestWithTopic) != 64 {
+		return fmt.Errorf("unknown feed create manifest hash format (topic): (%d) %s", len(out.String()), manifestWithTopic)
+	}
+	log.Debug("create topic feed", "manifest", manifestWithTopic)
+	out.Reset()
+
+	// create feed manifest, subtopic only
+	cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--name", subTopicHex, "--user", userHex)
+	cmd.Stdout = &out
+	log.Debug("create feed manifest subtopic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	manifestWithSubTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
+	if len(manifestWithSubTopic) != 64 {
+		return fmt.Errorf("unknown feed create manifest hash format (subtopic): (%d) %s", len(out.String()), manifestWithSubTopic)
+	}
+	log.Debug("create subtopic feed", "manifest", manifestWithTopic)
+	out.Reset()
+
+	// create feed manifest, merged topic
+	cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex)
+	cmd.Stdout = &out
+	log.Debug("create feed manifest mergetopic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		log.Error(err.Error())
+		return err
+	}
+	manifestWithMergedTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
+	if len(manifestWithMergedTopic) != 64 {
+		return fmt.Errorf("unknown feed create manifest hash format (mergedtopic): (%d) %s", len(out.String()), manifestWithMergedTopic)
+	}
+	log.Debug("create mergedtopic feed", "manifest", manifestWithMergedTopic)
+	out.Reset()
+
+	// create test data
+	data, err := generateRandomData(feedRandomDataLength)
+	if err != nil {
+		return err
+	}
+	h := md5.New()
+	h.Write(data)
+	dataHash := h.Sum(nil)
+	dataHex := hexutil.Encode(data)
+
+	// update with topic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, dataHex)
+	cmd.Stdout = &out
+	log.Debug("update feed manifest topic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update topic", "out", out)
+	out.Reset()
+
+	// update with subtopic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, dataHex)
+	cmd.Stdout = &out
+	log.Debug("update feed manifest subtopic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update subtopic", "out", out)
+	out.Reset()
+
+	// update with merged topic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex)
+	cmd.Stdout = &out
+	log.Debug("update feed manifest merged topic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update mergedtopic", "out", out)
+	out.Reset()
+
+	time.Sleep(3 * time.Second)
+
+	// retrieve the data
+	wg := sync.WaitGroup{}
+	for _, endpoint := range endpoints {
+		// raw retrieve, topic only
+		for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} {
+			wg.Add(1)
+			ruid := uuid.New()[:8]
+			go func(hex string, endpoint string, ruid string) {
+				for {
+					err := fetchFeed(hex, userHex, endpoint, dataHash, ruid)
+					if err != nil {
+						continue
+					}
+
+					wg.Done()
+					return
+				}
+			}(hex, endpoint, ruid)
+
+		}
+	}
+	wg.Wait()
+	log.Info("all endpoints synced random data successfully")
+
+	// upload test file
+	seed := int(time.Now().UnixNano() / 1e6)
+	log.Info("feed uploading to "+endpoints[0]+" and syncing", "seed", seed)
+
+	randomBytes := testutil.RandomBytes(seed, filesize*1000)
+
+	hash, err := upload(&randomBytes, endpoints[0])
+	if err != nil {
+		return err
+	}
+	hashBytes, err := hexutil.Decode("0x" + hash)
+	if err != nil {
+		return err
+	}
+	multihashHex := hexutil.Encode(hashBytes)
+	fileHash, err := digest(bytes.NewReader(randomBytes))
+	if err != nil {
+		return err
+	}
+
+	log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash))
+
+	// update file with topic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, multihashHex)
+	cmd.Stdout = &out
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update topic", "out", out)
+	out.Reset()
+
+	// update file with subtopic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, multihashHex)
+	cmd.Stdout = &out
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update subtopic", "out", out)
+	out.Reset()
+
+	// update file with merged topic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex)
+	cmd.Stdout = &out
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update mergedtopic", "out", out)
+	out.Reset()
+
+	time.Sleep(3 * time.Second)
+
+	for _, endpoint := range endpoints {
+
+		// manifest retrieve, topic only
+		for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} {
+			wg.Add(1)
+			ruid := uuid.New()[:8]
+			go func(url string, endpoint string, ruid string) {
+				for {
+					err := fetch(url, endpoint, fileHash, ruid)
+					if err != nil {
+						continue
+					}
+
+					wg.Done()
+					return
+				}
+			}(url, endpoint, ruid)
+		}
+
+	}
+	wg.Wait()
+	log.Info("all endpoints synced random file successfully")
+
+	return nil
+}
+
+func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error {
+	ctx, sp := spancontext.StartSpan(context.Background(), "feed-and-sync.fetch")
+	defer sp.Finish()
+
+	log.Trace("sleeping", "ruid", ruid)
+	time.Sleep(3 * time.Second)
+
+	log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user)
+
+	var tn time.Time
+	reqUri := endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user
+	req, _ := http.NewRequest("GET", reqUri, nil)
+
+	opentracing.GlobalTracer().Inject(
+		sp.Context(),
+		opentracing.HTTPHeaders,
+		opentracing.HTTPHeadersCarrier(req.Header))
+
+	trace := client.GetClientTrace("feed-and-sync - http get", "feed-and-sync", ruid, &tn)
+
+	req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
+	transport := http.DefaultTransport
+
+	//transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+
+	tn = time.Now()
+	res, err := transport.RoundTrip(req)
+	if err != nil {
+		log.Error(err.Error(), "ruid", ruid)
+		return err
+	}
+
+	log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength)
+
+	if res.StatusCode != 200 {
+		return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid)
+	}
+
+	defer res.Body.Close()
+
+	rdigest, err := digest(res.Body)
+	if err != nil {
+		log.Warn(err.Error(), "ruid", ruid)
+		return err
+	}
+
+	if !bytes.Equal(rdigest, original) {
+		err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
+		log.Warn(err.Error(), "ruid", ruid)
+		return err
+	}
+
+	log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
+
+	return nil
+}

cmd/swarm/swarm-smoke/main.go

@@ -17,28 +17,41 @@
 package main
 
 import (
+	"fmt"
 	"os"
 	"sort"
 
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	gethmetrics "github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/metrics/influxdb"
+	swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics"
+	"github.com/ethereum/go-ethereum/swarm/tracing"
+
 	"github.com/ethereum/go-ethereum/log"
-	colorable "github.com/mattn/go-colorable"
 
 	cli "gopkg.in/urfave/cli.v1"
 )
 
+var (
+	gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
+)
+
 var (
 	endpoints        []string
 	includeLocalhost bool
 	cluster          string
+	appName          string
 	scheme           string
 	filesize         int
+	syncDelay        int
 	from             int
 	to               int
+	verbosity        int
+	timeout          int
+	single           bool
 )
 
 func main() {
-	log.PrintOrigins(true)
-	log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
 
 	app := cli.NewApp()
 	app.Name = "smoke-test"
@@ -47,10 +60,16 @@ func main() {
 	app.Flags = []cli.Flag{
 		cli.StringFlag{
 			Name:        "cluster-endpoint",
-			Value:       "testing",
-			Usage:       "cluster to point to (local, open or testing)",
+			Value:       "prod",
+			Usage:       "cluster to point to (prod or a given namespace)",
 			Destination: &cluster,
 		},
+		cli.StringFlag{
+			Name:        "app",
+			Value:       "swarm",
+			Usage:       "application to point to (swarm or swarm-private)",
+			Destination: &appName,
+		},
 		cli.IntFlag{
 			Name:        "cluster-from",
 			Value:       8501,
@@ -80,8 +99,42 @@ func main() {
 			Usage:       "file size for generated random file in KB",
 			Destination: &filesize,
 		},
+		cli.IntFlag{
+			Name:        "sync-delay",
+			Value:       5,
+			Usage:       "duration of delay in seconds to wait for content to be synced",
+			Destination: &syncDelay,
+		},
+		cli.IntFlag{
+			Name:        "verbosity",
+			Value:       1,
+			Usage:       "verbosity",
+			Destination: &verbosity,
+		},
+		cli.IntFlag{
+			Name:        "timeout",
+			Value:       120,
+			Usage:       "timeout in seconds after which kill the process",
+			Destination: &timeout,
+		},
+		cli.BoolFlag{
+			Name:        "single",
+			Usage:       "whether to fetch content from a single node or from all nodes",
+			Destination: &single,
+		},
 	}
 
+	app.Flags = append(app.Flags, []cli.Flag{
+		utils.MetricsEnabledFlag,
+		swarmmetrics.MetricsInfluxDBEndpointFlag,
+		swarmmetrics.MetricsInfluxDBDatabaseFlag,
+		swarmmetrics.MetricsInfluxDBUsernameFlag,
+		swarmmetrics.MetricsInfluxDBPasswordFlag,
+		swarmmetrics.MetricsInfluxDBHostTagFlag,
+	}...)
+
+	app.Flags = append(app.Flags, tracing.Flags...)
+
 	app.Commands = []cli.Command{
 		{
 			Name:    "upload_and_sync",
@@ -89,13 +142,49 @@ func main() {
 			Usage:   "upload and sync",
 			Action:  cliUploadAndSync,
 		},
+		{
+			Name:    "feed_sync",
+			Aliases: []string{"f"},
+			Usage:   "feed update generate, upload and sync",
+			Action:  cliFeedUploadAndSync,
+		},
 	}
 
 	sort.Sort(cli.FlagsByName(app.Flags))
 	sort.Sort(cli.CommandsByName(app.Commands))
 
+	app.Before = func(ctx *cli.Context) error {
+		tracing.Setup(ctx)
+		return nil
+	}
+
+	app.After = func(ctx *cli.Context) error {
+		return emitMetrics(ctx)
+	}
+
 	err := app.Run(os.Args)
 	if err != nil {
 		log.Error(err.Error())
+
+		os.Exit(1)
 	}
 }
+
+func emitMetrics(ctx *cli.Context) error {
+	if gethmetrics.Enabled {
+		var (
+			endpoint = ctx.GlobalString(swarmmetrics.MetricsInfluxDBEndpointFlag.Name)
+			database = ctx.GlobalString(swarmmetrics.MetricsInfluxDBDatabaseFlag.Name)
+			username = ctx.GlobalString(swarmmetrics.MetricsInfluxDBUsernameFlag.Name)
+			password = ctx.GlobalString(swarmmetrics.MetricsInfluxDBPasswordFlag.Name)
+			hosttag  = ctx.GlobalString(swarmmetrics.MetricsInfluxDBHostTagFlag.Name)
+		)
+		return influxdb.InfluxDBWithTagsOnce(gethmetrics.DefaultRegistry, endpoint, database, username, password, "swarm-smoke.", map[string]string{
+			"host":     hosttag,
+			"version":  gitCommit,
+			"filesize": fmt.Sprintf("%v", filesize),
+		})
+	}
+
+	return nil
+}

cmd/swarm/swarm-smoke/upload_and_sync.go

@@ -18,38 +18,41 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"crypto/md5"
-	"crypto/rand"
+	crand "crypto/rand"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
+	"net/http/httptrace"
 	"os"
-	"os/exec"
-	"strings"
 	"sync"
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/api/client"
+	"github.com/ethereum/go-ethereum/swarm/spancontext"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
+	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/pborman/uuid"
 
 	cli "gopkg.in/urfave/cli.v1"
 )
 
-func generateEndpoints(scheme string, cluster string, from int, to int) {
+func generateEndpoints(scheme string, cluster string, app string, from int, to int) {
 	if cluster == "prod" {
-		cluster = ""
-	} else if cluster == "local" {
-		for port := from; port <= to; port++ {
-			endpoints = append(endpoints, fmt.Sprintf("%s://localhost:%v", scheme, port))
+		for port := from; port < to; port++ {
+			endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port))
 		}
-		return
 	} else {
-		cluster = cluster + "."
-	}
-
-	for port := from; port <= to; port++ {
-		endpoints = append(endpoints, fmt.Sprintf("%s://%v.%sswarm-gateways.net", scheme, port, cluster))
+		for port := from; port < to; port++ {
+			endpoints = append(endpoints, fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, port, cluster))
+		}
 	}
 
 	if includeLocalhost {
@@ -58,22 +61,51 @@ func generateEndpoints(scheme string, cluster string, from int, to int) {
 }
 
 func cliUploadAndSync(c *cli.Context) error {
-	defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now())
+	log.PrintOrigins(true)
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true))))
 
-	generateEndpoints(scheme, cluster, from, to)
+	metrics.GetOrRegisterCounter("upload-and-sync", nil).Inc(1)
 
-	log.Info("uploading to " + endpoints[0] + " and syncing")
+	errc := make(chan error)
+	go func() {
+		errc <- uploadAndSync(c)
+	}()
 
-	f, cleanup := generateRandomFile(filesize * 1000)
-	defer cleanup()
+	select {
+	case err := <-errc:
+		if err != nil {
+			metrics.GetOrRegisterCounter("upload-and-sync.fail", nil).Inc(1)
+		}
+		return err
+	case <-time.After(time.Duration(timeout) * time.Second):
+		metrics.GetOrRegisterCounter("upload-and-sync.timeout", nil).Inc(1)
+		return fmt.Errorf("timeout after %v sec", timeout)
+	}
+}
 
-	hash, err := upload(f, endpoints[0])
+func uploadAndSync(c *cli.Context) error {
+	defer func(now time.Time) {
+		totalTime := time.Since(now)
+
+		log.Info("total time", "time", totalTime, "kb", filesize)
+		metrics.GetOrRegisterCounter("upload-and-sync.total-time", nil).Inc(int64(totalTime))
+	}(time.Now())
+
+	generateEndpoints(scheme, cluster, appName, from, to)
+	seed := int(time.Now().UnixNano() / 1e6)
+	log.Info("uploading to "+endpoints[0]+" and syncing", "seed", seed)
+
+	randomBytes := testutil.RandomBytes(seed, filesize*1000)
+
+	t1 := time.Now()
+	hash, err := upload(&randomBytes, endpoints[0])
 	if err != nil {
 		log.Error(err.Error())
 		return err
 	}
+	metrics.GetOrRegisterCounter("upload-and-sync.upload-time", nil).Inc(int64(time.Since(t1)))
 
-	fhash, err := digest(f)
+	fhash, err := digest(bytes.NewReader(randomBytes))
 	if err != nil {
 		log.Error(err.Error())
 		return err
@@ -81,24 +113,47 @@ func cliUploadAndSync(c *cli.Context) error {
 
 	log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash))
 
-	time.Sleep(3 * time.Second)
+	time.Sleep(time.Duration(syncDelay) * time.Second)
 
 	wg := sync.WaitGroup{}
-	for _, endpoint := range endpoints {
-		endpoint := endpoint
+	if single {
+		rand.Seed(time.Now().UTC().UnixNano())
+		randIndex := 1 + rand.Intn(len(endpoints)-1)
 		ruid := uuid.New()[:8]
 		wg.Add(1)
 		go func(endpoint string, ruid string) {
 			for {
+				start := time.Now()
 				err := fetch(hash, endpoint, fhash, ruid)
+				fetchTime := time.Since(start)
 				if err != nil {
 					continue
 				}
 
+				metrics.GetOrRegisterMeter("upload-and-sync.single.fetch-time", nil).Mark(int64(fetchTime))
 				wg.Done()
 				return
 			}
-		}(endpoint, ruid)
+		}(endpoints[randIndex], ruid)
+	} else {
+		for _, endpoint := range endpoints {
+			ruid := uuid.New()[:8]
+			wg.Add(1)
+			go func(endpoint string, ruid string) {
+				for {
+					start := time.Now()
+					err := fetch(hash, endpoint, fhash, ruid)
+					fetchTime := time.Since(start)
+					if err != nil {
+						continue
+					}
+
+					metrics.GetOrRegisterMeter("upload-and-sync.each.fetch-time", nil).Mark(int64(fetchTime))
+					wg.Done()
+					return
+				}
+			}(endpoint, ruid)
+		}
 	}
 	wg.Wait()
 	log.Info("all endpoints synced random file successfully")
@@ -108,13 +163,33 @@ func cliUploadAndSync(c *cli.Context) error {
 
 // fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file
 func fetch(hash string, endpoint string, original []byte, ruid string) error {
+	ctx, sp := spancontext.StartSpan(context.Background(), "upload-and-sync.fetch")
+	defer sp.Finish()
+
 	log.Trace("sleeping", "ruid", ruid)
 	time.Sleep(3 * time.Second)
-
 	log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash)
-	res, err := http.Get(endpoint + "/bzz:/" + hash + "/")
+
+	var tn time.Time
+	reqUri := endpoint + "/bzz:/" + hash + "/"
+	req, _ := http.NewRequest("GET", reqUri, nil)
+
+	opentracing.GlobalTracer().Inject(
+		sp.Context(),
+		opentracing.HTTPHeaders,
+		opentracing.HTTPHeadersCarrier(req.Header))
+
+	trace := client.GetClientTrace("upload-and-sync - http get", "upload-and-sync", ruid, &tn)
+
+	req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
+	transport := http.DefaultTransport
+
+	//transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+
+	tn = time.Now()
+	res, err := transport.RoundTrip(req)
 	if err != nil {
-		log.Warn(err.Error(), "ruid", ruid)
+		log.Error(err.Error(), "ruid", ruid)
 		return err
 	}
 	log.Trace("http get response", "ruid", ruid, "api", endpoint, "hash", hash, "code", res.StatusCode, "len", res.ContentLength)
@@ -145,16 +220,19 @@ func fetch(hash string, endpoint string, original []byte, ruid string) error {
 }
 
 // upload is uploading a file `f` to `endpoint` via the `swarm up` cmd
-func upload(f *os.File, endpoint string) (string, error) {
-	var out bytes.Buffer
-	cmd := exec.Command("swarm", "--bzzapi", endpoint, "up", f.Name())
-	cmd.Stdout = &out
-	err := cmd.Run()
-	if err != nil {
-		return "", err
+func upload(dataBytes *[]byte, endpoint string) (string, error) {
+	swarm := client.NewClient(endpoint)
+	f := &client.File{
+		ReadCloser: ioutil.NopCloser(bytes.NewReader(*dataBytes)),
+		ManifestEntry: api.ManifestEntry{
+			ContentType: "text/plain",
+			Mode:        0660,
+			Size:        int64(len(*dataBytes)),
+		},
 	}
-	hash := strings.TrimRight(out.String(), "\r\n")
-	return hash, nil
+
+	// upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded.
+	return swarm.Upload(f, "", false)
 }
 
 func digest(r io.Reader) ([]byte, error) {
@@ -166,26 +244,14 @@ func digest(r io.Reader) ([]byte, error) {
 	return h.Sum(nil), nil
 }
 
-// generateRandomFile is creating a temporary file with the requested byte size
-func generateRandomFile(size int) (f *os.File, teardown func()) {
-	// create a tmp file
-	tmp, err := ioutil.TempFile("", "swarm-test")
+// generates random data in heap buffer
+func generateRandomData(datasize int) ([]byte, error) {
+	b := make([]byte, datasize)
+	c, err := crand.Read(b)
 	if err != nil {
-		panic(err)
+		return nil, err
+	} else if c != datasize {
+		return nil, errors.New("short read")
 	}
-
-	// callback for tmp file cleanup
-	teardown = func() {
-		tmp.Close()
-		os.Remove(tmp.Name())
-	}
-
-	buf := make([]byte, size)
-	_, err = rand.Read(buf)
-	if err != nil {
-		panic(err)
-	}
-	ioutil.WriteFile(tmp.Name(), buf, 0755)
-
-	return tmp, teardown
+	return b, nil
 }

cmd/swarm/upload.go

@@ -26,29 +26,47 @@ import (
 	"os/user"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 
+	"github.com/ethereum/go-ethereum/log"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"gopkg.in/urfave/cli.v1"
 )
 
-func upload(ctx *cli.Context) {
+var upCommand = cli.Command{
+	Action:             upload,
+	CustomHelpTemplate: helpTemplate,
+	Name:               "up",
+	Usage:              "uploads a file or directory to swarm using the HTTP API",
+	ArgsUsage:          "<file>",
+	Flags:              []cli.Flag{SwarmEncryptedFlag},
+	Description:        "uploads a file or directory to swarm using the HTTP API and prints the root hash",
+}
 
+func upload(ctx *cli.Context) {
 	args := ctx.Args()
 	var (
-		bzzapi       = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		recursive    = ctx.GlobalBool(SwarmRecursiveFlag.Name)
-		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		defaultPath  = ctx.GlobalString(SwarmUploadDefaultPath.Name)
-		fromStdin    = ctx.GlobalBool(SwarmUpFromStdinFlag.Name)
-		mimeType     = ctx.GlobalString(SwarmUploadMimeType.Name)
-		client       = swarm.NewClient(bzzapi)
-		toEncrypt    = ctx.Bool(SwarmEncryptedFlag.Name)
-		file         string
+		bzzapi          = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		recursive       = ctx.GlobalBool(SwarmRecursiveFlag.Name)
+		wantManifest    = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
+		defaultPath     = ctx.GlobalString(SwarmUploadDefaultPath.Name)
+		fromStdin       = ctx.GlobalBool(SwarmUpFromStdinFlag.Name)
+		mimeType        = ctx.GlobalString(SwarmUploadMimeType.Name)
+		client          = swarm.NewClient(bzzapi)
+		toEncrypt       = ctx.Bool(SwarmEncryptedFlag.Name)
+		autoDefaultPath = false
+		file            string
 	)
-
+	if autoDefaultPathString := os.Getenv(SWARM_AUTO_DEFAULTPATH); autoDefaultPathString != "" {
+		b, err := strconv.ParseBool(autoDefaultPathString)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_AUTO_DEFAULTPATH, err)
+		}
+		autoDefaultPath = b
+	}
 	if len(args) != 1 {
 		if fromStdin {
 			tmp, err := ioutil.TempFile("", "swarm-stdin")
@@ -97,6 +115,15 @@ func upload(ctx *cli.Context) {
 			if !recursive {
 				return "", errors.New("Argument is a directory and recursive upload is disabled")
 			}
+			if autoDefaultPath && defaultPath == "" {
+				defaultEntryCandidate := path.Join(file, "index.html")
+				log.Debug("trying to find default path", "path", defaultEntryCandidate)
+				defaultEntryStat, err := os.Stat(defaultEntryCandidate)
+				if err == nil && !defaultEntryStat.IsDir() {
+					log.Debug("setting auto detected default path", "path", defaultEntryCandidate)
+					defaultPath = defaultEntryCandidate
+				}
+			}
 			if defaultPath != "" {
 				// construct absolute default path
 				absDefaultPath, _ := filepath.Abs(defaultPath)

cmd/swarm/upload_test.go

@@ -31,7 +31,8 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
-	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	swarmapi "github.com/ethereum/go-ethereum/swarm/api/client"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 	"github.com/mattn/go-colorable"
 )
 
@@ -40,69 +41,66 @@ func init() {
 	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
 }
 
-// TestCLISwarmUp tests that running 'swarm up' makes the resulting file
+func TestSwarmUp(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
+	initCluster(t)
+
+	cases := []struct {
+		name string
+		f    func(t *testing.T)
+	}{
+		{"NoEncryption", testNoEncryption},
+		{"Encrypted", testEncrypted},
+		{"RecursiveNoEncryption", testRecursiveNoEncryption},
+		{"RecursiveEncrypted", testRecursiveEncrypted},
+		{"DefaultPathAll", testDefaultPathAll},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, tc.f)
+	}
+}
+
+// testNoEncryption tests that running 'swarm up' makes the resulting file
 // available from all nodes via the HTTP API
-func TestCLISwarmUp(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-
-	testCLISwarmUp(false, t)
-}
-func TestCLISwarmUpRecursive(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-	testCLISwarmUpRecursive(false, t)
+func testNoEncryption(t *testing.T) {
+	testDefault(false, t)
 }
 
-// TestCLISwarmUpEncrypted tests that running 'swarm encrypted-up' makes the resulting file
+// testEncrypted tests that running 'swarm up --encrypted' makes the resulting file
 // available from all nodes via the HTTP API
-func TestCLISwarmUpEncrypted(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-	testCLISwarmUp(true, t)
-}
-func TestCLISwarmUpEncryptedRecursive(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-	testCLISwarmUpRecursive(true, t)
+func testEncrypted(t *testing.T) {
+	testDefault(true, t)
 }
 
-func testCLISwarmUp(toEncrypt bool, t *testing.T) {
-	log.Info("starting 3 node cluster")
-	cluster := newTestCluster(t, 3)
-	defer cluster.Shutdown()
+func testRecursiveNoEncryption(t *testing.T) {
+	testRecursive(false, t)
+}
 
-	// create a tmp file
-	tmp, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer tmp.Close()
-	defer os.Remove(tmp.Name())
+func testRecursiveEncrypted(t *testing.T) {
+	testRecursive(true, t)
+}
+
+func testDefault(toEncrypt bool, t *testing.T) {
+	tmpFileName := testutil.TempFileWithContent(t, data)
+	defer os.Remove(tmpFileName)
 
 	// write data to file
-	data := "notsorandomdata"
-	_, err = io.WriteString(tmp, data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	hashRegexp := `[a-f\d]{64}`
 	flags := []string{
 		"--bzzapi", cluster.Nodes[0].URL,
 		"up",
-		tmp.Name()}
+		tmpFileName}
 	if toEncrypt {
 		hashRegexp = `[a-f\d]{128}`
 		flags = []string{
 			"--bzzapi", cluster.Nodes[0].URL,
 			"up",
 			"--encrypt",
-			tmp.Name()}
+			tmpFileName}
 	}
 	// upload the file with 'swarm up' and expect a hash
 	log.Info(fmt.Sprintf("uploading file with 'swarm up'"))
@@ -191,18 +189,13 @@ func testCLISwarmUp(toEncrypt bool, t *testing.T) {
 	}
 }
 
-func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
-	fmt.Println("starting 3 node cluster")
-	cluster := newTestCluster(t, 3)
-	defer cluster.Shutdown()
-
+func testRecursive(toEncrypt bool, t *testing.T) {
 	tmpUploadDir, err := ioutil.TempDir("", "swarm-test")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer os.RemoveAll(tmpUploadDir)
 	// create tmp files
-	data := "notsorandomdata"
 	for _, path := range []string{"tmp1", "tmp2"} {
 		if err := ioutil.WriteFile(filepath.Join(tmpUploadDir, path), bytes.NewBufferString(data).Bytes(), 0644); err != nil {
 			t.Fatal(err)
@@ -242,8 +235,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
 		}
 		defer os.RemoveAll(tmpDownload)
 		bzzLocator := "bzz:/" + hash
-		flagss := []string{}
-		flagss = []string{
+		flagss := []string{
 			"--bzzapi", cluster.Nodes[0].URL,
 			"down",
 			"--recursive",
@@ -264,7 +256,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
 
 			switch mode := fi.Mode(); {
 			case mode.IsRegular():
-				if file, err := swarm.Open(path.Join(tmpDownload, v.Name())); err != nil {
+				if file, err := swarmapi.Open(path.Join(tmpDownload, v.Name())); err != nil {
 					t.Fatalf("encountered an error opening the file returned from the CLI: %v", err)
 				} else {
 					ff := make([]byte, len(data))
@@ -285,22 +277,16 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
 	}
 }
 
-// TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute
+// testDefaultPathAll tests swarm recursive upload with relative and absolute
 // default paths and with encryption.
-func TestCLISwarmUpDefaultPath(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-	testCLISwarmUpDefaultPath(false, false, t)
-	testCLISwarmUpDefaultPath(false, true, t)
-	testCLISwarmUpDefaultPath(true, false, t)
-	testCLISwarmUpDefaultPath(true, true, t)
+func testDefaultPathAll(t *testing.T) {
+	testDefaultPath(false, false, t)
+	testDefaultPath(false, true, t)
+	testDefaultPath(true, false, t)
+	testDefaultPath(true, true, t)
 }
 
-func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
-	cluster := newTestCluster(t, 1)
-	defer cluster.Shutdown()
-
+func testDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
 	tmp, err := ioutil.TempDir("", "swarm-defaultpath-test")
 	if err != nil {
 		t.Fatal(err)
@@ -340,7 +326,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T
 	up.ExpectExit()
 	hash := matches[0]
 
-	client := swarm.NewClient(cluster.Nodes[0].URL)
+	client := swarmapi.NewClient(cluster.Nodes[0].URL)
 
 	m, isEncrypted, err := client.DownloadManifest(hash)
 	if err != nil {

cmd/utils/cmd.go

@@ -272,13 +272,13 @@ func ImportPreimages(db *ethdb.LDBDatabase, fn string) error {
 		// Accumulate the preimages and flush when enough ws gathered
 		preimages[crypto.Keccak256Hash(blob)] = common.CopyBytes(blob)
 		if len(preimages) > 1024 {
-			rawdb.WritePreimages(db, 0, preimages)
+			rawdb.WritePreimages(db, preimages)
 			preimages = make(map[common.Hash][]byte)
 		}
 	}
 	// Flush the last batch preimage data
 	if len(preimages) > 0 {
-		rawdb.WritePreimages(db, 0, preimages)
+		rawdb.WritePreimages(db, preimages)
 	}
 	return nil
 }

cmd/utils/flags.go

@@ -140,6 +140,10 @@ var (
 		Name:  "rinkeby",
 		Usage: "Rinkeby network: pre-configured proof-of-authority test network",
 	}
+	ConstantinopleOverrideFlag = cli.Uint64Flag{
+		Name:  "override.constantinople",
+		Usage: "Manually specify constantinople fork-block, overriding the bundled setting",
+	}
 	DeveloperFlag = cli.BoolFlag{
 		Name:  "dev",
 		Usage: "Ephemeral proof-of-authority network with a pre-funded developer account, mining enabled",
@@ -182,6 +186,10 @@ var (
 		Name:  "lightkdf",
 		Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
 	}
+	WhitelistFlag = cli.StringFlag{
+		Name:  "whitelist",
+		Usage: "Comma separated block number-to-hash mappings to enforce (<number>=<hash>)",
+	}
 	// Dashboard settings
 	DashboardEnabledFlag = cli.BoolFlag{
 		Name:  metrics.DashboardEnabledFlag,
@@ -295,7 +303,12 @@ var (
 	CacheDatabaseFlag = cli.IntFlag{
 		Name:  "cache.database",
 		Usage: "Percentage of cache memory allowance to use for database io",
-		Value: 75,
+		Value: 50,
+	}
+	CacheTrieFlag = cli.IntFlag{
+		Name:  "cache.trie",
+		Usage: "Percentage of cache memory allowance to use for trie caching",
+		Value: 25,
 	}
 	CacheGCFlag = cli.IntFlag{
 		Name:  "cache.gc",
@@ -819,17 +832,12 @@ func setIPC(ctx *cli.Context, cfg *node.Config) {
 // makeDatabaseHandles raises out the number of allowed file handles per process
 // for Geth and returns half of the allowance to assign to the database.
 func makeDatabaseHandles() int {
-	limit, err := fdlimit.Current()
+	limit, err := fdlimit.Maximum()
 	if err != nil {
 		Fatalf("Failed to retrieve file descriptor allowance: %v", err)
 	}
-	if limit < 2048 {
-		if err := fdlimit.Raise(2048); err != nil {
-			Fatalf("Failed to raise file descriptor allowance: %v", err)
-		}
-	}
-	if limit > 2048 { // cap database file descriptors even if more is available
-		limit = 2048
+	if err := fdlimit.Raise(uint64(limit)); err != nil {
+		Fatalf("Failed to raise file descriptor allowance: %v", err)
 	}
 	return limit / 2 // Leave half for networking and other stuff
 }
@@ -973,16 +981,7 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	setWS(ctx, cfg)
 	setNodeUserIdent(ctx, cfg)
 
-	switch {
-	case ctx.GlobalIsSet(DataDirFlag.Name):
-		cfg.DataDir = ctx.GlobalString(DataDirFlag.Name)
-	case ctx.GlobalBool(DeveloperFlag.Name):
-		cfg.DataDir = "" // unless explicitly requested, use memory databases
-	case ctx.GlobalBool(TestnetFlag.Name):
-		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
-	case ctx.GlobalBool(RinkebyFlag.Name):
-		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby")
-	}
+	setDataDir(ctx, cfg)
 
 	if ctx.GlobalIsSet(KeyStoreDirFlag.Name) {
 		cfg.KeyStoreDir = ctx.GlobalString(KeyStoreDirFlag.Name)
@@ -995,6 +994,19 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	}
 }
 
+func setDataDir(ctx *cli.Context, cfg *node.Config) {
+	switch {
+	case ctx.GlobalIsSet(DataDirFlag.Name):
+		cfg.DataDir = ctx.GlobalString(DataDirFlag.Name)
+	case ctx.GlobalBool(DeveloperFlag.Name):
+		cfg.DataDir = "" // unless explicitly requested, use memory databases
+	case ctx.GlobalBool(TestnetFlag.Name):
+		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "testnet")
+	case ctx.GlobalBool(RinkebyFlag.Name):
+		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby")
+	}
+}
+
 func setGPO(ctx *cli.Context, cfg *gasprice.Config) {
 	if ctx.GlobalIsSet(GpoBlocksFlag.Name) {
 		cfg.Blocks = ctx.GlobalInt(GpoBlocksFlag.Name)
@@ -1068,6 +1080,29 @@ func setEthash(ctx *cli.Context, cfg *eth.Config) {
 	}
 }
 
+func setWhitelist(ctx *cli.Context, cfg *eth.Config) {
+	whitelist := ctx.GlobalString(WhitelistFlag.Name)
+	if whitelist == "" {
+		return
+	}
+	cfg.Whitelist = make(map[uint64]common.Hash)
+	for _, entry := range strings.Split(whitelist, ",") {
+		parts := strings.Split(entry, "=")
+		if len(parts) != 2 {
+			Fatalf("Invalid whitelist entry: %s", entry)
+		}
+		number, err := strconv.ParseUint(parts[0], 0, 64)
+		if err != nil {
+			Fatalf("Invalid whitelist block number %s: %v", parts[0], err)
+		}
+		var hash common.Hash
+		if err = hash.UnmarshalText([]byte(parts[1])); err != nil {
+			Fatalf("Invalid whitelist hash %s: %v", parts[1], err)
+		}
+		cfg.Whitelist[number] = hash
+	}
+}
+
 // checkExclusive verifies that only a single instance of the provided flags was
 // set by the user. Each flag might optionally be followed by a string type to
 // specialize it further.
@@ -1133,6 +1168,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	setGPO(ctx, &cfg.GPO)
 	setTxPool(ctx, &cfg.TxPool)
 	setEthash(ctx, cfg)
+	setWhitelist(ctx, cfg)
 
 	if ctx.GlobalIsSet(SyncModeFlag.Name) {
 		cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode)
@@ -1146,7 +1182,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	if ctx.GlobalIsSet(NetworkIdFlag.Name) {
 		cfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name)
 	}
-
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheDatabaseFlag.Name) {
 		cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100
 	}
@@ -1157,8 +1192,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	}
 	cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive"
 
+	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
+		cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
+	}
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
-		cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
+		cfg.TrieDirtyCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
 	}
 	if ctx.GlobalIsSet(MinerNotifyFlag.Name) {
 		cfg.MinerNotify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",")
@@ -1368,7 +1406,6 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis {
 func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chainDb ethdb.Database) {
 	var err error
 	chainDb = MakeChainDatabase(ctx, stack)
-
 	config, _, err := core.SetupGenesisBlock(chainDb, MakeGenesis(ctx))
 	if err != nil {
 		Fatalf("%v", err)
@@ -1393,12 +1430,16 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 		Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
 	}
 	cache := &core.CacheConfig{
-		Disabled:      ctx.GlobalString(GCModeFlag.Name) == "archive",
-		TrieNodeLimit: eth.DefaultConfig.TrieCache,
-		TrieTimeLimit: eth.DefaultConfig.TrieTimeout,
+		Disabled:       ctx.GlobalString(GCModeFlag.Name) == "archive",
+		TrieCleanLimit: eth.DefaultConfig.TrieCleanCache,
+		TrieDirtyLimit: eth.DefaultConfig.TrieDirtyCache,
+		TrieTimeLimit:  eth.DefaultConfig.TrieTimeout,
+	}
+	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
+		cache.TrieCleanLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
 	}
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) {
-		cache.TrieNodeLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
+		cache.TrieDirtyLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100
 	}
 	vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
 	chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil)

common/bytes.go

@@ -31,6 +31,15 @@ func ToHex(b []byte) string {
 	return "0x" + hex
 }
 
+// ToHexArray creates a array of hex-string based on []byte
+func ToHexArray(b [][]byte) []string {
+	r := make([]string, len(b))
+	for i := range b {
+		r[i] = ToHex(b[i])
+	}
+	return r
+}
+
 // FromHex returns the bytes represented by the hexadecimal string s.
 // s may be prefixed with "0x".
 func FromHex(s string) []byte {

common/compiler/solidity.go

@@ -31,14 +31,15 @@ import (
 
 var versionRegexp = regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`)
 
-// Contract contains information about a compiled contract, alongside its code.
+// Contract contains information about a compiled contract, alongside its code and runtime code.
 type Contract struct {
-	Code string       `json:"code"`
-	Info ContractInfo `json:"info"`
+	Code        string       `json:"code"`
+	RuntimeCode string       `json:"runtime-code"`
+	Info        ContractInfo `json:"info"`
 }
 
 // ContractInfo contains information about a compiled contract, including access
-// to the ABI definition, user and developer docs, and metadata.
+// to the ABI definition, source mapping, user and developer docs, and metadata.
 //
 // Depending on the source, language version, compiler version, and compiler
 // options will provide information about how the contract was compiled.
@@ -48,6 +49,8 @@ type ContractInfo struct {
 	LanguageVersion string      `json:"languageVersion"`
 	CompilerVersion string      `json:"compilerVersion"`
 	CompilerOptions string      `json:"compilerOptions"`
+	SrcMap          string      `json:"srcMap"`
+	SrcMapRuntime   string      `json:"srcMapRuntime"`
 	AbiDefinition   interface{} `json:"abiDefinition"`
 	UserDoc         interface{} `json:"userDoc"`
 	DeveloperDoc    interface{} `json:"developerDoc"`
@@ -63,14 +66,16 @@ type Solidity struct {
 // --combined-output format
 type solcOutput struct {
 	Contracts map[string]struct {
-		Bin, Abi, Devdoc, Userdoc, Metadata string
+		BinRuntime                                  string `json:"bin-runtime"`
+		SrcMapRuntime                               string `json:"srcmap-runtime"`
+		Bin, SrcMap, Abi, Devdoc, Userdoc, Metadata string
 	}
 	Version string
 }
 
 func (s *Solidity) makeArgs() []string {
 	p := []string{
-		"--combined-json", "bin,abi,userdoc,devdoc",
+		"--combined-json", "bin,bin-runtime,srcmap,srcmap-runtime,abi,userdoc,devdoc",
 		"--optimize", // code optimizer switched on
 	}
 	if s.Major > 0 || s.Minor > 4 || s.Patch > 6 {
@@ -157,7 +162,7 @@ func (s *Solidity) run(cmd *exec.Cmd, source string) (map[string]*Contract, erro
 // provided source, language and compiler version, and compiler options are all
 // passed through into the Contract structs.
 //
-// The solc output is expected to contain ABI, user docs, and dev docs.
+// The solc output is expected to contain ABI, source mapping, user docs, and dev docs.
 //
 // Returns an error if the JSON is malformed or missing data, or if the JSON
 // embedded within the JSON is malformed.
@@ -184,13 +189,16 @@ func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion strin
 			return nil, fmt.Errorf("solc: error reading dev doc: %v", err)
 		}
 		contracts[name] = &Contract{
-			Code: "0x" + info.Bin,
+			Code:        "0x" + info.Bin,
+			RuntimeCode: "0x" + info.BinRuntime,
 			Info: ContractInfo{
 				Source:          source,
 				Language:        "Solidity",
 				LanguageVersion: languageVersion,
 				CompilerVersion: compilerVersion,
 				CompilerOptions: compilerOptions,
+				SrcMap:          info.SrcMap,
+				SrcMapRuntime:   info.SrcMapRuntime,
 				AbiDefinition:   abi,
 				UserDoc:         userdoc,
 				DeveloperDoc:    devdoc,

consensus/clique/clique.go

@@ -696,7 +696,7 @@ func (c *Clique) SealHash(header *types.Header) common.Hash {
 	return sigHash(header)
 }
 
-// Close implements consensus.Engine. It's a noop for clique as there is are no background threads.
+// Close implements consensus.Engine. It's a noop for clique as there are no background threads.
 func (c *Clique) Close() error {
 	return nil
 }

consensus/ethash/api.go

@@ -37,27 +37,28 @@ type API struct {
 //   result[0] - 32 bytes hex encoded current block header pow-hash
 //   result[1] - 32 bytes hex encoded seed hash used for DAG
 //   result[2] - 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty
-func (api *API) GetWork() ([3]string, error) {
+//   result[3] - hex encoded block number
+func (api *API) GetWork() ([4]string, error) {
 	if api.ethash.config.PowMode != ModeNormal && api.ethash.config.PowMode != ModeTest {
-		return [3]string{}, errors.New("not supported")
+		return [4]string{}, errors.New("not supported")
 	}
 
 	var (
-		workCh = make(chan [3]string, 1)
+		workCh = make(chan [4]string, 1)
 		errc   = make(chan error, 1)
 	)
 
 	select {
 	case api.ethash.fetchWorkCh <- &sealWork{errc: errc, res: workCh}:
 	case <-api.ethash.exitCh:
-		return [3]string{}, errEthashStopped
+		return [4]string{}, errEthashStopped
 	}
 
 	select {
 	case work := <-workCh:
 		return work, nil
 	case err := <-errc:
-		return [3]string{}, err
+		return [4]string{}, err
 	}
 }
 

consensus/ethash/ethash.go

@@ -432,7 +432,7 @@ type hashrate struct {
 // sealWork wraps a seal work package for remote sealer.
 type sealWork struct {
 	errc chan error
-	res  chan [3]string
+	res  chan [4]string
 }
 
 // Ethash is a consensus engine based on proof-of-work implementing the ethash

consensus/ethash/ethash_test.go

@@ -107,7 +107,7 @@ func TestRemoteSealer(t *testing.T) {
 	ethash.Seal(nil, block, results, nil)
 
 	var (
-		work [3]string
+		work [4]string
 		err  error
 	)
 	if work, err = api.GetWork(); err != nil || work[0] != sealhash.Hex() {

consensus/ethash/sealer.go

@@ -30,6 +30,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
@@ -193,7 +194,7 @@ func (ethash *Ethash) remote(notify []string, noverify bool) {
 
 		results      chan<- *types.Block
 		currentBlock *types.Block
-		currentWork  [3]string
+		currentWork  [4]string
 
 		notifyTransport = &http.Transport{}
 		notifyClient    = &http.Client{
@@ -234,12 +235,14 @@ func (ethash *Ethash) remote(notify []string, noverify bool) {
 	//   result[0], 32 bytes hex encoded current block header pow-hash
 	//   result[1], 32 bytes hex encoded seed hash used for DAG
 	//   result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty
+	//   result[3], hex encoded block number
 	makeWork := func(block *types.Block) {
 		hash := ethash.SealHash(block.Header())
 
 		currentWork[0] = hash.Hex()
 		currentWork[1] = common.BytesToHash(SeedHash(block.NumberU64())).Hex()
 		currentWork[2] = common.BytesToHash(new(big.Int).Div(two256, block.Difficulty()).Bytes()).Hex()
+		currentWork[3] = hexutil.EncodeBig(block.Number())
 
 		// Trace the seal work fetched by remote sealer.
 		currentBlock = block

core/block_validator.go

@@ -53,12 +53,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) {
 		return ErrKnownBlock
 	}
-	if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
-		if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
-			return consensus.ErrUnknownAncestor
-		}
-		return consensus.ErrPrunedAncestor
-	}
 	// Header validity is known at this point, check the uncles and transactions
 	header := block.Header()
 	if err := v.engine.VerifyUncles(v.bc, block); err != nil {
@@ -70,6 +64,12 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	if hash := types.DeriveSha(block.Transactions()); hash != header.TxHash {
 		return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash)
 	}
+	if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
+		if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
+			return consensus.ErrUnknownAncestor
+		}
+		return consensus.ErrPrunedAncestor
+	}
 	return nil
 }
 

core/blockchain.go

@@ -47,7 +47,10 @@ import (
 )
 
 var (
-	blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil)
+	blockInsertTimer     = metrics.NewRegisteredTimer("chain/inserts", nil)
+	blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil)
+	blockExecutionTimer  = metrics.NewRegisteredTimer("chain/execution", nil)
+	blockWriteTimer      = metrics.NewRegisteredTimer("chain/write", nil)
 
 	ErrNoGenesis = errors.New("Genesis not found in chain")
 )
@@ -68,9 +71,10 @@ const (
 // CacheConfig contains the configuration values for the trie caching/pruning
 // that's resident in a blockchain.
 type CacheConfig struct {
-	Disabled      bool          // Whether to disable trie write caching (archive node)
-	TrieNodeLimit int           // Memory limit (MB) at which to flush the current in-memory trie to disk
-	TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
+	Disabled       bool          // Whether to disable trie write caching (archive node)
+	TrieCleanLimit int           // Memory allowance (MB) to use for caching trie nodes in memory
+	TrieDirtyLimit int           // Memory limit (MB) at which to start flushing dirty trie nodes to disk
+	TrieTimeLimit  time.Duration // Time limit after which to flush the current in-memory trie to disk
 }
 
 // BlockChain represents the canonical chain given a database with a genesis
@@ -140,8 +144,9 @@ type BlockChain struct {
 func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool) (*BlockChain, error) {
 	if cacheConfig == nil {
 		cacheConfig = &CacheConfig{
-			TrieNodeLimit: 256 * 1024 * 1024,
-			TrieTimeLimit: 5 * time.Minute,
+			TrieCleanLimit: 256,
+			TrieDirtyLimit: 256,
+			TrieTimeLimit:  5 * time.Minute,
 		}
 	}
 	bodyCache, _ := lru.New(bodyCacheLimit)
@@ -156,7 +161,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 		cacheConfig:    cacheConfig,
 		db:             db,
 		triegc:         prque.New(nil),
-		stateCache:     state.NewDatabase(db),
+		stateCache:     state.NewDatabaseWithCache(db, cacheConfig.TrieCleanLimit),
 		quit:           make(chan struct{}),
 		shouldPreserve: shouldPreserve,
 		bodyCache:      bodyCache,
@@ -205,6 +210,11 @@ func (bc *BlockChain) getProcInterrupt() bool {
 	return atomic.LoadInt32(&bc.procInterrupt) == 1
 }
 
+// GetVMConfig returns the block chain VM config.
+func (bc *BlockChain) GetVMConfig() *vm.Config {
+	return &bc.vmConfig
+}
+
 // loadLastState loads the last known chain state from the database. This method
 // assumes that the chain manager mutex is held.
 func (bc *BlockChain) loadLastState() error {
@@ -393,6 +403,11 @@ func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) {
 	return state.New(root, bc.stateCache)
 }
 
+// StateCache returns the caching database underpinning the blockchain instance.
+func (bc *BlockChain) StateCache() state.Database {
+	return bc.stateCache
+}
+
 // Reset purges the entire blockchain, restoring it to its genesis state.
 func (bc *BlockChain) Reset() error {
 	return bc.ResetWithGenesisBlock(bc.genesisBlock)
@@ -438,7 +453,11 @@ func (bc *BlockChain) repair(head **types.Block) error {
 			return nil
 		}
 		// Otherwise rewind one block and recheck state availability there
-		(*head) = bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1)
+		block := bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1)
+		if block == nil {
+			return fmt.Errorf("missing block %d [%x]", (*head).NumberU64()-1, (*head).ParentHash())
+		}
+		(*head) = block
 	}
 }
 
@@ -554,6 +573,17 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool {
 	return rawdb.HasBody(bc.db, hash, number)
 }
 
+// HasFastBlock checks if a fast block is fully present in the database or not.
+func (bc *BlockChain) HasFastBlock(hash common.Hash, number uint64) bool {
+	if !bc.HasBlock(hash, number) {
+		return false
+	}
+	if bc.receiptsCache.Contains(hash) {
+		return true
+	}
+	return rawdb.HasReceipts(bc.db, hash, number)
+}
+
 // HasState checks if state trie is fully present in the database or not.
 func (bc *BlockChain) HasState(hash common.Hash) bool {
 	_, err := bc.stateCache.OpenTrie(hash)
@@ -611,12 +641,10 @@ func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
 	if receipts, ok := bc.receiptsCache.Get(hash); ok {
 		return receipts.(types.Receipts)
 	}
-
 	number := rawdb.ReadHeaderNumber(bc.db, hash)
 	if number == nil {
 		return nil
 	}
-
 	receipts := rawdb.ReadReceipts(bc.db, hash, *number)
 	bc.receiptsCache.Add(hash, receipts)
 	return receipts
@@ -938,7 +966,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 			// If we exceeded our memory allowance, flush matured singleton nodes to disk
 			var (
 				nodes, imgs = triedb.Size()
-				limit       = common.StorageSize(bc.cacheConfig.TrieNodeLimit) * 1024 * 1024
+				limit       = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024
 			)
 			if nodes > limit || imgs > 4*1024*1024 {
 				triedb.Cap(limit - ethdb.IdealBatchSize)
@@ -1002,7 +1030,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 		}
 		// Write the positional metadata for transaction/receipt lookups and preimages
 		rawdb.WriteTxLookupEntries(batch, block)
-		rawdb.WritePreimages(batch, block.NumberU64(), state.Preimages())
+		rawdb.WritePreimages(batch, state.Preimages())
 
 		status = CanonStatTy
 	} else {
@@ -1020,6 +1048,18 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 	return status, nil
 }
 
+// addFutureBlock checks if the block is within the max allowed window to get
+// accepted for future processing, and returns an error if the block is too far
+// ahead and was not added.
+func (bc *BlockChain) addFutureBlock(block *types.Block) error {
+	max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks)
+	if block.Time().Cmp(max) > 0 {
+		return fmt.Errorf("future block timestamp %v > allowed %v", block.Time(), max)
+	}
+	bc.futureBlocks.Add(block.Hash(), block)
+	return nil
+}
+
 // InsertChain attempts to insert the given batch of blocks in to the canonical
 // chain or, otherwise, create a fork. If an error is returned it will return
 // the index number of the failing block as well an error describing what went
@@ -1027,18 +1067,9 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 //
 // After insertion is done, all accumulated events will be fired.
 func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) {
-	n, events, logs, err := bc.insertChain(chain)
-	bc.PostChainEvents(events, logs)
-	return n, err
-}
-
-// insertChain will execute the actual chain insertion and event aggregation. The
-// only reason this method exists as a separate one is to make locking cleaner
-// with deferred statements.
-func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*types.Log, error) {
 	// Sanity check that we have something meaningful to import
 	if len(chain) == 0 {
-		return 0, nil, nil, nil
+		return 0, nil
 	}
 	// Do a sanity check that the provided chain is actually ordered and linked
 	for i := 1; i < len(chain); i++ {
@@ -1047,16 +1078,36 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 			log.Error("Non contiguous block insert", "number", chain[i].Number(), "hash", chain[i].Hash(),
 				"parent", chain[i].ParentHash(), "prevnumber", chain[i-1].Number(), "prevhash", chain[i-1].Hash())
 
-			return 0, nil, nil, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(),
+			return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(),
 				chain[i-1].Hash().Bytes()[:4], i, chain[i].NumberU64(), chain[i].Hash().Bytes()[:4], chain[i].ParentHash().Bytes()[:4])
 		}
 	}
 	// Pre-checks passed, start the full block imports
 	bc.wg.Add(1)
-	defer bc.wg.Done()
-
 	bc.chainmu.Lock()
-	defer bc.chainmu.Unlock()
+	n, events, logs, err := bc.insertChain(chain, true)
+	bc.chainmu.Unlock()
+	bc.wg.Done()
+
+	bc.PostChainEvents(events, logs)
+	return n, err
+}
+
+// insertChain is the internal implementation of insertChain, which assumes that
+// 1) chains are contiguous, and 2) The chain mutex is held.
+//
+// This method is split out so that import batches that require re-injecting
+// historical blocks can do so without releasing the lock, which could lead to
+// racey behaviour. If a sidechain import is in progress, and the historic state
+// is imported, but then new canon-head is added before the actual sidechain
+// completes, then the historic state could be pruned again
+func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []interface{}, []*types.Log, error) {
+	// If the chain is terminating, don't even bother starting u
+	if atomic.LoadInt32(&bc.procInterrupt) == 1 {
+		return 0, nil, nil, nil
+	}
+	// Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss)
+	senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number()), chain)
 
 	// A queued approach to delivering events. This is generally
 	// faster than direct delivery and requires much less mutex
@@ -1073,16 +1124,56 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 
 	for i, block := range chain {
 		headers[i] = block.Header()
-		seals[i] = true
+		seals[i] = verifySeals
 	}
 	abort, results := bc.engine.VerifyHeaders(bc, headers, seals)
 	defer close(abort)
 
-	// Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss)
-	senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number()), chain)
+	// Peek the error for the first block to decide the directing import logic
+	it := newInsertIterator(chain, results, bc.Validator())
 
-	// Iterate over the blocks and insert when the verifier permits
-	for i, block := range chain {
+	block, err := it.next()
+	switch {
+	// First block is pruned, insert as sidechain and reorg only if TD grows enough
+	case err == consensus.ErrPrunedAncestor:
+		return bc.insertSidechain(it)
+
+	// First block is future, shove it (and all children) to the future queue (unknown ancestor)
+	case err == consensus.ErrFutureBlock || (err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(it.first().ParentHash())):
+		for block != nil && (it.index == 0 || err == consensus.ErrUnknownAncestor) {
+			if err := bc.addFutureBlock(block); err != nil {
+				return it.index, events, coalescedLogs, err
+			}
+			block, err = it.next()
+		}
+		stats.queued += it.processed()
+		stats.ignored += it.remaining()
+
+		// If there are any still remaining, mark as ignored
+		return it.index, events, coalescedLogs, err
+
+	// First block (and state) is known
+	//   1. We did a roll-back, and should now do a re-import
+	//   2. The block is stored as a sidechain, and is lying about it's stateroot, and passes a stateroot
+	// 	    from the canonical chain, which has not been verified.
+	case err == ErrKnownBlock:
+		// Skip all known blocks that behind us
+		current := bc.CurrentBlock().NumberU64()
+
+		for block != nil && err == ErrKnownBlock && current >= block.NumberU64() {
+			stats.ignored++
+			block, err = it.next()
+		}
+		// Falls through to the block import
+
+	// Some other error occurred, abort
+	case err != nil:
+		stats.ignored += len(it.chain)
+		bc.reportBlock(block, nil, err)
+		return it.index, events, coalescedLogs, err
+	}
+	// No validation errors for the first block (or chain prefix skipped)
+	for ; block != nil && err == nil; block, err = it.next() {
 		// If the chain is terminating, stop processing blocks
 		if atomic.LoadInt32(&bc.procInterrupt) == 1 {
 			log.Debug("Premature abort during blocks processing")
@@ -1091,115 +1182,53 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 		// If the header is a banned one, straight out abort
 		if BadHashes[block.Hash()] {
 			bc.reportBlock(block, nil, ErrBlacklistedHash)
-			return i, events, coalescedLogs, ErrBlacklistedHash
+			return it.index, events, coalescedLogs, ErrBlacklistedHash
 		}
-		// Wait for the block's verification to complete
-		bstart := time.Now()
+		// Retrieve the parent block and it's state to execute on top
+		start := time.Now()
 
-		err := <-results
-		if err == nil {
-			err = bc.Validator().ValidateBody(block)
-		}
-		switch {
-		case err == ErrKnownBlock:
-			// Block and state both already known. However if the current block is below
-			// this number we did a rollback and we should reimport it nonetheless.
-			if bc.CurrentBlock().NumberU64() >= block.NumberU64() {
-				stats.ignored++
-				continue
-			}
-
-		case err == consensus.ErrFutureBlock:
-			// Allow up to MaxFuture second in the future blocks. If this limit is exceeded
-			// the chain is discarded and processed at a later time if given.
-			max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks)
-			if block.Time().Cmp(max) > 0 {
-				return i, events, coalescedLogs, fmt.Errorf("future block: %v > %v", block.Time(), max)
-			}
-			bc.futureBlocks.Add(block.Hash(), block)
-			stats.queued++
-			continue
-
-		case err == consensus.ErrUnknownAncestor && bc.futureBlocks.Contains(block.ParentHash()):
-			bc.futureBlocks.Add(block.Hash(), block)
-			stats.queued++
-			continue
-
-		case err == consensus.ErrPrunedAncestor:
-			// Block competing with the canonical chain, store in the db, but don't process
-			// until the competitor TD goes above the canonical TD
-			currentBlock := bc.CurrentBlock()
-			localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64())
-			externTd := new(big.Int).Add(bc.GetTd(block.ParentHash(), block.NumberU64()-1), block.Difficulty())
-			if localTd.Cmp(externTd) > 0 {
-				if err = bc.WriteBlockWithoutState(block, externTd); err != nil {
-					return i, events, coalescedLogs, err
-				}
-				continue
-			}
-			// Competitor chain beat canonical, gather all blocks from the common ancestor
-			var winner []*types.Block
-
-			parent := bc.GetBlock(block.ParentHash(), block.NumberU64()-1)
-			for !bc.HasState(parent.Root()) {
-				winner = append(winner, parent)
-				parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1)
-			}
-			for j := 0; j < len(winner)/2; j++ {
-				winner[j], winner[len(winner)-1-j] = winner[len(winner)-1-j], winner[j]
-			}
-			// Import all the pruned blocks to make the state available
-			bc.chainmu.Unlock()
-			_, evs, logs, err := bc.insertChain(winner)
-			bc.chainmu.Lock()
-			events, coalescedLogs = evs, logs
-
-			if err != nil {
-				return i, events, coalescedLogs, err
-			}
-
-		case err != nil:
-			bc.reportBlock(block, nil, err)
-			return i, events, coalescedLogs, err
-		}
-		// Create a new statedb using the parent block and report an
-		// error if it fails.
-		var parent *types.Block
-		if i == 0 {
+		parent := it.previous()
+		if parent == nil {
 			parent = bc.GetBlock(block.ParentHash(), block.NumberU64()-1)
-		} else {
-			parent = chain[i-1]
 		}
 		state, err := state.New(parent.Root(), bc.stateCache)
 		if err != nil {
-			return i, events, coalescedLogs, err
+			return it.index, events, coalescedLogs, err
 		}
 		// Process block using the parent state as reference point.
+		t0 := time.Now()
 		receipts, logs, usedGas, err := bc.processor.Process(block, state, bc.vmConfig)
+		t1 := time.Now()
 		if err != nil {
 			bc.reportBlock(block, receipts, err)
-			return i, events, coalescedLogs, err
+			return it.index, events, coalescedLogs, err
 		}
 		// Validate the state using the default validator
-		err = bc.Validator().ValidateState(block, parent, state, receipts, usedGas)
-		if err != nil {
+		if err := bc.Validator().ValidateState(block, parent, state, receipts, usedGas); err != nil {
 			bc.reportBlock(block, receipts, err)
-			return i, events, coalescedLogs, err
+			return it.index, events, coalescedLogs, err
 		}
-		proctime := time.Since(bstart)
+		t2 := time.Now()
+		proctime := time.Since(start)
 
 		// Write the block to the chain and get the status.
 		status, err := bc.WriteBlockWithState(block, receipts, state)
+		t3 := time.Now()
 		if err != nil {
-			return i, events, coalescedLogs, err
+			return it.index, events, coalescedLogs, err
 		}
+		blockInsertTimer.UpdateSince(start)
+		blockExecutionTimer.Update(t1.Sub(t0))
+		blockValidationTimer.Update(t2.Sub(t1))
+		blockWriteTimer.Update(t3.Sub(t2))
 		switch status {
 		case CanonStatTy:
-			log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(), "uncles", len(block.Uncles()),
-				"txs", len(block.Transactions()), "gas", block.GasUsed(), "elapsed", common.PrettyDuration(time.Since(bstart)))
+			log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(),
+				"uncles", len(block.Uncles()), "txs", len(block.Transactions()), "gas", block.GasUsed(),
+				"elapsed", common.PrettyDuration(time.Since(start)),
+				"root", block.Root())
 
 			coalescedLogs = append(coalescedLogs, logs...)
-			blockInsertTimer.UpdateSince(bstart)
 			events = append(events, ChainEvent{block, block.Hash(), logs})
 			lastCanon = block
 
@@ -1207,78 +1236,153 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty
 			bc.gcproc += proctime
 
 		case SideStatTy:
-			log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(), "diff", block.Difficulty(), "elapsed",
-				common.PrettyDuration(time.Since(bstart)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()))
-
-			blockInsertTimer.UpdateSince(bstart)
+			log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(),
+				"diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)),
+				"txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()),
+				"root", block.Root())
 			events = append(events, ChainSideEvent{block})
 		}
+		blockInsertTimer.UpdateSince(start)
 		stats.processed++
 		stats.usedGas += usedGas
 
 		cache, _ := bc.stateCache.TrieDB().Size()
-		stats.report(chain, i, cache)
+		stats.report(chain, it.index, cache)
 	}
+	// Any blocks remaining here? The only ones we care about are the future ones
+	if block != nil && err == consensus.ErrFutureBlock {
+		if err := bc.addFutureBlock(block); err != nil {
+			return it.index, events, coalescedLogs, err
+		}
+		block, err = it.next()
+
+		for ; block != nil && err == consensus.ErrUnknownAncestor; block, err = it.next() {
+			if err := bc.addFutureBlock(block); err != nil {
+				return it.index, events, coalescedLogs, err
+			}
+			stats.queued++
+		}
+	}
+	stats.ignored += it.remaining()
+
 	// Append a single chain head event if we've progressed the chain
 	if lastCanon != nil && bc.CurrentBlock().Hash() == lastCanon.Hash() {
 		events = append(events, ChainHeadEvent{lastCanon})
 	}
-	return 0, events, coalescedLogs, nil
+	return it.index, events, coalescedLogs, err
 }
 
-// insertStats tracks and reports on block insertion.
-type insertStats struct {
-	queued, processed, ignored int
-	usedGas                    uint64
-	lastIndex                  int
-	startTime                  mclock.AbsTime
-}
-
-// statsReportLimit is the time limit during import and export after which we
-// always print out progress. This avoids the user wondering what's going on.
-const statsReportLimit = 8 * time.Second
-
-// report prints statistics if some number of blocks have been processed
-// or more than a few seconds have passed since the last message.
-func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) {
-	// Fetch the timings for the batch
+// insertSidechain is called when an import batch hits upon a pruned ancestor
+// error, which happens when a sidechain with a sufficiently old fork-block is
+// found.
+//
+// The method writes all (header-and-body-valid) blocks to disk, then tries to
+// switch over to the new chain if the TD exceeded the current chain.
+func (bc *BlockChain) insertSidechain(it *insertIterator) (int, []interface{}, []*types.Log, error) {
 	var (
-		now     = mclock.Now()
-		elapsed = time.Duration(now) - time.Duration(st.startTime)
+		externTd *big.Int
+		current  = bc.CurrentBlock().NumberU64()
 	)
-	// If we're at the last block of the batch or report period reached, log
-	if index == len(chain)-1 || elapsed >= statsReportLimit {
-		var (
-			end = chain[index]
-			txs = countTransactions(chain[st.lastIndex : index+1])
-		)
-		context := []interface{}{
-			"blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000,
-			"elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
-			"number", end.Number(), "hash", end.Hash(),
-		}
-		if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute {
-			context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
-		}
-		context = append(context, []interface{}{"cache", cache}...)
+	// The first sidechain block error is already verified to be ErrPrunedAncestor.
+	// Since we don't import them here, we expect ErrUnknownAncestor for the remaining
+	// ones. Any other errors means that the block is invalid, and should not be written
+	// to disk.
+	block, err := it.current(), consensus.ErrPrunedAncestor
+	for ; block != nil && (err == consensus.ErrPrunedAncestor); block, err = it.next() {
+		// Check the canonical state root for that number
+		if number := block.NumberU64(); current >= number {
+			if canonical := bc.GetBlockByNumber(number); canonical != nil && canonical.Root() == block.Root() {
+				// This is most likely a shadow-state attack. When a fork is imported into the
+				// database, and it eventually reaches a block height which is not pruned, we
+				// just found that the state already exist! This means that the sidechain block
+				// refers to a state which already exists in our canon chain.
+				//
+				// If left unchecked, we would now proceed importing the blocks, without actually
+				// having verified the state of the previous blocks.
+				log.Warn("Sidechain ghost-state attack detected", "number", block.NumberU64(), "sideroot", block.Root(), "canonroot", canonical.Root())
 
-		if st.queued > 0 {
-			context = append(context, []interface{}{"queued", st.queued}...)
+				// If someone legitimately side-mines blocks, they would still be imported as usual. However,
+				// we cannot risk writing unverified blocks to disk when they obviously target the pruning
+				// mechanism.
+				return it.index, nil, nil, errors.New("sidechain ghost-state attack")
+			}
 		}
-		if st.ignored > 0 {
-			context = append(context, []interface{}{"ignored", st.ignored}...)
+		if externTd == nil {
+			externTd = bc.GetTd(block.ParentHash(), block.NumberU64()-1)
 		}
-		log.Info("Imported new chain segment", context...)
+		externTd = new(big.Int).Add(externTd, block.Difficulty())
 
-		*st = insertStats{startTime: now, lastIndex: index + 1}
+		if !bc.HasBlock(block.Hash(), block.NumberU64()) {
+			start := time.Now()
+			if err := bc.WriteBlockWithoutState(block, externTd); err != nil {
+				return it.index, nil, nil, err
+			}
+			log.Debug("Inserted sidechain block", "number", block.Number(), "hash", block.Hash(),
+				"diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)),
+				"txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()),
+				"root", block.Root())
+		}
 	}
-}
-
-func countTransactions(chain []*types.Block) (c int) {
-	for _, b := range chain {
-		c += len(b.Transactions())
+	// At this point, we've written all sidechain blocks to database. Loop ended
+	// either on some other error or all were processed. If there was some other
+	// error, we can ignore the rest of those blocks.
+	//
+	// If the externTd was larger than our local TD, we now need to reimport the previous
+	// blocks to regenerate the required state
+	localTd := bc.GetTd(bc.CurrentBlock().Hash(), current)
+	if localTd.Cmp(externTd) > 0 {
+		log.Info("Sidechain written to disk", "start", it.first().NumberU64(), "end", it.previous().NumberU64(), "sidetd", externTd, "localtd", localTd)
+		return it.index, nil, nil, err
 	}
-	return c
+	// Gather all the sidechain hashes (full blocks may be memory heavy)
+	var (
+		hashes  []common.Hash
+		numbers []uint64
+	)
+	parent := bc.GetHeader(it.previous().Hash(), it.previous().NumberU64())
+	for parent != nil && !bc.HasState(parent.Root) {
+		hashes = append(hashes, parent.Hash())
+		numbers = append(numbers, parent.Number.Uint64())
+
+		parent = bc.GetHeader(parent.ParentHash, parent.Number.Uint64()-1)
+	}
+	if parent == nil {
+		return it.index, nil, nil, errors.New("missing parent")
+	}
+	// Import all the pruned blocks to make the state available
+	var (
+		blocks []*types.Block
+		memory common.StorageSize
+	)
+	for i := len(hashes) - 1; i >= 0; i-- {
+		// Append the next block to our batch
+		block := bc.GetBlock(hashes[i], numbers[i])
+
+		blocks = append(blocks, block)
+		memory += block.Size()
+
+		// If memory use grew too large, import and continue. Sadly we need to discard
+		// all raised events and logs from notifications since we're too heavy on the
+		// memory here.
+		if len(blocks) >= 2048 || memory > 64*1024*1024 {
+			log.Info("Importing heavy sidechain segment", "blocks", len(blocks), "start", blocks[0].NumberU64(), "end", block.NumberU64())
+			if _, _, _, err := bc.insertChain(blocks, false); err != nil {
+				return 0, nil, nil, err
+			}
+			blocks, memory = blocks[:0], 0
+
+			// If the chain is terminating, stop processing blocks
+			if atomic.LoadInt32(&bc.procInterrupt) == 1 {
+				log.Debug("Premature abort during blocks processing")
+				return 0, nil, nil, nil
+			}
+		}
+	}
+	if len(blocks) > 0 {
+		log.Info("Importing sidechain segment", "start", blocks[0].NumberU64(), "end", blocks[len(blocks)-1].NumberU64())
+		return bc.insertChain(blocks, false)
+	}
+	return 0, nil, nil, nil
 }
 
 // reorgs takes two blocks, an old chain and a new chain and will reconstruct the blocks and inserts them
@@ -1453,8 +1557,10 @@ func (bc *BlockChain) reportBlock(block *types.Block, receipts types.Receipts, e
 	bc.addBadBlock(block)
 
 	var receiptString string
-	for _, receipt := range receipts {
-		receiptString += fmt.Sprintf("\t%v\n", receipt)
+	for i, receipt := range receipts {
+		receiptString += fmt.Sprintf("\t %d: cumulative: %v gas: %v contract: %v status: %v tx: %v logs: %v bloom: %x state: %x\n",
+			i, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.ContractAddress.Hex(),
+			receipt.Status, receipt.TxHash.Hex(), receipt.Logs, receipt.Bloom, receipt.PostState)
 	}
 	log.Error(fmt.Sprintf(`
 ########## BAD BLOCK #########

core/blockchain_insert.go

@@ -0,0 +1,143 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package core
+
+import (
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/mclock"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// insertStats tracks and reports on block insertion.
+type insertStats struct {
+	queued, processed, ignored int
+	usedGas                    uint64
+	lastIndex                  int
+	startTime                  mclock.AbsTime
+}
+
+// statsReportLimit is the time limit during import and export after which we
+// always print out progress. This avoids the user wondering what's going on.
+const statsReportLimit = 8 * time.Second
+
+// report prints statistics if some number of blocks have been processed
+// or more than a few seconds have passed since the last message.
+func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) {
+	// Fetch the timings for the batch
+	var (
+		now     = mclock.Now()
+		elapsed = time.Duration(now) - time.Duration(st.startTime)
+	)
+	// If we're at the last block of the batch or report period reached, log
+	if index == len(chain)-1 || elapsed >= statsReportLimit {
+		// Count the number of transactions in this segment
+		var txs int
+		for _, block := range chain[st.lastIndex : index+1] {
+			txs += len(block.Transactions())
+		}
+		end := chain[index]
+
+		// Assemble the log context and send it to the logger
+		context := []interface{}{
+			"blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000,
+			"elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
+			"number", end.Number(), "hash", end.Hash(),
+		}
+		if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute {
+			context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
+		}
+		context = append(context, []interface{}{"cache", cache}...)
+
+		if st.queued > 0 {
+			context = append(context, []interface{}{"queued", st.queued}...)
+		}
+		if st.ignored > 0 {
+			context = append(context, []interface{}{"ignored", st.ignored}...)
+		}
+		log.Info("Imported new chain segment", context...)
+
+		// Bump the stats reported to the next section
+		*st = insertStats{startTime: now, lastIndex: index + 1}
+	}
+}
+
+// insertIterator is a helper to assist during chain import.
+type insertIterator struct {
+	chain     types.Blocks
+	results   <-chan error
+	index     int
+	validator Validator
+}
+
+// newInsertIterator creates a new iterator based on the given blocks, which are
+// assumed to be a contiguous chain.
+func newInsertIterator(chain types.Blocks, results <-chan error, validator Validator) *insertIterator {
+	return &insertIterator{
+		chain:     chain,
+		results:   results,
+		index:     -1,
+		validator: validator,
+	}
+}
+
+// next returns the next block in the iterator, along with any potential validation
+// error for that block. When the end is reached, it will return (nil, nil).
+func (it *insertIterator) next() (*types.Block, error) {
+	if it.index+1 >= len(it.chain) {
+		it.index = len(it.chain)
+		return nil, nil
+	}
+	it.index++
+	if err := <-it.results; err != nil {
+		return it.chain[it.index], err
+	}
+	return it.chain[it.index], it.validator.ValidateBody(it.chain[it.index])
+}
+
+// current returns the current block that's being processed.
+func (it *insertIterator) current() *types.Block {
+	if it.index < 0 || it.index+1 >= len(it.chain) {
+		return nil
+	}
+	return it.chain[it.index]
+}
+
+// previous returns the previous block was being processed, or nil
+func (it *insertIterator) previous() *types.Block {
+	if it.index < 1 {
+		return nil
+	}
+	return it.chain[it.index-1]
+}
+
+// first returns the first block in the it.
+func (it *insertIterator) first() *types.Block {
+	return it.chain[0]
+}
+
+// remaining returns the number of remaining blocks.
+func (it *insertIterator) remaining() int {
+	return len(it.chain) - it.index
+}
+
+// processed returns the number of processed blocks.
+func (it *insertIterator) processed() int {
+	return it.index + 1
+}

core/blockchain_test.go

@@ -579,11 +579,11 @@ func testInsertNonceError(t *testing.T, full bool) {
 			blockchain.hc.engine = blockchain.engine
 			failRes, err = blockchain.InsertHeaderChain(headers, 1)
 		}
-		// Check that the returned error indicates the failure.
+		// Check that the returned error indicates the failure
 		if failRes != failAt {
-			t.Errorf("test %d: failure index mismatch: have %d, want %d", i, failRes, failAt)
+			t.Errorf("test %d: failure (%v) index mismatch: have %d, want %d", i, err, failRes, failAt)
 		}
-		// Check that all no blocks after the failing block have been inserted.
+		// Check that all blocks after the failing block have been inserted
 		for j := 0; j < i-failAt; j++ {
 			if full {
 				if block := blockchain.GetBlockByNumber(failNum + uint64(j)); block != nil {
@@ -1345,7 +1345,7 @@ func TestLargeReorgTrieGC(t *testing.T) {
 		t.Fatalf("failed to insert shared chain: %v", err)
 	}
 	if _, err := chain.InsertChain(original); err != nil {
-		t.Fatalf("failed to insert shared chain: %v", err)
+		t.Fatalf("failed to insert original chain: %v", err)
 	}
 	// Ensure that the state associated with the forking point is pruned away
 	if node, _ := chain.stateCache.TrieDB().Node(shared[len(shared)-1].Root()); node != nil {

core/chain_makers.go

@@ -33,12 +33,11 @@ import (
 // BlockGen creates blocks for testing.
 // See GenerateChain for a detailed explanation.
 type BlockGen struct {
-	i           int
-	parent      *types.Block
-	chain       []*types.Block
-	chainReader consensus.ChainReader
-	header      *types.Header
-	statedb     *state.StateDB
+	i       int
+	parent  *types.Block
+	chain   []*types.Block
+	header  *types.Header
+	statedb *state.StateDB
 
 	gasPool  *GasPool
 	txs      []*types.Transaction
@@ -138,7 +137,7 @@ func (b *BlockGen) AddUncle(h *types.Header) {
 // For index -1, PrevBlock returns the parent block given to GenerateChain.
 func (b *BlockGen) PrevBlock(index int) *types.Block {
 	if index >= b.i {
-		panic("block index out of range")
+		panic(fmt.Errorf("block index %d out of range (%d,%d)", index, -1, b.i))
 	}
 	if index == -1 {
 		return b.parent
@@ -154,7 +153,8 @@ func (b *BlockGen) OffsetTime(seconds int64) {
 	if b.header.Time.Cmp(b.parent.Header().Time) <= 0 {
 		panic("block time out of range")
 	}
-	b.header.Difficulty = b.engine.CalcDifficulty(b.chainReader, b.header.Time.Uint64(), b.parent.Header())
+	chainreader := &fakeChainReader{config: b.config}
+	b.header.Difficulty = b.engine.CalcDifficulty(chainreader, b.header.Time.Uint64(), b.parent.Header())
 }
 
 // GenerateChain creates a chain of n blocks. The first block's
@@ -174,14 +174,10 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		config = params.TestChainConfig
 	}
 	blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
+	chainreader := &fakeChainReader{config: config}
 	genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) {
-		// TODO(karalabe): This is needed for clique, which depends on multiple blocks.
-		// It's nonetheless ugly to spin up a blockchain here. Get rid of this somehow.
-		blockchain, _ := NewBlockChain(db, nil, config, engine, vm.Config{}, nil)
-		defer blockchain.Stop()
-
-		b := &BlockGen{i: i, parent: parent, chain: blocks, chainReader: blockchain, statedb: statedb, config: config, engine: engine}
-		b.header = makeHeader(b.chainReader, parent, statedb, b.engine)
+		b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine}
+		b.header = makeHeader(chainreader, parent, statedb, b.engine)
 
 		// Mutate the state and block according to any hard-fork specs
 		if daoBlock := config.DAOForkBlock; daoBlock != nil {
@@ -201,7 +197,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		}
 		if b.engine != nil {
 			// Finalize and seal the block
-			block, _ := b.engine.Finalize(b.chainReader, b.header, statedb, b.txs, b.uncles, b.receipts)
+			block, _ := b.engine.Finalize(chainreader, b.header, statedb, b.txs, b.uncles, b.receipts)
 
 			// Write state changes to db
 			root, err := statedb.Commit(config.IsEIP158(b.header.Number))
@@ -269,3 +265,19 @@ func makeBlockChain(parent *types.Block, n int, engine consensus.Engine, db ethd
 	})
 	return blocks
 }
+
+type fakeChainReader struct {
+	config  *params.ChainConfig
+	genesis *types.Block
+}
+
+// Config returns the chain configuration.
+func (cr *fakeChainReader) Config() *params.ChainConfig {
+	return cr.config
+}
+
+func (cr *fakeChainReader) CurrentHeader() *types.Header                            { return nil }
+func (cr *fakeChainReader) GetHeaderByNumber(number uint64) *types.Header           { return nil }
+func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *types.Header          { return nil }
+func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *types.Header { return nil }
+func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block   { return nil }

core/genesis.go

@@ -151,6 +151,9 @@ func (e *GenesisMismatchError) Error() string {
 //
 // The returned chain configuration is never nil.
 func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, error) {
+	return SetupGenesisBlockWithOverride(db, genesis, nil)
+}
+func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, constantinopleOverride *big.Int) (*params.ChainConfig, common.Hash, error) {
 	if genesis != nil && genesis.Config == nil {
 		return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig
 	}
@@ -178,6 +181,9 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig
 
 	// Get the existing chain configuration.
 	newcfg := genesis.configOrDefault(stored)
+	if constantinopleOverride != nil {
+		newcfg.ConstantinopleBlock = constantinopleOverride
+	}
 	storedcfg := rawdb.ReadChainConfig(db, stored)
 	if storedcfg == nil {
 		log.Warn("Found genesis block without chain config")

core/rawdb/accessors_chain.go

@@ -271,6 +271,15 @@ func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) {
 	}
 }
 
+// HasReceipts verifies the existence of all the transaction receipts belonging
+// to a block.
+func HasReceipts(db DatabaseReader, hash common.Hash, number uint64) bool {
+	if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil {
+		return false
+	}
+	return true
+}
+
 // ReadReceipts retrieves all the transaction receipts belonging to a block.
 func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts {
 	// Retrieve the flattened receipt slice
@@ -278,7 +287,7 @@ func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Rece
 	if len(data) == 0 {
 		return nil
 	}
-	// Convert the revceipts from their storage form to their internal representation
+	// Convert the receipts from their storage form to their internal representation
 	storageReceipts := []*types.ReceiptForStorage{}
 	if err := rlp.DecodeBytes(data, &storageReceipts); err != nil {
 		log.Error("Invalid receipt array RLP", "hash", hash, "err", err)

core/rawdb/accessors_metadata.go

@@ -77,9 +77,8 @@ func ReadPreimage(db DatabaseReader, hash common.Hash) []byte {
 	return data
 }
 
-// WritePreimages writes the provided set of preimages to the database. `number` is the
-// current block number, and is used for debug messages only.
-func WritePreimages(db DatabaseWriter, number uint64, preimages map[common.Hash][]byte) {
+// WritePreimages writes the provided set of preimages to the database.
+func WritePreimages(db DatabaseWriter, preimages map[common.Hash][]byte) {
 	for hash, preimage := range preimages {
 		if err := db.Put(preimageKey(hash), preimage); err != nil {
 			log.Crit("Failed to store trie preimage", "err", err)

core/rawdb/schema.go

@@ -35,7 +35,7 @@ var (
 	// headBlockKey tracks the latest know full block's hash.
 	headBlockKey = []byte("LastBlock")
 
-	// headFastBlockKey tracks the latest known incomplete block's hash duirng fast sync.
+	// headFastBlockKey tracks the latest known incomplete block's hash during fast sync.
 	headFastBlockKey = []byte("LastFast")
 
 	// fastTrieProgressKey tracks the number of trie entries imported during fast sync.

core/state/database.go

@@ -72,13 +72,19 @@ type Trie interface {
 }
 
 // NewDatabase creates a backing store for state. The returned database is safe for
-// concurrent use and retains cached trie nodes in memory. The pool is an optional
-// intermediate trie-node memory pool between the low level storage layer and the
-// high level trie abstraction.
+// concurrent use and retains a few recent expanded trie nodes in memory. To keep
+// more historical state in memory, use the NewDatabaseWithCache constructor.
 func NewDatabase(db ethdb.Database) Database {
+	return NewDatabaseWithCache(db, 0)
+}
+
+// NewDatabase creates a backing store for state. The returned database is safe for
+// concurrent use and retains both a few recent expanded trie nodes in memory, as
+// well as a lot of collapsed RLP trie nodes in a large memory cache.
+func NewDatabaseWithCache(db ethdb.Database, cache int) Database {
 	csc, _ := lru.New(codeSizeCacheSize)
 	return &cachingDB{
-		db:            trie.NewDatabase(db),
+		db:            trie.NewDatabaseWithCache(db, cache),
 		codeSizeCache: csc,
 	}
 }

core/state/statedb.go

@@ -18,10 +18,10 @@
 package state
 
 import (
+	"errors"
 	"fmt"
 	"math/big"
 	"sort"
-	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -44,6 +44,13 @@ var (
 	emptyCode = crypto.Keccak256Hash(nil)
 )
 
+type proofList [][]byte
+
+func (n *proofList) Put(key []byte, value []byte) error {
+	*n = append(*n, value)
+	return nil
+}
+
 // StateDBs within the ethereum protocol are used to store anything
 // within the merkle trie. StateDBs take care of caching and storing
 // nested states. It's the general query interface to retrieve:
@@ -79,8 +86,6 @@ type StateDB struct {
 	journal        *journal
 	validRevisions []revision
 	nextRevisionId int
-
-	lock sync.Mutex
 }
 
 // Create a new state from a given trie.
@@ -256,6 +261,24 @@ func (self *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash
 	return common.Hash{}
 }
 
+// GetProof returns the MerkleProof for a given Account
+func (self *StateDB) GetProof(a common.Address) ([][]byte, error) {
+	var proof proofList
+	err := self.trie.Prove(crypto.Keccak256(a.Bytes()), 0, &proof)
+	return [][]byte(proof), err
+}
+
+// GetProof returns the StorageProof for given key
+func (self *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, error) {
+	var proof proofList
+	trie := self.StorageTrie(a)
+	if trie == nil {
+		return proof, errors.New("storage trie for requested address does not exist")
+	}
+	err := trie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof)
+	return [][]byte(proof), err
+}
+
 // GetCommittedState retrieves a value from the given account's committed storage trie.
 func (self *StateDB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash {
 	stateObject := self.getStateObject(addr)
@@ -470,9 +493,6 @@ func (db *StateDB) ForEachStorage(addr common.Address, cb func(key, value common
 // Copy creates a deep, independent copy of the state.
 // Snapshots of the copied state cannot be applied to the copy.
 func (self *StateDB) Copy() *StateDB {
-	self.lock.Lock()
-	defer self.lock.Unlock()
-
 	// Copy all the basic fields, initialize the memory ones
 	state := &StateDB{
 		db:                self.db,

core/tx_pool.go

@@ -825,7 +825,7 @@ func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) []error {
 // addTxsLocked attempts to queue a batch of transactions if they are valid,
 // whilst assuming the transaction pool lock is already held.
 func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) []error {
-	// Add the batch of transaction, tracking the accepted ones
+	// Add the batch of transactions, tracking the accepted ones
 	dirty := make(map[common.Address]struct{})
 	errs := make([]error, len(txs))
 

core/types/block.go

@@ -81,8 +81,8 @@ type Header struct {
 	GasUsed     uint64         `json:"gasUsed"          gencodec:"required"`
 	Time        *big.Int       `json:"timestamp"        gencodec:"required"`
 	Extra       []byte         `json:"extraData"        gencodec:"required"`
-	MixDigest   common.Hash    `json:"mixHash"          gencodec:"required"`
-	Nonce       BlockNonce     `json:"nonce"            gencodec:"required"`
+	MixDigest   common.Hash    `json:"mixHash"`
+	Nonce       BlockNonce     `json:"nonce"`
 }
 
 // field type overrides for gencodec

core/types/gen_header_json.go

@@ -13,6 +13,7 @@ import (
 
 var _ = (*headerMarshaling)(nil)
 
+// MarshalJSON marshals as JSON.
 func (h Header) MarshalJSON() ([]byte, error) {
 	type Header struct {
 		ParentHash  common.Hash    `json:"parentHash"       gencodec:"required"`
@@ -28,8 +29,8 @@ func (h Header) MarshalJSON() ([]byte, error) {
 		GasUsed     hexutil.Uint64 `json:"gasUsed"          gencodec:"required"`
 		Time        *hexutil.Big   `json:"timestamp"        gencodec:"required"`
 		Extra       hexutil.Bytes  `json:"extraData"        gencodec:"required"`
-		MixDigest   common.Hash    `json:"mixHash"          gencodec:"required"`
-		Nonce       BlockNonce     `json:"nonce"            gencodec:"required"`
+		MixDigest   common.Hash    `json:"mixHash"`
+		Nonce       BlockNonce     `json:"nonce"`
 		Hash        common.Hash    `json:"hash"`
 	}
 	var enc Header
@@ -52,6 +53,7 @@ func (h Header) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&enc)
 }
 
+// UnmarshalJSON unmarshals from JSON.
 func (h *Header) UnmarshalJSON(input []byte) error {
 	type Header struct {
 		ParentHash  *common.Hash    `json:"parentHash"       gencodec:"required"`
@@ -67,8 +69,8 @@ func (h *Header) UnmarshalJSON(input []byte) error {
 		GasUsed     *hexutil.Uint64 `json:"gasUsed"          gencodec:"required"`
 		Time        *hexutil.Big    `json:"timestamp"        gencodec:"required"`
 		Extra       *hexutil.Bytes  `json:"extraData"        gencodec:"required"`
-		MixDigest   *common.Hash    `json:"mixHash"          gencodec:"required"`
-		Nonce       *BlockNonce     `json:"nonce"            gencodec:"required"`
+		MixDigest   *common.Hash    `json:"mixHash"`
+		Nonce       *BlockNonce     `json:"nonce"`
 	}
 	var dec Header
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -126,13 +128,11 @@ func (h *Header) UnmarshalJSON(input []byte) error {
 		return errors.New("missing required field 'extraData' for Header")
 	}
 	h.Extra = *dec.Extra
-	if dec.MixDigest == nil {
-		return errors.New("missing required field 'mixHash' for Header")
+	if dec.MixDigest != nil {
+		h.MixDigest = *dec.MixDigest
 	}
-	h.MixDigest = *dec.MixDigest
-	if dec.Nonce == nil {
-		return errors.New("missing required field 'nonce' for Header")
+	if dec.Nonce != nil {
+		h.Nonce = *dec.Nonce
 	}
-	h.Nonce = *dec.Nonce
 	return nil
 }

core/types/transaction_signing.go

@@ -136,7 +136,7 @@ func (s EIP155Signer) Sender(tx *Transaction) (common.Address, error) {
 	return recoverPlain(s.Hash(tx), tx.data.R, tx.data.S, V, true)
 }
 
-// WithSignature returns a new transaction with the given signature. This signature
+// SignatureValues returns signature values. This signature
 // needs to be in the [R || S || V] format where V is 0 or 1.
 func (s EIP155Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big.Int, err error) {
 	R, S, V, err = HomesteadSigner{}.SignatureValues(tx, sig)

core/vm/evm.go

@@ -339,6 +339,12 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte
 	contract := NewContract(caller, to, new(big.Int), gas)
 	contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
 
+	// We do an AddBalance of zero here, just in order to trigger a touch.
+	// This doesn't matter on Mainnet, where all empties are gone at the time of Byzantium,
+	// but is the correct thing to do and matters on other networks, in tests, and potential
+	// future scenarios
+	evm.StateDB.AddBalance(addr, bigZero)
+
 	// When an error was returned by the EVM or when setting the creation code
 	// above we revert to the snapshot and consume any gas remaining. Additionally
 	// when we're in Homestead this also counts for code storage gas errors.

core/vm/gen_structlog.go

@@ -13,20 +13,22 @@ import (
 
 var _ = (*structLogMarshaling)(nil)
 
+// MarshalJSON marshals as JSON.
 func (s StructLog) MarshalJSON() ([]byte, error) {
 	type StructLog struct {
-		Pc          uint64                      `json:"pc"`
-		Op          OpCode                      `json:"op"`
-		Gas         math.HexOrDecimal64         `json:"gas"`
-		GasCost     math.HexOrDecimal64         `json:"gasCost"`
-		Memory      hexutil.Bytes               `json:"memory"`
-		MemorySize  int                         `json:"memSize"`
-		Stack       []*math.HexOrDecimal256     `json:"stack"`
-		Storage     map[common.Hash]common.Hash `json:"-"`
-		Depth       int                         `json:"depth"`
-		Err         error                       `json:"-"`
-		OpName      string                      `json:"opName"`
-		ErrorString string                      `json:"error"`
+		Pc            uint64                      `json:"pc"`
+		Op            OpCode                      `json:"op"`
+		Gas           math.HexOrDecimal64         `json:"gas"`
+		GasCost       math.HexOrDecimal64         `json:"gasCost"`
+		Memory        hexutil.Bytes               `json:"memory"`
+		MemorySize    int                         `json:"memSize"`
+		Stack         []*math.HexOrDecimal256     `json:"stack"`
+		Storage       map[common.Hash]common.Hash `json:"-"`
+		Depth         int                         `json:"depth"`
+		RefundCounter uint64                      `json:"refund"`
+		Err           error                       `json:"-"`
+		OpName        string                      `json:"opName"`
+		ErrorString   string                      `json:"error"`
 	}
 	var enc StructLog
 	enc.Pc = s.Pc
@@ -43,24 +45,27 @@ func (s StructLog) MarshalJSON() ([]byte, error) {
 	}
 	enc.Storage = s.Storage
 	enc.Depth = s.Depth
+	enc.RefundCounter = s.RefundCounter
 	enc.Err = s.Err
 	enc.OpName = s.OpName()
 	enc.ErrorString = s.ErrorString()
 	return json.Marshal(&enc)
 }
 
+// UnmarshalJSON unmarshals from JSON.
 func (s *StructLog) UnmarshalJSON(input []byte) error {
 	type StructLog struct {
-		Pc         *uint64                     `json:"pc"`
-		Op         *OpCode                     `json:"op"`
-		Gas        *math.HexOrDecimal64        `json:"gas"`
-		GasCost    *math.HexOrDecimal64        `json:"gasCost"`
-		Memory     *hexutil.Bytes              `json:"memory"`
-		MemorySize *int                        `json:"memSize"`
-		Stack      []*math.HexOrDecimal256     `json:"stack"`
-		Storage    map[common.Hash]common.Hash `json:"-"`
-		Depth      *int                        `json:"depth"`
-		Err        error                       `json:"-"`
+		Pc            *uint64                     `json:"pc"`
+		Op            *OpCode                     `json:"op"`
+		Gas           *math.HexOrDecimal64        `json:"gas"`
+		GasCost       *math.HexOrDecimal64        `json:"gasCost"`
+		Memory        *hexutil.Bytes              `json:"memory"`
+		MemorySize    *int                        `json:"memSize"`
+		Stack         []*math.HexOrDecimal256     `json:"stack"`
+		Storage       map[common.Hash]common.Hash `json:"-"`
+		Depth         *int                        `json:"depth"`
+		RefundCounter *uint64                     `json:"refund"`
+		Err           error                       `json:"-"`
 	}
 	var dec StructLog
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -96,6 +101,9 @@ func (s *StructLog) UnmarshalJSON(input []byte) error {
 	if dec.Depth != nil {
 		s.Depth = *dec.Depth
 	}
+	if dec.RefundCounter != nil {
+		s.RefundCounter = *dec.RefundCounter
+	}
 	if dec.Err != nil {
 		s.Err = dec.Err
 	}