params, swarm: release Geth v1.8.18 and Swarm v0.3.6

- Replace "crypto/rand" to "math/rand" for files content generation
- Remove swarm/network_test.go.Shuffle and swarm/btm/btm_test.go.Shuffle - because go1.9 support dropped (see https://github.com/ethereum/go-ethereum/pull/17807 and comments to swarm/network_test.go.Shuffle)

.github/CODEOWNERS

@@ -9,23 +9,26 @@ les/                            @zsfelfoldi
 light/                          @zsfelfoldi
 mobile/                         @karalabe
 p2p/                            @fjl @zsfelfoldi
+p2p/simulations                 @lmars
+p2p/protocols                   @zelig
+swarm/api/http                  @justelad
 swarm/bmt                       @zelig
 swarm/dev                       @lmars
 swarm/fuse                      @jmozah @holisticode
 swarm/grafana_dashboards        @nonsense
 swarm/metrics                   @nonsense @holisticode
 swarm/multihash                 @nolash
-swarm/network/bitvector         @zelig @janos @gbalint
+swarm/network/bitvector         @zelig @janos 
-swarm/network/priorityqueue     @zelig @janos @gbalint
+swarm/network/priorityqueue     @zelig @janos 
-swarm/network/simulations       @zelig
+swarm/network/simulations       @zelig @janos
-swarm/network/stream            @janos @zelig @gbalint @holisticode @justelad
+swarm/network/stream            @janos @zelig @holisticode @justelad
 swarm/network/stream/intervals  @janos
 swarm/network/stream/testing    @zelig
 swarm/pot                       @zelig
 swarm/pss                       @nolash @zelig @nonsense
 swarm/services                  @zelig
 swarm/state                     @justelad
-swarm/storage/encryption        @gbalint @zelig @nagydani
+swarm/storage/encryption        @zelig @nagydani
 swarm/storage/mock              @janos
 swarm/storage/feed              @nolash @jpeletier
 swarm/testutil                  @lmars

.travis.yml

@@ -148,7 +148,7 @@ matrix:
       git:
         submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.11.1.linux-amd64.tar.gz | tar -xz
+        - curl https://storage.googleapis.com/golang/go1.11.2.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go

accounts/hd.go

@@ -30,8 +30,8 @@ import (
 var DefaultRootDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0}
 
 // DefaultBaseDerivationPath is the base path from which custom derivation endpoints
-// are incremented. As such, the first account will be at m/44'/60'/0'/0, the second
+// are incremented. As such, the first account will be at m/44'/60'/0'/0/0, the second
-// at m/44'/60'/0'/1, etc.
+// at m/44'/60'/0'/0/1, etc.
 var DefaultBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 + 60, 0x80000000 + 0, 0, 0}
 
 // DefaultLedgerBaseDerivationPath is the base path from which custom derivation endpoints

accounts/keystore/key.go

@@ -66,19 +66,19 @@ type plainKeyJSON struct {
 
 type encryptedKeyJSONV3 struct {
 	Address string     `json:"address"`
-	Crypto  cryptoJSON `json:"crypto"`
+	Crypto  CryptoJSON `json:"crypto"`
 	Id      string     `json:"id"`
 	Version int        `json:"version"`
 }
 
 type encryptedKeyJSONV1 struct {
 	Address string     `json:"address"`
-	Crypto  cryptoJSON `json:"crypto"`
+	Crypto  CryptoJSON `json:"crypto"`
 	Id      string     `json:"id"`
 	Version string     `json:"version"`
 }
 
-type cryptoJSON struct {
+type CryptoJSON struct {
 	Cipher       string                 `json:"cipher"`
 	CipherText   string                 `json:"ciphertext"`
 	CipherParams cipherparamsJSON       `json:"cipherparams"`

accounts/keystore/keystore_passphrase.go

@@ -135,29 +135,26 @@ func (ks keyStorePassphrase) JoinPath(filename string) string {
 	return filepath.Join(ks.keysDirPath, filename)
 }
 
-// EncryptKey encrypts a key using the specified scrypt parameters into a json
+// Encryptdata encrypts the data given as 'data' with the password 'auth'.
-// blob that can be decrypted later on.
+func EncryptDataV3(data, auth []byte, scryptN, scryptP int) (CryptoJSON, error) {
-func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
-	authArray := []byte(auth)
 
 	salt := make([]byte, 32)
 	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
 		panic("reading from crypto/rand failed: " + err.Error())
 	}
-	derivedKey, err := scrypt.Key(authArray, salt, scryptN, scryptR, scryptP, scryptDKLen)
+	derivedKey, err := scrypt.Key(auth, salt, scryptN, scryptR, scryptP, scryptDKLen)
 	if err != nil {
-		return nil, err
+		return CryptoJSON{}, err
 	}
 	encryptKey := derivedKey[:16]
-	keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
 
 	iv := make([]byte, aes.BlockSize) // 16
 	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
 		panic("reading from crypto/rand failed: " + err.Error())
 	}
-	cipherText, err := aesCTRXOR(encryptKey, keyBytes, iv)
+	cipherText, err := aesCTRXOR(encryptKey, data, iv)
 	if err != nil {
-		return nil, err
+		return CryptoJSON{}, err
 	}
 	mac := crypto.Keccak256(derivedKey[16:32], cipherText)
 
@@ -167,12 +164,11 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
 	scryptParamsJSON["p"] = scryptP
 	scryptParamsJSON["dklen"] = scryptDKLen
 	scryptParamsJSON["salt"] = hex.EncodeToString(salt)
-
 	cipherParamsJSON := cipherparamsJSON{
 		IV: hex.EncodeToString(iv),
 	}
 
-	cryptoStruct := cryptoJSON{
+	cryptoStruct := CryptoJSON{
 		Cipher:       "aes-128-ctr",
 		CipherText:   hex.EncodeToString(cipherText),
 		CipherParams: cipherParamsJSON,
@@ -180,6 +176,17 @@ func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
 		KDFParams:    scryptParamsJSON,
 		MAC:          hex.EncodeToString(mac),
 	}
+	return cryptoStruct, nil
+}
+
+// EncryptKey encrypts a key using the specified scrypt parameters into a json
+// blob that can be decrypted later on.
+func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
+	keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
+	cryptoStruct, err := EncryptDataV3(keyBytes, []byte(auth), scryptN, scryptP)
+	if err != nil {
+		return nil, err
+	}
 	encryptedKeyJSONV3 := encryptedKeyJSONV3{
 		hex.EncodeToString(key.Address[:]),
 		cryptoStruct,
@@ -226,43 +233,48 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) {
 		PrivateKey: key,
 	}, nil
 }
+func DecryptDataV3(cryptoJson CryptoJSON, auth string) ([]byte, error) {
+	if cryptoJson.Cipher != "aes-128-ctr" {
+		return nil, fmt.Errorf("Cipher not supported: %v", cryptoJson.Cipher)
+	}
+	mac, err := hex.DecodeString(cryptoJson.MAC)
+	if err != nil {
+		return nil, err
+	}
+
+	iv, err := hex.DecodeString(cryptoJson.CipherParams.IV)
+	if err != nil {
+		return nil, err
+	}
+
+	cipherText, err := hex.DecodeString(cryptoJson.CipherText)
+	if err != nil {
+		return nil, err
+	}
+
+	derivedKey, err := getKDFKey(cryptoJson, auth)
+	if err != nil {
+		return nil, err
+	}
+
+	calculatedMAC := crypto.Keccak256(derivedKey[16:32], cipherText)
+	if !bytes.Equal(calculatedMAC, mac) {
+		return nil, ErrDecrypt
+	}
+
+	plainText, err := aesCTRXOR(derivedKey[:16], cipherText, iv)
+	if err != nil {
+		return nil, err
+	}
+	return plainText, err
+}
 
 func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byte, keyId []byte, err error) {
 	if keyProtected.Version != version {
 		return nil, nil, fmt.Errorf("Version not supported: %v", keyProtected.Version)
 	}
-
-	if keyProtected.Crypto.Cipher != "aes-128-ctr" {
-		return nil, nil, fmt.Errorf("Cipher not supported: %v", keyProtected.Crypto.Cipher)
-	}
-
 	keyId = uuid.Parse(keyProtected.Id)
-	mac, err := hex.DecodeString(keyProtected.Crypto.MAC)
+	plainText, err := DecryptDataV3(keyProtected.Crypto, auth)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	iv, err := hex.DecodeString(keyProtected.Crypto.CipherParams.IV)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	cipherText, err := hex.DecodeString(keyProtected.Crypto.CipherText)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	derivedKey, err := getKDFKey(keyProtected.Crypto, auth)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	calculatedMAC := crypto.Keccak256(derivedKey[16:32], cipherText)
-	if !bytes.Equal(calculatedMAC, mac) {
-		return nil, nil, ErrDecrypt
-	}
-
-	plainText, err := aesCTRXOR(derivedKey[:16], cipherText, iv)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -303,7 +315,7 @@ func decryptKeyV1(keyProtected *encryptedKeyJSONV1, auth string) (keyBytes []byt
 	return plainText, keyId, err
 }
 
-func getKDFKey(cryptoJSON cryptoJSON, auth string) ([]byte, error) {
+func getKDFKey(cryptoJSON CryptoJSON, auth string) ([]byte, error) {
 	authArray := []byte(auth)
 	salt, err := hex.DecodeString(cryptoJSON.KDFParams["salt"].(string))
 	if err != nil {

accounts/usbwallet/ledger.go

@@ -350,7 +350,7 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction
 		signer = new(types.HomesteadSigner)
 	} else {
 		signer = types.NewEIP155Signer(chainID)
-		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
+		signature[64] -= byte(chainID.Uint64()*2 + 35)
 	}
 	signed, err := tx.WithSignature(signer, signature)
 	if err != nil {

accounts/usbwallet/trezor.go

@@ -221,7 +221,7 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction
 		signer = new(types.HomesteadSigner)
 	} else {
 		signer = types.NewEIP155Signer(chainID)
-		signature[64] = signature[64] - byte(chainID.Uint64()*2+35)
+		signature[64] -= byte(chainID.Uint64()*2 + 35)
 	}
 	// Inject the final signature into the transaction and sanity check the sender
 	signed, err := tx.WithSignature(signer, signature)

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.1.windows-%GETH_ARCH%.zip
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.2.windows-%GETH_ARCH%.zip
-  - 7z x go1.11.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - 7z x go1.11.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

cmd/bootnode/main.go

@@ -38,7 +38,7 @@ func main() {
 	var (
 		listenAddr  = flag.String("addr", ":30301", "listen address")
 		genKey      = flag.String("genkey", "", "generate a node key")
-		writeAddr   = flag.Bool("writeaddress", false, "write out the node's pubkey hash and quit")
+		writeAddr   = flag.Bool("writeaddress", false, "write out the node's public key and quit")
 		nodeKeyFile = flag.String("nodekey", "", "private key filename")
 		nodeKeyHex  = flag.String("nodekeyhex", "", "private key as hex (for testing)")
 		natdesc     = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|extip:<IP>)")
@@ -86,7 +86,7 @@ func main() {
 	}
 
 	if *writeAddr {
-		fmt.Printf("%v\n", enode.PubkeyToIDV4(&nodeKey.PublicKey))
+		fmt.Printf("%x\n", crypto.FromECDSAPub(&nodeKey.PublicKey)[1:])
 		os.Exit(0)
 	}
 
@@ -119,16 +119,17 @@ func main() {
 	}
 
 	if *runv5 {
-		if _, err := discv5.ListenUDP(nodeKey, conn, realaddr, "", restrictList); err != nil {
+		if _, err := discv5.ListenUDP(nodeKey, conn, "", restrictList); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	} else {
+		db, _ := enode.OpenDB("")
+		ln := enode.NewLocalNode(db, nodeKey)
 		cfg := discover.Config{
-			PrivateKey:   nodeKey,
+			PrivateKey:  nodeKey,
-			AnnounceAddr: realaddr,
+			NetRestrict: restrictList,
-			NetRestrict:  restrictList,
 		}
-		if _, err := discover.ListenUDP(conn, cfg); err != nil {
+		if _, err := discover.ListenUDP(conn, ln, cfg); err != nil {
 			utils.Fatalf("%v", err)
 		}
 	}

cmd/clef/README.md

@@ -91,7 +91,7 @@ invoking methods with the following info:
   * [x] Version info about the signer
   * [x] Address of API (http/ipc)
   * [ ] List of known accounts
-* [ ] Have a default timeout on signing operations, so that if the user has not answered withing e.g. 60 seconds, the request is rejected.
+* [ ] Have a default timeout on signing operations, so that if the user has not answered within e.g. 60 seconds, the request is rejected.
 * [ ] `account_signRawTransaction`
 * [ ] `account_bulkSignTransactions([] transactions)` should
    * only exist if enabled via config/flag
@@ -129,7 +129,7 @@ The signer listens to HTTP requests on `rpcaddr`:`rpcport`, with the same JSONRP
 expected to be JSON [jsonrpc 2.0 standard](http://www.jsonrpc.org/specification).
 
 Some of these call can require user interaction. Clients must be aware that responses
-may be delayed significanlty or may never be received if a users decides to ignore the confirmation request.
+may be delayed significantly or may never be received if a users decides to ignore the confirmation request.
 
 The External API is **untrusted** : it does not accept credentials over this api, nor does it expect
 that requests have any authority.
@@ -862,7 +862,7 @@ A UI should conform to the following rules.
 * A UI SHOULD inform the user about the `SHA256` or `MD5` hash of the binary being executed
 * A UI SHOULD NOT maintain a secondary storage of data, e.g. list of accounts
   * The signer provides accounts
-* A UI SHOULD, to the best extent possible, use static linking / bundling, so that requried libraries are bundled
+* A UI SHOULD, to the best extent possible, use static linking / bundling, so that required libraries are bundled
 along with the UI.
 
 

cmd/clef/intapi_changelog.md

@@ -1,5 +1,9 @@
 ### Changelog for internal API (ui-api)
 
+### 3.0.0
+
+* Make use of `OnInputRequired(info UserInputRequest)` for obtaining master password during startup
+
 ### 2.1.0
 
 * Add `OnInputRequired(info UserInputRequest)` to internal API. This method is used when Clef needs user input, e.g. passwords.
@@ -14,7 +18,6 @@ The following structures are used:
        UserInputResponse struct {
                Text string `json:"text"`
        }
-```
 
 ### 2.0.0
 

cmd/clef/main.go

@@ -35,8 +35,10 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
@@ -48,10 +50,10 @@ import (
 )
 
 // ExternalAPIVersion -- see extapi_changelog.md
-const ExternalAPIVersion = "3.0.0"
+const ExternalAPIVersion = "4.0.0"
 
 // InternalAPIVersion -- see intapi_changelog.md
-const InternalAPIVersion = "2.0.0"
+const InternalAPIVersion = "3.0.0"
 
 const legalWarning = `
 WARNING! 
@@ -91,7 +93,7 @@ var (
 	}
 	signerSecretFlag = cli.StringFlag{
 		Name:  "signersecret",
-		Usage: "A file containing the password used to encrypt Clef credentials, e.g. keystore credentials and ruleset hash",
+		Usage: "A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash",
 	}
 	dBFlag = cli.StringFlag{
 		Name:  "4bytedb",
@@ -155,18 +157,18 @@ Whenever you make an edit to the rule file, you need to use attestation to tell
 Clef that the file is 'safe' to execute.`,
 	}
 
-	addCredentialCommand = cli.Command{
+	setCredentialCommand = cli.Command{
-		Action:    utils.MigrateFlags(addCredential),
+		Action:    utils.MigrateFlags(setCredential),
-		Name:      "addpw",
+		Name:      "setpw",
 		Usage:     "Store a credential for a keystore file",
-		ArgsUsage: "<address> <password>",
+		ArgsUsage: "<address>",
 		Flags: []cli.Flag{
 			logLevelFlag,
 			configdirFlag,
 			signerSecretFlag,
 		},
 		Description: `
-The addpw command stores a password for a given address (keyfile). If you invoke it with only one parameter, it will 
+		The setpw command stores a password for a given address (keyfile). If you enter a blank passphrase, it will 
 remove any stored credential for that address (keyfile)
 `,
 	}
@@ -198,7 +200,7 @@ func init() {
 		advancedMode,
 	}
 	app.Action = signer
-	app.Commands = []cli.Command{initCommand, attestCommand, addCredentialCommand}
+	app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand}
 
 }
 func main() {
@@ -212,25 +214,45 @@ func initializeSecrets(c *cli.Context) error {
 	if err := initialize(c); err != nil {
 		return err
 	}
-	configDir := c.String(configdirFlag.Name)
+	configDir := c.GlobalString(configdirFlag.Name)
 
 	masterSeed := make([]byte, 256)
-	n, err := io.ReadFull(rand.Reader, masterSeed)
+	num, err := io.ReadFull(rand.Reader, masterSeed)
 	if err != nil {
 		return err
 	}
-	if n != len(masterSeed) {
+	if num != len(masterSeed) {
 		return fmt.Errorf("failed to read enough random")
 	}
+
+	n, p := keystore.StandardScryptN, keystore.StandardScryptP
+	if c.GlobalBool(utils.LightKDFFlag.Name) {
+		n, p = keystore.LightScryptN, keystore.LightScryptP
+	}
+	text := "The master seed of clef is locked with a password. Please give a password. Do not forget this password."
+	var password string
+	for {
+		password = getPassPhrase(text, true)
+		if err := core.ValidatePasswordFormat(password); err != nil {
+			fmt.Printf("invalid password: %v\n", err)
+		} else {
+			break
+		}
+	}
+	cipherSeed, err := encryptSeed(masterSeed, []byte(password), n, p)
+	if err != nil {
+		return fmt.Errorf("failed to encrypt master seed: %v", err)
+	}
+
 	err = os.Mkdir(configDir, 0700)
 	if err != nil && !os.IsExist(err) {
 		return err
 	}
-	location := filepath.Join(configDir, "secrets.dat")
+	location := filepath.Join(configDir, "masterseed.json")
 	if _, err := os.Stat(location); err == nil {
 		return fmt.Errorf("file %v already exists, will not overwrite", location)
 	}
-	err = ioutil.WriteFile(location, masterSeed, 0400)
+	err = ioutil.WriteFile(location, cipherSeed, 0400)
 	if err != nil {
 		return err
 	}
@@ -255,11 +277,11 @@ func attestFile(ctx *cli.Context) error {
 		return err
 	}
 
-	stretchedKey, err := readMasterKey(ctx)
+	stretchedKey, err := readMasterKey(ctx, nil)
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	configDir := ctx.String(configdirFlag.Name)
+	configDir := ctx.GlobalString(configdirFlag.Name)
 	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 	confKey := crypto.Keccak256([]byte("config"), stretchedKey)
 
@@ -271,38 +293,36 @@ func attestFile(ctx *cli.Context) error {
 	return nil
 }
 
-func addCredential(ctx *cli.Context) error {
+func setCredential(ctx *cli.Context) error {
 	if len(ctx.Args()) < 1 {
-		utils.Fatalf("This command requires at leaste one argument.")
+		utils.Fatalf("This command requires an address to be passed as an argument.")
 	}
 	if err := initialize(ctx); err != nil {
 		return err
 	}
 
-	stretchedKey, err := readMasterKey(ctx)
+	address := ctx.Args().First()
+	password := getPassPhrase("Enter a passphrase to store with this address.", true)
+
+	stretchedKey, err := readMasterKey(ctx, nil)
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	configDir := ctx.String(configdirFlag.Name)
+	configDir := ctx.GlobalString(configdirFlag.Name)
 	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10]))
 	pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey)
 
 	// Initialize the encrypted storages
 	pwStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "credentials.json"), pwkey)
-	key := ctx.Args().First()
+	pwStorage.Put(address, password)
-	value := ""
+	log.Info("Credential store updated", "key", address)
-	if len(ctx.Args()) > 1 {
-		value = ctx.Args().Get(1)
-	}
-	pwStorage.Put(key, value)
-	log.Info("Credential store updated", "key", key)
 	return nil
 }
 
 func initialize(c *cli.Context) error {
 	// Set up the logger to print everything
 	logOutput := os.Stdout
-	if c.Bool(stdiouiFlag.Name) {
+	if c.GlobalBool(stdiouiFlag.Name) {
 		logOutput = os.Stderr
 		// If using the stdioui, we can't do the 'confirm'-flow
 		fmt.Fprintf(logOutput, legalWarning)
@@ -323,26 +343,28 @@ func signer(c *cli.Context) error {
 	var (
 		ui core.SignerUI
 	)
-	if c.Bool(stdiouiFlag.Name) {
+	if c.GlobalBool(stdiouiFlag.Name) {
 		log.Info("Using stdin/stdout as UI-channel")
 		ui = core.NewStdIOUI()
 	} else {
 		log.Info("Using CLI as UI-channel")
 		ui = core.NewCommandlineUI()
 	}
-	db, err := core.NewAbiDBFromFiles(c.String(dBFlag.Name), c.String(customDBFlag.Name))
+	fourByteDb := c.GlobalString(dBFlag.Name)
+	fourByteLocal := c.GlobalString(customDBFlag.Name)
+	db, err := core.NewAbiDBFromFiles(fourByteDb, fourByteLocal)
 	if err != nil {
 		utils.Fatalf(err.Error())
 	}
-	log.Info("Loaded 4byte db", "signatures", db.Size(), "file", c.String("4bytedb"))
+	log.Info("Loaded 4byte db", "signatures", db.Size(), "file", fourByteDb, "local", fourByteLocal)
 
 	var (
 		api core.ExternalAPI
 	)
 
-	configDir := c.String(configdirFlag.Name)
+	configDir := c.GlobalString(configdirFlag.Name)
-	if stretchedKey, err := readMasterKey(c); err != nil {
+	if stretchedKey, err := readMasterKey(c, ui); err != nil {
-		log.Info("No master seed provided, rules disabled")
+		log.Info("No master seed provided, rules disabled", "error", err)
 	} else {
 
 		if err != nil {
@@ -361,7 +383,7 @@ func signer(c *cli.Context) error {
 		configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey)
 
 		//Do we have a rule-file?
-		ruleJS, err := ioutil.ReadFile(c.String(ruleFlag.Name))
+		ruleJS, err := ioutil.ReadFile(c.GlobalString(ruleFlag.Name))
 		if err != nil {
 			log.Info("Could not load rulefile, rules not enabled", "file", "rulefile")
 		} else {
@@ -385,17 +407,15 @@ func signer(c *cli.Context) error {
 	}
 
 	apiImpl := core.NewSignerAPI(
-		c.Int64(utils.NetworkIdFlag.Name),
+		c.GlobalInt64(utils.NetworkIdFlag.Name),
-		c.String(keystoreFlag.Name),
+		c.GlobalString(keystoreFlag.Name),
-		c.Bool(utils.NoUSBFlag.Name),
+		c.GlobalBool(utils.NoUSBFlag.Name),
 		ui, db,
-		c.Bool(utils.LightKDFFlag.Name),
+		c.GlobalBool(utils.LightKDFFlag.Name),
-		c.Bool(advancedMode.Name))
+		c.GlobalBool(advancedMode.Name))
-
 	api = apiImpl
-
 	// Audit logging
-	if logfile := c.String(auditLogFlag.Name); logfile != "" {
+	if logfile := c.GlobalString(auditLogFlag.Name); logfile != "" {
 		api, err = core.NewAuditLogger(logfile, api)
 		if err != nil {
 			utils.Fatalf(err.Error())
@@ -414,13 +434,13 @@ func signer(c *cli.Context) error {
 			Service:   api,
 			Version:   "1.0"},
 	}
-	if c.Bool(utils.RPCEnabledFlag.Name) {
+	if c.GlobalBool(utils.RPCEnabledFlag.Name) {
 
 		vhosts := splitAndTrim(c.GlobalString(utils.RPCVirtualHostsFlag.Name))
 		cors := splitAndTrim(c.GlobalString(utils.RPCCORSDomainFlag.Name))
 
 		// start http server
-		httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
+		httpEndpoint := fmt.Sprintf("%s:%d", c.GlobalString(utils.RPCListenAddrFlag.Name), c.Int(rpcPortFlag.Name))
 		listener, _, err := rpc.StartHTTPEndpoint(httpEndpoint, rpcAPI, []string{"account"}, cors, vhosts, rpc.DefaultHTTPTimeouts)
 		if err != nil {
 			utils.Fatalf("Could not start RPC api: %v", err)
@@ -434,9 +454,9 @@ func signer(c *cli.Context) error {
 		}()
 
 	}
-	if !c.Bool(utils.IPCDisabledFlag.Name) {
+	if !c.GlobalBool(utils.IPCDisabledFlag.Name) {
 		if c.IsSet(utils.IPCPathFlag.Name) {
-			ipcapiURL = c.String(utils.IPCPathFlag.Name)
+			ipcapiURL = c.GlobalString(utils.IPCPathFlag.Name)
 		} else {
 			ipcapiURL = filepath.Join(configDir, "clef.ipc")
 		}
@@ -453,7 +473,7 @@ func signer(c *cli.Context) error {
 
 	}
 
-	if c.Bool(testFlag.Name) {
+	if c.GlobalBool(testFlag.Name) {
 		log.Info("Performing UI test")
 		go testExternalUI(apiImpl)
 	}
@@ -512,36 +532,52 @@ func homeDir() string {
 	}
 	return ""
 }
-func readMasterKey(ctx *cli.Context) ([]byte, error) {
+func readMasterKey(ctx *cli.Context, ui core.SignerUI) ([]byte, error) {
 	var (
 		file      string
-		configDir = ctx.String(configdirFlag.Name)
+		configDir = ctx.GlobalString(configdirFlag.Name)
 	)
-	if ctx.IsSet(signerSecretFlag.Name) {
+	if ctx.GlobalIsSet(signerSecretFlag.Name) {
-		file = ctx.String(signerSecretFlag.Name)
+		file = ctx.GlobalString(signerSecretFlag.Name)
 	} else {
-		file = filepath.Join(configDir, "secrets.dat")
+		file = filepath.Join(configDir, "masterseed.json")
 	}
 	if err := checkFile(file); err != nil {
 		return nil, err
 	}
-	masterKey, err := ioutil.ReadFile(file)
+	cipherKey, err := ioutil.ReadFile(file)
 	if err != nil {
 		return nil, err
 	}
-	if len(masterKey) < 256 {
+	var password string
-		return nil, fmt.Errorf("master key of insufficient length, expected >255 bytes, got %d", len(masterKey))
+	// If ui is not nil, get the password from ui.
+	if ui != nil {
+		resp, err := ui.OnInputRequired(core.UserInputRequest{
+			Title:      "Master Password",
+			Prompt:     "Please enter the password to decrypt the master seed",
+			IsPassword: true})
+		if err != nil {
+			return nil, err
+		}
+		password = resp.Text
+	} else {
+		password = getPassPhrase("Decrypt master seed of clef", false)
 	}
+	masterSeed, err := decryptSeed(cipherKey, password)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decrypt the master seed of clef")
+	}
+	if len(masterSeed) < 256 {
+		return nil, fmt.Errorf("master seed of insufficient length, expected >255 bytes, got %d", len(masterSeed))
+	}
+
 	// Create vault location
-	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterKey)[:10]))
+	vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), masterSeed)[:10]))
 	err = os.Mkdir(vaultLocation, 0700)
 	if err != nil && !os.IsExist(err) {
 		return nil, err
 	}
-	//!TODO, use KDF to stretch the master key
+	return masterSeed, nil
-	//			stretched_key := stretch_key(master_key)
-
-	return masterKey, nil
 }
 
 // checkFile is a convenience function to check if a file
@@ -619,6 +655,59 @@ func testExternalUI(api *core.SignerAPI) {
 
 }
 
+// getPassPhrase retrieves the password associated with clef, either fetched
+// from a list of preloaded passphrases, or requested interactively from the user.
+// TODO: there are many `getPassPhrase` functions, it will be better to abstract them into one.
+func getPassPhrase(prompt string, confirmation bool) string {
+	fmt.Println(prompt)
+	password, err := console.Stdin.PromptPassword("Passphrase: ")
+	if err != nil {
+		utils.Fatalf("Failed to read passphrase: %v", err)
+	}
+	if confirmation {
+		confirm, err := console.Stdin.PromptPassword("Repeat passphrase: ")
+		if err != nil {
+			utils.Fatalf("Failed to read passphrase confirmation: %v", err)
+		}
+		if password != confirm {
+			utils.Fatalf("Passphrases do not match")
+		}
+	}
+	return password
+}
+
+type encryptedSeedStorage struct {
+	Description string              `json:"description"`
+	Version     int                 `json:"version"`
+	Params      keystore.CryptoJSON `json:"params"`
+}
+
+// encryptSeed uses a similar scheme as the keystore uses, but with a different wrapping,
+// to encrypt the master seed
+func encryptSeed(seed []byte, auth []byte, scryptN, scryptP int) ([]byte, error) {
+	cryptoStruct, err := keystore.EncryptDataV3(seed, auth, scryptN, scryptP)
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(&encryptedSeedStorage{"Clef seed", 1, cryptoStruct})
+}
+
+// decryptSeed decrypts the master seed
+func decryptSeed(keyjson []byte, auth string) ([]byte, error) {
+	var encSeed encryptedSeedStorage
+	if err := json.Unmarshal(keyjson, &encSeed); err != nil {
+		return nil, err
+	}
+	if encSeed.Version != 1 {
+		log.Warn(fmt.Sprintf("unsupported encryption format of seed: %d, operation will likely fail", encSeed.Version))
+	}
+	seed, err := keystore.DecryptDataV3(encSeed.Params, auth)
+	if err != nil {
+		return nil, err
+	}
+	return seed, err
+}
+
 /**
 //Create Account
 

cmd/evm/json_logger.go

@@ -45,14 +45,15 @@ func (l *JSONLogger) CaptureStart(from common.Address, to common.Address, create
 // CaptureState outputs state information on the logger.
 func (l *JSONLogger) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error {
 	log := vm.StructLog{
-		Pc:         pc,
+		Pc:            pc,
-		Op:         op,
+		Op:            op,
-		Gas:        gas,
+		Gas:           gas,
-		GasCost:    cost,
+		GasCost:       cost,
-		MemorySize: memory.Len(),
+		MemorySize:    memory.Len(),
-		Storage:    nil,
+		Storage:       nil,
-		Depth:      depth,
+		Depth:         depth,
-		Err:        err,
+		RefundCounter: env.StateDB.GetRefund(),
+		Err:           err,
 	}
 	if !l.cfg.DisableMemory {
 		log.Memory = memory.Data()

cmd/geth/main.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"math"
 	"os"
-	"runtime"
 	godebug "runtime/debug"
 	"sort"
 	"strconv"
@@ -209,8 +208,6 @@ func init() {
 	app.Flags = append(app.Flags, metricsFlags...)
 
 	app.Before = func(ctx *cli.Context) error {
-		runtime.GOMAXPROCS(runtime.NumCPU())
-
 		logdir := ""
 		if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) {
 			logdir = (&node.Config{DataDir: utils.MakeDataDir(ctx)}).ResolvePath("logs")

cmd/swarm/access.go

@@ -29,7 +29,65 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
-var salt = make([]byte, 32)
+var (
+	salt          = make([]byte, 32)
+	accessCommand = cli.Command{
+		CustomHelpTemplate: helpTemplate,
+		Name:               "access",
+		Usage:              "encrypts a reference and embeds it into a root manifest",
+		ArgsUsage:          "<ref>",
+		Description:        "encrypts a reference and embeds it into a root manifest",
+		Subcommands: []cli.Command{
+			{
+				CustomHelpTemplate: helpTemplate,
+				Name:               "new",
+				Usage:              "encrypts a reference and embeds it into a root manifest",
+				ArgsUsage:          "<ref>",
+				Description:        "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+				Subcommands: []cli.Command{
+					{
+						Action:             accessNewPass,
+						CustomHelpTemplate: helpTemplate,
+						Flags: []cli.Flag{
+							utils.PasswordFileFlag,
+							SwarmDryRunFlag,
+						},
+						Name:        "pass",
+						Usage:       "encrypts a reference with a password and embeds it into a root manifest",
+						ArgsUsage:   "<ref>",
+						Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+					},
+					{
+						Action:             accessNewPK,
+						CustomHelpTemplate: helpTemplate,
+						Flags: []cli.Flag{
+							utils.PasswordFileFlag,
+							SwarmDryRunFlag,
+							SwarmAccessGrantKeyFlag,
+						},
+						Name:        "pk",
+						Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
+						ArgsUsage:   "<ref>",
+						Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+					},
+					{
+						Action:             accessNewACT,
+						CustomHelpTemplate: helpTemplate,
+						Flags: []cli.Flag{
+							SwarmAccessGrantKeysFlag,
+							SwarmDryRunFlag,
+							utils.PasswordFileFlag,
+						},
+						Name:        "act",
+						Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
+						ArgsUsage:   "<ref>",
+						Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
+					},
+				},
+			},
+		},
+	}
+)
 
 func init() {
 	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
@@ -56,6 +114,9 @@ func accessNewPass(ctx *cli.Context) {
 		utils.Fatalf("error getting session key: %v", err)
 	}
 	m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae)
+	if err != nil {
+		utils.Fatalf("had an error generating the manifest: %v", err)
+	}
 	if dryRun {
 		err = printManifests(m, nil)
 		if err != nil {
@@ -89,6 +150,9 @@ func accessNewPK(ctx *cli.Context) {
 		utils.Fatalf("error getting session key: %v", err)
 	}
 	m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae)
+	if err != nil {
+		utils.Fatalf("had an error generating the manifest: %v", err)
+	}
 	if dryRun {
 		err = printManifests(m, nil)
 		if err != nil {

cmd/swarm/access_test.go

@@ -38,6 +38,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
@@ -54,9 +55,8 @@ var DefaultCurve = crypto.S256()
 // is then fetched through 2nd node. since the tested code is not key-aware - we can just
 // fetch from the 2nd node using HTTP BasicAuth
 func TestAccessPassword(t *testing.T) {
-	cluster := newTestCluster(t, 1)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
-	defer cluster.Shutdown()
+	defer srv.Close()
-	proxyNode := cluster.Nodes[0]
 
 	dataFilename := testutil.TempFileWithContent(t, data)
 	defer os.RemoveAll(dataFilename)
@@ -64,7 +64,7 @@ func TestAccessPassword(t *testing.T) {
 	// upload the file with 'swarm up' and expect a hash
 	up := runSwarm(t,
 		"--bzzapi",
-		proxyNode.URL, //it doesn't matter through which node we upload content
+		srv.URL, //it doesn't matter through which node we upload content
 		"up",
 		"--encrypt",
 		dataFilename)
@@ -138,7 +138,7 @@ func TestAccessPassword(t *testing.T) {
 	if a.Publisher != "" {
 		t.Fatal("should be empty")
 	}
-	client := swarm.NewClient(cluster.Nodes[0].URL)
+	client := swarm.NewClient(srv.URL)
 
 	hash, err := client.UploadManifest(&m, false)
 	if err != nil {
@@ -147,7 +147,7 @@ func TestAccessPassword(t *testing.T) {
 
 	httpClient := &http.Client{}
 
-	url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash
+	url := srv.URL + "/" + "bzz:/" + hash
 	response, err := httpClient.Get(url)
 	if err != nil {
 		t.Fatal(err)
@@ -189,7 +189,7 @@ func TestAccessPassword(t *testing.T) {
 	//download file with 'swarm down' with wrong password
 	up = runSwarm(t,
 		"--bzzapi",
-		proxyNode.URL,
+		srv.URL,
 		"down",
 		"bzz:/"+hash,
 		tmp,

cmd/swarm/config.go

@@ -80,6 +80,7 @@ const (
 	SWARM_ENV_STORE_CAPACITY          = "SWARM_STORE_CAPACITY"
 	SWARM_ENV_STORE_CACHE_CAPACITY    = "SWARM_STORE_CACHE_CAPACITY"
 	SWARM_ACCESS_PASSWORD             = "SWARM_ACCESS_PASSWORD"
+	SWARM_AUTO_DEFAULTPATH            = "SWARM_AUTO_DEFAULTPATH"
 	GETH_ENV_DATADIR                  = "GETH_DATADIR"
 )
 

cmd/swarm/db.go

@@ -29,6 +29,48 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var dbCommand = cli.Command{
+	Name:               "db",
+	CustomHelpTemplate: helpTemplate,
+	Usage:              "manage the local chunk database",
+	ArgsUsage:          "db COMMAND",
+	Description:        "Manage the local chunk database",
+	Subcommands: []cli.Command{
+		{
+			Action:             dbExport,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "export",
+			Usage:              "export a local chunk database as a tar archive (use - to send to stdout)",
+			ArgsUsage:          "<chunkdb> <file>",
+			Description: `
+Export a local chunk database as a tar archive (use - to send to stdout).
+
+    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
+
+The export may be quite large, consider piping the output through the Unix
+pv(1) tool to get a progress bar:
+
+    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks - | pv > chunks.tar
+`,
+		},
+		{
+			Action:             dbImport,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "import",
+			Usage:              "import chunks from a tar archive into a local chunk database (use - to read from stdin)",
+			ArgsUsage:          "<chunkdb> <file>",
+			Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin).
+
+    swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
+
+The import may be quite large, consider piping the input through the Unix
+pv(1) tool to get a progress bar:
+
+    pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`,
+		},
+	},
+}
+
 func dbExport(ctx *cli.Context) {
 	args := ctx.Args()
 	if len(args) != 3 {

cmd/swarm/download.go

@@ -28,6 +28,15 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var downloadCommand = cli.Command{
+	Action:      download,
+	Name:        "down",
+	Flags:       []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag},
+	Usage:       "downloads a swarm manifest or a file inside a manifest",
+	ArgsUsage:   " <uri> [<dir>]",
+	Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`,
+}
+
 func download(ctx *cli.Context) {
 	log.Debug("downloading content using swarm down")
 	args := ctx.Args()

cmd/swarm/export_test.go

@@ -19,9 +19,7 @@ package main
 import (
 	"bytes"
 	"crypto/md5"
-	"crypto/rand"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"runtime"
@@ -29,6 +27,7 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/swarm"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 // TestCLISwarmExportImport perform the following test:
@@ -45,11 +44,12 @@ func TestCLISwarmExportImport(t *testing.T) {
 	cluster := newTestCluster(t, 1)
 
 	// generate random 10mb file
-	f, cleanup := generateRandomFile(t, 10000000)
+	content := testutil.RandomBytes(1, 10000000)
-	defer cleanup()
+	fileName := testutil.TempFileWithContent(t, string(content))
+	defer os.Remove(fileName)
 
 	// upload the file with 'swarm up' and expect a hash
-	up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", f.Name())
+	up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", fileName)
 	_, matches := up.ExpectRegexp(`[a-f\d]{64}`)
 	up.ExpectExit()
 	hash := matches[0]
@@ -96,7 +96,7 @@ func TestCLISwarmExportImport(t *testing.T) {
 	}
 
 	// compare downloaded file with the generated random file
-	mustEqualFiles(t, f, res.Body)
+	mustEqualFiles(t, bytes.NewReader(content), res.Body)
 }
 
 func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
@@ -117,27 +117,3 @@ func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
 		t.Fatalf("downloaded imported file md5=%x (length %v) is not the same as the generated one mp5=%x (length %v)", downHash, downLen, upHash, upLen)
 	}
 }
-
-func generateRandomFile(t *testing.T, size int) (f *os.File, teardown func()) {
-	// create a tmp file
-	tmp, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// callback for tmp file cleanup
-	teardown = func() {
-		tmp.Close()
-		os.Remove(tmp.Name())
-	}
-
-	// write 10mb random data to file
-	buf := make([]byte, 10000000)
-	_, err = rand.Read(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ioutil.WriteFile(tmp.Name(), buf, 0755)
-
-	return tmp, teardown
-}

cmd/swarm/feeds.go

@@ -31,6 +31,68 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var feedCommand = cli.Command{
+	CustomHelpTemplate: helpTemplate,
+	Name:               "feed",
+	Usage:              "(Advanced) Create and update Swarm Feeds",
+	ArgsUsage:          "<create|update|info>",
+	Description:        "Works with Swarm Feeds",
+	Subcommands: []cli.Command{
+		{
+			Action:             feedCreateManifest,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "create",
+			Usage:              "creates and publishes a new feed manifest",
+			Description: `creates and publishes a new feed manifest pointing to a specified user's updates about a particular topic.
+					The feed topic can be built in the following ways:
+					* use --topic to set the topic to an arbitrary binary hex string.
+					* use --name to set the topic to a human-readable name.
+					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
+					* use both --topic and --name to create named subtopics. 
+						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
+						this feed tracks a discussion about that contract.
+					The --user flag allows to have this manifest refer to a user other than yourself. If not specified,
+					it will then default to your local account (--bzzaccount)`,
+			Flags: []cli.Flag{SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
+		},
+		{
+			Action:             feedUpdate,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "update",
+			Usage:              "updates the content of an existing Swarm Feed",
+			ArgsUsage:          "<0x Hex data>",
+			Description: `publishes a new update on the specified topic
+					The feed topic can be built in the following ways:
+					* use --topic to set the topic to an arbitrary binary hex string.
+					* use --name to set the topic to a human-readable name.
+					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
+					* use both --topic and --name to create named subtopics. 
+						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
+						this feed tracks a discussion about that contract.
+					
+					If you have a manifest, you can specify it with --manifest to refer to the feed,
+					instead of using --topic / --name
+					`,
+			Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag},
+		},
+		{
+			Action:             feedInfo,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "info",
+			Usage:              "obtains information about an existing Swarm feed",
+			Description: `obtains information about an existing Swarm feed
+					The topic can be specified directly with the --topic flag as an hex string
+					If no topic is specified, the default topic (zero) will be used
+					The --name flag can be used to specify subtopics with a specific name.
+					The --user flag allows to refer to a user other than yourself. If not specified,
+					it will then default to your local account (--bzzaccount)
+					If you have a manifest, you can specify it with --manifest instead of --topic / --name / ---user
+					to refer to the feed`,
+			Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
+		},
+	},
+}
+
 func NewGenericSigner(ctx *cli.Context) feed.Signer {
 	return feed.NewGenericSigner(getPrivKey(ctx))
 }

cmd/swarm/feeds_test.go

@@ -20,49 +20,37 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/swarm/api"
-	"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
-	"github.com/ethereum/go-ethereum/swarm/testutil"
-
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/swarm/storage/feed"
-
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 func TestCLIFeedUpdate(t *testing.T) {
 
-	srv := testutil.NewTestSwarmServer(t, func(api *api.API) testutil.TestServer {
+	srv := swarmhttp.NewTestSwarmServer(t, func(api *api.API) swarmhttp.TestServer {
 		return swarmhttp.NewServer(api, "")
 	}, nil)
 	log.Info("starting a test swarm server")
 	defer srv.Close()
 
 	// create a private key file for signing
-	pkfile, err := ioutil.TempFile("", "swarm-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer pkfile.Close()
-	defer os.Remove(pkfile.Name())
 
 	privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979"
 	privKey, _ := crypto.HexToECDSA(privkeyHex)
 	address := crypto.PubkeyToAddress(privKey.PublicKey)
 
-	// save the private key to a file
+	pkFileName := testutil.TempFileWithContent(t, privkeyHex)
-	_, err = io.WriteString(pkfile, privkeyHex)
+	defer os.Remove(pkFileName)
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	// compose a topic. We'll be doing quotes about Miguel de Cervantes
 	var topic feed.Topic
@@ -76,7 +64,7 @@ func TestCLIFeedUpdate(t *testing.T) {
 
 	flags := []string{
 		"--bzzapi", srv.URL,
-		"--bzzaccount", pkfile.Name(),
+		"--bzzaccount", pkFileName,
 		"feed", "update",
 		"--topic", topic.Hex(),
 		"--name", name,
@@ -89,13 +77,10 @@ func TestCLIFeedUpdate(t *testing.T) {
 
 	// now try to get the update using the client
 	client := swarm.NewClient(srv.URL)
-	if err != nil {
-		t.Fatal(err)
-	}
 
 	// build the same topic as before, this time
 	// we use NewTopic to create a topic automatically.
-	topic, err = feed.NewTopic(name, subject)
+	topic, err := feed.NewTopic(name, subject)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -153,7 +138,7 @@ func TestCLIFeedUpdate(t *testing.T) {
 	// test publishing a manifest
 	flags = []string{
 		"--bzzapi", srv.URL,
-		"--bzzaccount", pkfile.Name(),
+		"--bzzaccount", pkFileName,
 		"feed", "create",
 		"--topic", topic.Hex(),
 	}

cmd/swarm/flags.go

@@ -0,0 +1,179 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// Command feed allows the user to create and update signed Swarm feeds
+package main
+
+import cli "gopkg.in/urfave/cli.v1"
+
+var (
+	ChequebookAddrFlag = cli.StringFlag{
+		Name:   "chequebook",
+		Usage:  "chequebook contract address",
+		EnvVar: SWARM_ENV_CHEQUEBOOK_ADDR,
+	}
+	SwarmAccountFlag = cli.StringFlag{
+		Name:   "bzzaccount",
+		Usage:  "Swarm account key file",
+		EnvVar: SWARM_ENV_ACCOUNT,
+	}
+	SwarmListenAddrFlag = cli.StringFlag{
+		Name:   "httpaddr",
+		Usage:  "Swarm HTTP API listening interface",
+		EnvVar: SWARM_ENV_LISTEN_ADDR,
+	}
+	SwarmPortFlag = cli.StringFlag{
+		Name:   "bzzport",
+		Usage:  "Swarm local http api port",
+		EnvVar: SWARM_ENV_PORT,
+	}
+	SwarmNetworkIdFlag = cli.IntFlag{
+		Name:   "bzznetworkid",
+		Usage:  "Network identifier (integer, default 3=swarm testnet)",
+		EnvVar: SWARM_ENV_NETWORK_ID,
+	}
+	SwarmSwapEnabledFlag = cli.BoolFlag{
+		Name:   "swap",
+		Usage:  "Swarm SWAP enabled (default false)",
+		EnvVar: SWARM_ENV_SWAP_ENABLE,
+	}
+	SwarmSwapAPIFlag = cli.StringFlag{
+		Name:   "swap-api",
+		Usage:  "URL of the Ethereum API provider to use to settle SWAP payments",
+		EnvVar: SWARM_ENV_SWAP_API,
+	}
+	SwarmSyncDisabledFlag = cli.BoolTFlag{
+		Name:   "nosync",
+		Usage:  "Disable swarm syncing",
+		EnvVar: SWARM_ENV_SYNC_DISABLE,
+	}
+	SwarmSyncUpdateDelay = cli.DurationFlag{
+		Name:   "sync-update-delay",
+		Usage:  "Duration for sync subscriptions update after no new peers are added (default 15s)",
+		EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY,
+	}
+	SwarmMaxStreamPeerServersFlag = cli.IntFlag{
+		Name:   "max-stream-peer-servers",
+		Usage:  "Limit of Stream peer servers, 0 denotes unlimited",
+		EnvVar: SWARM_ENV_MAX_STREAM_PEER_SERVERS,
+		Value:  10000, // A very large default value is possible as stream servers have very small memory footprint
+	}
+	SwarmLightNodeEnabled = cli.BoolFlag{
+		Name:   "lightnode",
+		Usage:  "Enable Swarm LightNode (default false)",
+		EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE,
+	}
+	SwarmDeliverySkipCheckFlag = cli.BoolFlag{
+		Name:   "delivery-skip-check",
+		Usage:  "Skip chunk delivery check (default false)",
+		EnvVar: SWARM_ENV_DELIVERY_SKIP_CHECK,
+	}
+	EnsAPIFlag = cli.StringSliceFlag{
+		Name:   "ens-api",
+		Usage:  "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url",
+		EnvVar: SWARM_ENV_ENS_API,
+	}
+	SwarmApiFlag = cli.StringFlag{
+		Name:  "bzzapi",
+		Usage: "Specifies the Swarm HTTP endpoint to connect to",
+		Value: "http://127.0.0.1:8500",
+	}
+	SwarmRecursiveFlag = cli.BoolFlag{
+		Name:  "recursive",
+		Usage: "Upload directories recursively",
+	}
+	SwarmWantManifestFlag = cli.BoolTFlag{
+		Name:  "manifest",
+		Usage: "Automatic manifest upload (default true)",
+	}
+	SwarmUploadDefaultPath = cli.StringFlag{
+		Name:  "defaultpath",
+		Usage: "path to file served for empty url path (none)",
+	}
+	SwarmAccessGrantKeyFlag = cli.StringFlag{
+		Name:  "grant-key",
+		Usage: "grants a given public key access to an ACT",
+	}
+	SwarmAccessGrantKeysFlag = cli.StringFlag{
+		Name:  "grant-keys",
+		Usage: "grants a given list of public keys in the following file (separated by line breaks) access to an ACT",
+	}
+	SwarmUpFromStdinFlag = cli.BoolFlag{
+		Name:  "stdin",
+		Usage: "reads data to be uploaded from stdin",
+	}
+	SwarmUploadMimeType = cli.StringFlag{
+		Name:  "mime",
+		Usage: "Manually specify MIME type",
+	}
+	SwarmEncryptedFlag = cli.BoolFlag{
+		Name:  "encrypt",
+		Usage: "use encrypted upload",
+	}
+	SwarmAccessPasswordFlag = cli.StringFlag{
+		Name:   "password",
+		Usage:  "Password",
+		EnvVar: SWARM_ACCESS_PASSWORD,
+	}
+	SwarmDryRunFlag = cli.BoolFlag{
+		Name:  "dry-run",
+		Usage: "dry-run",
+	}
+	CorsStringFlag = cli.StringFlag{
+		Name:   "corsdomain",
+		Usage:  "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
+		EnvVar: SWARM_ENV_CORS,
+	}
+	SwarmStorePath = cli.StringFlag{
+		Name:   "store.path",
+		Usage:  "Path to leveldb chunk DB (default <$GETH_ENV_DIR>/swarm/bzz-<$BZZ_KEY>/chunks)",
+		EnvVar: SWARM_ENV_STORE_PATH,
+	}
+	SwarmStoreCapacity = cli.Uint64Flag{
+		Name:   "store.size",
+		Usage:  "Number of chunks (5M is roughly 20-25GB) (default 5000000)",
+		EnvVar: SWARM_ENV_STORE_CAPACITY,
+	}
+	SwarmStoreCacheCapacity = cli.UintFlag{
+		Name:   "store.cache.size",
+		Usage:  "Number of recent chunks cached in memory (default 5000)",
+		EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
+	}
+	SwarmCompressedFlag = cli.BoolFlag{
+		Name:  "compressed",
+		Usage: "Prints encryption keys in compressed form",
+	}
+	SwarmFeedNameFlag = cli.StringFlag{
+		Name:  "name",
+		Usage: "User-defined name for the new feed, limited to 32 characters. If combined with topic, it will refer to a subtopic with this name",
+	}
+	SwarmFeedTopicFlag = cli.StringFlag{
+		Name:  "topic",
+		Usage: "User-defined topic this feed is tracking, hex encoded. Limited to 64 hexadecimal characters",
+	}
+	SwarmFeedDataOnCreateFlag = cli.StringFlag{
+		Name:  "data",
+		Usage: "Initializes the feed with the given hex-encoded data. Data must be prefixed by 0x",
+	}
+	SwarmFeedManifestFlag = cli.StringFlag{
+		Name:  "manifest",
+		Usage: "Refers to the feed through a manifest",
+	}
+	SwarmFeedUserFlag = cli.StringFlag{
+		Name:  "user",
+		Usage: "Indicates the user who updates the feed",
+	}
+)

cmd/swarm/fs.go

@@ -30,6 +30,43 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var fsCommand = cli.Command{
+	Name:               "fs",
+	CustomHelpTemplate: helpTemplate,
+	Usage:              "perform FUSE operations",
+	ArgsUsage:          "fs COMMAND",
+	Description:        "Performs FUSE operations by mounting/unmounting/listing mount points. This assumes you already have a Swarm node running locally. For all operation you must reference the correct path to bzzd.ipc in order to communicate with the node",
+	Subcommands: []cli.Command{
+		{
+			Action:             mount,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "mount",
+			Flags:              []cli.Flag{utils.IPCPathFlag},
+			Usage:              "mount a swarm hash to a mount point",
+			ArgsUsage:          "swarm fs mount --ipcpath <path to bzzd.ipc> <manifest hash> <mount point>",
+			Description:        "Mounts a Swarm manifest hash to a given mount point. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
+		},
+		{
+			Action:             unmount,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "unmount",
+			Flags:              []cli.Flag{utils.IPCPathFlag},
+			Usage:              "unmount a swarmfs mount",
+			ArgsUsage:          "swarm fs unmount --ipcpath <path to bzzd.ipc> <mount point>",
+			Description:        "Unmounts a swarmfs mount residing at <mount point>. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
+		},
+		{
+			Action:             listMounts,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "list",
+			Flags:              []cli.Flag{utils.IPCPathFlag},
+			Usage:              "list swarmfs mounts",
+			ArgsUsage:          "swarm fs list --ipcpath <path to bzzd.ipc>",
+			Description:        "Lists all mounted swarmfs volumes. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
+		},
+	},
+}
+
 func mount(cliContext *cli.Context) {
 	args := cliContext.Args()
 	if len(args) < 2 {

cmd/swarm/fs_test.go

@@ -80,6 +80,9 @@ func TestCLISwarmFs(t *testing.T) {
 		t.Fatal(err)
 	}
 	dirPath2, err := createDirInDir(dirPath, "AnotherTestSubDir")
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	dummyContent := "somerandomtestcontentthatshouldbeasserted"
 	dirs := []string{

cmd/swarm/hash.go

@@ -27,6 +27,15 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var hashCommand = cli.Command{
+	Action:             hash,
+	CustomHelpTemplate: helpTemplate,
+	Name:               "hash",
+	Usage:              "print the swarm hash of a file or directory",
+	ArgsUsage:          "<file>",
+	Description:        "Prints the swarm hash of file or directory",
+}
+
 func hash(ctx *cli.Context) {
 	args := ctx.Args()
 	if len(args) < 1 {

cmd/swarm/list.go

@@ -27,6 +27,15 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var listCommand = cli.Command{
+	Action:             list,
+	CustomHelpTemplate: helpTemplate,
+	Name:               "ls",
+	Usage:              "list files and directories contained in a manifest",
+	ArgsUsage:          "<manifest> [<prefix>]",
+	Description:        "Lists files and directories contained in a manifest",
+}
+
 func list(ctx *cli.Context) {
 	args := ctx.Args()
 

cmd/swarm/main.go

@@ -70,165 +70,6 @@ var (
 	gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
 )
 
-var (
-	ChequebookAddrFlag = cli.StringFlag{
-		Name:   "chequebook",
-		Usage:  "chequebook contract address",
-		EnvVar: SWARM_ENV_CHEQUEBOOK_ADDR,
-	}
-	SwarmAccountFlag = cli.StringFlag{
-		Name:   "bzzaccount",
-		Usage:  "Swarm account key file",
-		EnvVar: SWARM_ENV_ACCOUNT,
-	}
-	SwarmListenAddrFlag = cli.StringFlag{
-		Name:   "httpaddr",
-		Usage:  "Swarm HTTP API listening interface",
-		EnvVar: SWARM_ENV_LISTEN_ADDR,
-	}
-	SwarmPortFlag = cli.StringFlag{
-		Name:   "bzzport",
-		Usage:  "Swarm local http api port",
-		EnvVar: SWARM_ENV_PORT,
-	}
-	SwarmNetworkIdFlag = cli.IntFlag{
-		Name:   "bzznetworkid",
-		Usage:  "Network identifier (integer, default 3=swarm testnet)",
-		EnvVar: SWARM_ENV_NETWORK_ID,
-	}
-	SwarmSwapEnabledFlag = cli.BoolFlag{
-		Name:   "swap",
-		Usage:  "Swarm SWAP enabled (default false)",
-		EnvVar: SWARM_ENV_SWAP_ENABLE,
-	}
-	SwarmSwapAPIFlag = cli.StringFlag{
-		Name:   "swap-api",
-		Usage:  "URL of the Ethereum API provider to use to settle SWAP payments",
-		EnvVar: SWARM_ENV_SWAP_API,
-	}
-	SwarmSyncDisabledFlag = cli.BoolTFlag{
-		Name:   "nosync",
-		Usage:  "Disable swarm syncing",
-		EnvVar: SWARM_ENV_SYNC_DISABLE,
-	}
-	SwarmSyncUpdateDelay = cli.DurationFlag{
-		Name:   "sync-update-delay",
-		Usage:  "Duration for sync subscriptions update after no new peers are added (default 15s)",
-		EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY,
-	}
-	SwarmMaxStreamPeerServersFlag = cli.IntFlag{
-		Name:   "max-stream-peer-servers",
-		Usage:  "Limit of Stream peer servers, 0 denotes unlimited",
-		EnvVar: SWARM_ENV_MAX_STREAM_PEER_SERVERS,
-		Value:  10000, // A very large default value is possible as stream servers have very small memory footprint
-	}
-	SwarmLightNodeEnabled = cli.BoolFlag{
-		Name:   "lightnode",
-		Usage:  "Enable Swarm LightNode (default false)",
-		EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE,
-	}
-	SwarmDeliverySkipCheckFlag = cli.BoolFlag{
-		Name:   "delivery-skip-check",
-		Usage:  "Skip chunk delivery check (default false)",
-		EnvVar: SWARM_ENV_DELIVERY_SKIP_CHECK,
-	}
-	EnsAPIFlag = cli.StringSliceFlag{
-		Name:   "ens-api",
-		Usage:  "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url",
-		EnvVar: SWARM_ENV_ENS_API,
-	}
-	SwarmApiFlag = cli.StringFlag{
-		Name:  "bzzapi",
-		Usage: "Swarm HTTP endpoint",
-		Value: "http://127.0.0.1:8500",
-	}
-	SwarmRecursiveFlag = cli.BoolFlag{
-		Name:  "recursive",
-		Usage: "Upload directories recursively",
-	}
-	SwarmWantManifestFlag = cli.BoolTFlag{
-		Name:  "manifest",
-		Usage: "Automatic manifest upload (default true)",
-	}
-	SwarmUploadDefaultPath = cli.StringFlag{
-		Name:  "defaultpath",
-		Usage: "path to file served for empty url path (none)",
-	}
-	SwarmAccessGrantKeyFlag = cli.StringFlag{
-		Name:  "grant-key",
-		Usage: "grants a given public key access to an ACT",
-	}
-	SwarmAccessGrantKeysFlag = cli.StringFlag{
-		Name:  "grant-keys",
-		Usage: "grants a given list of public keys in the following file (separated by line breaks) access to an ACT",
-	}
-	SwarmUpFromStdinFlag = cli.BoolFlag{
-		Name:  "stdin",
-		Usage: "reads data to be uploaded from stdin",
-	}
-	SwarmUploadMimeType = cli.StringFlag{
-		Name:  "mime",
-		Usage: "Manually specify MIME type",
-	}
-	SwarmEncryptedFlag = cli.BoolFlag{
-		Name:  "encrypt",
-		Usage: "use encrypted upload",
-	}
-	SwarmAccessPasswordFlag = cli.StringFlag{
-		Name:   "password",
-		Usage:  "Password",
-		EnvVar: SWARM_ACCESS_PASSWORD,
-	}
-	SwarmDryRunFlag = cli.BoolFlag{
-		Name:  "dry-run",
-		Usage: "dry-run",
-	}
-	CorsStringFlag = cli.StringFlag{
-		Name:   "corsdomain",
-		Usage:  "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
-		EnvVar: SWARM_ENV_CORS,
-	}
-	SwarmStorePath = cli.StringFlag{
-		Name:   "store.path",
-		Usage:  "Path to leveldb chunk DB (default <$GETH_ENV_DIR>/swarm/bzz-<$BZZ_KEY>/chunks)",
-		EnvVar: SWARM_ENV_STORE_PATH,
-	}
-	SwarmStoreCapacity = cli.Uint64Flag{
-		Name:   "store.size",
-		Usage:  "Number of chunks (5M is roughly 20-25GB) (default 5000000)",
-		EnvVar: SWARM_ENV_STORE_CAPACITY,
-	}
-	SwarmStoreCacheCapacity = cli.UintFlag{
-		Name:   "store.cache.size",
-		Usage:  "Number of recent chunks cached in memory (default 5000)",
-		EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
-	}
-	SwarmCompressedFlag = cli.BoolFlag{
-		Name:  "compressed",
-		Usage: "Prints encryption keys in compressed form",
-	}
-	SwarmFeedNameFlag = cli.StringFlag{
-		Name:  "name",
-		Usage: "User-defined name for the new feed, limited to 32 characters. If combined with topic, it will refer to a subtopic with this name",
-	}
-	SwarmFeedTopicFlag = cli.StringFlag{
-		Name:  "topic",
-		Usage: "User-defined topic this feed is tracking, hex encoded. Limited to 64 hexadecimal characters",
-	}
-	SwarmFeedDataOnCreateFlag = cli.StringFlag{
-		Name:  "data",
-		Usage: "Initializes the feed with the given hex-encoded data. Data must be prefixed by 0x",
-	}
-	SwarmFeedManifestFlag = cli.StringFlag{
-		Name:  "manifest",
-		Usage: "Refers to the feed through a manifest",
-	}
-	SwarmFeedUserFlag = cli.StringFlag{
-		Name:  "user",
-		Usage: "Indicates the user who updates the feed",
-	}
-)
-
 //declare a few constant error messages, useful for later error check comparisons in test
 var (
 	SWARM_ERR_NO_BZZACCOUNT   = "bzzaccount option is required but not set; check your config file, command line or environment variables"
@@ -279,267 +120,24 @@ func init() {
 			Usage:              "Print public key information",
 			Description:        "The output of this command is supposed to be machine-readable",
 		},
-		{
+		// See upload.go
-			Action:             upload,
+		upCommand,
-			CustomHelpTemplate: helpTemplate,
+		// See access.go
-			Name:               "up",
+		accessCommand,
-			Usage:              "uploads a file or directory to swarm using the HTTP API",
+		// See feeds.go
-			ArgsUsage:          "<file>",
+		feedCommand,
-			Flags:              []cli.Flag{SwarmEncryptedFlag},
+		// See list.go
-			Description:        "uploads a file or directory to swarm using the HTTP API and prints the root hash",
+		listCommand,
-		},
+		// See hash.go
-		{
+		hashCommand,
-			CustomHelpTemplate: helpTemplate,
+		// See download.go
-			Name:               "access",
+		downloadCommand,
-			Usage:              "encrypts a reference and embeds it into a root manifest",
+		// See manifest.go
-			ArgsUsage:          "<ref>",
+		manifestCommand,
-			Description:        "encrypts a reference and embeds it into a root manifest",
+		// See fs.go
-			Subcommands: []cli.Command{
+		fsCommand,
-				{
+		// See db.go
-					CustomHelpTemplate: helpTemplate,
+		dbCommand,
-					Name:               "new",
-					Usage:              "encrypts a reference and embeds it into a root manifest",
-					ArgsUsage:          "<ref>",
-					Description:        "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
-					Subcommands: []cli.Command{
-						{
-							Action:             accessNewPass,
-							CustomHelpTemplate: helpTemplate,
-							Flags: []cli.Flag{
-								utils.PasswordFileFlag,
-								SwarmDryRunFlag,
-							},
-							Name:        "pass",
-							Usage:       "encrypts a reference with a password and embeds it into a root manifest",
-							ArgsUsage:   "<ref>",
-							Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
-						},
-						{
-							Action:             accessNewPK,
-							CustomHelpTemplate: helpTemplate,
-							Flags: []cli.Flag{
-								utils.PasswordFileFlag,
-								SwarmDryRunFlag,
-								SwarmAccessGrantKeyFlag,
-							},
-							Name:        "pk",
-							Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
-							ArgsUsage:   "<ref>",
-							Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
-						},
-						{
-							Action:             accessNewACT,
-							CustomHelpTemplate: helpTemplate,
-							Flags: []cli.Flag{
-								SwarmAccessGrantKeysFlag,
-								SwarmDryRunFlag,
-								utils.PasswordFileFlag,
-							},
-							Name:        "act",
-							Usage:       "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
-							ArgsUsage:   "<ref>",
-							Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
-						},
-					},
-				},
-			},
-		},
-		{
-			CustomHelpTemplate: helpTemplate,
-			Name:               "feed",
-			Usage:              "(Advanced) Create and update Swarm Feeds",
-			ArgsUsage:          "<create|update|info>",
-			Description:        "Works with Swarm Feeds",
-			Subcommands: []cli.Command{
-				{
-					Action:             feedCreateManifest,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "create",
-					Usage:              "creates and publishes a new feed manifest",
-					Description: `creates and publishes a new feed manifest pointing to a specified user's updates about a particular topic.
-					The feed topic can be built in the following ways:
-					* use --topic to set the topic to an arbitrary binary hex string.
-					* use --name to set the topic to a human-readable name.
-					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
-					* use both --topic and --name to create named subtopics. 
-						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
-						this feed tracks a discussion about that contract.
-					The --user flag allows to have this manifest refer to a user other than yourself. If not specified,
-					it will then default to your local account (--bzzaccount)`,
-					Flags: []cli.Flag{SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
-				},
-				{
-					Action:             feedUpdate,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "update",
-					Usage:              "updates the content of an existing Swarm Feed",
-					ArgsUsage:          "<0x Hex data>",
-					Description: `publishes a new update on the specified topic
-					The feed topic can be built in the following ways:
-					* use --topic to set the topic to an arbitrary binary hex string.
-					* use --name to set the topic to a human-readable name.
-					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
-					* use both --topic and --name to create named subtopics. 
-						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
-						this feed tracks a discussion about that contract.
-					
-					If you have a manifest, you can specify it with --manifest to refer to the feed,
-					instead of using --topic / --name
-					`,
-					Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag},
-				},
-				{
-					Action:             feedInfo,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "info",
-					Usage:              "obtains information about an existing Swarm feed",
-					Description: `obtains information about an existing Swarm feed
-					The topic can be specified directly with the --topic flag as an hex string
-					If no topic is specified, the default topic (zero) will be used
-					The --name flag can be used to specify subtopics with a specific name.
-					The --user flag allows to refer to a user other than yourself. If not specified,
-					it will then default to your local account (--bzzaccount)
-					If you have a manifest, you can specify it with --manifest instead of --topic / --name / ---user
-					to refer to the feed`,
-					Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
-				},
-			},
-		},
-		{
-			Action:             list,
-			CustomHelpTemplate: helpTemplate,
-			Name:               "ls",
-			Usage:              "list files and directories contained in a manifest",
-			ArgsUsage:          "<manifest> [<prefix>]",
-			Description:        "Lists files and directories contained in a manifest",
-		},
-		{
-			Action:             hash,
-			CustomHelpTemplate: helpTemplate,
-			Name:               "hash",
-			Usage:              "print the swarm hash of a file or directory",
-			ArgsUsage:          "<file>",
-			Description:        "Prints the swarm hash of file or directory",
-		},
-		{
-			Action:      download,
-			Name:        "down",
-			Flags:       []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag},
-			Usage:       "downloads a swarm manifest or a file inside a manifest",
-			ArgsUsage:   " <uri> [<dir>]",
-			Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`,
-		},
-		{
-			Name:               "manifest",
-			CustomHelpTemplate: helpTemplate,
-			Usage:              "perform operations on swarm manifests",
-			ArgsUsage:          "COMMAND",
-			Description:        "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove",
-			Subcommands: []cli.Command{
-				{
-					Action:             manifestAdd,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "add",
-					Usage:              "add a new path to the manifest",
-					ArgsUsage:          "<MANIFEST> <path> <hash>",
-					Description:        "Adds a new path to the manifest",
-				},
-				{
-					Action:             manifestUpdate,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "update",
-					Usage:              "update the hash for an already existing path in the manifest",
-					ArgsUsage:          "<MANIFEST> <path> <newhash>",
-					Description:        "Update the hash for an already existing path in the manifest",
-				},
-				{
-					Action:             manifestRemove,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "remove",
-					Usage:              "removes a path from the manifest",
-					ArgsUsage:          "<MANIFEST> <path>",
-					Description:        "Removes a path from the manifest",
-				},
-			},
-		},
-		{
-			Name:               "fs",
-			CustomHelpTemplate: helpTemplate,
-			Usage:              "perform FUSE operations",
-			ArgsUsage:          "fs COMMAND",
-			Description:        "Performs FUSE operations by mounting/unmounting/listing mount points. This assumes you already have a Swarm node running locally. For all operation you must reference the correct path to bzzd.ipc in order to communicate with the node",
-			Subcommands: []cli.Command{
-				{
-					Action:             mount,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "mount",
-					Flags:              []cli.Flag{utils.IPCPathFlag},
-					Usage:              "mount a swarm hash to a mount point",
-					ArgsUsage:          "swarm fs mount --ipcpath <path to bzzd.ipc> <manifest hash> <mount point>",
-					Description:        "Mounts a Swarm manifest hash to a given mount point. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
-				},
-				{
-					Action:             unmount,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "unmount",
-					Flags:              []cli.Flag{utils.IPCPathFlag},
-					Usage:              "unmount a swarmfs mount",
-					ArgsUsage:          "swarm fs unmount --ipcpath <path to bzzd.ipc> <mount point>",
-					Description:        "Unmounts a swarmfs mount residing at <mount point>. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
-				},
-				{
-					Action:             listMounts,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "list",
-					Flags:              []cli.Flag{utils.IPCPathFlag},
-					Usage:              "list swarmfs mounts",
-					ArgsUsage:          "swarm fs list --ipcpath <path to bzzd.ipc>",
-					Description:        "Lists all mounted swarmfs volumes. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
-				},
-			},
-		},
-		{
-			Name:               "db",
-			CustomHelpTemplate: helpTemplate,
-			Usage:              "manage the local chunk database",
-			ArgsUsage:          "db COMMAND",
-			Description:        "Manage the local chunk database",
-			Subcommands: []cli.Command{
-				{
-					Action:             dbExport,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "export",
-					Usage:              "export a local chunk database as a tar archive (use - to send to stdout)",
-					ArgsUsage:          "<chunkdb> <file>",
-					Description: `
-Export a local chunk database as a tar archive (use - to send to stdout).
-
-    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
-
-The export may be quite large, consider piping the output through the Unix
-pv(1) tool to get a progress bar:
-
-    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks - | pv > chunks.tar
-`,
-				},
-				{
-					Action:             dbImport,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "import",
-					Usage:              "import chunks from a tar archive into a local chunk database (use - to read from stdin)",
-					ArgsUsage:          "<chunkdb> <file>",
-					Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin).
-
-    swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
-
-The import may be quite large, consider piping the input through the Unix
-pv(1) tool to get a progress bar:
-
-    pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`,
-				},
-			},
-		},
-
 		// See config.go
 		DumpConfigCommand,
 	}

cmd/swarm/manifest.go

@@ -28,6 +28,40 @@ import (
 	"gopkg.in/urfave/cli.v1"
 )
 
+var manifestCommand = cli.Command{
+	Name:               "manifest",
+	CustomHelpTemplate: helpTemplate,
+	Usage:              "perform operations on swarm manifests",
+	ArgsUsage:          "COMMAND",
+	Description:        "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove",
+	Subcommands: []cli.Command{
+		{
+			Action:             manifestAdd,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "add",
+			Usage:              "add a new path to the manifest",
+			ArgsUsage:          "<MANIFEST> <path> <hash>",
+			Description:        "Adds a new path to the manifest",
+		},
+		{
+			Action:             manifestUpdate,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "update",
+			Usage:              "update the hash for an already existing path in the manifest",
+			ArgsUsage:          "<MANIFEST> <path> <newhash>",
+			Description:        "Update the hash for an already existing path in the manifest",
+		},
+		{
+			Action:             manifestRemove,
+			CustomHelpTemplate: helpTemplate,
+			Name:               "remove",
+			Usage:              "removes a path from the manifest",
+			ArgsUsage:          "<MANIFEST> <path>",
+			Description:        "Removes a path from the manifest",
+		},
+	},
+}
+
 // manifestAdd adds a new entry to the manifest at the given path.
 // New entry hash, the last argument, must be the hash of a manifest
 // with only one entry, which meta-data will be added to the original manifest.

cmd/swarm/manifest_test.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
 )
 
 // TestManifestChange tests manifest add, update and remove
@@ -57,8 +58,8 @@ func TestManifestChangeEncrypted(t *testing.T) {
 // Argument encrypt controls whether to use encryption or not.
 func testManifestChange(t *testing.T, encrypt bool) {
 	t.Parallel()
-	cluster := newTestCluster(t, 1)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
-	defer cluster.Shutdown()
+	defer srv.Close()
 
 	tmp, err := ioutil.TempDir("", "swarm-manifest-test")
 	if err != nil {
@@ -94,7 +95,7 @@ func testManifestChange(t *testing.T, encrypt bool) {
 
 	args := []string{
 		"--bzzapi",
-		cluster.Nodes[0].URL,
+		srv.URL,
 		"--recursive",
 		"--defaultpath",
 		indexDataFilename,
@@ -109,7 +110,7 @@ func testManifestChange(t *testing.T, encrypt bool) {
 
 	checkHashLength(t, origManifestHash, encrypt)
 
-	client := swarm.NewClient(cluster.Nodes[0].URL)
+	client := swarm.NewClient(srv.URL)
 
 	// upload a new file and use its manifest to add it the original manifest.
 	t.Run("add", func(t *testing.T) {
@@ -122,14 +123,14 @@ func testManifestChange(t *testing.T, encrypt bool) {
 
 		humansManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"up",
 			humansDataFilename,
 		)
 
 		newManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"manifest",
 			"add",
 			origManifestHash,
@@ -177,14 +178,14 @@ func testManifestChange(t *testing.T, encrypt bool) {
 
 		robotsManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"up",
 			robotsDataFilename,
 		)
 
 		newManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"manifest",
 			"add",
 			origManifestHash,
@@ -237,14 +238,14 @@ func testManifestChange(t *testing.T, encrypt bool) {
 
 		indexManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"up",
 			indexDataFilename,
 		)
 
 		newManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"manifest",
 			"update",
 			origManifestHash,
@@ -295,14 +296,14 @@ func testManifestChange(t *testing.T, encrypt bool) {
 
 		humansManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"up",
 			robotsDataFilename,
 		)
 
 		newManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"manifest",
 			"update",
 			origManifestHash,
@@ -348,7 +349,7 @@ func testManifestChange(t *testing.T, encrypt bool) {
 	t.Run("remove", func(t *testing.T) {
 		newManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"manifest",
 			"remove",
 			origManifestHash,
@@ -376,7 +377,7 @@ func testManifestChange(t *testing.T, encrypt bool) {
 	t.Run("remove nested", func(t *testing.T) {
 		newManifestHash := runSwarmExpectHash(t,
 			"--bzzapi",
-			cluster.Nodes[0].URL,
+			srv.URL,
 			"manifest",
 			"remove",
 			origManifestHash,
@@ -429,8 +430,8 @@ func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) {
 
 func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
 	t.Parallel()
-	cluster := newTestCluster(t, 1)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
-	defer cluster.Shutdown()
+	defer srv.Close()
 
 	tmp, err := ioutil.TempDir("", "swarm-manifest-test")
 	if err != nil {
@@ -458,7 +459,7 @@ func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
 
 	args := []string{
 		"--bzzapi",
-		cluster.Nodes[0].URL,
+		srv.URL,
 		"--recursive",
 		"--defaultpath",
 		indexDataFilename,
@@ -473,7 +474,7 @@ func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
 
 	checkHashLength(t, origManifestHash, encrypt)
 
-	client := swarm.NewClient(cluster.Nodes[0].URL)
+	client := swarm.NewClient(srv.URL)
 
 	newIndexData := []byte("<h1>Ethereum Swarm</h1>")
 	newIndexDataFilename := filepath.Join(tmp, "index.html")
@@ -484,14 +485,14 @@ func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
 
 	newIndexManifestHash := runSwarmExpectHash(t,
 		"--bzzapi",
-		cluster.Nodes[0].URL,
+		srv.URL,
 		"up",
 		newIndexDataFilename,
 	)
 
 	newManifestHash := runSwarmExpectHash(t,
 		"--bzzapi",
-		cluster.Nodes[0].URL,
+		srv.URL,
 		"manifest",
 		"update",
 		origManifestHash,

cmd/swarm/run_test.go

@@ -40,6 +40,8 @@ import (
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm"
+	"github.com/ethereum/go-ethereum/swarm/api"
+	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
 )
 
 var loglevel = flag.Int("loglevel", 3, "verbosity of logs")
@@ -55,6 +57,9 @@ func init() {
 	})
 }
 
+func serverFunc(api *api.API) swarmhttp.TestServer {
+	return swarmhttp.NewServer(api, "")
+}
 func TestMain(m *testing.M) {
 	// check if we have been reexec'd
 	if reexec.Init() {

cmd/swarm/swarm-smoke/feed_upload_and_sync.go

@@ -0,0 +1,320 @@
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/multihash"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed"
+	colorable "github.com/mattn/go-colorable"
+	"github.com/pborman/uuid"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+const (
+	feedRandomDataLength = 8
+)
+
+// TODO: retrieve with manifest + extract repeating code
+func cliFeedUploadAndSync(c *cli.Context) error {
+
+	log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))))
+
+	defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now())
+
+	generateEndpoints(scheme, cluster, from, to)
+
+	log.Info("generating and uploading MRUs to " + endpoints[0] + " and syncing")
+
+	// create a random private key to sign updates with and derive the address
+	pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test")
+	if err != nil {
+		return err
+	}
+	defer pkFile.Close()
+	defer os.Remove(pkFile.Name())
+
+	privkeyHex := "0000000000000000000000000000000000000000000000000000000000001976"
+	privKey, err := crypto.HexToECDSA(privkeyHex)
+	if err != nil {
+		return err
+	}
+	user := crypto.PubkeyToAddress(privKey.PublicKey)
+	userHex := hexutil.Encode(user.Bytes())
+
+	// save the private key to a file
+	_, err = io.WriteString(pkFile, privkeyHex)
+	if err != nil {
+		return err
+	}
+
+	// keep hex strings for topic and subtopic
+	var topicHex string
+	var subTopicHex string
+
+	// and create combination hex topics for bzz-feed retrieval
+	// xor'ed with topic (zero-value topic if no topic)
+	var subTopicOnlyHex string
+	var mergedSubTopicHex string
+
+	// generate random topic and subtopic and put a hex on them
+	topicBytes, err := generateRandomData(feed.TopicLength)
+	topicHex = hexutil.Encode(topicBytes)
+	subTopicBytes, err := generateRandomData(8)
+	subTopicHex = hexutil.Encode(subTopicBytes)
+	if err != nil {
+		return err
+	}
+	mergedSubTopic, err := feed.NewTopic(subTopicHex, topicBytes)
+	if err != nil {
+		return err
+	}
+	mergedSubTopicHex = hexutil.Encode(mergedSubTopic[:])
+	subTopicOnlyBytes, err := feed.NewTopic(subTopicHex, nil)
+	if err != nil {
+		return err
+	}
+	subTopicOnlyHex = hexutil.Encode(subTopicOnlyBytes[:])
+
+	// create feed manifest, topic only
+	var out bytes.Buffer
+	cmd := exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--user", userHex)
+	cmd.Stdout = &out
+	log.Debug("create feed manifest topic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	manifestWithTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
+	if len(manifestWithTopic) != 64 {
+		return fmt.Errorf("unknown feed create manifest hash format (topic): (%d) %s", len(out.String()), manifestWithTopic)
+	}
+	log.Debug("create topic feed", "manifest", manifestWithTopic)
+	out.Reset()
+
+	// create feed manifest, subtopic only
+	cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--name", subTopicHex, "--user", userHex)
+	cmd.Stdout = &out
+	log.Debug("create feed manifest subtopic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	manifestWithSubTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
+	if len(manifestWithSubTopic) != 64 {
+		return fmt.Errorf("unknown feed create manifest hash format (subtopic): (%d) %s", len(out.String()), manifestWithSubTopic)
+	}
+	log.Debug("create subtopic feed", "manifest", manifestWithTopic)
+	out.Reset()
+
+	// create feed manifest, merged topic
+	cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex)
+	cmd.Stdout = &out
+	log.Debug("create feed manifest mergetopic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		log.Error(err.Error())
+		return err
+	}
+	manifestWithMergedTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
+	if len(manifestWithMergedTopic) != 64 {
+		return fmt.Errorf("unknown feed create manifest hash format (mergedtopic): (%d) %s", len(out.String()), manifestWithMergedTopic)
+	}
+	log.Debug("create mergedtopic feed", "manifest", manifestWithMergedTopic)
+	out.Reset()
+
+	// create test data
+	data, err := generateRandomData(feedRandomDataLength)
+	if err != nil {
+		return err
+	}
+	h := md5.New()
+	h.Write(data)
+	dataHash := h.Sum(nil)
+	dataHex := hexutil.Encode(data)
+
+	// update with topic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, dataHex)
+	cmd.Stdout = &out
+	log.Debug("update feed manifest topic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update topic", "out", out)
+	out.Reset()
+
+	// update with subtopic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, dataHex)
+	cmd.Stdout = &out
+	log.Debug("update feed manifest subtopic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update subtopic", "out", out)
+	out.Reset()
+
+	// update with merged topic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex)
+	cmd.Stdout = &out
+	log.Debug("update feed manifest merged topic cmd", "cmd", cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update mergedtopic", "out", out)
+	out.Reset()
+
+	time.Sleep(3 * time.Second)
+
+	// retrieve the data
+	wg := sync.WaitGroup{}
+	for _, endpoint := range endpoints {
+		// raw retrieve, topic only
+		for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} {
+			wg.Add(1)
+			ruid := uuid.New()[:8]
+			go func(hex string, endpoint string, ruid string) {
+				for {
+					err := fetchFeed(hex, userHex, endpoint, dataHash, ruid)
+					if err != nil {
+						continue
+					}
+
+					wg.Done()
+					return
+				}
+			}(hex, endpoint, ruid)
+
+		}
+	}
+	wg.Wait()
+	log.Info("all endpoints synced random data successfully")
+
+	// upload test file
+	log.Info("uploading to " + endpoints[0] + " and syncing")
+
+	f, cleanup := generateRandomFile(filesize * 1000)
+	defer cleanup()
+
+	hash, err := upload(f, endpoints[0])
+	if err != nil {
+		return err
+	}
+	hashBytes, err := hexutil.Decode("0x" + hash)
+	if err != nil {
+		return err
+	}
+	multihashHex := hexutil.Encode(multihash.ToMultihash(hashBytes))
+
+	fileHash, err := digest(f)
+	if err != nil {
+		return err
+	}
+
+	log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash))
+
+	// update file with topic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, multihashHex)
+	cmd.Stdout = &out
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update topic", "out", out)
+	out.Reset()
+
+	// update file with subtopic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, multihashHex)
+	cmd.Stdout = &out
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update subtopic", "out", out)
+	out.Reset()
+
+	// update file with merged topic
+	cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex)
+	cmd.Stdout = &out
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	log.Debug("feed update mergedtopic", "out", out)
+	out.Reset()
+
+	time.Sleep(3 * time.Second)
+
+	for _, endpoint := range endpoints {
+
+		// manifest retrieve, topic only
+		for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} {
+			wg.Add(1)
+			ruid := uuid.New()[:8]
+			go func(url string, endpoint string, ruid string) {
+				for {
+					err := fetch(url, endpoint, fileHash, ruid)
+					if err != nil {
+						continue
+					}
+
+					wg.Done()
+					return
+				}
+			}(url, endpoint, ruid)
+		}
+
+	}
+	wg.Wait()
+	log.Info("all endpoints synced random file successfully")
+
+	return nil
+}
+
+func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error {
+	log.Trace("sleeping", "ruid", ruid)
+	time.Sleep(3 * time.Second)
+
+	log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user)
+	res, err := http.Get(endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user)
+	if err != nil {
+		return err
+	}
+	log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength)
+
+	if res.StatusCode != 200 {
+		return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid)
+	}
+
+	defer res.Body.Close()
+
+	rdigest, err := digest(res.Body)
+	if err != nil {
+		log.Warn(err.Error(), "ruid", ruid)
+		return err
+	}
+
+	if !bytes.Equal(rdigest, original) {
+		err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
+		log.Warn(err.Error(), "ruid", ruid)
+		return err
+	}
+
+	log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
+
+	return nil
+}

cmd/swarm/swarm-smoke/main.go

@@ -21,7 +21,6 @@ import (
 	"sort"
 
 	"github.com/ethereum/go-ethereum/log"
-	colorable "github.com/mattn/go-colorable"
 
 	cli "gopkg.in/urfave/cli.v1"
 )
@@ -34,11 +33,10 @@ var (
 	filesize         int
 	from             int
 	to               int
+	verbosity        int
 )
 
 func main() {
-	log.PrintOrigins(true)
-	log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
 
 	app := cli.NewApp()
 	app.Name = "smoke-test"
@@ -80,6 +78,12 @@ func main() {
 			Usage:       "file size for generated random file in KB",
 			Destination: &filesize,
 		},
+		cli.IntFlag{
+			Name:        "verbosity",
+			Value:       1,
+			Usage:       "verbosity",
+			Destination: &verbosity,
+		},
 	}
 
 	app.Commands = []cli.Command{
@@ -89,6 +93,12 @@ func main() {
 			Usage:   "upload and sync",
 			Action:  cliUploadAndSync,
 		},
+		{
+			Name:    "feed_sync",
+			Aliases: []string{"f"},
+			Usage:   "feed update generate, upload and sync",
+			Action:  cliFeedUploadAndSync,
+		},
 	}
 
 	sort.Sort(cli.FlagsByName(app.Flags))

cmd/swarm/swarm-smoke/upload_and_sync.go

@@ -19,7 +19,8 @@ package main
 import (
 	"bytes"
 	"crypto/md5"
-	"crypto/rand"
+	crand "crypto/rand"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -85,7 +86,6 @@ func cliUploadAndSync(c *cli.Context) error {
 
 	wg := sync.WaitGroup{}
 	for _, endpoint := range endpoints {
-		endpoint := endpoint
 		ruid := uuid.New()[:8]
 		wg.Add(1)
 		go func(endpoint string, ruid string) {
@@ -166,6 +166,18 @@ func digest(r io.Reader) ([]byte, error) {
 	return h.Sum(nil), nil
 }
 
+// generates random data in heap buffer
+func generateRandomData(datasize int) ([]byte, error) {
+	b := make([]byte, datasize)
+	c, err := crand.Read(b)
+	if err != nil {
+		return nil, err
+	} else if c != datasize {
+		return nil, errors.New("short read")
+	}
+	return b, nil
+}
+
 // generateRandomFile is creating a temporary file with the requested byte size
 func generateRandomFile(size int) (f *os.File, teardown func()) {
 	// create a tmp file
@@ -181,7 +193,7 @@ func generateRandomFile(size int) (f *os.File, teardown func()) {
 	}
 
 	buf := make([]byte, size)
-	_, err = rand.Read(buf)
+	_, err = crand.Read(buf)
 	if err != nil {
 		panic(err)
 	}

cmd/swarm/upload.go

@@ -26,29 +26,47 @@ import (
 	"os/user"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 
+	"github.com/ethereum/go-ethereum/log"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"gopkg.in/urfave/cli.v1"
 )
 
-func upload(ctx *cli.Context) {
+var upCommand = cli.Command{
+	Action:             upload,
+	CustomHelpTemplate: helpTemplate,
+	Name:               "up",
+	Usage:              "uploads a file or directory to swarm using the HTTP API",
+	ArgsUsage:          "<file>",
+	Flags:              []cli.Flag{SwarmEncryptedFlag},
+	Description:        "uploads a file or directory to swarm using the HTTP API and prints the root hash",
+}
 
+func upload(ctx *cli.Context) {
 	args := ctx.Args()
 	var (
-		bzzapi       = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		bzzapi          = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		recursive    = ctx.GlobalBool(SwarmRecursiveFlag.Name)
+		recursive       = ctx.GlobalBool(SwarmRecursiveFlag.Name)
-		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
+		wantManifest    = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		defaultPath  = ctx.GlobalString(SwarmUploadDefaultPath.Name)
+		defaultPath     = ctx.GlobalString(SwarmUploadDefaultPath.Name)
-		fromStdin    = ctx.GlobalBool(SwarmUpFromStdinFlag.Name)
+		fromStdin       = ctx.GlobalBool(SwarmUpFromStdinFlag.Name)
-		mimeType     = ctx.GlobalString(SwarmUploadMimeType.Name)
+		mimeType        = ctx.GlobalString(SwarmUploadMimeType.Name)
-		client       = swarm.NewClient(bzzapi)
+		client          = swarm.NewClient(bzzapi)
-		toEncrypt    = ctx.Bool(SwarmEncryptedFlag.Name)
+		toEncrypt       = ctx.Bool(SwarmEncryptedFlag.Name)
-		file         string
+		autoDefaultPath = false
+		file            string
 	)
-
+	if autoDefaultPathString := os.Getenv(SWARM_AUTO_DEFAULTPATH); autoDefaultPathString != "" {
+		b, err := strconv.ParseBool(autoDefaultPathString)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_AUTO_DEFAULTPATH, err)
+		}
+		autoDefaultPath = b
+	}
 	if len(args) != 1 {
 		if fromStdin {
 			tmp, err := ioutil.TempFile("", "swarm-stdin")
@@ -97,6 +115,15 @@ func upload(ctx *cli.Context) {
 			if !recursive {
 				return "", errors.New("Argument is a directory and recursive upload is disabled")
 			}
+			if autoDefaultPath && defaultPath == "" {
+				defaultEntryCandidate := path.Join(file, "index.html")
+				log.Debug("trying to find default path", "path", defaultEntryCandidate)
+				defaultEntryStat, err := os.Stat(defaultEntryCandidate)
+				if err == nil && !defaultEntryStat.IsDir() {
+					log.Debug("setting auto detected default path", "path", defaultEntryCandidate)
+					defaultPath = defaultEntryCandidate
+				}
+			}
 			if defaultPath != "" {
 				// construct absolute default path
 				absDefaultPath, _ := filepath.Abs(defaultPath)

cmd/swarm/upload_test.go

@@ -32,6 +32,8 @@ import (
 
 	"github.com/ethereum/go-ethereum/log"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 	"github.com/mattn/go-colorable"
 )
 
@@ -76,33 +78,22 @@ func testCLISwarmUp(toEncrypt bool, t *testing.T) {
 	cluster := newTestCluster(t, 3)
 	defer cluster.Shutdown()
 
-	// create a tmp file
+	tmpFileName := testutil.TempFileWithContent(t, data)
-	tmp, err := ioutil.TempFile("", "swarm-test")
+	defer os.Remove(tmpFileName)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer tmp.Close()
-	defer os.Remove(tmp.Name())
 
 	// write data to file
-	data := "notsorandomdata"
-	_, err = io.WriteString(tmp, data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	hashRegexp := `[a-f\d]{64}`
 	flags := []string{
 		"--bzzapi", cluster.Nodes[0].URL,
 		"up",
-		tmp.Name()}
+		tmpFileName}
 	if toEncrypt {
 		hashRegexp = `[a-f\d]{128}`
 		flags = []string{
 			"--bzzapi", cluster.Nodes[0].URL,
 			"up",
 			"--encrypt",
-			tmp.Name()}
+			tmpFileName}
 	}
 	// upload the file with 'swarm up' and expect a hash
 	log.Info(fmt.Sprintf("uploading file with 'swarm up'"))
@@ -202,7 +193,6 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
 	}
 	defer os.RemoveAll(tmpUploadDir)
 	// create tmp files
-	data := "notsorandomdata"
 	for _, path := range []string{"tmp1", "tmp2"} {
 		if err := ioutil.WriteFile(filepath.Join(tmpUploadDir, path), bytes.NewBufferString(data).Bytes(), 0644); err != nil {
 			t.Fatal(err)
@@ -242,8 +232,7 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
 		}
 		defer os.RemoveAll(tmpDownload)
 		bzzLocator := "bzz:/" + hash
-		flagss := []string{}
+		flagss := []string{
-		flagss = []string{
 			"--bzzapi", cluster.Nodes[0].URL,
 			"down",
 			"--recursive",
@@ -298,8 +287,8 @@ func TestCLISwarmUpDefaultPath(t *testing.T) {
 }
 
 func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
-	cluster := newTestCluster(t, 1)
+	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
-	defer cluster.Shutdown()
+	defer srv.Close()
 
 	tmp, err := ioutil.TempDir("", "swarm-defaultpath-test")
 	if err != nil {
@@ -323,7 +312,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T
 
 	args := []string{
 		"--bzzapi",
-		cluster.Nodes[0].URL,
+		srv.URL,
 		"--recursive",
 		"--defaultpath",
 		defaultPath,
@@ -340,7 +329,7 @@ func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T
 	up.ExpectExit()
 	hash := matches[0]
 
-	client := swarm.NewClient(cluster.Nodes[0].URL)
+	client := swarm.NewClient(srv.URL)
 
 	m, isEncrypted, err := client.DownloadManifest(hash)
 	if err != nil {

cmd/utils/cmd.go

@@ -272,13 +272,13 @@ func ImportPreimages(db *ethdb.LDBDatabase, fn string) error {
 		// Accumulate the preimages and flush when enough ws gathered
 		preimages[crypto.Keccak256Hash(blob)] = common.CopyBytes(blob)
 		if len(preimages) > 1024 {
-			rawdb.WritePreimages(db, 0, preimages)
+			rawdb.WritePreimages(db, preimages)
 			preimages = make(map[common.Hash][]byte)
 		}
 	}
 	// Flush the last batch preimage data
 	if len(preimages) > 0 {
-		rawdb.WritePreimages(db, 0, preimages)
+		rawdb.WritePreimages(db, preimages)
 	}
 	return nil
 }

common/bytes.go

@@ -31,6 +31,15 @@ func ToHex(b []byte) string {
 	return "0x" + hex
 }
 
+// ToHexArray creates a array of hex-string based on []byte
+func ToHexArray(b [][]byte) []string {
+	r := make([]string, len(b))
+	for i := range b {
+		r[i] = ToHex(b[i])
+	}
+	return r
+}
+
 // FromHex returns the bytes represented by the hexadecimal string s.
 // s may be prefixed with "0x".
 func FromHex(s string) []byte {

common/compiler/solidity.go

@@ -31,14 +31,15 @@ import (
 
 var versionRegexp = regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`)
 
-// Contract contains information about a compiled contract, alongside its code.
+// Contract contains information about a compiled contract, alongside its code and runtime code.
 type Contract struct {
-	Code string       `json:"code"`
+	Code        string       `json:"code"`
-	Info ContractInfo `json:"info"`
+	RuntimeCode string       `json:"runtime-code"`
+	Info        ContractInfo `json:"info"`
 }
 
 // ContractInfo contains information about a compiled contract, including access
-// to the ABI definition, user and developer docs, and metadata.
+// to the ABI definition, source mapping, user and developer docs, and metadata.
 //
 // Depending on the source, language version, compiler version, and compiler
 // options will provide information about how the contract was compiled.
@@ -48,6 +49,8 @@ type ContractInfo struct {
 	LanguageVersion string      `json:"languageVersion"`
 	CompilerVersion string      `json:"compilerVersion"`
 	CompilerOptions string      `json:"compilerOptions"`
+	SrcMap          string      `json:"srcMap"`
+	SrcMapRuntime   string      `json:"srcMapRuntime"`
 	AbiDefinition   interface{} `json:"abiDefinition"`
 	UserDoc         interface{} `json:"userDoc"`
 	DeveloperDoc    interface{} `json:"developerDoc"`
@@ -63,14 +66,16 @@ type Solidity struct {
 // --combined-output format
 type solcOutput struct {
 	Contracts map[string]struct {
-		Bin, Abi, Devdoc, Userdoc, Metadata string
+		BinRuntime                                  string `json:"bin-runtime"`
+		SrcMapRuntime                               string `json:"srcmap-runtime"`
+		Bin, SrcMap, Abi, Devdoc, Userdoc, Metadata string
 	}
 	Version string
 }
 
 func (s *Solidity) makeArgs() []string {
 	p := []string{
-		"--combined-json", "bin,abi,userdoc,devdoc",
+		"--combined-json", "bin,bin-runtime,srcmap,srcmap-runtime,abi,userdoc,devdoc",
 		"--optimize", // code optimizer switched on
 	}
 	if s.Major > 0 || s.Minor > 4 || s.Patch > 6 {
@@ -157,7 +162,7 @@ func (s *Solidity) run(cmd *exec.Cmd, source string) (map[string]*Contract, erro
 // provided source, language and compiler version, and compiler options are all
 // passed through into the Contract structs.
 //
-// The solc output is expected to contain ABI, user docs, and dev docs.
+// The solc output is expected to contain ABI, source mapping, user docs, and dev docs.
 //
 // Returns an error if the JSON is malformed or missing data, or if the JSON
 // embedded within the JSON is malformed.
@@ -184,13 +189,16 @@ func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion strin
 			return nil, fmt.Errorf("solc: error reading dev doc: %v", err)
 		}
 		contracts[name] = &Contract{
-			Code: "0x" + info.Bin,
+			Code:        "0x" + info.Bin,
+			RuntimeCode: "0x" + info.BinRuntime,
 			Info: ContractInfo{
 				Source:          source,
 				Language:        "Solidity",
 				LanguageVersion: languageVersion,
 				CompilerVersion: compilerVersion,
 				CompilerOptions: compilerOptions,
+				SrcMap:          info.SrcMap,
+				SrcMapRuntime:   info.SrcMapRuntime,
 				AbiDefinition:   abi,
 				UserDoc:         userdoc,
 				DeveloperDoc:    devdoc,

consensus/ethash/api.go

@@ -37,27 +37,28 @@ type API struct {
 //   result[0] - 32 bytes hex encoded current block header pow-hash
 //   result[1] - 32 bytes hex encoded seed hash used for DAG
 //   result[2] - 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty
-func (api *API) GetWork() ([3]string, error) {
+//   result[3] - hex encoded block number
+func (api *API) GetWork() ([4]string, error) {
 	if api.ethash.config.PowMode != ModeNormal && api.ethash.config.PowMode != ModeTest {
-		return [3]string{}, errors.New("not supported")
+		return [4]string{}, errors.New("not supported")
 	}
 
 	var (
-		workCh = make(chan [3]string, 1)
+		workCh = make(chan [4]string, 1)
 		errc   = make(chan error, 1)
 	)
 
 	select {
 	case api.ethash.fetchWorkCh <- &sealWork{errc: errc, res: workCh}:
 	case <-api.ethash.exitCh:
-		return [3]string{}, errEthashStopped
+		return [4]string{}, errEthashStopped
 	}
 
 	select {
 	case work := <-workCh:
 		return work, nil
 	case err := <-errc:
-		return [3]string{}, err
+		return [4]string{}, err
 	}
 }
 

consensus/ethash/ethash.go

@@ -432,7 +432,7 @@ type hashrate struct {
 // sealWork wraps a seal work package for remote sealer.
 type sealWork struct {
 	errc chan error
-	res  chan [3]string
+	res  chan [4]string
 }
 
 // Ethash is a consensus engine based on proof-of-work implementing the ethash

consensus/ethash/ethash_test.go

@@ -107,7 +107,7 @@ func TestRemoteSealer(t *testing.T) {
 	ethash.Seal(nil, block, results, nil)
 
 	var (
-		work [3]string
+		work [4]string
 		err  error
 	)
 	if work, err = api.GetWork(); err != nil || work[0] != sealhash.Hex() {

consensus/ethash/sealer.go

@@ -30,6 +30,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
@@ -193,7 +194,7 @@ func (ethash *Ethash) remote(notify []string, noverify bool) {
 
 		results      chan<- *types.Block
 		currentBlock *types.Block
-		currentWork  [3]string
+		currentWork  [4]string
 
 		notifyTransport = &http.Transport{}
 		notifyClient    = &http.Client{
@@ -234,12 +235,14 @@ func (ethash *Ethash) remote(notify []string, noverify bool) {
 	//   result[0], 32 bytes hex encoded current block header pow-hash
 	//   result[1], 32 bytes hex encoded seed hash used for DAG
 	//   result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty
+	//   result[3], hex encoded block number
 	makeWork := func(block *types.Block) {
 		hash := ethash.SealHash(block.Header())
 
 		currentWork[0] = hash.Hex()
 		currentWork[1] = common.BytesToHash(SeedHash(block.NumberU64())).Hex()
 		currentWork[2] = common.BytesToHash(new(big.Int).Div(two256, block.Difficulty()).Bytes()).Hex()
+		currentWork[3] = hexutil.EncodeBig(block.Number())
 
 		// Trace the seal work fetched by remote sealer.
 		currentBlock = block

core/block_validator.go

@@ -53,12 +53,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) {
 		return ErrKnownBlock
 	}
-	if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
-		if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
-			return consensus.ErrUnknownAncestor
-		}
-		return consensus.ErrPrunedAncestor
-	}
 	// Header validity is known at this point, check the uncles and transactions
 	header := block.Header()
 	if err := v.engine.VerifyUncles(v.bc, block); err != nil {
@@ -70,6 +64,12 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	if hash := types.DeriveSha(block.Transactions()); hash != header.TxHash {
 		return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash)
 	}
+	if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
+		if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
+			return consensus.ErrUnknownAncestor
+		}
+		return consensus.ErrPrunedAncestor
+	}
 	return nil
 }
 

core/blockchain.go

@@ -140,7 +140,7 @@ type BlockChain struct {
 func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(block *types.Block) bool) (*BlockChain, error) {
 	if cacheConfig == nil {
 		cacheConfig = &CacheConfig{
-			TrieNodeLimit: 256 * 1024 * 1024,
+			TrieNodeLimit: 256,
 			TrieTimeLimit: 5 * time.Minute,
 		}
 	}
@@ -1002,7 +1002,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.
 		}
 		// Write the positional metadata for transaction/receipt lookups and preimages
 		rawdb.WriteTxLookupEntries(batch, block)
-		rawdb.WritePreimages(batch, block.NumberU64(), state.Preimages())
+		rawdb.WritePreimages(batch, state.Preimages())
 
 		status = CanonStatTy
 	} else {

core/chain_makers.go

@@ -33,12 +33,11 @@ import (
 // BlockGen creates blocks for testing.
 // See GenerateChain for a detailed explanation.
 type BlockGen struct {
-	i           int
+	i       int
-	parent      *types.Block
+	parent  *types.Block
-	chain       []*types.Block
+	chain   []*types.Block
-	chainReader consensus.ChainReader
+	header  *types.Header
-	header      *types.Header
+	statedb *state.StateDB
-	statedb     *state.StateDB
 
 	gasPool  *GasPool
 	txs      []*types.Transaction
@@ -138,7 +137,7 @@ func (b *BlockGen) AddUncle(h *types.Header) {
 // For index -1, PrevBlock returns the parent block given to GenerateChain.
 func (b *BlockGen) PrevBlock(index int) *types.Block {
 	if index >= b.i {
-		panic("block index out of range")
+		panic(fmt.Errorf("block index %d out of range (%d,%d)", index, -1, b.i))
 	}
 	if index == -1 {
 		return b.parent
@@ -154,7 +153,8 @@ func (b *BlockGen) OffsetTime(seconds int64) {
 	if b.header.Time.Cmp(b.parent.Header().Time) <= 0 {
 		panic("block time out of range")
 	}
-	b.header.Difficulty = b.engine.CalcDifficulty(b.chainReader, b.header.Time.Uint64(), b.parent.Header())
+	chainreader := &fakeChainReader{config: b.config}
+	b.header.Difficulty = b.engine.CalcDifficulty(chainreader, b.header.Time.Uint64(), b.parent.Header())
 }
 
 // GenerateChain creates a chain of n blocks. The first block's
@@ -174,14 +174,10 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		config = params.TestChainConfig
 	}
 	blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
+	chainreader := &fakeChainReader{config: config}
 	genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) {
-		// TODO(karalabe): This is needed for clique, which depends on multiple blocks.
+		b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine}
-		// It's nonetheless ugly to spin up a blockchain here. Get rid of this somehow.
+		b.header = makeHeader(chainreader, parent, statedb, b.engine)
-		blockchain, _ := NewBlockChain(db, nil, config, engine, vm.Config{}, nil)
-		defer blockchain.Stop()
-
-		b := &BlockGen{i: i, parent: parent, chain: blocks, chainReader: blockchain, statedb: statedb, config: config, engine: engine}
-		b.header = makeHeader(b.chainReader, parent, statedb, b.engine)
 
 		// Mutate the state and block according to any hard-fork specs
 		if daoBlock := config.DAOForkBlock; daoBlock != nil {
@@ -201,7 +197,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		}
 		if b.engine != nil {
 			// Finalize and seal the block
-			block, _ := b.engine.Finalize(b.chainReader, b.header, statedb, b.txs, b.uncles, b.receipts)
+			block, _ := b.engine.Finalize(chainreader, b.header, statedb, b.txs, b.uncles, b.receipts)
 
 			// Write state changes to db
 			root, err := statedb.Commit(config.IsEIP158(b.header.Number))
@@ -269,3 +265,19 @@ func makeBlockChain(parent *types.Block, n int, engine consensus.Engine, db ethd
 	})
 	return blocks
 }
+
+type fakeChainReader struct {
+	config  *params.ChainConfig
+	genesis *types.Block
+}
+
+// Config returns the chain configuration.
+func (cr *fakeChainReader) Config() *params.ChainConfig {
+	return cr.config
+}
+
+func (cr *fakeChainReader) CurrentHeader() *types.Header                            { return nil }
+func (cr *fakeChainReader) GetHeaderByNumber(number uint64) *types.Header           { return nil }
+func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *types.Header          { return nil }
+func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *types.Header { return nil }
+func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block   { return nil }

core/rawdb/accessors_chain.go

@@ -278,7 +278,7 @@ func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Rece
 	if len(data) == 0 {
 		return nil
 	}
-	// Convert the revceipts from their storage form to their internal representation
+	// Convert the receipts from their storage form to their internal representation
 	storageReceipts := []*types.ReceiptForStorage{}
 	if err := rlp.DecodeBytes(data, &storageReceipts); err != nil {
 		log.Error("Invalid receipt array RLP", "hash", hash, "err", err)

core/rawdb/accessors_metadata.go

@@ -77,9 +77,8 @@ func ReadPreimage(db DatabaseReader, hash common.Hash) []byte {
 	return data
 }
 
-// WritePreimages writes the provided set of preimages to the database. `number` is the
+// WritePreimages writes the provided set of preimages to the database.
-// current block number, and is used for debug messages only.
+func WritePreimages(db DatabaseWriter, preimages map[common.Hash][]byte) {
-func WritePreimages(db DatabaseWriter, number uint64, preimages map[common.Hash][]byte) {
 	for hash, preimage := range preimages {
 		if err := db.Put(preimageKey(hash), preimage); err != nil {
 			log.Crit("Failed to store trie preimage", "err", err)

core/rawdb/schema.go

@@ -35,7 +35,7 @@ var (
 	// headBlockKey tracks the latest know full block's hash.
 	headBlockKey = []byte("LastBlock")
 
-	// headFastBlockKey tracks the latest known incomplete block's hash duirng fast sync.
+	// headFastBlockKey tracks the latest known incomplete block's hash during fast sync.
 	headFastBlockKey = []byte("LastFast")
 
 	// fastTrieProgressKey tracks the number of trie entries imported during fast sync.

core/state/statedb.go

@@ -18,10 +18,10 @@
 package state
 
 import (
+	"errors"
 	"fmt"
 	"math/big"
 	"sort"
-	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -44,6 +44,13 @@ var (
 	emptyCode = crypto.Keccak256Hash(nil)
 )
 
+type proofList [][]byte
+
+func (n *proofList) Put(key []byte, value []byte) error {
+	*n = append(*n, value)
+	return nil
+}
+
 // StateDBs within the ethereum protocol are used to store anything
 // within the merkle trie. StateDBs take care of caching and storing
 // nested states. It's the general query interface to retrieve:
@@ -79,8 +86,6 @@ type StateDB struct {
 	journal        *journal
 	validRevisions []revision
 	nextRevisionId int
-
-	lock sync.Mutex
 }
 
 // Create a new state from a given trie.
@@ -256,6 +261,24 @@ func (self *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash
 	return common.Hash{}
 }
 
+// GetProof returns the MerkleProof for a given Account
+func (self *StateDB) GetProof(a common.Address) ([][]byte, error) {
+	var proof proofList
+	err := self.trie.Prove(crypto.Keccak256(a.Bytes()), 0, &proof)
+	return [][]byte(proof), err
+}
+
+// GetProof returns the StorageProof for given key
+func (self *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, error) {
+	var proof proofList
+	trie := self.StorageTrie(a)
+	if trie == nil {
+		return proof, errors.New("storage trie for requested address does not exist")
+	}
+	err := trie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof)
+	return [][]byte(proof), err
+}
+
 // GetCommittedState retrieves a value from the given account's committed storage trie.
 func (self *StateDB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash {
 	stateObject := self.getStateObject(addr)
@@ -470,9 +493,6 @@ func (db *StateDB) ForEachStorage(addr common.Address, cb func(key, value common
 // Copy creates a deep, independent copy of the state.
 // Snapshots of the copied state cannot be applied to the copy.
 func (self *StateDB) Copy() *StateDB {
-	self.lock.Lock()
-	defer self.lock.Unlock()
-
 	// Copy all the basic fields, initialize the memory ones
 	state := &StateDB{
 		db:                self.db,

core/types/transaction_signing.go

@@ -136,7 +136,7 @@ func (s EIP155Signer) Sender(tx *Transaction) (common.Address, error) {
 	return recoverPlain(s.Hash(tx), tx.data.R, tx.data.S, V, true)
 }
 
-// WithSignature returns a new transaction with the given signature. This signature
+// SignatureValues returns signature values. This signature
 // needs to be in the [R || S || V] format where V is 0 or 1.
 func (s EIP155Signer) SignatureValues(tx *Transaction, sig []byte) (R, S, V *big.Int, err error) {
 	R, S, V, err = HomesteadSigner{}.SignatureValues(tx, sig)

core/vm/gen_structlog.go

@@ -13,20 +13,22 @@ import (
 
 var _ = (*structLogMarshaling)(nil)
 
+// MarshalJSON marshals as JSON.
 func (s StructLog) MarshalJSON() ([]byte, error) {
 	type StructLog struct {
-		Pc          uint64                      `json:"pc"`
+		Pc            uint64                      `json:"pc"`
-		Op          OpCode                      `json:"op"`
+		Op            OpCode                      `json:"op"`
-		Gas         math.HexOrDecimal64         `json:"gas"`
+		Gas           math.HexOrDecimal64         `json:"gas"`
-		GasCost     math.HexOrDecimal64         `json:"gasCost"`
+		GasCost       math.HexOrDecimal64         `json:"gasCost"`
-		Memory      hexutil.Bytes               `json:"memory"`
+		Memory        hexutil.Bytes               `json:"memory"`
-		MemorySize  int                         `json:"memSize"`
+		MemorySize    int                         `json:"memSize"`
-		Stack       []*math.HexOrDecimal256     `json:"stack"`
+		Stack         []*math.HexOrDecimal256     `json:"stack"`
-		Storage     map[common.Hash]common.Hash `json:"-"`
+		Storage       map[common.Hash]common.Hash `json:"-"`
-		Depth       int                         `json:"depth"`
+		Depth         int                         `json:"depth"`
-		Err         error                       `json:"-"`
+		RefundCounter uint64                      `json:"refund"`
-		OpName      string                      `json:"opName"`
+		Err           error                       `json:"-"`
-		ErrorString string                      `json:"error"`
+		OpName        string                      `json:"opName"`
+		ErrorString   string                      `json:"error"`
 	}
 	var enc StructLog
 	enc.Pc = s.Pc
@@ -43,24 +45,27 @@ func (s StructLog) MarshalJSON() ([]byte, error) {
 	}
 	enc.Storage = s.Storage
 	enc.Depth = s.Depth
+	enc.RefundCounter = s.RefundCounter
 	enc.Err = s.Err
 	enc.OpName = s.OpName()
 	enc.ErrorString = s.ErrorString()
 	return json.Marshal(&enc)
 }
 
+// UnmarshalJSON unmarshals from JSON.
 func (s *StructLog) UnmarshalJSON(input []byte) error {
 	type StructLog struct {
-		Pc         *uint64                     `json:"pc"`
+		Pc            *uint64                     `json:"pc"`
-		Op         *OpCode                     `json:"op"`
+		Op            *OpCode                     `json:"op"`
-		Gas        *math.HexOrDecimal64        `json:"gas"`
+		Gas           *math.HexOrDecimal64        `json:"gas"`
-		GasCost    *math.HexOrDecimal64        `json:"gasCost"`
+		GasCost       *math.HexOrDecimal64        `json:"gasCost"`
-		Memory     *hexutil.Bytes              `json:"memory"`
+		Memory        *hexutil.Bytes              `json:"memory"`
-		MemorySize *int                        `json:"memSize"`
+		MemorySize    *int                        `json:"memSize"`
-		Stack      []*math.HexOrDecimal256     `json:"stack"`
+		Stack         []*math.HexOrDecimal256     `json:"stack"`
-		Storage    map[common.Hash]common.Hash `json:"-"`
+		Storage       map[common.Hash]common.Hash `json:"-"`
-		Depth      *int                        `json:"depth"`
+		Depth         *int                        `json:"depth"`
-		Err        error                       `json:"-"`
+		RefundCounter *uint64                     `json:"refund"`
+		Err           error                       `json:"-"`
 	}
 	var dec StructLog
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -96,6 +101,9 @@ func (s *StructLog) UnmarshalJSON(input []byte) error {
 	if dec.Depth != nil {
 		s.Depth = *dec.Depth
 	}
+	if dec.RefundCounter != nil {
+		s.RefundCounter = *dec.RefundCounter
+	}
 	if dec.Err != nil {
 		s.Err = dec.Err
 	}

core/vm/instructions.go

@@ -124,10 +124,22 @@ func opSmod(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory
 
 func opExp(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	base, exponent := stack.pop(), stack.pop()
-	stack.push(math.Exp(base, exponent))
+	// some shortcuts
-
+	cmpToOne := exponent.Cmp(big1)
-	interpreter.intPool.put(base, exponent)
+	if cmpToOne < 0 { // Exponent is zero
-
+		// x ^ 0 == 1
+		stack.push(base.SetUint64(1))
+	} else if base.Sign() == 0 {
+		// 0 ^ y, if y != 0, == 0
+		stack.push(base.SetUint64(0))
+	} else if cmpToOne == 0 { // Exponent is one
+		// x ^ 1 == x
+		stack.push(base)
+	} else {
+		stack.push(math.Exp(base, exponent))
+		interpreter.intPool.put(base)
+	}
+	interpreter.intPool.put(exponent)
 	return nil, nil
 }
 
@@ -532,7 +544,12 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, contract *Contract,
 // this account should be regarded as a non-existent account and zero should be returned.
 func opExtCodeHash(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	slot := stack.peek()
-	slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(common.BigToAddress(slot)).Bytes())
+	address := common.BigToAddress(slot)
+	if interpreter.evm.StateDB.Empty(address) {
+		slot.SetUint64(0)
+	} else {
+		slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(address).Bytes())
+	}
 	return nil, nil
 }
 

core/vm/logger.go

@@ -56,16 +56,17 @@ type LogConfig struct {
 // StructLog is emitted to the EVM each cycle and lists information about the current internal state
 // prior to the execution of the statement.
 type StructLog struct {
-	Pc         uint64                      `json:"pc"`
+	Pc            uint64                      `json:"pc"`
-	Op         OpCode                      `json:"op"`
+	Op            OpCode                      `json:"op"`
-	Gas        uint64                      `json:"gas"`
+	Gas           uint64                      `json:"gas"`
-	GasCost    uint64                      `json:"gasCost"`
+	GasCost       uint64                      `json:"gasCost"`
-	Memory     []byte                      `json:"memory"`
+	Memory        []byte                      `json:"memory"`
-	MemorySize int                         `json:"memSize"`
+	MemorySize    int                         `json:"memSize"`
-	Stack      []*big.Int                  `json:"stack"`
+	Stack         []*big.Int                  `json:"stack"`
-	Storage    map[common.Hash]common.Hash `json:"-"`
+	Storage       map[common.Hash]common.Hash `json:"-"`
-	Depth      int                         `json:"depth"`
+	Depth         int                         `json:"depth"`
-	Err        error                       `json:"-"`
+	RefundCounter uint64                      `json:"refund"`
+	Err           error                       `json:"-"`
 }
 
 // overrides for gencodec
@@ -177,7 +178,7 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost ui
 		storage = l.changedValues[contract.Address()].Copy()
 	}
 	// create a new snaptshot of the EVM.
-	log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, storage, depth, err}
+	log := StructLog{pc, op, gas, cost, mem, memory.Len(), stck, storage, depth, env.StateDB.GetRefund(), err}
 
 	l.logs = append(l.logs, log)
 	return nil

core/vm/logger_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -41,9 +42,15 @@ func (d *dummyContractRef) SetBalance(*big.Int)        {}
 func (d *dummyContractRef) SetNonce(uint64)            {}
 func (d *dummyContractRef) Balance() *big.Int          { return new(big.Int) }
 
+type dummyStatedb struct {
+	state.StateDB
+}
+
+func (*dummyStatedb) GetRefund() uint64 { return 1337 }
+
 func TestStoreCapture(t *testing.T) {
 	var (
-		env      = NewEVM(Context{}, nil, params.TestChainConfig, Config{})
+		env      = NewEVM(Context{}, &dummyStatedb{}, params.TestChainConfig, Config{})
 		logger   = NewStructLogger(nil)
 		mem      = NewMemory()
 		stack    = newstack()
@@ -51,9 +58,7 @@ func TestStoreCapture(t *testing.T) {
 	)
 	stack.push(big.NewInt(1))
 	stack.push(big.NewInt(0))
-
 	var index common.Hash
-
 	logger.CaptureState(env, 0, SSTORE, 0, 0, mem, stack, contract, 0, nil)
 	if len(logger.changedValues[contract.Address()]) == 0 {
 		t.Fatalf("expected exactly 1 changed value on address %x, got %d", contract.Address(), len(logger.changedValues[contract.Address()]))

eth/config.go

@@ -122,7 +122,7 @@ type Config struct {
 	// Miscellaneous options
 	DocRoot string `toml:"-"`
 
-	// Type of the EWASM interpreter ("" for detault)
+	// Type of the EWASM interpreter ("" for default)
 	EWASMInterpreter string
 	// Type of the EVM interpreter ("" for default)
 	EVMInterpreter string

eth/downloader/downloader.go

@@ -740,6 +740,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 					return 0, errBadPeer
 				}
 				start = check
+				hash = h
 
 			case <-timeout:
 				p.log.Debug("Waiting for search header timed out", "elapsed", ttl)
@@ -1246,7 +1247,7 @@ func (d *Downloader) processHeaders(origin uint64, pivot uint64, td *big.Int) er
 				}
 				// If no headers were retrieved at all, the peer violated its TD promise that it had a
 				// better chain compared to ours. The only exception is if its promised blocks were
-				// already imported by other means (e.g. fecher):
+				// already imported by other means (e.g. fetcher):
 				//
 				// R <remote peer>, L <local node>: Both at block 10
 				// R: Mine block 11, and propagate it to L
@@ -1415,7 +1416,7 @@ func (d *Downloader) processFastSyncContent(latest *types.Header) error {
 	defer stateSync.Cancel()
 	go func() {
 		if err := stateSync.Wait(); err != nil && err != errCancelStateFetch {
-			d.queue.Close() // wake up WaitResults
+			d.queue.Close() // wake up Results
 		}
 	}()
 	// Figure out the ideal pivot block. Note, that this goalpost may move if the
@@ -1473,7 +1474,7 @@ func (d *Downloader) processFastSyncContent(latest *types.Header) error {
 				defer stateSync.Cancel()
 				go func() {
 					if err := stateSync.Wait(); err != nil && err != errCancelStateFetch {
-						d.queue.Close() // wake up WaitResults
+						d.queue.Close() // wake up Results
 					}
 				}()
 				oldPivot = P

eth/downloader/peer.go

@@ -229,13 +229,6 @@ func (p *peerConnection) SetHeadersIdle(delivered int) {
 	p.setIdle(p.headerStarted, delivered, &p.headerThroughput, &p.headerIdle)
 }
 
-// SetBlocksIdle sets the peer to idle, allowing it to execute new block retrieval
-// requests. Its estimated block retrieval throughput is updated with that measured
-// just now.
-func (p *peerConnection) SetBlocksIdle(delivered int) {
-	p.setIdle(p.blockStarted, delivered, &p.blockThroughput, &p.blockIdle)
-}
-
 // SetBodiesIdle sets the peer to idle, allowing it to execute block body retrieval
 // requests. Its estimated body retrieval throughput is updated with that measured
 // just now.

eth/downloader/queue.go

@@ -143,7 +143,7 @@ func (q *queue) Reset() {
 	q.resultOffset = 0
 }
 
-// Close marks the end of the sync, unblocking WaitResults.
+// Close marks the end of the sync, unblocking Results.
 // It may be called even if the queue is already closed.
 func (q *queue) Close() {
 	q.lock.Lock()
@@ -545,7 +545,7 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common
 		taskQueue.Push(header, -int64(header.Number.Uint64()))
 	}
 	if progress {
-		// Wake WaitResults, resultCache was modified
+		// Wake Results, resultCache was modified
 		q.active.Signal()
 	}
 	// Assemble and return the block download request
@@ -664,12 +664,11 @@ func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest,
 			}
 			// Add the peer to the expiry report along the number of failed requests
 			expiries[id] = len(request.Headers)
+
+			// Remove the expired requests from the pending pool directly
+			delete(pendPool, id)
 		}
 	}
-	// Remove the expired requests from the pending pool
-	for id := range expiries {
-		delete(pendPool, id)
-	}
 	return expiries
 }
 
@@ -857,7 +856,7 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQ
 			taskQueue.Push(header, -int64(header.Number.Uint64()))
 		}
 	}
-	// Wake up WaitResults
+	// Wake up Results
 	if accepted > 0 {
 		q.active.Signal()
 	}

eth/downloader/statesync.go

@@ -313,11 +313,12 @@ func (s *stateSync) loop() (err error) {
 				s.d.dropPeer(req.peer.id)
 			}
 			// Process all the received blobs and check for stale delivery
-			if err = s.process(req); err != nil {
+			delivered, err := s.process(req)
+			if err != nil {
 				log.Warn("Node data write error", "err", err)
 				return err
 			}
-			req.peer.SetNodeDataIdle(len(req.response))
+			req.peer.SetNodeDataIdle(delivered)
 		}
 	}
 	return nil
@@ -398,9 +399,11 @@ func (s *stateSync) fillTasks(n int, req *stateReq) {
 // process iterates over a batch of delivered state data, injecting each item
 // into a running state sync, re-queuing any items that were requested but not
 // delivered.
-func (s *stateSync) process(req *stateReq) error {
+// Returns whether the peer actually managed to deliver anything of value,
+// and any error that occurred
+func (s *stateSync) process(req *stateReq) (int, error) {
 	// Collect processing stats and update progress if valid data was received
-	duplicate, unexpected := 0, 0
+	duplicate, unexpected, successful := 0, 0, 0
 
 	defer func(start time.Time) {
 		if duplicate > 0 || unexpected > 0 {
@@ -410,7 +413,6 @@ func (s *stateSync) process(req *stateReq) error {
 
 	// Iterate over all the delivered data and inject one-by-one into the trie
 	progress := false
-
 	for _, blob := range req.response {
 		prog, hash, err := s.processNodeData(blob)
 		switch err {
@@ -418,12 +420,13 @@ func (s *stateSync) process(req *stateReq) error {
 			s.numUncommitted++
 			s.bytesUncommitted += len(blob)
 			progress = progress || prog
+			successful++
 		case trie.ErrNotRequested:
 			unexpected++
 		case trie.ErrAlreadyProcessed:
 			duplicate++
 		default:
-			return fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err)
+			return successful, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err)
 		}
 		if _, ok := req.tasks[hash]; ok {
 			delete(req.tasks, hash)
@@ -441,12 +444,12 @@ func (s *stateSync) process(req *stateReq) error {
 		// If we've requested the node too many times already, it may be a malicious
 		// sync where nobody has the right data. Abort.
 		if len(task.attempts) >= npeers {
-			return fmt.Errorf("state node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers)
+			return successful, fmt.Errorf("state node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers)
 		}
 		// Missing item, place into the retry queue.
 		s.tasks[hash] = task
 	}
-	return nil
+	return successful, nil
 }
 
 // processNodeData tries to inject a trie node data blob delivered from a remote

eth/downloader/testchain_test.go

@@ -0,0 +1,221 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package downloader
+
+import (
+	"fmt"
+	"math/big"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+// Test chain parameters.
+var (
+	testKey, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+	testAddress = crypto.PubkeyToAddress(testKey.PublicKey)
+	testDB      = ethdb.NewMemDatabase()
+	testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000))
+)
+
+// The common prefix of all test chains:
+var testChainBase = newTestChain(blockCacheItems+200, testGenesis)
+
+// Different forks on top of the base chain:
+var testChainForkLightA, testChainForkLightB, testChainForkHeavy *testChain
+
+func init() {
+	var forkLen = int(MaxForkAncestry + 50)
+	var wg sync.WaitGroup
+	wg.Add(3)
+	go func() { testChainForkLightA = testChainBase.makeFork(forkLen, false, 1); wg.Done() }()
+	go func() { testChainForkLightB = testChainBase.makeFork(forkLen, false, 2); wg.Done() }()
+	go func() { testChainForkHeavy = testChainBase.makeFork(forkLen, true, 3); wg.Done() }()
+	wg.Wait()
+}
+
+type testChain struct {
+	genesis  *types.Block
+	chain    []common.Hash
+	headerm  map[common.Hash]*types.Header
+	blockm   map[common.Hash]*types.Block
+	receiptm map[common.Hash][]*types.Receipt
+	tdm      map[common.Hash]*big.Int
+}
+
+// newTestChain creates a blockchain of the given length.
+func newTestChain(length int, genesis *types.Block) *testChain {
+	tc := new(testChain).copy(length)
+	tc.genesis = genesis
+	tc.chain = append(tc.chain, genesis.Hash())
+	tc.headerm[tc.genesis.Hash()] = tc.genesis.Header()
+	tc.tdm[tc.genesis.Hash()] = tc.genesis.Difficulty()
+	tc.blockm[tc.genesis.Hash()] = tc.genesis
+	tc.generate(length-1, 0, genesis, false)
+	return tc
+}
+
+// makeFork creates a fork on top of the test chain.
+func (tc *testChain) makeFork(length int, heavy bool, seed byte) *testChain {
+	fork := tc.copy(tc.len() + length)
+	fork.generate(length, seed, tc.headBlock(), heavy)
+	return fork
+}
+
+// shorten creates a copy of the chain with the given length. It panics if the
+// length is longer than the number of available blocks.
+func (tc *testChain) shorten(length int) *testChain {
+	if length > tc.len() {
+		panic(fmt.Errorf("can't shorten test chain to %d blocks, it's only %d blocks long", length, tc.len()))
+	}
+	return tc.copy(length)
+}
+
+func (tc *testChain) copy(newlen int) *testChain {
+	cpy := &testChain{
+		genesis:  tc.genesis,
+		headerm:  make(map[common.Hash]*types.Header, newlen),
+		blockm:   make(map[common.Hash]*types.Block, newlen),
+		receiptm: make(map[common.Hash][]*types.Receipt, newlen),
+		tdm:      make(map[common.Hash]*big.Int, newlen),
+	}
+	for i := 0; i < len(tc.chain) && i < newlen; i++ {
+		hash := tc.chain[i]
+		cpy.chain = append(cpy.chain, tc.chain[i])
+		cpy.tdm[hash] = tc.tdm[hash]
+		cpy.blockm[hash] = tc.blockm[hash]
+		cpy.headerm[hash] = tc.headerm[hash]
+		cpy.receiptm[hash] = tc.receiptm[hash]
+	}
+	return cpy
+}
+
+// generate creates a chain of n blocks starting at and including parent.
+// the returned hash chain is ordered head->parent. In addition, every 22th block
+// contains a transaction and every 5th an uncle to allow testing correct block
+// reassembly.
+func (tc *testChain) generate(n int, seed byte, parent *types.Block, heavy bool) {
+	// start := time.Now()
+	// defer func() { fmt.Printf("test chain generated in %v\n", time.Since(start)) }()
+
+	blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) {
+		block.SetCoinbase(common.Address{seed})
+		// If a heavy chain is requested, delay blocks to raise difficulty
+		if heavy {
+			block.OffsetTime(-1)
+		}
+		// Include transactions to the miner to make blocks more interesting.
+		if parent == tc.genesis && i%22 == 0 {
+			signer := types.MakeSigner(params.TestChainConfig, block.Number())
+			tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey)
+			if err != nil {
+				panic(err)
+			}
+			block.AddTx(tx)
+		}
+		// if the block number is a multiple of 5, add a bonus uncle to the block
+		if i > 0 && i%5 == 0 {
+			block.AddUncle(&types.Header{
+				ParentHash: block.PrevBlock(i - 1).Hash(),
+				Number:     big.NewInt(block.Number().Int64() - 1),
+			})
+		}
+	})
+
+	// Convert the block-chain into a hash-chain and header/block maps
+	td := new(big.Int).Set(tc.td(parent.Hash()))
+	for i, b := range blocks {
+		td := td.Add(td, b.Difficulty())
+		hash := b.Hash()
+		tc.chain = append(tc.chain, hash)
+		tc.blockm[hash] = b
+		tc.headerm[hash] = b.Header()
+		tc.receiptm[hash] = receipts[i]
+		tc.tdm[hash] = new(big.Int).Set(td)
+	}
+}
+
+// len returns the total number of blocks in the chain.
+func (tc *testChain) len() int {
+	return len(tc.chain)
+}
+
+// headBlock returns the head of the chain.
+func (tc *testChain) headBlock() *types.Block {
+	return tc.blockm[tc.chain[len(tc.chain)-1]]
+}
+
+// td returns the total difficulty of the given block.
+func (tc *testChain) td(hash common.Hash) *big.Int {
+	return tc.tdm[hash]
+}
+
+// headersByHash returns headers in ascending order from the given hash.
+func (tc *testChain) headersByHash(origin common.Hash, amount int, skip int) []*types.Header {
+	num, _ := tc.hashToNumber(origin)
+	return tc.headersByNumber(num, amount, skip)
+}
+
+// headersByNumber returns headers in ascending order from the given number.
+func (tc *testChain) headersByNumber(origin uint64, amount int, skip int) []*types.Header {
+	result := make([]*types.Header, 0, amount)
+	for num := origin; num < uint64(len(tc.chain)) && len(result) < amount; num += uint64(skip) + 1 {
+		if header, ok := tc.headerm[tc.chain[int(num)]]; ok {
+			result = append(result, header)
+		}
+	}
+	return result
+}
+
+// receipts returns the receipts of the given block hashes.
+func (tc *testChain) receipts(hashes []common.Hash) [][]*types.Receipt {
+	results := make([][]*types.Receipt, 0, len(hashes))
+	for _, hash := range hashes {
+		if receipt, ok := tc.receiptm[hash]; ok {
+			results = append(results, receipt)
+		}
+	}
+	return results
+}
+
+// bodies returns the block bodies of the given block hashes.
+func (tc *testChain) bodies(hashes []common.Hash) ([][]*types.Transaction, [][]*types.Header) {
+	transactions := make([][]*types.Transaction, 0, len(hashes))
+	uncles := make([][]*types.Header, 0, len(hashes))
+	for _, hash := range hashes {
+		if block, ok := tc.blockm[hash]; ok {
+			transactions = append(transactions, block.Transactions())
+			uncles = append(uncles, block.Uncles())
+		}
+	}
+	return transactions, uncles
+}
+
+func (tc *testChain) hashToNumber(target common.Hash) (uint64, bool) {
+	for num, hash := range tc.chain {
+		if hash == target {
+			return uint64(num), true
+		}
+	}
+	return 0, false
+}

eth/handler.go

@@ -653,7 +653,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 			trueHead = request.Block.ParentHash()
 			trueTD   = new(big.Int).Sub(request.TD, request.Block.Difficulty())
 		)
-		// Update the peers total difficulty if better than the previous
+		// Update the peer's total difficulty if better than the previous
 		if _, td := p.Head(); trueTD.Cmp(td) > 0 {
 			p.SetHead(trueHead, trueTD)
 

eth/tracers/tracer.go

@@ -290,11 +290,12 @@ type Tracer struct {
 	contractWrapper *contractWrapper // Wrapper around the contract object
 	dbWrapper       *dbWrapper       // Wrapper around the VM environment
 
-	pcValue    *uint   // Swappable pc value wrapped by a log accessor
+	pcValue     *uint   // Swappable pc value wrapped by a log accessor
-	gasValue   *uint   // Swappable gas value wrapped by a log accessor
+	gasValue    *uint   // Swappable gas value wrapped by a log accessor
-	costValue  *uint   // Swappable cost value wrapped by a log accessor
+	costValue   *uint   // Swappable cost value wrapped by a log accessor
-	depthValue *uint   // Swappable depth value wrapped by a log accessor
+	depthValue  *uint   // Swappable depth value wrapped by a log accessor
-	errorValue *string // Swappable error value wrapped by a log accessor
+	errorValue  *string // Swappable error value wrapped by a log accessor
+	refundValue *uint   // Swappable refund value wrapped by a log accessor
 
 	ctx map[string]interface{} // Transaction context gathered throughout execution
 	err error                  // Error, if one has occurred
@@ -323,6 +324,7 @@ func New(code string) (*Tracer, error) {
 		gasValue:        new(uint),
 		costValue:       new(uint),
 		depthValue:      new(uint),
+		refundValue:     new(uint),
 	}
 	// Set up builtins for this environment
 	tracer.vm.PushGlobalGoFunction("toHex", func(ctx *duktape.Context) int {
@@ -442,6 +444,9 @@ func New(code string) (*Tracer, error) {
 	tracer.vm.PushGoFunction(func(ctx *duktape.Context) int { ctx.PushUint(*tracer.depthValue); return 1 })
 	tracer.vm.PutPropString(logObject, "getDepth")
 
+	tracer.vm.PushGoFunction(func(ctx *duktape.Context) int { ctx.PushUint(*tracer.refundValue); return 1 })
+	tracer.vm.PutPropString(logObject, "getRefund")
+
 	tracer.vm.PushGoFunction(func(ctx *duktape.Context) int {
 		if tracer.errorValue != nil {
 			ctx.PushString(*tracer.errorValue)
@@ -527,6 +532,7 @@ func (jst *Tracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost
 		*jst.gasValue = uint(gas)
 		*jst.costValue = uint(cost)
 		*jst.depthValue = uint(depth)
+		*jst.refundValue = uint(env.StateDB.GetRefund())
 
 		jst.errorValue = nil
 		if err != nil {

eth/tracers/tracer_test.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/params"
 )
@@ -43,8 +44,14 @@ func (account) ReturnGas(*big.Int)                                  {}
 func (account) SetCode(common.Hash, []byte)                         {}
 func (account) ForEachStorage(cb func(key, value common.Hash) bool) {}
 
+type dummyStatedb struct {
+	state.StateDB
+}
+
+func (*dummyStatedb) GetRefund() uint64 { return 1337 }
+
 func runTrace(tracer *Tracer) (json.RawMessage, error) {
-	env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, nil, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer})
+	env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer})
 
 	contract := vm.NewContract(account{}, account{}, big.NewInt(0), 10000)
 	contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0}
@@ -126,7 +133,7 @@ func TestHaltBetweenSteps(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, nil, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer})
+	env := vm.NewEVM(vm.Context{BlockNumber: big.NewInt(1)}, &dummyStatedb{}, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer})
 	contract := vm.NewContract(&account{}, &account{}, big.NewInt(0), 0)
 
 	tracer.CaptureState(env, 0, 0, 0, 0, nil, nil, contract, 0, nil)

ethclient/ethclient.go

@@ -365,26 +365,42 @@ func (ec *Client) NonceAt(ctx context.Context, account common.Address, blockNumb
 // FilterLogs executes a filter query.
 func (ec *Client) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) {
 	var result []types.Log
-	err := ec.c.CallContext(ctx, &result, "eth_getLogs", toFilterArg(q))
+	arg, err := toFilterArg(q)
+	if err != nil {
+		return nil, err
+	}
+	err = ec.c.CallContext(ctx, &result, "eth_getLogs", arg)
 	return result, err
 }
 
 // SubscribeFilterLogs subscribes to the results of a streaming filter query.
 func (ec *Client) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- types.Log) (ethereum.Subscription, error) {
-	return ec.c.EthSubscribe(ctx, ch, "logs", toFilterArg(q))
+	arg, err := toFilterArg(q)
+	if err != nil {
+		return nil, err
+	}
+	return ec.c.EthSubscribe(ctx, ch, "logs", arg)
 }
 
-func toFilterArg(q ethereum.FilterQuery) interface{} {
+func toFilterArg(q ethereum.FilterQuery) (interface{}, error) {
 	arg := map[string]interface{}{
-		"fromBlock": toBlockNumArg(q.FromBlock),
+		"address": q.Addresses,
-		"toBlock":   toBlockNumArg(q.ToBlock),
+		"topics":  q.Topics,
-		"address":   q.Addresses,
-		"topics":    q.Topics,
 	}
-	if q.FromBlock == nil {
+	if q.BlockHash != nil {
-		arg["fromBlock"] = "0x0"
+		arg["blockHash"] = *q.BlockHash
+		if q.FromBlock != nil || q.ToBlock != nil {
+			return nil, fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock")
+		}
+	} else {
+		if q.FromBlock == nil {
+			arg["fromBlock"] = "0x0"
+		} else {
+			arg["fromBlock"] = toBlockNumArg(q.FromBlock)
+		}
+		arg["toBlock"] = toBlockNumArg(q.ToBlock)
 	}
-	return arg
+	return arg, nil
 }
 
 // Pending State

ethclient/ethclient_test.go

@@ -16,7 +16,15 @@
 
 package ethclient
 
-import "github.com/ethereum/go-ethereum"
+import (
+	"fmt"
+	"math/big"
+	"reflect"
+	"testing"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+)
 
 // Verify that Client implements the ethereum interfaces.
 var (
@@ -32,3 +40,113 @@ var (
 	// _ = ethereum.PendingStateEventer(&Client{})
 	_ = ethereum.PendingContractCaller(&Client{})
 )
+
+func TestToFilterArg(t *testing.T) {
+	blockHashErr := fmt.Errorf("cannot specify both BlockHash and FromBlock/ToBlock")
+	addresses := []common.Address{
+		common.HexToAddress("0xD36722ADeC3EdCB29c8e7b5a47f352D701393462"),
+	}
+	blockHash := common.HexToHash(
+		"0xeb94bb7d78b73657a9d7a99792413f50c0a45c51fc62bdcb08a53f18e9a2b4eb",
+	)
+
+	for _, testCase := range []struct {
+		name   string
+		input  ethereum.FilterQuery
+		output interface{}
+		err    error
+	}{
+		{
+			"without BlockHash",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				FromBlock: big.NewInt(1),
+				ToBlock:   big.NewInt(2),
+				Topics:    [][]common.Hash{},
+			},
+			map[string]interface{}{
+				"address":   addresses,
+				"fromBlock": "0x1",
+				"toBlock":   "0x2",
+				"topics":    [][]common.Hash{},
+			},
+			nil,
+		},
+		{
+			"with nil fromBlock and nil toBlock",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				Topics:    [][]common.Hash{},
+			},
+			map[string]interface{}{
+				"address":   addresses,
+				"fromBlock": "0x0",
+				"toBlock":   "latest",
+				"topics":    [][]common.Hash{},
+			},
+			nil,
+		},
+		{
+			"with blockhash",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				BlockHash: &blockHash,
+				Topics:    [][]common.Hash{},
+			},
+			map[string]interface{}{
+				"address":   addresses,
+				"blockHash": blockHash,
+				"topics":    [][]common.Hash{},
+			},
+			nil,
+		},
+		{
+			"with blockhash and from block",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				BlockHash: &blockHash,
+				FromBlock: big.NewInt(1),
+				Topics:    [][]common.Hash{},
+			},
+			nil,
+			blockHashErr,
+		},
+		{
+			"with blockhash and to block",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				BlockHash: &blockHash,
+				ToBlock:   big.NewInt(1),
+				Topics:    [][]common.Hash{},
+			},
+			nil,
+			blockHashErr,
+		},
+		{
+			"with blockhash and both from / to block",
+			ethereum.FilterQuery{
+				Addresses: addresses,
+				BlockHash: &blockHash,
+				FromBlock: big.NewInt(1),
+				ToBlock:   big.NewInt(2),
+				Topics:    [][]common.Hash{},
+			},
+			nil,
+			blockHashErr,
+		},
+	} {
+		t.Run(testCase.name, func(t *testing.T) {
+			output, err := toFilterArg(testCase.input)
+			if (testCase.err == nil) != (err == nil) {
+				t.Fatalf("expected error %v but got %v", testCase.err, err)
+			}
+			if testCase.err != nil {
+				if testCase.err.Error() != err.Error() {
+					t.Fatalf("expected error %v but got %v", testCase.err, err)
+				}
+			} else if !reflect.DeepEqual(testCase.output, output) {
+				t.Fatalf("expected filter arg %v but got %v", testCase.output, output)
+			}
+		})
+	}
+}

ethdb/database.go

@@ -14,6 +14,8 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
+// +build !js
+
 package ethdb
 
 import (
@@ -380,71 +382,3 @@ func (b *ldbBatch) Reset() {
 	b.b.Reset()
 	b.size = 0
 }
-
-type table struct {
-	db     Database
-	prefix string
-}
-
-// NewTable returns a Database object that prefixes all keys with a given
-// string.
-func NewTable(db Database, prefix string) Database {
-	return &table{
-		db:     db,
-		prefix: prefix,
-	}
-}
-
-func (dt *table) Put(key []byte, value []byte) error {
-	return dt.db.Put(append([]byte(dt.prefix), key...), value)
-}
-
-func (dt *table) Has(key []byte) (bool, error) {
-	return dt.db.Has(append([]byte(dt.prefix), key...))
-}
-
-func (dt *table) Get(key []byte) ([]byte, error) {
-	return dt.db.Get(append([]byte(dt.prefix), key...))
-}
-
-func (dt *table) Delete(key []byte) error {
-	return dt.db.Delete(append([]byte(dt.prefix), key...))
-}
-
-func (dt *table) Close() {
-	// Do nothing; don't close the underlying DB.
-}
-
-type tableBatch struct {
-	batch  Batch
-	prefix string
-}
-
-// NewTableBatch returns a Batch object which prefixes all keys with a given string.
-func NewTableBatch(db Database, prefix string) Batch {
-	return &tableBatch{db.NewBatch(), prefix}
-}
-
-func (dt *table) NewBatch() Batch {
-	return &tableBatch{dt.db.NewBatch(), dt.prefix}
-}
-
-func (tb *tableBatch) Put(key, value []byte) error {
-	return tb.batch.Put(append([]byte(tb.prefix), key...), value)
-}
-
-func (tb *tableBatch) Delete(key []byte) error {
-	return tb.batch.Delete(append([]byte(tb.prefix), key...))
-}
-
-func (tb *tableBatch) Write() error {
-	return tb.batch.Write()
-}
-
-func (tb *tableBatch) ValueSize() int {
-	return tb.batch.ValueSize()
-}
-
-func (tb *tableBatch) Reset() {
-	tb.batch.Reset()
-}

ethdb/database_js.go

@@ -0,0 +1,68 @@
+// Copyright 2014 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// +build js
+
+package ethdb
+
+import (
+	"errors"
+)
+
+var errNotSupported = errors.New("ethdb: not supported")
+
+type LDBDatabase struct {
+}
+
+// NewLDBDatabase returns a LevelDB wrapped object.
+func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) {
+	return nil, errNotSupported
+}
+
+// Path returns the path to the database directory.
+func (db *LDBDatabase) Path() string {
+	return ""
+}
+
+// Put puts the given key / value to the queue
+func (db *LDBDatabase) Put(key []byte, value []byte) error {
+	return errNotSupported
+}
+
+func (db *LDBDatabase) Has(key []byte) (bool, error) {
+	return false, errNotSupported
+}
+
+// Get returns the given key if it's present.
+func (db *LDBDatabase) Get(key []byte) ([]byte, error) {
+	return nil, errNotSupported
+}
+
+// Delete deletes the key from the queue and database
+func (db *LDBDatabase) Delete(key []byte) error {
+	return errNotSupported
+}
+
+func (db *LDBDatabase) Close() {
+}
+
+// Meter configures the database metrics collectors and
+func (db *LDBDatabase) Meter(prefix string) {
+}
+
+func (db *LDBDatabase) NewBatch() Batch {
+	return nil
+}

ethdb/database_js_test.go

@@ -14,35 +14,12 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package filter
+// +build js
 
-type Generic struct {
+package ethdb_test
-	Str1, Str2, Str3 string
-	Data             map[string]struct{}
 
-	Fn func(data interface{})
+import (
-}
+	"github.com/ethereum/go-ethereum/ethdb"
+)
 
-// self = registered, f = incoming
+var _ ethdb.Database = &ethdb.LDBDatabase{}
-func (self Generic) Compare(f Filter) bool {
-	var strMatch, dataMatch = true, true
-
-	filter := f.(Generic)
-	if (len(self.Str1) > 0 && filter.Str1 != self.Str1) ||
-		(len(self.Str2) > 0 && filter.Str2 != self.Str2) ||
-		(len(self.Str3) > 0 && filter.Str3 != self.Str3) {
-		strMatch = false
-	}
-
-	for k := range self.Data {
-		if _, ok := filter.Data[k]; !ok {
-			return false
-		}
-	}
-
-	return strMatch && dataMatch
-}
-
-func (self Generic) Trigger(data interface{}) {
-	self.Fn(data)
-}

ethdb/database_test.go

@@ -14,6 +14,8 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
+// +build !js
+
 package ethdb_test
 
 import (

ethdb/table.go

@@ -0,0 +1,51 @@
+// Copyright 2014 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethdb
+
+type table struct {
+	db     Database
+	prefix string
+}
+
+// NewTable returns a Database object that prefixes all keys with a given
+// string.
+func NewTable(db Database, prefix string) Database {
+	return &table{
+		db:     db,
+		prefix: prefix,
+	}
+}
+
+func (dt *table) Put(key []byte, value []byte) error {
+	return dt.db.Put(append([]byte(dt.prefix), key...), value)
+}
+
+func (dt *table) Has(key []byte) (bool, error) {
+	return dt.db.Has(append([]byte(dt.prefix), key...))
+}
+
+func (dt *table) Get(key []byte) ([]byte, error) {
+	return dt.db.Get(append([]byte(dt.prefix), key...))
+}
+
+func (dt *table) Delete(key []byte) error {
+	return dt.db.Delete(append([]byte(dt.prefix), key...))
+}
+
+func (dt *table) Close() {
+	// Do nothing; don't close the underlying DB.
+}

ethdb/table_batch.go

@@ -0,0 +1,51 @@
+// Copyright 2014 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethdb
+
+type tableBatch struct {
+	batch  Batch
+	prefix string
+}
+
+// NewTableBatch returns a Batch object which prefixes all keys with a given string.
+func NewTableBatch(db Database, prefix string) Batch {
+	return &tableBatch{db.NewBatch(), prefix}
+}
+
+func (dt *table) NewBatch() Batch {
+	return &tableBatch{dt.db.NewBatch(), dt.prefix}
+}
+
+func (tb *tableBatch) Put(key, value []byte) error {
+	return tb.batch.Put(append([]byte(tb.prefix), key...), value)
+}
+
+func (tb *tableBatch) Delete(key []byte) error {
+	return tb.batch.Delete(append([]byte(tb.prefix), key...))
+}
+
+func (tb *tableBatch) Write() error {
+	return tb.batch.Write()
+}
+
+func (tb *tableBatch) ValueSize() int {
+	return tb.batch.ValueSize()
+}
+
+func (tb *tableBatch) Reset() {
+	tb.batch.Reset()
+}

event/event_test.go

@@ -141,7 +141,7 @@ func TestMuxConcurrent(t *testing.T) {
 	}
 }
 
-func emptySubscriber(mux *TypeMux, types ...interface{}) {
+func emptySubscriber(mux *TypeMux) {
 	s := mux.Subscribe(testEvent(0))
 	go func() {
 		for range s.Chan() {
@@ -182,9 +182,9 @@ func BenchmarkPost1000(b *testing.B) {
 func BenchmarkPostConcurrent(b *testing.B) {
 	var mux = new(TypeMux)
 	defer mux.Stop()
-	emptySubscriber(mux, testEvent(0))
+	emptySubscriber(mux)
-	emptySubscriber(mux, testEvent(0))
+	emptySubscriber(mux)
-	emptySubscriber(mux, testEvent(0))
+	emptySubscriber(mux)
 
 	var wg sync.WaitGroup
 	poster := func() {

event/filter/filter.go

@@ -1,95 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Package filter implements event filters.
-package filter
-
-import "reflect"
-
-type Filter interface {
-	Compare(Filter) bool
-	Trigger(data interface{})
-}
-
-type FilterEvent struct {
-	filter Filter
-	data   interface{}
-}
-
-type Filters struct {
-	id       int
-	watchers map[int]Filter
-	ch       chan FilterEvent
-
-	quit chan struct{}
-}
-
-func New() *Filters {
-	return &Filters{
-		ch:       make(chan FilterEvent),
-		watchers: make(map[int]Filter),
-		quit:     make(chan struct{}),
-	}
-}
-
-func (f *Filters) Start() {
-	go f.loop()
-}
-
-func (f *Filters) Stop() {
-	close(f.quit)
-}
-
-func (f *Filters) Notify(filter Filter, data interface{}) {
-	f.ch <- FilterEvent{filter, data}
-}
-
-func (f *Filters) Install(watcher Filter) int {
-	f.watchers[f.id] = watcher
-	f.id++
-
-	return f.id - 1
-}
-
-func (f *Filters) Uninstall(id int) {
-	delete(f.watchers, id)
-}
-
-func (f *Filters) loop() {
-out:
-	for {
-		select {
-		case <-f.quit:
-			break out
-		case event := <-f.ch:
-			for _, watcher := range f.watchers {
-				if reflect.TypeOf(watcher) == reflect.TypeOf(event.filter) {
-					if watcher.Compare(event.filter) {
-						watcher.Trigger(event.data)
-					}
-				}
-			}
-		}
-	}
-}
-
-func (f *Filters) Match(a, b Filter) bool {
-	return reflect.TypeOf(a) == reflect.TypeOf(b) && a.Compare(b)
-}
-
-func (f *Filters) Get(i int) Filter {
-	return f.watchers[i]
-}

event/filter/filter_test.go

@@ -1,60 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package filter
-
-import (
-	"testing"
-	"time"
-)
-
-// Simple test to check if baseline matching/mismatching filtering works.
-func TestFilters(t *testing.T) {
-	fm := New()
-	fm.Start()
-
-	// Register two filters to catch posted data
-	first := make(chan struct{})
-	fm.Install(Generic{
-		Str1: "hello",
-		Fn: func(data interface{}) {
-			first <- struct{}{}
-		},
-	})
-	second := make(chan struct{})
-	fm.Install(Generic{
-		Str1: "hello1",
-		Str2: "hello",
-		Fn: func(data interface{}) {
-			second <- struct{}{}
-		},
-	})
-	// Post an event that should only match the first filter
-	fm.Notify(Generic{Str1: "hello"}, true)
-	fm.Stop()
-
-	// Ensure only the mathcing filters fire
-	select {
-	case <-first:
-	case <-time.After(100 * time.Millisecond):
-		t.Error("matching filter timed out")
-	}
-	select {
-	case <-second:
-		t.Error("mismatching filter fired")
-	case <-time.After(100 * time.Millisecond):
-	}
-}

internal/ethapi/api.go

@@ -328,6 +328,9 @@ func (s *PrivateAccountAPI) UnlockAccount(addr common.Address, password string,
 		d = time.Duration(*duration) * time.Second
 	}
 	err := fetchKeystore(s.am).TimedUnlock(accounts.Account{Address: addr}, password, d)
+	if err != nil {
+		log.Warn("Failed account unlock attempt", "address", addr, "err", err)
+	}
 	return err == nil, err
 }
 
@@ -339,7 +342,7 @@ func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool {
 // signTransactions sets defaults and signs the given transaction
 // NOTE: the caller needs to ensure that the nonceLock is held, if applicable,
 // and release it after the transaction has been submitted to the tx pool
-func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args SendTxArgs, passwd string) (*types.Transaction, error) {
+func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *SendTxArgs, passwd string) (*types.Transaction, error) {
 	// Look up the wallet containing the requested signer
 	account := accounts.Account{Address: args.From}
 	wallet, err := s.am.Find(account)
@@ -370,8 +373,9 @@ func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args SendTxArgs
 		s.nonceLock.LockAddr(args.From)
 		defer s.nonceLock.UnlockAddr(args.From)
 	}
-	signed, err := s.signTransaction(ctx, args, passwd)
+	signed, err := s.signTransaction(ctx, &args, passwd)
 	if err != nil {
+		log.Warn("Failed transaction send attempt", "from", args.From, "to", args.To, "value", args.Value.ToInt(), "err", err)
 		return common.Hash{}, err
 	}
 	return submitTransaction(ctx, s.b, signed)
@@ -393,8 +397,9 @@ func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args SendTxArgs
 	if args.Nonce == nil {
 		return nil, fmt.Errorf("nonce not specified")
 	}
-	signed, err := s.signTransaction(ctx, args, passwd)
+	signed, err := s.signTransaction(ctx, &args, passwd)
 	if err != nil {
+		log.Warn("Failed transaction sign attempt", "from", args.From, "to", args.To, "value", args.Value.ToInt(), "err", err)
 		return nil, err
 	}
 	data, err := rlp.EncodeToBytes(signed)
@@ -436,6 +441,7 @@ func (s *PrivateAccountAPI) Sign(ctx context.Context, data hexutil.Bytes, addr c
 	// Assemble sign the data with the wallet
 	signature, err := wallet.SignHashWithPassphrase(account, passwd, signHash(data))
 	if err != nil {
+		log.Warn("Failed data sign attempt", "address", addr, "err", err)
 		return nil, err
 	}
 	signature[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper
@@ -502,6 +508,72 @@ func (s *PublicBlockChainAPI) GetBalance(ctx context.Context, address common.Add
 	return (*hexutil.Big)(state.GetBalance(address)), state.Error()
 }
 
+// Result structs for GetProof
+type AccountResult struct {
+	Address      common.Address  `json:"address"`
+	AccountProof []string        `json:"accountProof"`
+	Balance      *hexutil.Big    `json:"balance"`
+	CodeHash     common.Hash     `json:"codeHash"`
+	Nonce        hexutil.Uint64  `json:"nonce"`
+	StorageHash  common.Hash     `json:"storageHash"`
+	StorageProof []StorageResult `json:"storageProof"`
+}
+type StorageResult struct {
+	Key   string       `json:"key"`
+	Value *hexutil.Big `json:"value"`
+	Proof []string     `json:"proof"`
+}
+
+// GetProof returns the Merkle-proof for a given account and optionally some storage keys.
+func (s *PublicBlockChainAPI) GetProof(ctx context.Context, address common.Address, storageKeys []string, blockNr rpc.BlockNumber) (*AccountResult, error) {
+	state, _, err := s.b.StateAndHeaderByNumber(ctx, blockNr)
+	if state == nil || err != nil {
+		return nil, err
+	}
+
+	storageTrie := state.StorageTrie(address)
+	storageHash := types.EmptyRootHash
+	codeHash := state.GetCodeHash(address)
+	storageProof := make([]StorageResult, len(storageKeys))
+
+	// if we have a storageTrie, (which means the account exists), we can update the storagehash
+	if storageTrie != nil {
+		storageHash = storageTrie.Hash()
+	} else {
+		// no storageTrie means the account does not exist, so the codeHash is the hash of an empty bytearray.
+		codeHash = crypto.Keccak256Hash(nil)
+	}
+
+	// create the proof for the storageKeys
+	for i, key := range storageKeys {
+		if storageTrie != nil {
+			proof, storageError := state.GetStorageProof(address, common.HexToHash(key))
+			if storageError != nil {
+				return nil, storageError
+			}
+			storageProof[i] = StorageResult{key, (*hexutil.Big)(state.GetState(address, common.HexToHash(key)).Big()), common.ToHexArray(proof)}
+		} else {
+			storageProof[i] = StorageResult{key, &hexutil.Big{}, []string{}}
+		}
+	}
+
+	// create the accountProof
+	accountProof, proofErr := state.GetProof(address)
+	if proofErr != nil {
+		return nil, proofErr
+	}
+
+	return &AccountResult{
+		Address:      address,
+		AccountProof: common.ToHexArray(accountProof),
+		Balance:      (*hexutil.Big)(state.GetBalance(address)),
+		CodeHash:     codeHash,
+		Nonce:        hexutil.Uint64(state.GetNonce(address)),
+		StorageHash:  storageHash,
+		StorageProof: storageProof,
+	}, state.Error()
+}
+
 // GetBlockByNumber returns the requested block. When blockNr is -1 the chain head is returned. When fullTx is true all
 // transactions in the block are returned in full detail, otherwise only the transaction hash is returned.
 func (s *PublicBlockChainAPI) GetBlockByNumber(ctx context.Context, blockNr rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) {

internal/web3ext/web3ext.go

@@ -481,6 +481,12 @@ web3._extend({
 			params: 2,
 			inputFormatter: [web3._extend.formatters.inputBlockNumberFormatter, web3._extend.utils.toHex]
 		}),
+		new web3._extend.Method({
+			name: 'getProof',
+			call: 'eth_getProof',
+			params: 3,
+			inputFormatter: [web3._extend.formatters.inputAddressFormatter, null, web3._extend.formatters.inputBlockNumberFormatter]
+		}),
 	],
 	properties: [
 		new web3._extend.Property({

metrics/counter.go

@@ -1,6 +1,8 @@
 package metrics
 
-import "sync/atomic"
+import (
+	"sync/atomic"
+)
 
 // Counters hold an int64 value that can be incremented and decremented.
 type Counter interface {
@@ -20,6 +22,17 @@ func GetOrRegisterCounter(name string, r Registry) Counter {
 	return r.GetOrRegister(name, NewCounter).(Counter)
 }
 
+// GetOrRegisterCounterForced returns an existing Counter or constructs and registers a
+// new Counter no matter the global switch is enabled or not.
+// Be sure to unregister the counter from the registry once it is of no use to
+// allow for garbage collection.
+func GetOrRegisterCounterForced(name string, r Registry) Counter {
+	if nil == r {
+		r = DefaultRegistry
+	}
+	return r.GetOrRegister(name, NewCounterForced).(Counter)
+}
+
 // NewCounter constructs a new StandardCounter.
 func NewCounter() Counter {
 	if !Enabled {
@@ -28,6 +41,12 @@ func NewCounter() Counter {
 	return &StandardCounter{0}
 }
 
+// NewCounterForced constructs a new StandardCounter and returns it no matter if
+// the global switch is enabled or not.
+func NewCounterForced() Counter {
+	return &StandardCounter{0}
+}
+
 // NewRegisteredCounter constructs and registers a new StandardCounter.
 func NewRegisteredCounter(name string, r Registry) Counter {
 	c := NewCounter()
@@ -38,6 +57,19 @@ func NewRegisteredCounter(name string, r Registry) Counter {
 	return c
 }
 
+// NewRegisteredCounterForced constructs and registers a new StandardCounter
+// and launches a goroutine no matter the global switch is enabled or not.
+// Be sure to unregister the counter from the registry once it is of no use to
+// allow for garbage collection.
+func NewRegisteredCounterForced(name string, r Registry) Counter {
+	c := NewCounterForced()
+	if nil == r {
+		r = DefaultRegistry
+	}
+	r.Register(name, c)
+	return c
+}
+
 // CounterSnapshot is a read-only copy of another Counter.
 type CounterSnapshot int64
 

metrics/registry.go

@@ -311,7 +311,10 @@ func (r *PrefixedRegistry) UnregisterAll() {
 	r.underlying.UnregisterAll()
 }
 
-var DefaultRegistry Registry = NewRegistry()
+var (
+	DefaultRegistry   = NewRegistry()
+	EphemeralRegistry = NewRegistry()
+)
 
 // Call the given function for each registered metric.
 func Each(f func(string, interface{})) {

miner/stress_clique.go

@@ -22,7 +22,6 @@ package main
 import (
 	"bytes"
 	"crypto/ecdsa"
-	"fmt"
 	"io/ioutil"
 	"math/big"
 	"math/rand"
@@ -40,7 +39,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -62,11 +61,11 @@ func main() {
 
 	var (
 		nodes  []*node.Node
-		enodes []string
+		enodes []*enode.Node
 	)
 	for _, sealer := range sealers {
 		// Start the node and wait until it's up
-		node, err := makeSealer(genesis, enodes)
+		node, err := makeSealer(genesis)
 		if err != nil {
 			panic(err)
 		}
@@ -76,18 +75,12 @@ func main() {
 			time.Sleep(250 * time.Millisecond)
 		}
 		// Connect the node to al the previous ones
-		for _, enode := range enodes {
+		for _, n := range enodes {
-			enode, err := discover.ParseNode(enode)
+			node.Server().AddPeer(n)
-			if err != nil {
-				panic(err)
-			}
-			node.Server().AddPeer(enode)
 		}
-		// Start tracking the node and it's enode url
+		// Start tracking the node and it's enode
 		nodes = append(nodes, node)
-
+		enodes = append(enodes, node.Server().Self())
-		enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener)
-		enodes = append(enodes, enode)
 
 		// Inject the signer key and start sealing with it
 		store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@@ -177,7 +170,7 @@ func makeGenesis(faucets []*ecdsa.PrivateKey, sealers []*ecdsa.PrivateKey) *core
 	return genesis
 }
 
-func makeSealer(genesis *core.Genesis, nodes []string) (*node.Node, error) {
+func makeSealer(genesis *core.Genesis) (*node.Node, error) {
 	// Define the basic configurations for the Ethereum node
 	datadir, _ := ioutil.TempDir("", "")
 

miner/stress_ethash.go

@@ -21,7 +21,6 @@ package main
 
 import (
 	"crypto/ecdsa"
-	"fmt"
 	"io/ioutil"
 	"math/big"
 	"math/rand"
@@ -41,7 +40,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -62,11 +61,11 @@ func main() {
 
 	var (
 		nodes  []*node.Node
-		enodes []string
+		enodes []*enode.Node
 	)
 	for i := 0; i < 4; i++ {
 		// Start the node and wait until it's up
-		node, err := makeMiner(genesis, enodes)
+		node, err := makeMiner(genesis)
 		if err != nil {
 			panic(err)
 		}
@@ -76,18 +75,12 @@ func main() {
 			time.Sleep(250 * time.Millisecond)
 		}
 		// Connect the node to al the previous ones
-		for _, enode := range enodes {
+		for _, n := range enodes {
-			enode, err := discover.ParseNode(enode)
+			node.Server().AddPeer(n)
-			if err != nil {
-				panic(err)
-			}
-			node.Server().AddPeer(enode)
 		}
-		// Start tracking the node and it's enode url
+		// Start tracking the node and it's enode
 		nodes = append(nodes, node)
-
+		enodes = append(enodes, node.Server().Self())
-		enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener)
-		enodes = append(enodes, enode)
 
 		// Inject the signer key and start sealing with it
 		store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
@@ -155,7 +148,7 @@ func makeGenesis(faucets []*ecdsa.PrivateKey) *core.Genesis {
 	return genesis
 }
 
-func makeMiner(genesis *core.Genesis, nodes []string) (*node.Node, error) {
+func makeMiner(genesis *core.Genesis) (*node.Node, error) {
 	// Define the basic configurations for the Ethereum node
 	datadir, _ := ioutil.TempDir("", "")
 

node/node.go

@@ -549,11 +549,23 @@ func (n *Node) IPCEndpoint() string {
 
 // HTTPEndpoint retrieves the current HTTP endpoint used by the protocol stack.
 func (n *Node) HTTPEndpoint() string {
+	n.lock.Lock()
+	defer n.lock.Unlock()
+
+	if n.httpListener != nil {
+		return n.httpListener.Addr().String()
+	}
 	return n.httpEndpoint
 }
 
 // WSEndpoint retrieves the current WS endpoint used by the protocol stack.
 func (n *Node) WSEndpoint() string {
+	n.lock.Lock()
+	defer n.lock.Unlock()
+
+	if n.wsListener != nil {
+		return n.wsListener.Addr().String()
+	}
 	return n.wsEndpoint
 }
 

node/node_test.go

@@ -454,9 +454,9 @@ func TestProtocolGather(t *testing.T) {
 		Count int
 		Maker InstrumentingWrapper
 	}{
-		"Zero Protocols":  {0, InstrumentedServiceMakerA},
+		"zero": {0, InstrumentedServiceMakerA},
-		"Single Protocol": {1, InstrumentedServiceMakerB},
+		"one":  {1, InstrumentedServiceMakerB},
-		"Many Protocols":  {25, InstrumentedServiceMakerC},
+		"many": {10, InstrumentedServiceMakerC},
 	}
 	for id, config := range services {
 		protocols := make([]p2p.Protocol, config.Count)
@@ -480,7 +480,7 @@ func TestProtocolGather(t *testing.T) {
 	defer stack.Stop()
 
 	protocols := stack.Server().Protocols
-	if len(protocols) != 26 {
+	if len(protocols) != 11 {
 		t.Fatalf("mismatching number of protocols launched: have %d, want %d", len(protocols), 26)
 	}
 	for id, config := range services {

p2p/dial.go

@@ -71,6 +71,7 @@ type dialstate struct {
 	maxDynDials int
 	ntab        discoverTable
 	netrestrict *netutil.Netlist
+	self        enode.ID
 
 	lookupRunning bool
 	dialing       map[enode.ID]connFlag
@@ -84,7 +85,6 @@ type dialstate struct {
 }
 
 type discoverTable interface {
-	Self() *enode.Node
 	Close()
 	Resolve(*enode.Node) *enode.Node
 	LookupRandom() []*enode.Node
@@ -126,10 +126,11 @@ type waitExpireTask struct {
 	time.Duration
 }
 
-func newDialState(static []*enode.Node, bootnodes []*enode.Node, ntab discoverTable, maxdyn int, netrestrict *netutil.Netlist) *dialstate {
+func newDialState(self enode.ID, static []*enode.Node, bootnodes []*enode.Node, ntab discoverTable, maxdyn int, netrestrict *netutil.Netlist) *dialstate {
 	s := &dialstate{
 		maxDynDials: maxdyn,
 		ntab:        ntab,
+		self:        self,
 		netrestrict: netrestrict,
 		static:      make(map[enode.ID]*dialTask),
 		dialing:     make(map[enode.ID]connFlag),
@@ -266,7 +267,7 @@ func (s *dialstate) checkDial(n *enode.Node, peers map[enode.ID]*Peer) error {
 		return errAlreadyDialing
 	case peers[n.ID()] != nil:
 		return errAlreadyConnected
-	case s.ntab != nil && n.ID() == s.ntab.Self().ID():
+	case n.ID() == s.self:
 		return errSelf
 	case s.netrestrict != nil && !s.netrestrict.Contains(n.IP()):
 		return errNotWhitelisted
@@ -349,7 +350,7 @@ func (t *dialTask) dial(srv *Server, dest *enode.Node) error {
 	if err != nil {
 		return &dialError{err}
 	}
-	mfd := newMeteredConn(fd, false)
+	mfd := newMeteredConn(fd, false, dest.IP())
 	return srv.SetupConn(mfd, t.flags, dest)
 }
 

p2p/dial_test.go

@@ -89,7 +89,7 @@ func (t fakeTable) ReadRandomNodes(buf []*enode.Node) int { return copy(buf, t)
 // This test checks that dynamic dials are launched from discovery results.
 func TestDialStateDynDial(t *testing.T) {
 	runDialTest(t, dialtest{
-		init: newDialState(nil, nil, fakeTable{}, 5, nil),
+		init: newDialState(enode.ID{}, nil, nil, fakeTable{}, 5, nil),
 		rounds: []round{
 			// A discovery query is launched.
 			{
@@ -236,7 +236,7 @@ func TestDialStateDynDialBootnode(t *testing.T) {
 		newNode(uintID(8), nil),
 	}
 	runDialTest(t, dialtest{
-		init: newDialState(nil, bootnodes, table, 5, nil),
+		init: newDialState(enode.ID{}, nil, bootnodes, table, 5, nil),
 		rounds: []round{
 			// 2 dynamic dials attempted, bootnodes pending fallback interval
 			{
@@ -324,7 +324,7 @@ func TestDialStateDynDialFromTable(t *testing.T) {
 	}
 
 	runDialTest(t, dialtest{
-		init: newDialState(nil, nil, table, 10, nil),
+		init: newDialState(enode.ID{}, nil, nil, table, 10, nil),
 		rounds: []round{
 			// 5 out of 8 of the nodes returned by ReadRandomNodes are dialed.
 			{
@@ -430,7 +430,7 @@ func TestDialStateNetRestrict(t *testing.T) {
 	restrict.Add("127.0.2.0/24")
 
 	runDialTest(t, dialtest{
-		init: newDialState(nil, nil, table, 10, restrict),
+		init: newDialState(enode.ID{}, nil, nil, table, 10, restrict),
 		rounds: []round{
 			{
 				new: []task{
@@ -453,7 +453,7 @@ func TestDialStateStaticDial(t *testing.T) {
 	}
 
 	runDialTest(t, dialtest{
-		init: newDialState(wantStatic, nil, fakeTable{}, 0, nil),
+		init: newDialState(enode.ID{}, wantStatic, nil, fakeTable{}, 0, nil),
 		rounds: []round{
 			// Static dials are launched for the nodes that
 			// aren't yet connected.
@@ -557,7 +557,7 @@ func TestDialStaticAfterReset(t *testing.T) {
 		},
 	}
 	dTest := dialtest{
-		init:   newDialState(wantStatic, nil, fakeTable{}, 0, nil),
+		init:   newDialState(enode.ID{}, wantStatic, nil, fakeTable{}, 0, nil),
 		rounds: rounds,
 	}
 	runDialTest(t, dTest)
@@ -578,7 +578,7 @@ func TestDialStateCache(t *testing.T) {
 	}
 
 	runDialTest(t, dialtest{
-		init: newDialState(wantStatic, nil, fakeTable{}, 0, nil),
+		init: newDialState(enode.ID{}, wantStatic, nil, fakeTable{}, 0, nil),
 		rounds: []round{
 			// Static dials are launched for the nodes that
 			// aren't yet connected.
@@ -640,7 +640,7 @@ func TestDialStateCache(t *testing.T) {
 func TestDialResolve(t *testing.T) {
 	resolved := newNode(uintID(1), net.IP{127, 0, 55, 234})
 	table := &resolveMock{answer: resolved}
-	state := newDialState(nil, nil, table, 0, nil)
+	state := newDialState(enode.ID{}, nil, nil, table, 0, nil)
 
 	// Check that the task is generated with an incomplete ID.
 	dest := newNode(uintID(1), nil)

p2p/discover/table.go

@@ -72,21 +72,20 @@ type Table struct {
 	ips     netutil.DistinctNetSet
 
 	db         *enode.DB // database of known nodes
+	net        transport
 	refreshReq chan chan struct{}
 	initDone   chan struct{}
 	closeReq   chan struct{}
 	closed     chan struct{}
 
 	nodeAddedHook func(*node) // for testing
-
-	net  transport
-	self *node // metadata of the local node
 }
 
 // transport is implemented by the UDP transport.
 // it is an interface so we can test without opening lots of UDP
 // sockets and without generating a private key.
 type transport interface {
+	self() *enode.Node
 	ping(enode.ID, *net.UDPAddr) error
 	findnode(toid enode.ID, addr *net.UDPAddr, target encPubkey) ([]*node, error)
 	close()
@@ -100,11 +99,10 @@ type bucket struct {
 	ips          netutil.DistinctNetSet
 }
 
-func newTable(t transport, self *enode.Node, db *enode.DB, bootnodes []*enode.Node) (*Table, error) {
+func newTable(t transport, db *enode.DB, bootnodes []*enode.Node) (*Table, error) {
 	tab := &Table{
 		net:        t,
 		db:         db,
-		self:       wrapNode(self),
 		refreshReq: make(chan chan struct{}),
 		initDone:   make(chan struct{}),
 		closeReq:   make(chan struct{}),
@@ -127,6 +125,10 @@ func newTable(t transport, self *enode.Node, db *enode.DB, bootnodes []*enode.No
 	return tab, nil
 }
 
+func (tab *Table) self() *enode.Node {
+	return tab.net.self()
+}
+
 func (tab *Table) seedRand() {
 	var b [8]byte
 	crand.Read(b[:])
@@ -136,11 +138,6 @@ func (tab *Table) seedRand() {
 	tab.mutex.Unlock()
 }
 
-// Self returns the local node.
-func (tab *Table) Self() *enode.Node {
-	return unwrapNode(tab.self)
-}
-
 // ReadRandomNodes fills the given slice with random nodes from the table. The results
 // are guaranteed to be unique for a single invocation, no node will appear twice.
 func (tab *Table) ReadRandomNodes(buf []*enode.Node) (n int) {
@@ -183,6 +180,10 @@ func (tab *Table) ReadRandomNodes(buf []*enode.Node) (n int) {
 
 // Close terminates the network listener and flushes the node database.
 func (tab *Table) Close() {
+	if tab.net != nil {
+		tab.net.close()
+	}
+
 	select {
 	case <-tab.closed:
 		// already closed.
@@ -257,7 +258,7 @@ func (tab *Table) lookup(targetKey encPubkey, refreshIfEmpty bool) []*node {
 	)
 	// don't query further if we hit ourself.
 	// unlikely to happen often in practice.
-	asked[tab.self.ID()] = true
+	asked[tab.self().ID()] = true
 
 	for {
 		tab.mutex.Lock()
@@ -340,8 +341,8 @@ func (tab *Table) loop() {
 		revalidate     = time.NewTimer(tab.nextRevalidateTime())
 		refresh        = time.NewTicker(refreshInterval)
 		copyNodes      = time.NewTicker(copyNodesInterval)
-		revalidateDone = make(chan struct{})
 		refreshDone    = make(chan struct{})           // where doRefresh reports completion
+		revalidateDone chan struct{}                   // where doRevalidate reports completion
 		waiting        = []chan struct{}{tab.initDone} // holds waiting callers while doRefresh runs
 	)
 	defer refresh.Stop()
@@ -372,9 +373,11 @@ loop:
 			}
 			waiting, refreshDone = nil, nil
 		case <-revalidate.C:
+			revalidateDone = make(chan struct{})
 			go tab.doRevalidate(revalidateDone)
 		case <-revalidateDone:
 			revalidate.Reset(tab.nextRevalidateTime())
+			revalidateDone = nil
 		case <-copyNodes.C:
 			go tab.copyLiveNodes()
 		case <-tab.closeReq:
@@ -382,15 +385,15 @@ loop:
 		}
 	}
 
-	if tab.net != nil {
-		tab.net.close()
-	}
 	if refreshDone != nil {
 		<-refreshDone
 	}
 	for _, ch := range waiting {
 		close(ch)
 	}
+	if revalidateDone != nil {
+		<-revalidateDone
+	}
 	close(tab.closed)
 }
 
@@ -408,7 +411,7 @@ func (tab *Table) doRefresh(done chan struct{}) {
 	// Run self lookup to discover new neighbor nodes.
 	// We can only do this if we have a secp256k1 identity.
 	var key ecdsa.PublicKey
-	if err := tab.self.Load((*enode.Secp256k1)(&key)); err == nil {
+	if err := tab.self().Load((*enode.Secp256k1)(&key)); err == nil {
 		tab.lookup(encodePubkey(&key), false)
 	}
 
@@ -530,7 +533,7 @@ func (tab *Table) len() (n int) {
 
 // bucket returns the bucket for the given node ID hash.
 func (tab *Table) bucket(id enode.ID) *bucket {
-	d := enode.LogDist(tab.self.ID(), id)
+	d := enode.LogDist(tab.self().ID(), id)
 	if d <= bucketMinDistance {
 		return tab.buckets[0]
 	}
@@ -543,7 +546,7 @@ func (tab *Table) bucket(id enode.ID) *bucket {
 //
 // The caller must not hold tab.mutex.
 func (tab *Table) add(n *node) {
-	if n.ID() == tab.self.ID() {
+	if n.ID() == tab.self().ID() {
 		return
 	}
 
@@ -576,7 +579,7 @@ func (tab *Table) stuff(nodes []*node) {
 	defer tab.mutex.Unlock()
 
 	for _, n := range nodes {
-		if n.ID() == tab.self.ID() {
+		if n.ID() == tab.self().ID() {
 			continue // don't add self
 		}
 		b := tab.bucket(n.ID())

p2p/discover/table_test.go

@@ -141,7 +141,7 @@ func TestTable_IPLimit(t *testing.T) {
 	defer db.Close()
 
 	for i := 0; i < tableIPLimit+1; i++ {
-		n := nodeAtDistance(tab.self.ID(), i, net.IP{172, 0, 1, byte(i)})
+		n := nodeAtDistance(tab.self().ID(), i, net.IP{172, 0, 1, byte(i)})
 		tab.add(n)
 	}
 	if tab.len() > tableIPLimit {
@@ -158,7 +158,7 @@ func TestTable_BucketIPLimit(t *testing.T) {
 
 	d := 3
 	for i := 0; i < bucketIPLimit+1; i++ {
-		n := nodeAtDistance(tab.self.ID(), d, net.IP{172, 0, 1, byte(i)})
+		n := nodeAtDistance(tab.self().ID(), d, net.IP{172, 0, 1, byte(i)})
 		tab.add(n)
 	}
 	if tab.len() > bucketIPLimit {
@@ -240,7 +240,7 @@ func TestTable_ReadRandomNodesGetAll(t *testing.T) {
 
 		for i := 0; i < len(buf); i++ {
 			ld := cfg.Rand.Intn(len(tab.buckets))
-			tab.stuff([]*node{nodeAtDistance(tab.self.ID(), ld, intIP(ld))})
+			tab.stuff([]*node{nodeAtDistance(tab.self().ID(), ld, intIP(ld))})
 		}
 		gotN := tab.ReadRandomNodes(buf)
 		if gotN != tab.len() {
@@ -510,6 +510,10 @@ type preminedTestnet struct {
 	dists     [hashBits + 1][]encPubkey
 }
 
+func (tn *preminedTestnet) self() *enode.Node {
+	return nullNode
+}
+
 func (tn *preminedTestnet) findnode(toid enode.ID, toaddr *net.UDPAddr, target encPubkey) ([]*node, error) {
 	// current log distance is encoded in port number
 	// fmt.Println("findnode query at dist", toaddr.Port)

p2p/discover/table_util_test.go

@@ -28,12 +28,17 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/enr"
 )
 
-func newTestTable(t transport) (*Table, *enode.DB) {
+var nullNode *enode.Node
+
+func init() {
 	var r enr.Record
 	r.Set(enr.IP{0, 0, 0, 0})
-	n := enode.SignNull(&r, enode.ID{})
+	nullNode = enode.SignNull(&r, enode.ID{})
+}
+
+func newTestTable(t transport) (*Table, *enode.DB) {
 	db, _ := enode.OpenDB("")
-	tab, _ := newTable(t, n, db, nil)
+	tab, _ := newTable(t, db, nil)
 	return tab, db
 }
 
@@ -70,10 +75,10 @@ func intIP(i int) net.IP {
 
 // fillBucket inserts nodes into the given bucket until it is full.
 func fillBucket(tab *Table, n *node) (last *node) {
-	ld := enode.LogDist(tab.self.ID(), n.ID())
+	ld := enode.LogDist(tab.self().ID(), n.ID())
 	b := tab.bucket(n.ID())
 	for len(b.entries) < bucketSize {
-		b.entries = append(b.entries, nodeAtDistance(tab.self.ID(), ld, intIP(ld)))
+		b.entries = append(b.entries, nodeAtDistance(tab.self().ID(), ld, intIP(ld)))
 	}
 	return b.entries[bucketSize-1]
 }
@@ -81,15 +86,25 @@ func fillBucket(tab *Table, n *node) (last *node) {
 type pingRecorder struct {
 	mu           sync.Mutex
 	dead, pinged map[enode.ID]bool
+	n            *enode.Node
 }
 
 func newPingRecorder() *pingRecorder {
+	var r enr.Record
+	r.Set(enr.IP{0, 0, 0, 0})
+	n := enode.SignNull(&r, enode.ID{})
+
 	return &pingRecorder{
 		dead:   make(map[enode.ID]bool),
 		pinged: make(map[enode.ID]bool),
+		n:      n,
 	}
 }
 
+func (t *pingRecorder) self() *enode.Node {
+	return nullNode
+}
+
 func (t *pingRecorder) findnode(toid enode.ID, toaddr *net.UDPAddr, target encPubkey) ([]*node, error) {
 	return nil, nil
 }

p2p/discover/udp.go

@@ -23,12 +23,12 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"sync"
 	"time"
 
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/enode"
-	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 	"github.com/ethereum/go-ethereum/rlp"
 )
@@ -118,9 +118,11 @@ type (
 )
 
 func makeEndpoint(addr *net.UDPAddr, tcpPort uint16) rpcEndpoint {
-	ip := addr.IP.To4()
+	ip := net.IP{}
-	if ip == nil {
+	if ip4 := addr.IP.To4(); ip4 != nil {
-		ip = addr.IP.To16()
+		ip = ip4
+	} else if ip6 := addr.IP.To16(); ip6 != nil {
+		ip = ip6
 	}
 	return rpcEndpoint{IP: ip, UDP: uint16(addr.Port), TCP: tcpPort}
 }
@@ -165,20 +167,19 @@ type conn interface {
 	LocalAddr() net.Addr
 }
 
-// udp implements the RPC protocol.
+// udp implements the discovery v4 UDP wire protocol.
 type udp struct {
 	conn        conn
 	netrestrict *netutil.Netlist
 	priv        *ecdsa.PrivateKey
-	ourEndpoint rpcEndpoint
+	localNode   *enode.LocalNode
+	db          *enode.DB
+	tab         *Table
+	wg          sync.WaitGroup
 
 	addpending chan *pending
 	gotreply   chan reply
-
+	closing    chan struct{}
-	closing chan struct{}
-	nat     nat.Interface
-
-	*Table
 }
 
 // pending represents a pending reply.
@@ -230,60 +231,57 @@ type Config struct {
 	PrivateKey *ecdsa.PrivateKey
 
 	// These settings are optional:
-	AnnounceAddr *net.UDPAddr      // local address announced in the DHT
+	NetRestrict *netutil.Netlist  // network whitelist
-	NodeDBPath   string            // if set, the node database is stored at this filesystem location
+	Bootnodes   []*enode.Node     // list of bootstrap nodes
-	NetRestrict  *netutil.Netlist  // network whitelist
+	Unhandled   chan<- ReadPacket // unhandled packets are sent on this channel
-	Bootnodes    []*enode.Node     // list of bootstrap nodes
-	Unhandled    chan<- ReadPacket // unhandled packets are sent on this channel
 }
 
 // ListenUDP returns a new table that listens for UDP packets on laddr.
-func ListenUDP(c conn, cfg Config) (*Table, error) {
+func ListenUDP(c conn, ln *enode.LocalNode, cfg Config) (*Table, error) {
-	tab, _, err := newUDP(c, cfg)
+	tab, _, err := newUDP(c, ln, cfg)
 	if err != nil {
 		return nil, err
 	}
-	log.Info("UDP listener up", "self", tab.self)
 	return tab, nil
 }
 
-func newUDP(c conn, cfg Config) (*Table, *udp, error) {
+func newUDP(c conn, ln *enode.LocalNode, cfg Config) (*Table, *udp, error) {
-	realaddr := c.LocalAddr().(*net.UDPAddr)
-	if cfg.AnnounceAddr != nil {
-		realaddr = cfg.AnnounceAddr
-	}
-	self := enode.NewV4(&cfg.PrivateKey.PublicKey, realaddr.IP, realaddr.Port, realaddr.Port)
-	db, err := enode.OpenDB(cfg.NodeDBPath)
-	if err != nil {
-		return nil, nil, err
-	}
-
 	udp := &udp{
 		conn:        c,
 		priv:        cfg.PrivateKey,
 		netrestrict: cfg.NetRestrict,
+		localNode:   ln,
+		db:          ln.Database(),
 		closing:     make(chan struct{}),
 		gotreply:    make(chan reply),
 		addpending:  make(chan *pending),
 	}
-	// TODO: separate TCP port
+	tab, err := newTable(udp, ln.Database(), cfg.Bootnodes)
-	udp.ourEndpoint = makeEndpoint(realaddr, uint16(realaddr.Port))
-	tab, err := newTable(udp, self, db, cfg.Bootnodes)
 	if err != nil {
 		return nil, nil, err
 	}
-	udp.Table = tab
+	udp.tab = tab
 
+	udp.wg.Add(2)
 	go udp.loop()
 	go udp.readLoop(cfg.Unhandled)
-	return udp.Table, udp, nil
+	return udp.tab, udp, nil
+}
+
+func (t *udp) self() *enode.Node {
+	return t.localNode.Node()
 }
 
 func (t *udp) close() {
 	close(t.closing)
 	t.conn.Close()
-	t.db.Close()
+	t.wg.Wait()
-	// TODO: wait for the loops to end.
+}
+
+func (t *udp) ourEndpoint() rpcEndpoint {
+	n := t.self()
+	a := &net.UDPAddr{IP: n.IP(), Port: n.UDP()}
+	return makeEndpoint(a, uint16(n.TCP()))
 }
 
 // ping sends a ping message to the given node and waits for a reply.
@@ -296,7 +294,7 @@ func (t *udp) ping(toid enode.ID, toaddr *net.UDPAddr) error {
 func (t *udp) sendPing(toid enode.ID, toaddr *net.UDPAddr, callback func()) <-chan error {
 	req := &ping{
 		Version:    4,
-		From:       t.ourEndpoint,
+		From:       t.ourEndpoint(),
 		To:         makeEndpoint(toaddr, 0), // TODO: maybe use known TCP port from DB
 		Expiration: uint64(time.Now().Add(expiration).Unix()),
 	}
@@ -313,6 +311,7 @@ func (t *udp) sendPing(toid enode.ID, toaddr *net.UDPAddr, callback func()) <-ch
 		}
 		return ok
 	})
+	t.localNode.UDPContact(toaddr)
 	t.write(toaddr, req.name(), packet)
 	return errc
 }
@@ -381,6 +380,8 @@ func (t *udp) handleReply(from enode.ID, ptype byte, req packet) bool {
 // loop runs in its own goroutine. it keeps track of
 // the refresh timer and the pending reply queue.
 func (t *udp) loop() {
+	defer t.wg.Done()
+
 	var (
 		plist        = list.New()
 		timeout      = time.NewTimer(0)
@@ -542,10 +543,11 @@ func encodePacket(priv *ecdsa.PrivateKey, ptype byte, req interface{}) (packet,
 
 // readLoop runs in its own goroutine. it handles incoming UDP packets.
 func (t *udp) readLoop(unhandled chan<- ReadPacket) {
-	defer t.conn.Close()
+	defer t.wg.Done()
 	if unhandled != nil {
 		defer close(unhandled)
 	}
+
 	// Discovery packets are defined to be no larger than 1280 bytes.
 	// Packets larger than this size will be cut at the end and treated
 	// as invalid because their hash won't match.
@@ -629,10 +631,11 @@ func (req *ping) handle(t *udp, from *net.UDPAddr, fromKey encPubkey, mac []byte
 	n := wrapNode(enode.NewV4(key, from.IP, int(req.From.TCP), from.Port))
 	t.handleReply(n.ID(), pingPacket, req)
 	if time.Since(t.db.LastPongReceived(n.ID())) > bondExpiration {
-		t.sendPing(n.ID(), from, func() { t.addThroughPing(n) })
+		t.sendPing(n.ID(), from, func() { t.tab.addThroughPing(n) })
 	} else {
-		t.addThroughPing(n)
+		t.tab.addThroughPing(n)
 	}
+	t.localNode.UDPEndpointStatement(from, &net.UDPAddr{IP: req.To.IP, Port: int(req.To.UDP)})
 	t.db.UpdateLastPingReceived(n.ID(), time.Now())
 	return nil
 }
@@ -647,6 +650,7 @@ func (req *pong) handle(t *udp, from *net.UDPAddr, fromKey encPubkey, mac []byte
 	if !t.handleReply(fromID, pongPacket, req) {
 		return errUnsolicitedReply
 	}
+	t.localNode.UDPEndpointStatement(from, &net.UDPAddr{IP: req.To.IP, Port: int(req.To.UDP)})
 	t.db.UpdateLastPongReceived(fromID, time.Now())
 	return nil
 }
@@ -668,9 +672,9 @@ func (req *findnode) handle(t *udp, from *net.UDPAddr, fromKey encPubkey, mac []
 		return errUnknownNode
 	}
 	target := enode.ID(crypto.Keccak256Hash(req.Target[:]))
-	t.mutex.Lock()
+	t.tab.mutex.Lock()
-	closest := t.closest(target, bucketSize).entries
+	closest := t.tab.closest(target, bucketSize).entries
-	t.mutex.Unlock()
+	t.tab.mutex.Unlock()
 
 	p := neighbors{Expiration: uint64(time.Now().Add(expiration).Unix())}
 	var sent bool

p2p/discover/udp_test.go

@@ -71,7 +71,9 @@ func newUDPTest(t *testing.T) *udpTest {
 		remotekey:  newkey(),
 		remoteaddr: &net.UDPAddr{IP: net.IP{10, 0, 1, 99}, Port: 30303},
 	}
-	test.table, test.udp, _ = newUDP(test.pipe, Config{PrivateKey: test.localkey})
+	db, _ := enode.OpenDB("")
+	ln := enode.NewLocalNode(db, test.localkey)
+	test.table, test.udp, _ = newUDP(test.pipe, ln, Config{PrivateKey: test.localkey})
 	// Wait for initial refresh so the table doesn't send unexpected findnode.
 	<-test.table.initDone
 	return test
@@ -355,12 +357,13 @@ func TestUDP_successfulPing(t *testing.T) {
 
 	// remote is unknown, the table pings back.
 	hash, _ := test.waitPacketOut(func(p *ping) error {
-		if !reflect.DeepEqual(p.From, test.udp.ourEndpoint) {
+		if !reflect.DeepEqual(p.From, test.udp.ourEndpoint()) {
-			t.Errorf("got ping.From %v, want %v", p.From, test.udp.ourEndpoint)
+			t.Errorf("got ping.From %#v, want %#v", p.From, test.udp.ourEndpoint())
 		}
 		wantTo := rpcEndpoint{
 			// The mirrored UDP address is the UDP packet sender.
-			IP: test.remoteaddr.IP, UDP: uint16(test.remoteaddr.Port),
+			IP:  test.remoteaddr.IP,
+			UDP: uint16(test.remoteaddr.Port),
 			TCP: 0,
 		}
 		if !reflect.DeepEqual(p.To, wantTo) {

p2p/discv5/udp.go

@@ -230,7 +230,8 @@ type udp struct {
 }
 
 // ListenUDP returns a new table that listens for UDP packets on laddr.
-func ListenUDP(priv *ecdsa.PrivateKey, conn conn, realaddr *net.UDPAddr, nodeDBPath string, netrestrict *netutil.Netlist) (*Network, error) {
+func ListenUDP(priv *ecdsa.PrivateKey, conn conn, nodeDBPath string, netrestrict *netutil.Netlist) (*Network, error) {
+	realaddr := conn.LocalAddr().(*net.UDPAddr)
 	transport, err := listenUDP(priv, conn, realaddr)
 	if err != nil {
 		return nil, err

p2p/enode/localnode.go

@@ -0,0 +1,246 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package enode
+
+import (
+	"crypto/ecdsa"
+	"fmt"
+	"net"
+	"reflect"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/p2p/enr"
+	"github.com/ethereum/go-ethereum/p2p/netutil"
+)
+
+const (
+	// IP tracker configuration
+	iptrackMinStatements = 10
+	iptrackWindow        = 5 * time.Minute
+	iptrackContactWindow = 10 * time.Minute
+)
+
+// LocalNode produces the signed node record of a local node, i.e. a node run in the
+// current process. Setting ENR entries via the Set method updates the record. A new version
+// of the record is signed on demand when the Node method is called.
+type LocalNode struct {
+	cur atomic.Value // holds a non-nil node pointer while the record is up-to-date.
+	id  ID
+	key *ecdsa.PrivateKey
+	db  *DB
+
+	// everything below is protected by a lock
+	mu          sync.Mutex
+	seq         uint64
+	entries     map[string]enr.Entry
+	udpTrack    *netutil.IPTracker // predicts external UDP endpoint
+	staticIP    net.IP
+	fallbackIP  net.IP
+	fallbackUDP int
+}
+
+// NewLocalNode creates a local node.
+func NewLocalNode(db *DB, key *ecdsa.PrivateKey) *LocalNode {
+	ln := &LocalNode{
+		id:       PubkeyToIDV4(&key.PublicKey),
+		db:       db,
+		key:      key,
+		udpTrack: netutil.NewIPTracker(iptrackWindow, iptrackContactWindow, iptrackMinStatements),
+		entries:  make(map[string]enr.Entry),
+	}
+	ln.seq = db.localSeq(ln.id)
+	ln.invalidate()
+	return ln
+}
+
+// Database returns the node database associated with the local node.
+func (ln *LocalNode) Database() *DB {
+	return ln.db
+}
+
+// Node returns the current version of the local node record.
+func (ln *LocalNode) Node() *Node {
+	n := ln.cur.Load().(*Node)
+	if n != nil {
+		return n
+	}
+	// Record was invalidated, sign a new copy.
+	ln.mu.Lock()
+	defer ln.mu.Unlock()
+	ln.sign()
+	return ln.cur.Load().(*Node)
+}
+
+// ID returns the local node ID.
+func (ln *LocalNode) ID() ID {
+	return ln.id
+}
+
+// Set puts the given entry into the local record, overwriting
+// any existing value.
+func (ln *LocalNode) Set(e enr.Entry) {
+	ln.mu.Lock()
+	defer ln.mu.Unlock()
+
+	ln.set(e)
+}
+
+func (ln *LocalNode) set(e enr.Entry) {
+	val, exists := ln.entries[e.ENRKey()]
+	if !exists || !reflect.DeepEqual(val, e) {
+		ln.entries[e.ENRKey()] = e
+		ln.invalidate()
+	}
+}
+
+// Delete removes the given entry from the local record.
+func (ln *LocalNode) Delete(e enr.Entry) {
+	ln.mu.Lock()
+	defer ln.mu.Unlock()
+
+	ln.delete(e)
+}
+
+func (ln *LocalNode) delete(e enr.Entry) {
+	_, exists := ln.entries[e.ENRKey()]
+	if exists {
+		delete(ln.entries, e.ENRKey())
+		ln.invalidate()
+	}
+}
+
+// SetStaticIP sets the local IP to the given one unconditionally.
+// This disables endpoint prediction.
+func (ln *LocalNode) SetStaticIP(ip net.IP) {
+	ln.mu.Lock()
+	defer ln.mu.Unlock()
+
+	ln.staticIP = ip
+	ln.updateEndpoints()
+}
+
+// SetFallbackIP sets the last-resort IP address. This address is used
+// if no endpoint prediction can be made and no static IP is set.
+func (ln *LocalNode) SetFallbackIP(ip net.IP) {
+	ln.mu.Lock()
+	defer ln.mu.Unlock()
+
+	ln.fallbackIP = ip
+	ln.updateEndpoints()
+}
+
+// SetFallbackUDP sets the last-resort UDP port. This port is used
+// if no endpoint prediction can be made.
+func (ln *LocalNode) SetFallbackUDP(port int) {
+	ln.mu.Lock()
+	defer ln.mu.Unlock()
+
+	ln.fallbackUDP = port
+	ln.updateEndpoints()
+}
+
+// UDPEndpointStatement should be called whenever a statement about the local node's
+// UDP endpoint is received. It feeds the local endpoint predictor.
+func (ln *LocalNode) UDPEndpointStatement(fromaddr, endpoint *net.UDPAddr) {
+	ln.mu.Lock()
+	defer ln.mu.Unlock()
+
+	ln.udpTrack.AddStatement(fromaddr.String(), endpoint.String())
+	ln.updateEndpoints()
+}
+
+// UDPContact should be called whenever the local node has announced itself to another node
+// via UDP. It feeds the local endpoint predictor.
+func (ln *LocalNode) UDPContact(toaddr *net.UDPAddr) {
+	ln.mu.Lock()
+	defer ln.mu.Unlock()
+
+	ln.udpTrack.AddContact(toaddr.String())
+	ln.updateEndpoints()
+}
+
+func (ln *LocalNode) updateEndpoints() {
+	// Determine the endpoints.
+	newIP := ln.fallbackIP
+	newUDP := ln.fallbackUDP
+	if ln.staticIP != nil {
+		newIP = ln.staticIP
+	} else if ip, port := predictAddr(ln.udpTrack); ip != nil {
+		newIP = ip
+		newUDP = port
+	}
+
+	// Update the record.
+	if newIP != nil && !newIP.IsUnspecified() {
+		ln.set(enr.IP(newIP))
+		if newUDP != 0 {
+			ln.set(enr.UDP(newUDP))
+		} else {
+			ln.delete(enr.UDP(0))
+		}
+	} else {
+		ln.delete(enr.IP{})
+	}
+}
+
+// predictAddr wraps IPTracker.PredictEndpoint, converting from its string-based
+// endpoint representation to IP and port types.
+func predictAddr(t *netutil.IPTracker) (net.IP, int) {
+	ep := t.PredictEndpoint()
+	if ep == "" {
+		return nil, 0
+	}
+	ipString, portString, _ := net.SplitHostPort(ep)
+	ip := net.ParseIP(ipString)
+	port, _ := strconv.Atoi(portString)
+	return ip, port
+}
+
+func (ln *LocalNode) invalidate() {
+	ln.cur.Store((*Node)(nil))
+}
+
+func (ln *LocalNode) sign() {
+	if n := ln.cur.Load().(*Node); n != nil {
+		return // no changes
+	}
+
+	var r enr.Record
+	for _, e := range ln.entries {
+		r.Set(e)
+	}
+	ln.bumpSeq()
+	r.SetSeq(ln.seq)
+	if err := SignV4(&r, ln.key); err != nil {
+		panic(fmt.Errorf("enode: can't sign record: %v", err))
+	}
+	n, err := New(ValidSchemes, &r)
+	if err != nil {
+		panic(fmt.Errorf("enode: can't verify local record: %v", err))
+	}
+	ln.cur.Store(n)
+	log.Info("New local node record", "seq", ln.seq, "id", n.ID(), "ip", n.IP(), "udp", n.UDP(), "tcp", n.TCP())
+}
+
+func (ln *LocalNode) bumpSeq() {
+	ln.seq++
+	ln.db.storeLocalSeq(ln.id, ln.seq)
+}

p2p/enode/localnode_test.go

@@ -0,0 +1,76 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package enode
+
+import (
+	"testing"
+
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/p2p/enr"
+)
+
+func newLocalNodeForTesting() (*LocalNode, *DB) {
+	db, _ := OpenDB("")
+	key, _ := crypto.GenerateKey()
+	return NewLocalNode(db, key), db
+}
+
+func TestLocalNode(t *testing.T) {
+	ln, db := newLocalNodeForTesting()
+	defer db.Close()
+
+	if ln.Node().ID() != ln.ID() {
+		t.Fatal("inconsistent ID")
+	}
+
+	ln.Set(enr.WithEntry("x", uint(3)))
+	var x uint
+	if err := ln.Node().Load(enr.WithEntry("x", &x)); err != nil {
+		t.Fatal("can't load entry 'x':", err)
+	} else if x != 3 {
+		t.Fatal("wrong value for entry 'x':", x)
+	}
+}
+
+func TestLocalNodeSeqPersist(t *testing.T) {
+	ln, db := newLocalNodeForTesting()
+	defer db.Close()
+
+	if s := ln.Node().Seq(); s != 1 {
+		t.Fatalf("wrong initial seq %d, want 1", s)
+	}
+	ln.Set(enr.WithEntry("x", uint(1)))
+	if s := ln.Node().Seq(); s != 2 {
+		t.Fatalf("wrong seq %d after set, want 2", s)
+	}
+
+	// Create a new instance, it should reload the sequence number.
+	// The number increases just after that because a new record is
+	// created without the "x" entry.
+	ln2 := NewLocalNode(db, ln.key)
+	if s := ln2.Node().Seq(); s != 3 {
+		t.Fatalf("wrong seq %d on new instance, want 3", s)
+	}
+
+	// Create a new instance with a different node key on the same database.
+	// This should reset the sequence number.
+	key, _ := crypto.GenerateKey()
+	ln3 := NewLocalNode(db, key)
+	if s := ln3.Node().Seq(); s != 1 {
+		t.Fatalf("wrong seq %d on instance with changed key, want 1", s)
+	}
+}

p2p/enode/node.go

@@ -98,6 +98,13 @@ func (n *Node) Pubkey() *ecdsa.PublicKey {
 	return &key
 }
 
+// Record returns the node's record. The return value is a copy and may
+// be modified by the caller.
+func (n *Node) Record() *enr.Record {
+	cpy := n.r
+	return &cpy
+}
+
 // checks whether n is a valid complete node.
 func (n *Node) ValidateComplete() error {
 	if n.Incomplete() {

p2p/enode/nodedb.go

@@ -35,11 +35,24 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
+// Keys in the node database.
+const (
+	dbVersionKey = "version" // Version of the database to flush if changes
+	dbItemPrefix = "n:"      // Identifier to prefix node entries with
+
+	dbDiscoverRoot      = ":discover"
+	dbDiscoverSeq       = dbDiscoverRoot + ":seq"
+	dbDiscoverPing      = dbDiscoverRoot + ":lastping"
+	dbDiscoverPong      = dbDiscoverRoot + ":lastpong"
+	dbDiscoverFindFails = dbDiscoverRoot + ":findfail"
+	dbLocalRoot         = ":local"
+	dbLocalSeq          = dbLocalRoot + ":seq"
+)
+
 var (
-	nodeDBNilID          = ID{}           // Special node ID to use as a nil element.
+	dbNodeExpiration = 24 * time.Hour // Time after which an unseen node should be dropped.
-	nodeDBNodeExpiration = 24 * time.Hour // Time after which an unseen node should be dropped.
+	dbCleanupCycle   = time.Hour      // Time period for running the expiration task.
-	nodeDBCleanupCycle   = time.Hour      // Time period for running the expiration task.
+	dbVersion        = 7
-	nodeDBVersion        = 6
 )
 
 // DB is the node database, storing previously seen nodes and any collected metadata about
@@ -50,17 +63,6 @@ type DB struct {
 	quit   chan struct{} // Channel to signal the expiring thread to stop
 }
 
-// Schema layout for the node database
-var (
-	nodeDBVersionKey = []byte("version") // Version of the database to flush if changes
-	nodeDBItemPrefix = []byte("n:")      // Identifier to prefix node entries with
-
-	nodeDBDiscoverRoot      = ":discover"
-	nodeDBDiscoverPing      = nodeDBDiscoverRoot + ":lastping"
-	nodeDBDiscoverPong      = nodeDBDiscoverRoot + ":lastpong"
-	nodeDBDiscoverFindFails = nodeDBDiscoverRoot + ":findfail"
-)
-
 // OpenDB opens a node database for storing and retrieving infos about known peers in the
 // network. If no path is given an in-memory, temporary database is constructed.
 func OpenDB(path string) (*DB, error) {
@@ -93,13 +95,13 @@ func newPersistentDB(path string) (*DB, error) {
 	// The nodes contained in the cache correspond to a certain protocol version.
 	// Flush all nodes if the version doesn't match.
 	currentVer := make([]byte, binary.MaxVarintLen64)
-	currentVer = currentVer[:binary.PutVarint(currentVer, int64(nodeDBVersion))]
+	currentVer = currentVer[:binary.PutVarint(currentVer, int64(dbVersion))]
 
-	blob, err := db.Get(nodeDBVersionKey, nil)
+	blob, err := db.Get([]byte(dbVersionKey), nil)
 	switch err {
 	case leveldb.ErrNotFound:
 		// Version not found (i.e. empty cache), insert it
-		if err := db.Put(nodeDBVersionKey, currentVer, nil); err != nil {
+		if err := db.Put([]byte(dbVersionKey), currentVer, nil); err != nil {
 			db.Close()
 			return nil, err
 		}
@@ -120,28 +122,27 @@ func newPersistentDB(path string) (*DB, error) {
 // makeKey generates the leveldb key-blob from a node id and its particular
 // field of interest.
 func makeKey(id ID, field string) []byte {
-	if bytes.Equal(id[:], nodeDBNilID[:]) {
+	if (id == ID{}) {
 		return []byte(field)
 	}
-	return append(nodeDBItemPrefix, append(id[:], field...)...)
+	return append([]byte(dbItemPrefix), append(id[:], field...)...)
 }
 
 // splitKey tries to split a database key into a node id and a field part.
 func splitKey(key []byte) (id ID, field string) {
 	// If the key is not of a node, return it plainly
-	if !bytes.HasPrefix(key, nodeDBItemPrefix) {
+	if !bytes.HasPrefix(key, []byte(dbItemPrefix)) {
 		return ID{}, string(key)
 	}
 	// Otherwise split the id and field
-	item := key[len(nodeDBItemPrefix):]
+	item := key[len(dbItemPrefix):]
 	copy(id[:], item[:len(id)])
 	field = string(item[len(id):])
 
 	return id, field
 }
 
-// fetchInt64 retrieves an integer instance associated with a particular
+// fetchInt64 retrieves an integer associated with a particular key.
-// database key.
 func (db *DB) fetchInt64(key []byte) int64 {
 	blob, err := db.lvl.Get(key, nil)
 	if err != nil {
@@ -154,18 +155,33 @@ func (db *DB) fetchInt64(key []byte) int64 {
 	return val
 }
 
-// storeInt64 update a specific database entry to the current time instance as a
+// storeInt64 stores an integer in the given key.
-// unix timestamp.
 func (db *DB) storeInt64(key []byte, n int64) error {
 	blob := make([]byte, binary.MaxVarintLen64)
 	blob = blob[:binary.PutVarint(blob, n)]
+	return db.lvl.Put(key, blob, nil)
+}
 
+// fetchUint64 retrieves an integer associated with a particular key.
+func (db *DB) fetchUint64(key []byte) uint64 {
+	blob, err := db.lvl.Get(key, nil)
+	if err != nil {
+		return 0
+	}
+	val, _ := binary.Uvarint(blob)
+	return val
+}
+
+// storeUint64 stores an integer in the given key.
+func (db *DB) storeUint64(key []byte, n uint64) error {
+	blob := make([]byte, binary.MaxVarintLen64)
+	blob = blob[:binary.PutUvarint(blob, n)]
 	return db.lvl.Put(key, blob, nil)
 }
 
 // Node retrieves a node with a given id from the database.
 func (db *DB) Node(id ID) *Node {
-	blob, err := db.lvl.Get(makeKey(id, nodeDBDiscoverRoot), nil)
+	blob, err := db.lvl.Get(makeKey(id, dbDiscoverRoot), nil)
 	if err != nil {
 		return nil
 	}
@@ -184,11 +200,31 @@ func mustDecodeNode(id, data []byte) *Node {
 
 // UpdateNode inserts - potentially overwriting - a node into the peer database.
 func (db *DB) UpdateNode(node *Node) error {
+	if node.Seq() < db.NodeSeq(node.ID()) {
+		return nil
+	}
 	blob, err := rlp.EncodeToBytes(&node.r)
 	if err != nil {
 		return err
 	}
-	return db.lvl.Put(makeKey(node.ID(), nodeDBDiscoverRoot), blob, nil)
+	if err := db.lvl.Put(makeKey(node.ID(), dbDiscoverRoot), blob, nil); err != nil {
+		return err
+	}
+	return db.storeUint64(makeKey(node.ID(), dbDiscoverSeq), node.Seq())
+}
+
+// NodeSeq returns the stored record sequence number of the given node.
+func (db *DB) NodeSeq(id ID) uint64 {
+	return db.fetchUint64(makeKey(id, dbDiscoverSeq))
+}
+
+// Resolve returns the stored record of the node if it has a larger sequence
+// number than n.
+func (db *DB) Resolve(n *Node) *Node {
+	if n.Seq() > db.NodeSeq(n.ID()) {
+		return n
+	}
+	return db.Node(n.ID())
 }
 
 // DeleteNode deletes all information/keys associated with a node.
@@ -218,7 +254,7 @@ func (db *DB) ensureExpirer() {
 // expirer should be started in a go routine, and is responsible for looping ad
 // infinitum and dropping stale data from the database.
 func (db *DB) expirer() {
-	tick := time.NewTicker(nodeDBCleanupCycle)
+	tick := time.NewTicker(dbCleanupCycle)
 	defer tick.Stop()
 	for {
 		select {
@@ -235,7 +271,7 @@ func (db *DB) expirer() {
 // expireNodes iterates over the database and deletes all nodes that have not
 // been seen (i.e. received a pong from) for some allotted time.
 func (db *DB) expireNodes() error {
-	threshold := time.Now().Add(-nodeDBNodeExpiration)
+	threshold := time.Now().Add(-dbNodeExpiration)
 
 	// Find discovered nodes that are older than the allowance
 	it := db.lvl.NewIterator(nil, nil)
@@ -244,7 +280,7 @@ func (db *DB) expireNodes() error {
 	for it.Next() {
 		// Skip the item if not a discovery node
 		id, field := splitKey(it.Key())
-		if field != nodeDBDiscoverRoot {
+		if field != dbDiscoverRoot {
 			continue
 		}
 		// Skip the node if not expired yet (and not self)
@@ -260,34 +296,44 @@ func (db *DB) expireNodes() error {
 // LastPingReceived retrieves the time of the last ping packet received from
 // a remote node.
 func (db *DB) LastPingReceived(id ID) time.Time {
-	return time.Unix(db.fetchInt64(makeKey(id, nodeDBDiscoverPing)), 0)
+	return time.Unix(db.fetchInt64(makeKey(id, dbDiscoverPing)), 0)
 }
 
 // UpdateLastPingReceived updates the last time we tried contacting a remote node.
 func (db *DB) UpdateLastPingReceived(id ID, instance time.Time) error {
-	return db.storeInt64(makeKey(id, nodeDBDiscoverPing), instance.Unix())
+	return db.storeInt64(makeKey(id, dbDiscoverPing), instance.Unix())
 }
 
 // LastPongReceived retrieves the time of the last successful pong from remote node.
 func (db *DB) LastPongReceived(id ID) time.Time {
 	// Launch expirer
 	db.ensureExpirer()
-	return time.Unix(db.fetchInt64(makeKey(id, nodeDBDiscoverPong)), 0)
+	return time.Unix(db.fetchInt64(makeKey(id, dbDiscoverPong)), 0)
 }
 
 // UpdateLastPongReceived updates the last pong time of a node.
 func (db *DB) UpdateLastPongReceived(id ID, instance time.Time) error {
-	return db.storeInt64(makeKey(id, nodeDBDiscoverPong), instance.Unix())
+	return db.storeInt64(makeKey(id, dbDiscoverPong), instance.Unix())
 }
 
 // FindFails retrieves the number of findnode failures since bonding.
 func (db *DB) FindFails(id ID) int {
-	return int(db.fetchInt64(makeKey(id, nodeDBDiscoverFindFails)))
+	return int(db.fetchInt64(makeKey(id, dbDiscoverFindFails)))
 }
 
 // UpdateFindFails updates the number of findnode failures since bonding.
 func (db *DB) UpdateFindFails(id ID, fails int) error {
-	return db.storeInt64(makeKey(id, nodeDBDiscoverFindFails), int64(fails))
+	return db.storeInt64(makeKey(id, dbDiscoverFindFails), int64(fails))
+}
+
+// LocalSeq retrieves the local record sequence counter.
+func (db *DB) localSeq(id ID) uint64 {
+	return db.fetchUint64(makeKey(id, dbLocalSeq))
+}
+
+// storeLocalSeq stores the local record sequence counter.
+func (db *DB) storeLocalSeq(id ID, n uint64) {
+	db.storeUint64(makeKey(id, dbLocalSeq), n)
 }
 
 // QuerySeeds retrieves random nodes to be used as potential seed nodes
@@ -309,7 +355,7 @@ seek:
 		ctr := id[0]
 		rand.Read(id[:])
 		id[0] = ctr + id[0]%16
-		it.Seek(makeKey(id, nodeDBDiscoverRoot))
+		it.Seek(makeKey(id, dbDiscoverRoot))
 
 		n := nextNode(it)
 		if n == nil {
@@ -334,7 +380,7 @@ seek:
 func nextNode(it iterator.Iterator) *Node {
 	for end := false; !end; end = !it.Next() {
 		id, field := splitKey(it.Key())
-		if field != nodeDBDiscoverRoot {
+		if field != dbDiscoverRoot {
 			continue
 		}
 		return mustDecodeNode(id[:], it.Value())

p2p/enode/nodedb_test.go

@@ -332,7 +332,7 @@ var nodeDBExpirationNodes = []struct {
 			30303,
 			30303,
 		),
-		pong: time.Now().Add(-nodeDBNodeExpiration + time.Minute),
+		pong: time.Now().Add(-dbNodeExpiration + time.Minute),
 		exp:  false,
 	}, {
 		node: NewV4(
@@ -341,7 +341,7 @@ var nodeDBExpirationNodes = []struct {
 			30303,
 			30303,
 		),
-		pong: time.Now().Add(-nodeDBNodeExpiration - time.Minute),
+		pong: time.Now().Add(-dbNodeExpiration - time.Minute),
 		exp:  true,
 	},
 }

p2p/enr/enr.go

@@ -156,7 +156,7 @@ func (r *Record) Set(e Entry) {
 }
 
 func (r *Record) invalidate() {
-	if r.signature == nil {
+	if r.signature != nil {
 		r.seq++
 	}
 	r.signature = nil

p2p/enr/enr_test.go

@@ -169,6 +169,18 @@ func TestDirty(t *testing.T) {
 	}
 }
 
+func TestSeq(t *testing.T) {
+	var r Record
+
+	assert.Equal(t, uint64(0), r.Seq())
+	r.Set(UDP(1))
+	assert.Equal(t, uint64(0), r.Seq())
+	signTest([]byte{5}, &r)
+	assert.Equal(t, uint64(0), r.Seq())
+	r.Set(UDP(2))
+	assert.Equal(t, uint64(1), r.Seq())
+}
+
 // TestGetSetOverwrite tests value overwrite when setting a new value with an existing key in record.
 func TestGetSetOverwrite(t *testing.T) {
 	var r Record

p2p/metrics.go

@@ -19,53 +19,214 @@
 package p2p
 
 import (
+	"fmt"
 	"net"
+	"sync"
+	"sync/atomic"
+	"time"
 
+	"github.com/ethereum/go-ethereum/p2p/enode"
+
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 )
 
-var (
+const (
-	ingressConnectMeter = metrics.NewRegisteredMeter("p2p/InboundConnects", nil)
+	MetricsInboundConnects  = "p2p/InboundConnects"  // Name for the registered inbound connects meter
-	ingressTrafficMeter = metrics.NewRegisteredMeter("p2p/InboundTraffic", nil)
+	MetricsInboundTraffic   = "p2p/InboundTraffic"   // Name for the registered inbound traffic meter
-	egressConnectMeter  = metrics.NewRegisteredMeter("p2p/OutboundConnects", nil)
+	MetricsOutboundConnects = "p2p/OutboundConnects" // Name for the registered outbound connects meter
-	egressTrafficMeter  = metrics.NewRegisteredMeter("p2p/OutboundTraffic", nil)
+	MetricsOutboundTraffic  = "p2p/OutboundTraffic"  // Name for the registered outbound traffic meter
+
+	MeteredPeerLimit = 1024 // This amount of peers are individually metered
 )
 
+var (
+	ingressConnectMeter = metrics.NewRegisteredMeter(MetricsInboundConnects, nil)  // Meter counting the ingress connections
+	ingressTrafficMeter = metrics.NewRegisteredMeter(MetricsInboundTraffic, nil)   // Meter metering the cumulative ingress traffic
+	egressConnectMeter  = metrics.NewRegisteredMeter(MetricsOutboundConnects, nil) // Meter counting the egress connections
+	egressTrafficMeter  = metrics.NewRegisteredMeter(MetricsOutboundTraffic, nil)  // Meter metering the cumulative egress traffic
+
+	PeerIngressRegistry = metrics.NewPrefixedChildRegistry(metrics.EphemeralRegistry, MetricsInboundTraffic+"/")  // Registry containing the peer ingress
+	PeerEgressRegistry  = metrics.NewPrefixedChildRegistry(metrics.EphemeralRegistry, MetricsOutboundTraffic+"/") // Registry containing the peer egress
+
+	meteredPeerFeed  event.Feed // Event feed for peer metrics
+	meteredPeerCount int32      // Actually stored peer connection count
+)
+
+// MeteredPeerEventType is the type of peer events emitted by a metered connection.
+type MeteredPeerEventType int
+
+const (
+	// PeerConnected is the type of event emitted when a peer successfully
+	// made the handshake.
+	PeerConnected MeteredPeerEventType = iota
+
+	// PeerDisconnected is the type of event emitted when a peer disconnects.
+	PeerDisconnected
+
+	// PeerHandshakeFailed is the type of event emitted when a peer fails to
+	// make the handshake or disconnects before the handshake.
+	PeerHandshakeFailed
+)
+
+// MeteredPeerEvent is an event emitted when peers connect or disconnect.
+type MeteredPeerEvent struct {
+	Type    MeteredPeerEventType // Type of peer event
+	IP      net.IP               // IP address of the peer
+	ID      enode.ID             // NodeID of the peer
+	Elapsed time.Duration        // Time elapsed between the connection and the handshake/disconnection
+	Ingress uint64               // Ingress count at the moment of the event
+	Egress  uint64               // Egress count at the moment of the event
+}
+
+// SubscribeMeteredPeerEvent registers a subscription for peer life-cycle events
+// if metrics collection is enabled.
+func SubscribeMeteredPeerEvent(ch chan<- MeteredPeerEvent) event.Subscription {
+	return meteredPeerFeed.Subscribe(ch)
+}
+
 // meteredConn is a wrapper around a net.Conn that meters both the
 // inbound and outbound network traffic.
 type meteredConn struct {
 	net.Conn // Network connection to wrap with metering
+
+	connected time.Time // Connection time of the peer
+	ip        net.IP    // IP address of the peer
+	id        enode.ID  // NodeID of the peer
+
+	// trafficMetered denotes if the peer is registered in the traffic registries.
+	// Its value is true if the metered peer count doesn't reach the limit in the
+	// moment of the peer's connection.
+	trafficMetered bool
+	ingressMeter   metrics.Meter // Meter for the read bytes of the peer
+	egressMeter    metrics.Meter // Meter for the written bytes of the peer
+
+	lock sync.RWMutex // Lock protecting the metered connection's internals
 }
 
-// newMeteredConn creates a new metered connection, also bumping the ingress or
+// newMeteredConn creates a new metered connection, bumps the ingress or egress
-// egress connection meter. If the metrics system is disabled, this function
+// connection meter and also increases the metered peer count. If the metrics
-// returns the original object.
+// system is disabled or the IP address is unspecified, this function returns
-func newMeteredConn(conn net.Conn, ingress bool) net.Conn {
+// the original object.
+func newMeteredConn(conn net.Conn, ingress bool, ip net.IP) net.Conn {
 	// Short circuit if metrics are disabled
 	if !metrics.Enabled {
 		return conn
 	}
-	// Otherwise bump the connection counters and wrap the connection
+	if ip.IsUnspecified() {
+		log.Warn("Peer IP is unspecified")
+		return conn
+	}
+	// Bump the connection counters and wrap the connection
 	if ingress {
 		ingressConnectMeter.Mark(1)
 	} else {
 		egressConnectMeter.Mark(1)
 	}
-	return &meteredConn{Conn: conn}
+	return &meteredConn{
+		Conn:      conn,
+		ip:        ip,
+		connected: time.Now(),
+	}
 }
 
-// Read delegates a network read to the underlying connection, bumping the ingress
+// Read delegates a network read to the underlying connection, bumping the common
-// traffic meter along the way.
+// and the peer ingress traffic meters along the way.
 func (c *meteredConn) Read(b []byte) (n int, err error) {
 	n, err = c.Conn.Read(b)
 	ingressTrafficMeter.Mark(int64(n))
-	return
+	c.lock.RLock()
+	if c.trafficMetered {
+		c.ingressMeter.Mark(int64(n))
+	}
+	c.lock.RUnlock()
+	return n, err
 }
 
-// Write delegates a network write to the underlying connection, bumping the
+// Write delegates a network write to the underlying connection, bumping the common
-// egress traffic meter along the way.
+// and the peer egress traffic meters along the way.
 func (c *meteredConn) Write(b []byte) (n int, err error) {
 	n, err = c.Conn.Write(b)
 	egressTrafficMeter.Mark(int64(n))
-	return
+	c.lock.RLock()
+	if c.trafficMetered {
+		c.egressMeter.Mark(int64(n))
+	}
+	c.lock.RUnlock()
+	return n, err
+}
+
+// handshakeDone is called when a peer handshake is done. Registers the peer to
+// the ingress and the egress traffic registries using the peer's IP and node ID,
+// also emits connect event.
+func (c *meteredConn) handshakeDone(id enode.ID) {
+	if atomic.AddInt32(&meteredPeerCount, 1) >= MeteredPeerLimit {
+		// Don't register the peer in the traffic registries.
+		atomic.AddInt32(&meteredPeerCount, -1)
+		c.lock.Lock()
+		c.id, c.trafficMetered = id, false
+		c.lock.Unlock()
+		log.Warn("Metered peer count reached the limit")
+	} else {
+		key := fmt.Sprintf("%s/%s", c.ip, id.String())
+		c.lock.Lock()
+		c.id, c.trafficMetered = id, true
+		c.ingressMeter = metrics.NewRegisteredMeter(key, PeerIngressRegistry)
+		c.egressMeter = metrics.NewRegisteredMeter(key, PeerEgressRegistry)
+		c.lock.Unlock()
+	}
+	meteredPeerFeed.Send(MeteredPeerEvent{
+		Type:    PeerConnected,
+		IP:      c.ip,
+		ID:      id,
+		Elapsed: time.Since(c.connected),
+	})
+}
+
+// Close delegates a close operation to the underlying connection, unregisters
+// the peer from the traffic registries and emits close event.
+func (c *meteredConn) Close() error {
+	err := c.Conn.Close()
+	c.lock.RLock()
+	if c.id == (enode.ID{}) {
+		// If the peer disconnects before the handshake.
+		c.lock.RUnlock()
+		meteredPeerFeed.Send(MeteredPeerEvent{
+			Type:    PeerHandshakeFailed,
+			IP:      c.ip,
+			Elapsed: time.Since(c.connected),
+		})
+		return err
+	}
+	id := c.id
+	if !c.trafficMetered {
+		// If the peer isn't registered in the traffic registries.
+		c.lock.RUnlock()
+		meteredPeerFeed.Send(MeteredPeerEvent{
+			Type: PeerDisconnected,
+			IP:   c.ip,
+			ID:   id,
+		})
+		return err
+	}
+	ingress, egress := uint64(c.ingressMeter.Count()), uint64(c.egressMeter.Count())
+	c.lock.RUnlock()
+
+	// Decrement the metered peer count
+	atomic.AddInt32(&meteredPeerCount, -1)
+
+	// Unregister the peer from the traffic registries
+	key := fmt.Sprintf("%s/%s", c.ip, id)
+	PeerIngressRegistry.Unregister(key)
+	PeerEgressRegistry.Unregister(key)
+
+	meteredPeerFeed.Send(MeteredPeerEvent{
+		Type:    PeerDisconnected,
+		IP:      c.ip,
+		ID:      id,
+		Ingress: ingress,
+		Egress:  egress,
+	})
+	return err
 }