params, swarm: release Geth v1.8.17 and Swar v0.3.5

* swarm/network/stream: introduced visualized snapshot sync test

* swarm/network/stream: non-existing hash visualization sim

* swarm/network/stream: fixed retrieval tests; new backend for visualization

* swarm/network/stream: cleanup of visualized_snapshot_sync_sim_test.go

* swarm/network/stream: rebased PR on master

* swarm/network/stream: fixed loop logic in retrieval tests

* swarm/network/stream: fixed iterations for snapshot tests

* swarm/network/stream: address PR comments

* swarm/network/stream: addressed PR comments

.github/CODEOWNERS

@@ -27,6 +27,6 @@ swarm/services                  @zelig
 swarm/state                     @justelad
 swarm/storage/encryption        @gbalint @zelig @nagydani
 swarm/storage/mock              @janos
-swarm/storage/mru               @nolash
+swarm/storage/feed              @nolash @jpeletier
 swarm/testutil                  @lmars
 whisper/                        @gballet @gluk256

.github/CONTRIBUTING.md

@@ -1,16 +1,40 @@
+# Contributing
+
+Thank you for considering to help out with the source code! We welcome 
+contributions from anyone on the internet, and are grateful for even the 
+smallest of fixes!
+
+If you'd like to contribute to go-ethereum, please fork, fix, commit and send a 
+pull request for the maintainers to review and merge into the main code base. If
+you wish to submit more complex changes though, please check up with the core 
+devs first on [our gitter channel](https://gitter.im/ethereum/go-ethereum) to 
+ensure those changes are in line with the general philosophy of the project 
+and/or get some early feedback which can make both your efforts much lighter as
+well as our review and merge procedures quick and simple.
+
+## Coding guidelines
+
+Please make sure your contributions adhere to our coding guidelines:
+
+ * Code must adhere to the official Go 
+[formatting](https://golang.org/doc/effective_go.html#formatting) guidelines 
+(i.e. uses [gofmt](https://golang.org/cmd/gofmt/)).
+ * Code must be documented adhering to the official Go 
+[commentary](https://golang.org/doc/effective_go.html#commentary) guidelines.
+ * Pull requests need to be based on and opened against the `master` branch.
+ * Commit messages should be prefixed with the package(s) they modify.
+   * E.g. "eth, rpc: make trace configs optional"
+
 ## Can I have feature X
 
-Before you do a feature request please check and make sure that it isn't possible
-through some other means. The JavaScript enabled console is a powerful feature
-in the right hands. Please check our [Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info
+Before you submit a feature request, please check and make sure that it isn't 
+possible through some other means. The JavaScript-enabled console is a powerful 
+feature in the right hands. Please check our 
+[Wiki page](https://github.com/ethereum/go-ethereum/wiki) for more info
 and help.
 
-## Contributing
+## Configuration, dependencies, and tests
 
-If you'd like to contribute to go-ethereum please fork, fix, commit and
-send a pull request. Commits which do not comply with the coding standards
-are ignored (use gofmt!).
-
-See [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
-for more details on configuring your environment, testing, and
-dependency management.
+Please see the [Developers' Guide](https://github.com/ethereum/go-ethereum/wiki/Developers'-Guide)
+for more details on configuring your environment, managing project dependencies
+and testing procedures.

.travis.yml

@@ -3,17 +3,6 @@ go_import_path: github.com/ethereum/go-ethereum
 sudo: false
 matrix:
   include:
-    - os: linux
-      dist: trusty
-      sudo: required
-      go: 1.9.x
-      script:
-        - sudo modprobe fuse
-        - sudo chmod 666 /dev/fuse
-        - sudo chown root:$USER /etc/fuse.conf
-        - go run build/ci.go install
-        - go run build/ci.go test -coverage $TEST_PACKAGES
-
     - os: linux
       dist: trusty
       sudo: required
@@ -56,7 +45,8 @@ matrix:
         - go run build/ci.go lint
 
     # This builder does the Ubuntu PPA upload
-    - os: linux
+    - if: type = push
+      os: linux
       dist: trusty
       go: 1.11.x
       env:
@@ -74,7 +64,8 @@ matrix:
         - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
 
     # This builder does the Linux Azure uploads
-    - os: linux
+    - if: type = push
+      os: linux
       dist: trusty
       sudo: required
       go: 1.11.x
@@ -107,7 +98,8 @@ matrix:
         - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
     # This builder does the Linux Azure MIPS xgo uploads
-    - os: linux
+    - if: type = push
+      os: linux
       dist: trusty
       services:
         - docker
@@ -134,7 +126,8 @@ matrix:
         - go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
 
     # This builder does the Android Maven and Azure uploads
-    - os: linux
+    - if: type = push
+      os: linux
       dist: trusty
       addons:
         apt:
@@ -155,7 +148,7 @@ matrix:
       git:
         submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.11.linux-amd64.tar.gz | tar -xz
+        - curl https://storage.googleapis.com/golang/go1.11.1.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go
@@ -171,7 +164,8 @@ matrix:
         - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds
 
     # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
-    - os: osx
+    - if: type = push
+      os: osx
       go: 1.11.x
       env:
         - azure-osx
@@ -199,7 +193,8 @@ matrix:
         - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds
 
     # This builder does the Azure archive purges to avoid accumulating junk
-    - os: linux
+    - if: type = cron
+      os: linux
       dist: trusty
       go: 1.11.x
       env:
@@ -208,10 +203,3 @@ matrix:
         submodules: false # avoid cloning ethereum/tests
       script:
         - go run build/ci.go purge -store gethstore/builds -days 14
-
-notifications:
-  webhooks:
-    urls:
-      - https://webhooks.gitter.im/e/e09ccdce1048c5e03445
-    on_success: change
-    on_failure: always

Makefile

@@ -57,6 +57,9 @@ devtools:
 	@type "solc" 2> /dev/null || echo 'Please install solc'
 	@type "protoc" 2> /dev/null || echo 'Please install protoc'
 
+swarm-devtools:
+	env GOBIN= go install ./cmd/swarm/mimegen
+
 # Cross Compilation Targets (xgo)
 
 geth-cross: geth-linux geth-darwin geth-windows geth-android geth-ios

accounts/abi/abi.go

@@ -137,6 +137,9 @@ func (abi *ABI) UnmarshalJSON(data []byte) error {
 // MethodById looks up a method by the 4-byte id
 // returns nil if none found
 func (abi *ABI) MethodById(sigdata []byte) (*Method, error) {
+	if len(sigdata) < 4 {
+		return nil, fmt.Errorf("data too short (% bytes) for abi method lookup", len(sigdata))
+	}
 	for _, method := range abi.Methods {
 		if bytes.Equal(method.Id(), sigdata[:4]) {
 			return &method, nil

accounts/abi/abi_test.go

@@ -711,5 +711,14 @@ func TestABI_MethodById(t *testing.T) {
 			t.Errorf("Method %v (id %v) not 'findable' by id in ABI", name, common.ToHex(m.Id()))
 		}
 	}
-
+	// Also test empty
+	if _, err := abi.MethodById([]byte{0x00}); err == nil {
+		t.Errorf("Expected error, too short to decode data")
+	}
+	if _, err := abi.MethodById([]byte{}); err == nil {
+		t.Errorf("Expected error, too short to decode data")
+	}
+	if _, err := abi.MethodById(nil); err == nil {
+		t.Errorf("Expected error, nil is short to decode data")
+	}
 }

accounts/abi/bind/backends/simulated.go

@@ -208,7 +208,7 @@ func (b *SimulatedBackend) PendingNonceAt(ctx context.Context, account common.Ad
 }
 
 // SuggestGasPrice implements ContractTransactor.SuggestGasPrice. Since the simulated
-// chain doens't have miners, we just return a gas price of 1 for any call.
+// chain doesn't have miners, we just return a gas price of 1 for any call.
 func (b *SimulatedBackend) SuggestGasPrice(ctx context.Context) (*big.Int, error) {
 	return big.NewInt(1), nil
 }

accounts/abi/bind/bind.go

@@ -23,13 +23,13 @@ package bind
 import (
 	"bytes"
 	"fmt"
+	"go/format"
 	"regexp"
 	"strings"
 	"text/template"
 	"unicode"
 
 	"github.com/ethereum/go-ethereum/accounts/abi"
-	"golang.org/x/tools/imports"
 )
 
 // Lang is a target programming language selector to generate bindings for.
@@ -145,9 +145,9 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 	if err := tmpl.Execute(buffer, data); err != nil {
 		return "", err
 	}
-	// For Go bindings pass the code through goimports to clean it up and double check
+	// For Go bindings pass the code through gofmt to clean it up
 	if lang == LangGo {
-		code, err := imports.Process(".", buffer.Bytes(), nil)
+		code, err := format.Source(buffer.Bytes())
 		if err != nil {
 			return "", fmt.Errorf("%v\n%s", err, buffer)
 		}

accounts/abi/bind/template.go

@@ -64,6 +64,30 @@ const tmplSourceGo = `
 
 package {{.Package}}
 
+import (
+	"math/big"
+	"strings"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/event"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var (
+	_ = big.NewInt
+	_ = strings.NewReader
+	_ = ethereum.NotFound
+	_ = abi.U256
+	_ = bind.Bind
+	_ = common.Big1
+	_ = types.BloomLookup
+	_ = event.NewSubscription
+)
+
 {{range $contract := .Contracts}}
 	// {{.Type}}ABI is the input ABI used to generate the binding from.
 	const {{.Type}}ABI = "{{.InputABI}}"

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.windows-%GETH_ARCH%.zip
-  - 7z x go1.11.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.1.windows-%GETH_ARCH%.zip
+  - 7z x go1.11.1.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

build/ci.go

@@ -320,9 +320,7 @@ func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd
 // "tests" also includes static analysis tools such as vet.
 
 func doTest(cmdline []string) {
-	var (
-		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
-	)
+	coverage := flag.Bool("coverage", false, "Whether to record code coverage")
 	flag.CommandLine.Parse(cmdline)
 	env := build.Env()
 
@@ -332,14 +330,11 @@ func doTest(cmdline []string) {
 	}
 	packages = build.ExpandPackagesNoVendor(packages)
 
-	// Run analysis tools before the tests.
-	build.MustRun(goTool("vet", packages...))
-
 	// Run the actual tests.
-	gotest := goTool("test", buildFlags(env)...)
 	// Test a single package at a time. CI builders are slow
 	// and some tests run into timeouts under load.
-	gotest.Args = append(gotest.Args, "-p", "1")
+	gotest := goTool("test", buildFlags(env)...)
+	gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m")
 	if *coverage {
 		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
 	}
@@ -1040,7 +1035,7 @@ func xgoTool(args []string) *exec.Cmd {
 func doPurge(cmdline []string) {
 	var (
 		store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`)
-		limit = flag.Int("days", 30, `Age threshold above which to delete unstalbe archives`)
+		limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`)
 	)
 	flag.CommandLine.Parse(cmdline)
 

cmd/abigen/main.go

@@ -75,7 +75,7 @@ func main() {
 		bins  []string
 		types []string
 	)
-	if *solFlag != "" || *abiFlag == "-" {
+	if *solFlag != "" || (*abiFlag == "-" && *pkgFlag == "") {
 		// Generate the list of types to exclude from binding
 		exclude := make(map[string]bool)
 		for _, kind := range strings.Split(*excFlag, ",") {
@@ -111,7 +111,13 @@ func main() {
 		}
 	} else {
 		// Otherwise load up the ABI, optional bytecode and type name from the parameters
-		abi, err := ioutil.ReadFile(*abiFlag)
+		var abi []byte
+		var err error
+		if *abiFlag == "-" {
+			abi, err = ioutil.ReadAll(os.Stdin)
+		} else {
+			abi, err = ioutil.ReadFile(*abiFlag)
+		}
 		if err != nil {
 			fmt.Printf("Failed to read input ABI: %v\n", err)
 			os.Exit(-1)
@@ -155,6 +161,5 @@ func contractsFromStdin() (map[string]*compiler.Contract, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	return compiler.ParseCombinedJSON(bytes, "", "", "", "")
 }

cmd/bootnode/main.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/discv5"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 )
@@ -85,7 +86,7 @@ func main() {
 	}
 
 	if *writeAddr {
-		fmt.Printf("%v\n", discover.PubkeyID(&nodeKey.PublicKey))
+		fmt.Printf("%v\n", enode.PubkeyToIDV4(&nodeKey.PublicKey))
 		os.Exit(0)
 	}
 

cmd/clef/README.md

@@ -875,3 +875,4 @@ There are a couple of implementation for a UI. We'll try to keep this list up to
 | QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: |  :+1: (partially)|
 | GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: |  :x: |
 | Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: |  :x: |
+| Clef UI| https://github.com/kyokan/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: |  :+1: (approve tx only)|

cmd/clef/extapi_changelog.md

@@ -1,6 +1,13 @@
 ### Changelog for external API
 
+#### 4.0.0
 
+* The external `account_Ecrecover`-method was removed. 
+* The external `account_Import`-method was removed.
+
+#### 3.0.0
+
+* The external `account_List`-method was changed to not expose `url`, which contained info about the local filesystem. It now returns only a list of addresses. 
 
 #### 2.0.0
 

cmd/clef/intapi_changelog.md

@@ -1,5 +1,21 @@
 ### Changelog for internal API (ui-api)
 
+### 2.1.0
+
+* Add `OnInputRequired(info UserInputRequest)` to internal API. This method is used when Clef needs user input, e.g. passwords.
+
+The following structures are used:
+```golang
+       UserInputRequest struct {
+               Prompt     string `json:"prompt"`
+               Title      string `json:"title"`
+               IsPassword bool   `json:"isPassword"`
+       }
+       UserInputResponse struct {
+               Text string `json:"text"`
+       }
+```
+
 ### 2.0.0
 
 * Modify how `call_info` on a transaction is conveyed. New format:

cmd/clef/main.go

@@ -48,7 +48,7 @@ import (
 )
 
 // ExternalAPIVersion -- see extapi_changelog.md
-const ExternalAPIVersion = "2.0.0"
+const ExternalAPIVersion = "3.0.0"
 
 // InternalAPIVersion -- see intapi_changelog.md
 const InternalAPIVersion = "2.0.0"
@@ -70,6 +70,10 @@ var (
 		Value: 4,
 		Usage: "log level to emit to the screen",
 	}
+	advancedMode = cli.BoolFlag{
+		Name:  "advanced",
+		Usage: "If enabled, issues warnings instead of rejections for suspicious requests. Default off",
+	}
 	keystoreFlag = cli.StringFlag{
 		Name:  "keystore",
 		Value: filepath.Join(node.DefaultDataDir(), "keystore"),
@@ -191,6 +195,7 @@ func init() {
 		ruleFlag,
 		stdiouiFlag,
 		testFlag,
+		advancedMode,
 	}
 	app.Action = signer
 	app.Commands = []cli.Command{initCommand, attestCommand, addCredentialCommand}
@@ -384,7 +389,8 @@ func signer(c *cli.Context) error {
 		c.String(keystoreFlag.Name),
 		c.Bool(utils.NoUSBFlag.Name),
 		ui, db,
-		c.Bool(utils.LightKDFFlag.Name))
+		c.Bool(utils.LightKDFFlag.Name),
+		c.Bool(advancedMode.Name))
 
 	api = apiImpl
 

cmd/clef/tutorial.md

@@ -31,43 +31,51 @@ NOTE: This file does not contain your accounts. Those need to be backed up separ
 
 ## Creating rules
 
-Now, you can create a rule-file.
+Now, you can create a rule-file. Note that it is not mandatory to use predefined rules, but it's really handy.
 
 ```javascript
 function ApproveListing(){
     return "Approve"
 }
 ```
-Get the `sha256` hash....
+
+Get the `sha256` hash. If you have openssl, you can do `openssl sha256 rules.js`...
 ```text
 #sha256sum rules.js
 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72  rules.js
 ```
-...And then `attest` the file:
+...now `attest` the file...
 ```text
 #./signer attest 6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
 
 INFO [02-21|12:14:38] Ruleset attestation updated              sha256=6c21d1737429d6d4f2e55146da0797782f3c0a0355227f19d702df377c165d72
 ```
-At this point, we then start the signer with the rule-file:
 
+...and (this is required only for non-production versions) load a mock-up `4byte.json` by copying the file from the source to your current working directory:
 ```text
-#./signer --rules rules.json
+#cp $GOPATH/src/github.com/ethereum/go-ethereum/cmd/clef/4byte.json $PWD
+```
 
-INFO [02-21|12:15:18] Using CLI as UI-channel
-INFO [02-21|12:15:18] Loaded 4byte db                          signatures=5509 file=./4byte.json
-INFO [02-21|12:15:18] Could not load rulefile, rules not enabled file=rulefile
-DEBUG[02-21|12:15:18] FS scan times                            list=35.335µs set=5.536µs diff=5.073µs
-DEBUG[02-21|12:15:18] Ledger support enabled
-DEBUG[02-21|12:15:18] Trezor support enabled
-INFO [02-21|12:15:18] Audit logs configured                    file=audit.log
-INFO [02-21|12:15:18] HTTP endpoint opened                     url=http://localhost:8550
+At this point, we can start the signer with the rule-file:
+```text
+#./signer --rules rules.js --rpc
+
+INFO [09-25|20:28:11.866] Using CLI as UI-channel 
+INFO [09-25|20:28:11.876] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [09-25|20:28:11.877] Rule engine configured                   file=./rules.js
+DEBUG[09-25|20:28:11.877] FS scan times                            list=100.781µs set=13.253µs diff=5.761µs
+DEBUG[09-25|20:28:11.884] Ledger support enabled 
+DEBUG[09-25|20:28:11.888] Trezor support enabled 
+INFO [09-25|20:28:11.888] Audit logs configured                    file=audit.log
+DEBUG[09-25|20:28:11.888] HTTP registered                          namespace=account
+INFO [09-25|20:28:11.890] HTTP endpoint opened                     url=http://localhost:8550
+DEBUG[09-25|20:28:11.890] IPC registered                           namespace=account
+INFO [09-25|20:28:11.890] IPC endpoint opened                      url=<nil>
 ------- Signer info -------
+* extapi_version : 2.0.0
+* intapi_version : 2.0.0
 * extapi_http : http://localhost:8550
 * extapi_ipc : <nil>
-* extapi_version : 2.0.0
-* intapi_version : 1.2.0
-
 ```
 
 Any list-requests will now be auto-approved by our rule-file.
@@ -107,16 +115,16 @@ The `master_seed` was then used to derive a few other things:
 
 ## Adding credentials
 
-In order to make more useful rules; sign transactions, the signer needs access to the passwords needed to unlock keystores.
+In order to make more useful rules like signing transactions, the signer needs access to the passwords needed to unlock keystores.
 
 ```text
-#./signer addpw 0x694267f14675d7e1b9494fd8d72fefe1755710fa test
+#./signer addpw "0x694267f14675d7e1b9494fd8d72fefe1755710fa" "test_password"
 
 INFO [02-21|13:43:21] Credential store updated                 key=0x694267f14675d7e1b9494fd8d72fefe1755710fa
 ```
 ## More advanced rules
 
-Now let's update the rules to make use of credentials
+Now let's update the rules to make use of credentials:
 
 ```javascript
 function ApproveListing(){
@@ -134,13 +142,15 @@ function ApproveSignData(r){
 }
 
 ```
-In this example,
-* any requests to sign data with the account `0x694...` will be
-    * auto-approved if the message contains with `bazonk`,
-    * and auto-rejected if it does not.
-    * Any other signing-requests will be passed along for manual approve/reject.
+In this example:
+* Any requests to sign data with the account `0x694...` will be
+    * auto-approved if the message contains with `bazonk`
+    * auto-rejected if it does not.
+* Any other signing-requests will be passed along for manual approve/reject.
 
-..attest the new file
+_Note: make sure that `0x694...` is an account you have access to. You can create it either via the clef or the traditional account cli tool. If the latter was chosen, make sure both clef and geth use the same keystore by specifing `--keystore path/to/your/keystore` when running clef._
+
+Attest the new file...
 ```text
 #sha256sum rules.js
 2a0cb661dacfc804b6e95d935d813fd17c0997a7170e4092ffbc34ca976acd9f  rules.js
@@ -153,23 +163,26 @@ INFO [02-21|14:36:30] Ruleset attestation updated              sha256=2a0cb661da
 And start the signer:
 
 ```
-#./signer --rules rules.js
+#./signer --rules rules.js --rpc
 
-INFO [02-21|14:41:56] Using CLI as UI-channel
-INFO [02-21|14:41:56] Loaded 4byte db                          signatures=5509 file=./4byte.json
-INFO [02-21|14:41:56] Rule engine configured                   file=rules.js
-DEBUG[02-21|14:41:56] FS scan times                            list=34.607µs set=4.509µs diff=4.87µs
-DEBUG[02-21|14:41:56] Ledger support enabled
-DEBUG[02-21|14:41:56] Trezor support enabled
-INFO [02-21|14:41:56] Audit logs configured                    file=audit.log
-INFO [02-21|14:41:56] HTTP endpoint opened                     url=http://localhost:8550
+INFO [09-25|21:02:16.450] Using CLI as UI-channel 
+INFO [09-25|21:02:16.466] Loaded 4byte db                          signatures=5509 file=./4byte.json
+INFO [09-25|21:02:16.467] Rule engine configured                   file=./rules.js
+DEBUG[09-25|21:02:16.468] FS scan times                            list=1.45262ms set=21.926µs diff=6.944µs
+DEBUG[09-25|21:02:16.473] Ledger support enabled 
+DEBUG[09-25|21:02:16.475] Trezor support enabled 
+INFO [09-25|21:02:16.476] Audit logs configured                    file=audit.log
+DEBUG[09-25|21:02:16.476] HTTP registered                          namespace=account
+INFO [09-25|21:02:16.478] HTTP endpoint opened                     url=http://localhost:8550
+DEBUG[09-25|21:02:16.478] IPC registered                           namespace=account
+INFO [09-25|21:02:16.478] IPC endpoint opened                      url=<nil>
 ------- Signer info -------
 * extapi_version : 2.0.0
-* intapi_version : 1.2.0
+* intapi_version : 2.0.0
 * extapi_http : http://localhost:8550
 * extapi_ipc : <nil>
-INFO [02-21|14:41:56] error occurred during execution          error="ReferenceError: 'OnSignerStartup' is not defined"
 ```
+
 And then test signing, once with `bazonk` and once without:
 
 ```
@@ -190,9 +203,9 @@ INFO [02-21|14:42:56] Op rejected
 The signer also stores all traffic over the external API in a log file. The last 4 lines shows the two requests and their responses:
 
 ```text
-#tail audit.log -n 4
+#tail -n 4 audit.log
 t=2018-02-21T14:42:41+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49706\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=202062617a6f6e6b2062617a2067617a0a
 t=2018-02-21T14:42:42+0100 lvl=info msg=Sign       api=signer type=response data=93e6161840c3ae1efc26dc68dedab6e8fc233bb3fefa1b4645dbf6609b93dace160572ea4ab33240256bb6d3dadb60dcd9c515d6374d3cf614ee897408d41d541c error=nil
 t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=request  metadata="{\"remote\":\"127.0.0.1:49708\",\"local\":\"localhost:8550\",\"scheme\":\"HTTP/1.1\"}" addr="0x694267f14675d7e1b9494fd8d72fefe1755710fa [chksum INVALID]" data=2020626f6e6b2062617a2067617a0a
 t=2018-02-21T14:42:56+0100 lvl=info msg=Sign       api=signer type=response data=                                                                                                                                   error="Request denied"
-```
+```

cmd/evm/staterunner.go

@@ -97,6 +97,10 @@ func stateTestCmd(ctx *cli.Context) error {
 			// Run the test and aggregate the result
 			result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
 			state, err := test.Run(st, cfg)
+			// print state root for evmlab tracing
+			if ctx.GlobalBool(MachineFlag.Name) && state != nil {
+				fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))
+			}
 			if err != nil {
 				// Test failed, mark as so and dump any state to aid debugging
 				result.Pass, result.Error = false, err.Error()
@@ -105,10 +109,6 @@ func stateTestCmd(ctx *cli.Context) error {
 					result.State = &dump
 				}
 			}
-			// print state root for evmlab tracing (already committed above, so no need to delete objects again
-			if ctx.GlobalBool(MachineFlag.Name) && state != nil {
-				fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))
-			}
 
 			results = append(results, *result)
 

cmd/faucet/faucet.go

@@ -54,8 +54,8 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/discv5"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/params"
 	"golang.org/x/net/websocket"
@@ -255,8 +255,10 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u
 		return nil, err
 	}
 	for _, boot := range enodes {
-		old, _ := discover.ParseNode(boot.String())
-		stack.Server().AddPeer(old)
+		old, err := enode.ParseV4(boot.String())
+		if err != nil {
+			stack.Server().AddPeer(old)
+		}
 	}
 	// Attach to the client and retrieve and interesting metadatas
 	api, err := stack.Attach()

cmd/p2psim/main.go

@@ -47,7 +47,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/simulations"
 	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -285,7 +285,7 @@ func createNode(ctx *cli.Context) error {
 		if err != nil {
 			return err
 		}
-		config.ID = discover.PubkeyID(&privKey.PublicKey)
+		config.ID = enode.PubkeyToIDV4(&privKey.PublicKey)
 		config.PrivateKey = privKey
 	}
 	if services := ctx.String("services"); services != "" {

cmd/puppeth/module_node.go

@@ -42,7 +42,7 @@ ADD genesis.json /genesis.json
 RUN \
   echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}}
 	echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
-	echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh
+	echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --nat extip:{{.IP}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh
 
 ENTRYPOINT ["/bin/sh", "geth.sh"]
 `
@@ -99,6 +99,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n
 	template.Must(template.New("").Parse(nodeDockerfile)).Execute(dockerfile, map[string]interface{}{
 		"NetworkID": config.network,
 		"Port":      config.port,
+		"IP":        client.address,
 		"Peers":     config.peersTotal,
 		"LightFlag": lightFlag,
 		"Bootnodes": strings.Join(bootnodes, ","),
@@ -227,10 +228,10 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
 
 	// Container available, retrieve its node ID and its genesis json
 	var out []byte
-	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.id --cache=16 attach", network, kind)); err != nil {
+	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 geth --exec admin.nodeInfo.enode --cache=16 attach", network, kind)); err != nil {
 		return nil, ErrServiceUnreachable
 	}
-	id := bytes.Trim(bytes.TrimSpace(out), "\"")
+	enode := bytes.Trim(bytes.TrimSpace(out), "\"")
 
 	if out, err = client.Run(fmt.Sprintf("docker exec %s_%s_1 cat /genesis.json", network, kind)); err != nil {
 		return nil, ErrServiceUnreachable
@@ -265,7 +266,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error)
 		gasLimit:   gasLimit,
 		gasPrice:   gasPrice,
 	}
-	stats.enode = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.port)
+	stats.enode = string(enode)
 
 	return stats, nil
 }

cmd/swarm/access.go

@@ -130,7 +130,7 @@ func accessNewACT(ctx *cli.Context) {
 		if err != nil {
 			utils.Fatalf("had an error reading the grantee public key list")
 		}
-		pkGrantees = strings.Split(string(bytes), "\n")
+		pkGrantees = strings.Split(strings.Trim(string(bytes), "\n"), "\n")
 	}
 
 	if passGranteesFilename != "" {
@@ -138,7 +138,7 @@ func accessNewACT(ctx *cli.Context) {
 		if err != nil {
 			utils.Fatalf("could not read password filename: %v", err)
 		}
-		passGrantees = strings.Split(string(bytes), "\n")
+		passGrantees = strings.Split(strings.Trim(string(bytes), "\n"), "\n")
 	}
 	accessKey, ae, actManifest, err = api.DoACT(ctx, privateKey, salt, pkGrantees, passGrantees)
 	if err != nil {

cmd/swarm/config.go

@@ -59,27 +59,28 @@ var (
 
 //constants for environment variables
 const (
-	SWARM_ENV_CHEQUEBOOK_ADDR      = "SWARM_CHEQUEBOOK_ADDR"
-	SWARM_ENV_ACCOUNT              = "SWARM_ACCOUNT"
-	SWARM_ENV_LISTEN_ADDR          = "SWARM_LISTEN_ADDR"
-	SWARM_ENV_PORT                 = "SWARM_PORT"
-	SWARM_ENV_NETWORK_ID           = "SWARM_NETWORK_ID"
-	SWARM_ENV_SWAP_ENABLE          = "SWARM_SWAP_ENABLE"
-	SWARM_ENV_SWAP_API             = "SWARM_SWAP_API"
-	SWARM_ENV_SYNC_DISABLE         = "SWARM_SYNC_DISABLE"
-	SWARM_ENV_SYNC_UPDATE_DELAY    = "SWARM_ENV_SYNC_UPDATE_DELAY"
-	SWARM_ENV_LIGHT_NODE_ENABLE    = "SWARM_LIGHT_NODE_ENABLE"
-	SWARM_ENV_DELIVERY_SKIP_CHECK  = "SWARM_DELIVERY_SKIP_CHECK"
-	SWARM_ENV_ENS_API              = "SWARM_ENS_API"
-	SWARM_ENV_ENS_ADDR             = "SWARM_ENS_ADDR"
-	SWARM_ENV_CORS                 = "SWARM_CORS"
-	SWARM_ENV_BOOTNODES            = "SWARM_BOOTNODES"
-	SWARM_ENV_PSS_ENABLE           = "SWARM_PSS_ENABLE"
-	SWARM_ENV_STORE_PATH           = "SWARM_STORE_PATH"
-	SWARM_ENV_STORE_CAPACITY       = "SWARM_STORE_CAPACITY"
-	SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY"
-	SWARM_ACCESS_PASSWORD          = "SWARM_ACCESS_PASSWORD"
-	GETH_ENV_DATADIR               = "GETH_DATADIR"
+	SWARM_ENV_CHEQUEBOOK_ADDR         = "SWARM_CHEQUEBOOK_ADDR"
+	SWARM_ENV_ACCOUNT                 = "SWARM_ACCOUNT"
+	SWARM_ENV_LISTEN_ADDR             = "SWARM_LISTEN_ADDR"
+	SWARM_ENV_PORT                    = "SWARM_PORT"
+	SWARM_ENV_NETWORK_ID              = "SWARM_NETWORK_ID"
+	SWARM_ENV_SWAP_ENABLE             = "SWARM_SWAP_ENABLE"
+	SWARM_ENV_SWAP_API                = "SWARM_SWAP_API"
+	SWARM_ENV_SYNC_DISABLE            = "SWARM_SYNC_DISABLE"
+	SWARM_ENV_SYNC_UPDATE_DELAY       = "SWARM_ENV_SYNC_UPDATE_DELAY"
+	SWARM_ENV_MAX_STREAM_PEER_SERVERS = "SWARM_ENV_MAX_STREAM_PEER_SERVERS"
+	SWARM_ENV_LIGHT_NODE_ENABLE       = "SWARM_LIGHT_NODE_ENABLE"
+	SWARM_ENV_DELIVERY_SKIP_CHECK     = "SWARM_DELIVERY_SKIP_CHECK"
+	SWARM_ENV_ENS_API                 = "SWARM_ENS_API"
+	SWARM_ENV_ENS_ADDR                = "SWARM_ENS_ADDR"
+	SWARM_ENV_CORS                    = "SWARM_CORS"
+	SWARM_ENV_BOOTNODES               = "SWARM_BOOTNODES"
+	SWARM_ENV_PSS_ENABLE              = "SWARM_PSS_ENABLE"
+	SWARM_ENV_STORE_PATH              = "SWARM_STORE_PATH"
+	SWARM_ENV_STORE_CAPACITY          = "SWARM_STORE_CAPACITY"
+	SWARM_ENV_STORE_CACHE_CAPACITY    = "SWARM_STORE_CACHE_CAPACITY"
+	SWARM_ACCESS_PASSWORD             = "SWARM_ACCESS_PASSWORD"
+	GETH_ENV_DATADIR                  = "GETH_DATADIR"
 )
 
 // These settings ensure that TOML keys use the same names as Go struct fields.
@@ -124,7 +125,7 @@ func initSwarmNode(config *bzzapi.Config, stack *node.Node, ctx *cli.Context) {
 	//get the account for the provided swarm account
 	prvkey := getAccount(config.BzzAccount, ctx, stack)
 	//set the resolved config path (geth --datadir)
-	config.Path = stack.InstanceDir()
+	config.Path = expandPath(stack.InstanceDir())
 	//finally, initialize the configuration
 	config.Init(prvkey)
 	//configuration phase completed here
@@ -175,14 +176,18 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 	}
 
 	if networkid := ctx.GlobalString(SwarmNetworkIdFlag.Name); networkid != "" {
-		if id, _ := strconv.Atoi(networkid); id != 0 {
-			currentConfig.NetworkID = uint64(id)
+		id, err := strconv.ParseUint(networkid, 10, 64)
+		if err != nil {
+			utils.Fatalf("invalid cli flag %s: %v", SwarmNetworkIdFlag.Name, err)
+		}
+		if id != 0 {
+			currentConfig.NetworkID = id
 		}
 	}
 
 	if ctx.GlobalIsSet(utils.DataDirFlag.Name) {
 		if datadir := ctx.GlobalString(utils.DataDirFlag.Name); datadir != "" {
-			currentConfig.Path = datadir
+			currentConfig.Path = expandPath(datadir)
 		}
 	}
 
@@ -207,6 +212,9 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		currentConfig.SyncUpdateDelay = d
 	}
 
+	// any value including 0 is acceptable
+	currentConfig.MaxStreamPeerServers = ctx.GlobalInt(SwarmMaxStreamPeerServersFlag.Name)
+
 	if ctx.GlobalIsSet(SwarmLightNodeEnabled.Name) {
 		currentConfig.LightNodeEnabled = true
 	}
@@ -226,6 +234,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		if len(ensAPIs) == 1 && ensAPIs[0] == "" {
 			ensAPIs = nil
 		}
+		for i := range ensAPIs {
+			ensAPIs[i] = expandPath(ensAPIs[i])
+		}
+
 		currentConfig.EnsAPIs = ensAPIs
 	}
 
@@ -262,13 +274,17 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
 	}
 
 	if networkid := os.Getenv(SWARM_ENV_NETWORK_ID); networkid != "" {
-		if id, _ := strconv.Atoi(networkid); id != 0 {
-			currentConfig.NetworkID = uint64(id)
+		id, err := strconv.ParseUint(networkid, 10, 64)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_NETWORK_ID, err)
+		}
+		if id != 0 {
+			currentConfig.NetworkID = id
 		}
 	}
 
 	if datadir := os.Getenv(GETH_ENV_DATADIR); datadir != "" {
-		currentConfig.Path = datadir
+		currentConfig.Path = expandPath(datadir)
 	}
 
 	bzzport := os.Getenv(SWARM_ENV_PORT)
@@ -281,33 +297,50 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
 	}
 
 	if swapenable := os.Getenv(SWARM_ENV_SWAP_ENABLE); swapenable != "" {
-		if swap, err := strconv.ParseBool(swapenable); err != nil {
-			currentConfig.SwapEnabled = swap
+		swap, err := strconv.ParseBool(swapenable)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SWAP_ENABLE, err)
 		}
+		currentConfig.SwapEnabled = swap
 	}
 
 	if syncdisable := os.Getenv(SWARM_ENV_SYNC_DISABLE); syncdisable != "" {
-		if sync, err := strconv.ParseBool(syncdisable); err != nil {
-			currentConfig.SyncEnabled = !sync
+		sync, err := strconv.ParseBool(syncdisable)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_DISABLE, err)
 		}
+		currentConfig.SyncEnabled = !sync
 	}
 
 	if v := os.Getenv(SWARM_ENV_DELIVERY_SKIP_CHECK); v != "" {
-		if skipCheck, err := strconv.ParseBool(v); err != nil {
+		skipCheck, err := strconv.ParseBool(v)
+		if err != nil {
 			currentConfig.DeliverySkipCheck = skipCheck
 		}
 	}
 
 	if v := os.Getenv(SWARM_ENV_SYNC_UPDATE_DELAY); v != "" {
-		if d, err := time.ParseDuration(v); err != nil {
-			currentConfig.SyncUpdateDelay = d
+		d, err := time.ParseDuration(v)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_UPDATE_DELAY, err)
 		}
+		currentConfig.SyncUpdateDelay = d
+	}
+
+	if max := os.Getenv(SWARM_ENV_MAX_STREAM_PEER_SERVERS); max != "" {
+		m, err := strconv.Atoi(max)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_MAX_STREAM_PEER_SERVERS, err)
+		}
+		currentConfig.MaxStreamPeerServers = m
 	}
 
 	if lne := os.Getenv(SWARM_ENV_LIGHT_NODE_ENABLE); lne != "" {
-		if lightnode, err := strconv.ParseBool(lne); err != nil {
-			currentConfig.LightNodeEnabled = lightnode
+		lightnode, err := strconv.ParseBool(lne)
+		if err != nil {
+			utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_LIGHT_NODE_ENABLE, err)
 		}
+		currentConfig.LightNodeEnabled = lightnode
 	}
 
 	if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" {

cmd/swarm/db.go

@@ -93,21 +93,6 @@ func dbImport(ctx *cli.Context) {
 	log.Info(fmt.Sprintf("successfully imported %d chunks", count))
 }
 
-func dbClean(ctx *cli.Context) {
-	args := ctx.Args()
-	if len(args) != 2 {
-		utils.Fatalf("invalid arguments, please specify <chunkdb> (path to a local chunk database) and the base key")
-	}
-
-	store, err := openLDBStore(args[0], common.Hex2Bytes(args[1]))
-	if err != nil {
-		utils.Fatalf("error opening local chunk database: %s", err)
-	}
-	defer store.Close()
-
-	store.Cleanup()
-}
-
 func openLDBStore(path string, basekey []byte) (*storage.LDBStore, error) {
 	if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
 		return nil, fmt.Errorf("invalid chunkdb path: %s", err)

cmd/swarm/export_test.go

@@ -24,6 +24,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"runtime"
 	"strings"
 	"testing"
 
@@ -38,6 +39,9 @@ import (
 // 5. imports the exported datastore
 // 6. fetches the uploaded random file from the second node
 func TestCLISwarmExportImport(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
 	cluster := newTestCluster(t, 1)
 
 	// generate random 10mb file

cmd/swarm/feeds.go

@@ -0,0 +1,172 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+// Command feed allows the user to create and update signed Swarm feeds
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed"
+	"gopkg.in/urfave/cli.v1"
+)
+
+func NewGenericSigner(ctx *cli.Context) feed.Signer {
+	return feed.NewGenericSigner(getPrivKey(ctx))
+}
+
+func getTopic(ctx *cli.Context) (topic feed.Topic) {
+	var name = ctx.String(SwarmFeedNameFlag.Name)
+	var relatedTopic = ctx.String(SwarmFeedTopicFlag.Name)
+	var relatedTopicBytes []byte
+	var err error
+
+	if relatedTopic != "" {
+		relatedTopicBytes, err = hexutil.Decode(relatedTopic)
+		if err != nil {
+			utils.Fatalf("Error parsing topic: %s", err)
+		}
+	}
+
+	topic, err = feed.NewTopic(name, relatedTopicBytes)
+	if err != nil {
+		utils.Fatalf("Error parsing topic: %s", err)
+	}
+	return topic
+}
+
+// swarm feed create <frequency> [--name <name>] [--data <0x Hexdata> [--multihash=false]]
+// swarm feed update <Manifest Address or ENS domain> <0x Hexdata> [--multihash=false]
+// swarm feed info <Manifest Address or ENS domain>
+
+func feedCreateManifest(ctx *cli.Context) {
+	var (
+		bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client = swarm.NewClient(bzzapi)
+	)
+
+	newFeedUpdateRequest := feed.NewFirstRequest(getTopic(ctx))
+	newFeedUpdateRequest.Feed.User = feedGetUser(ctx)
+
+	manifestAddress, err := client.CreateFeedWithManifest(newFeedUpdateRequest)
+	if err != nil {
+		utils.Fatalf("Error creating feed manifest: %s", err.Error())
+		return
+	}
+	fmt.Println(manifestAddress) // output manifest address to the user in a single line (useful for other commands to pick up)
+
+}
+
+func feedUpdate(ctx *cli.Context) {
+	args := ctx.Args()
+
+	var (
+		bzzapi                  = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client                  = swarm.NewClient(bzzapi)
+		manifestAddressOrDomain = ctx.String(SwarmFeedManifestFlag.Name)
+	)
+
+	if len(args) < 1 {
+		fmt.Println("Incorrect number of arguments")
+		cli.ShowCommandHelpAndExit(ctx, "update", 1)
+		return
+	}
+
+	signer := NewGenericSigner(ctx)
+
+	data, err := hexutil.Decode(args[0])
+	if err != nil {
+		utils.Fatalf("Error parsing data: %s", err.Error())
+		return
+	}
+
+	var updateRequest *feed.Request
+	var query *feed.Query
+
+	if manifestAddressOrDomain == "" {
+		query = new(feed.Query)
+		query.User = signer.Address()
+		query.Topic = getTopic(ctx)
+
+	}
+
+	// Retrieve a feed update request
+	updateRequest, err = client.GetFeedRequest(query, manifestAddressOrDomain)
+	if err != nil {
+		utils.Fatalf("Error retrieving feed status: %s", err.Error())
+	}
+
+	// set the new data
+	updateRequest.SetData(data)
+
+	// sign update
+	if err = updateRequest.Sign(signer); err != nil {
+		utils.Fatalf("Error signing feed update: %s", err.Error())
+	}
+
+	// post update
+	err = client.UpdateFeed(updateRequest)
+	if err != nil {
+		utils.Fatalf("Error updating feed: %s", err.Error())
+		return
+	}
+}
+
+func feedInfo(ctx *cli.Context) {
+	var (
+		bzzapi                  = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client                  = swarm.NewClient(bzzapi)
+		manifestAddressOrDomain = ctx.String(SwarmFeedManifestFlag.Name)
+	)
+
+	var query *feed.Query
+	if manifestAddressOrDomain == "" {
+		query = new(feed.Query)
+		query.Topic = getTopic(ctx)
+		query.User = feedGetUser(ctx)
+	}
+
+	metadata, err := client.GetFeedRequest(query, manifestAddressOrDomain)
+	if err != nil {
+		utils.Fatalf("Error retrieving feed metadata: %s", err.Error())
+		return
+	}
+	encodedMetadata, err := metadata.MarshalJSON()
+	if err != nil {
+		utils.Fatalf("Error encoding metadata to JSON for display:%s", err)
+	}
+	fmt.Println(string(encodedMetadata))
+}
+
+func feedGetUser(ctx *cli.Context) common.Address {
+	var user = ctx.String(SwarmFeedUserFlag.Name)
+	if user != "" {
+		return common.HexToAddress(user)
+	}
+	pk := getPrivKey(ctx)
+	if pk == nil {
+		utils.Fatalf("Cannot read private key. Must specify --user or --bzzaccount")
+	}
+	return crypto.PubkeyToAddress(pk.PublicKey)
+
+}

cmd/swarm/feeds_test.go

@@ -0,0 +1,182 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
+
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/swarm/storage/feed"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/log"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
+)
+
+func TestCLIFeedUpdate(t *testing.T) {
+
+	srv := testutil.NewTestSwarmServer(t, func(api *api.API) testutil.TestServer {
+		return swarmhttp.NewServer(api, "")
+	}, nil)
+	log.Info("starting a test swarm server")
+	defer srv.Close()
+
+	// create a private key file for signing
+	pkfile, err := ioutil.TempFile("", "swarm-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer pkfile.Close()
+	defer os.Remove(pkfile.Name())
+
+	privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979"
+	privKey, _ := crypto.HexToECDSA(privkeyHex)
+	address := crypto.PubkeyToAddress(privKey.PublicKey)
+
+	// save the private key to a file
+	_, err = io.WriteString(pkfile, privkeyHex)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// compose a topic. We'll be doing quotes about Miguel de Cervantes
+	var topic feed.Topic
+	subject := []byte("Miguel de Cervantes")
+	copy(topic[:], subject[:])
+	name := "quotes"
+
+	// prepare some data for the update
+	data := []byte("En boca cerrada no entran moscas")
+	hexData := hexutil.Encode(data)
+
+	flags := []string{
+		"--bzzapi", srv.URL,
+		"--bzzaccount", pkfile.Name(),
+		"feed", "update",
+		"--topic", topic.Hex(),
+		"--name", name,
+		hexData}
+
+	// create an update and expect an exit without errors
+	log.Info(fmt.Sprintf("updating a feed with 'swarm feed update'"))
+	cmd := runSwarm(t, flags...)
+	cmd.ExpectExit()
+
+	// now try to get the update using the client
+	client := swarm.NewClient(srv.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// build the same topic as before, this time
+	// we use NewTopic to create a topic automatically.
+	topic, err = feed.NewTopic(name, subject)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Feed configures whose updates we will be looking up.
+	fd := feed.Feed{
+		Topic: topic,
+		User:  address,
+	}
+
+	// Build a query to get the latest update
+	query := feed.NewQueryLatest(&fd, lookup.NoClue)
+
+	// retrieve content!
+	reader, err := client.QueryFeed(query, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	retrieved, err := ioutil.ReadAll(reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// check we retrieved the sent information
+	if !bytes.Equal(data, retrieved) {
+		t.Fatalf("Received %s, expected %s", retrieved, data)
+	}
+
+	// Now retrieve info for the next update
+	flags = []string{
+		"--bzzapi", srv.URL,
+		"feed", "info",
+		"--topic", topic.Hex(),
+		"--user", address.Hex(),
+	}
+
+	log.Info(fmt.Sprintf("getting feed info with 'swarm feed info'"))
+	cmd = runSwarm(t, flags...)
+	_, matches := cmd.ExpectRegexp(`.*`) // regex hack to extract stdout
+	cmd.ExpectExit()
+
+	// verify we can deserialize the result as a valid JSON
+	var request feed.Request
+	err = json.Unmarshal([]byte(matches[0]), &request)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// make sure the retrieved feed is the same
+	if request.Feed != fd {
+		t.Fatalf("Expected feed to be: %s, got %s", fd, request.Feed)
+	}
+
+	// test publishing a manifest
+	flags = []string{
+		"--bzzapi", srv.URL,
+		"--bzzaccount", pkfile.Name(),
+		"feed", "create",
+		"--topic", topic.Hex(),
+	}
+
+	log.Info(fmt.Sprintf("Publishing manifest with 'swarm feed create'"))
+	cmd = runSwarm(t, flags...)
+	_, matches = cmd.ExpectRegexp(`[a-f\d]{64}`) // regex hack to extract stdout
+	cmd.ExpectExit()
+
+	manifestAddress := matches[0] // read the received feed manifest
+
+	// now attempt to lookup the latest update using a manifest instead
+	reader, err = client.QueryFeed(nil, manifestAddress)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	retrieved, err = ioutil.ReadAll(reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !bytes.Equal(data, retrieved) {
+		t.Fatalf("Received %s, expected %s", retrieved, data)
+	}
+}

cmd/swarm/main.go

@@ -38,7 +38,7 @@ import (
 	"github.com/ethereum/go-ethereum/internal/debug"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/swarm"
 	bzzapi "github.com/ethereum/go-ethereum/swarm/api"
 	swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics"
@@ -116,6 +116,12 @@ var (
 		Usage:  "Duration for sync subscriptions update after no new peers are added (default 15s)",
 		EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY,
 	}
+	SwarmMaxStreamPeerServersFlag = cli.IntFlag{
+		Name:   "max-stream-peer-servers",
+		Usage:  "Limit of Stream peer servers, 0 denotes unlimited",
+		EnvVar: SWARM_ENV_MAX_STREAM_PEER_SERVERS,
+		Value:  10000, // A very large default value is possible as stream servers have very small memory footprint
+	}
 	SwarmLightNodeEnabled = cli.BoolFlag{
 		Name:   "lightnode",
 		Usage:  "Enable Swarm LightNode (default false)",
@@ -197,22 +203,30 @@ var (
 		Usage:  "Number of recent chunks cached in memory (default 5000)",
 		EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
 	}
-	SwarmResourceMultihashFlag = cli.BoolFlag{
-		Name:  "multihash",
-		Usage: "Determines how to interpret data for a resource update. If not present, data will be interpreted as raw, literal data that will be included in the resource",
-	}
-	SwarmResourceNameFlag = cli.StringFlag{
-		Name:  "name",
-		Usage: "User-defined name for the new resource",
-	}
-	SwarmResourceDataOnCreateFlag = cli.StringFlag{
-		Name:  "data",
-		Usage: "Initializes the resource with the given hex-encoded data. Data must be prefixed by 0x",
-	}
 	SwarmCompressedFlag = cli.BoolFlag{
 		Name:  "compressed",
 		Usage: "Prints encryption keys in compressed form",
 	}
+	SwarmFeedNameFlag = cli.StringFlag{
+		Name:  "name",
+		Usage: "User-defined name for the new feed, limited to 32 characters. If combined with topic, it will refer to a subtopic with this name",
+	}
+	SwarmFeedTopicFlag = cli.StringFlag{
+		Name:  "topic",
+		Usage: "User-defined topic this feed is tracking, hex encoded. Limited to 64 hexadecimal characters",
+	}
+	SwarmFeedDataOnCreateFlag = cli.StringFlag{
+		Name:  "data",
+		Usage: "Initializes the feed with the given hex-encoded data. Data must be prefixed by 0x",
+	}
+	SwarmFeedManifestFlag = cli.StringFlag{
+		Name:  "manifest",
+		Usage: "Refers to the feed through a manifest",
+	}
+	SwarmFeedUserFlag = cli.StringFlag{
+		Name:  "user",
+		Usage: "Indicates the user who updates the feed",
+	}
 )
 
 //declare a few constant error messages, useful for later error check comparisons in test
@@ -242,12 +256,12 @@ func init() {
 	utils.ListenPortFlag.Value = 30399
 }
 
-var app = utils.NewApp(gitCommit, "Ethereum Swarm")
+var app = utils.NewApp("", "Ethereum Swarm")
 
 // This init function creates the cli.App.
 func init() {
 	app.Action = bzzd
-	app.HideVersion = true // we have a command to print the version
+	app.Version = sv.ArchiveVersion(gitCommit)
 	app.Copyright = "Copyright 2013-2016 The go-ethereum Authors"
 	app.Commands = []cli.Command{
 		{
@@ -332,36 +346,62 @@ func init() {
 		},
 		{
 			CustomHelpTemplate: helpTemplate,
-			Name:               "resource",
-			Usage:              "(Advanced) Create and update Mutable Resources",
+			Name:               "feed",
+			Usage:              "(Advanced) Create and update Swarm Feeds",
 			ArgsUsage:          "<create|update|info>",
-			Description:        "Works with Mutable Resource Updates",
+			Description:        "Works with Swarm Feeds",
 			Subcommands: []cli.Command{
 				{
-					Action:             resourceCreate,
+					Action:             feedCreateManifest,
 					CustomHelpTemplate: helpTemplate,
 					Name:               "create",
-					Usage:              "creates a new Mutable Resource",
-					ArgsUsage:          "<frequency>",
-					Description:        "creates a new Mutable Resource",
-					Flags:              []cli.Flag{SwarmResourceNameFlag, SwarmResourceDataOnCreateFlag, SwarmResourceMultihashFlag},
+					Usage:              "creates and publishes a new feed manifest",
+					Description: `creates and publishes a new feed manifest pointing to a specified user's updates about a particular topic.
+					The feed topic can be built in the following ways:
+					* use --topic to set the topic to an arbitrary binary hex string.
+					* use --name to set the topic to a human-readable name.
+					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
+					* use both --topic and --name to create named subtopics. 
+						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
+						this feed tracks a discussion about that contract.
+					The --user flag allows to have this manifest refer to a user other than yourself. If not specified,
+					it will then default to your local account (--bzzaccount)`,
+					Flags: []cli.Flag{SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
 				},
 				{
-					Action:             resourceUpdate,
+					Action:             feedUpdate,
 					CustomHelpTemplate: helpTemplate,
 					Name:               "update",
-					Usage:              "updates the content of an existing Mutable Resource",
-					ArgsUsage:          "<Manifest Address or ENS domain> <0x Hex data>",
-					Description:        "updates the content of an existing Mutable Resource",
-					Flags:              []cli.Flag{SwarmResourceMultihashFlag},
+					Usage:              "updates the content of an existing Swarm Feed",
+					ArgsUsage:          "<0x Hex data>",
+					Description: `publishes a new update on the specified topic
+					The feed topic can be built in the following ways:
+					* use --topic to set the topic to an arbitrary binary hex string.
+					* use --name to set the topic to a human-readable name.
+					    For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
+					* use both --topic and --name to create named subtopics. 
+						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
+						this feed tracks a discussion about that contract.
+					
+					If you have a manifest, you can specify it with --manifest to refer to the feed,
+					instead of using --topic / --name
+					`,
+					Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag},
 				},
 				{
-					Action:             resourceInfo,
+					Action:             feedInfo,
 					CustomHelpTemplate: helpTemplate,
 					Name:               "info",
-					Usage:              "obtains information about an existing Mutable Resource",
-					ArgsUsage:          "<Manifest Address or ENS domain>",
-					Description:        "obtains information about an existing Mutable Resource",
+					Usage:              "obtains information about an existing Swarm feed",
+					Description: `obtains information about an existing Swarm feed
+					The topic can be specified directly with the --topic flag as an hex string
+					If no topic is specified, the default topic (zero) will be used
+					The --name flag can be used to specify subtopics with a specific name.
+					The --user flag allows to refer to a user other than yourself. If not specified,
+					it will then default to your local account (--bzzaccount)
+					If you have a manifest, you can specify it with --manifest instead of --topic / --name / ---user
+					to refer to the feed`,
+					Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
 				},
 			},
 		},
@@ -497,14 +537,6 @@ pv(1) tool to get a progress bar:
 
     pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`,
 				},
-				{
-					Action:             dbClean,
-					CustomHelpTemplate: helpTemplate,
-					Name:               "clean",
-					Usage:              "remove corrupt entries from a local chunk database",
-					ArgsUsage:          "<chunkdb>",
-					Description:        "Remove corrupt entries from a local chunk database",
-				},
 			},
 		},
 
@@ -542,6 +574,7 @@ pv(1) tool to get a progress bar:
 		SwarmSwapAPIFlag,
 		SwarmSyncDisabledFlag,
 		SwarmSyncUpdateDelay,
+		SwarmMaxStreamPeerServersFlag,
 		SwarmLightNodeEnabled,
 		SwarmDeliverySkipCheckFlag,
 		SwarmListenAddrFlag,
@@ -697,7 +730,7 @@ func getAccount(bzzaccount string, ctx *cli.Context, stack *node.Node) *ecdsa.Pr
 }
 
 // getPrivKey returns the private key of the specified bzzaccount
-// Used only by client commands, such as `resource`
+// Used only by client commands, such as `feed`
 func getPrivKey(ctx *cli.Context) *ecdsa.PrivateKey {
 	// booting up the swarm node just as we do in bzzd action
 	bzzconfig, err := buildConfig(ctx)
@@ -788,10 +821,10 @@ func setSwarmBootstrapNodes(ctx *cli.Context, cfg *node.Config) {
 		return
 	}
 
-	cfg.P2P.BootstrapNodes = []*discover.Node{}
+	cfg.P2P.BootstrapNodes = []*enode.Node{}
 
 	for _, url := range SwarmBootnodes {
-		node, err := discover.ParseNode(url)
+		node, err := enode.ParseV4(url)
 		if err != nil {
 			log.Error("Bootstrap URL invalid", "enode", url, "err", err)
 		}

cmd/swarm/manifest_test.go

@@ -21,6 +21,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/swarm/api"
@@ -30,12 +31,20 @@ import (
 // TestManifestChange tests manifest add, update and remove
 // cli commands without encryption.
 func TestManifestChange(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
 	testManifestChange(t, false)
 }
 
 // TestManifestChange tests manifest add, update and remove
 // cli commands with encryption enabled.
 func TestManifestChangeEncrypted(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
 	testManifestChange(t, true)
 }
 
@@ -400,6 +409,10 @@ func testManifestChange(t *testing.T, encrypt bool) {
 // TestNestedDefaultEntryUpdate tests if the default entry is updated
 // if the file in nested manifest used for it is also updated.
 func TestNestedDefaultEntryUpdate(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
 	testNestedDefaultEntryUpdate(t, false)
 }
 
@@ -407,6 +420,10 @@ func TestNestedDefaultEntryUpdate(t *testing.T) {
 // of encrypted upload is updated if the file in nested manifest
 // used for it is also updated.
 func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
 	testNestedDefaultEntryUpdate(t, true)
 }
 

cmd/swarm/mimegen/generator.go

@@ -0,0 +1,124 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+package main
+
+// Standard "mime" package rely on system-settings, see mime.osInitMime
+// Swarm will run on many OS/Platform/Docker and must behave similar
+// This command generates code to add common mime types based on mime.types file
+//
+// mime.types file provided by mailcap, which follow https://www.iana.org/assignments/media-types/media-types.xhtml
+//
+// Get last version of mime.types file by:
+// docker run --rm -v $(pwd):/tmp alpine:edge /bin/sh -c "apk add -U mailcap; mv /etc/mime.types /tmp"
+
+import (
+	"bufio"
+	"bytes"
+	"flag"
+	"html/template"
+	"io/ioutil"
+	"strings"
+
+	"log"
+)
+
+var (
+	typesFlag   = flag.String("types", "", "Input mime.types file")
+	packageFlag = flag.String("package", "", "Golang package in output file")
+	outFlag     = flag.String("out", "", "Output file name for the generated mime types")
+)
+
+type mime struct {
+	Name string
+	Exts []string
+}
+
+type templateParams struct {
+	PackageName string
+	Mimes       []mime
+}
+
+func main() {
+	// Parse and ensure all needed inputs are specified
+	flag.Parse()
+	if *typesFlag == "" {
+		log.Fatalf("--types is required")
+	}
+	if *packageFlag == "" {
+		log.Fatalf("--types is required")
+	}
+	if *outFlag == "" {
+		log.Fatalf("--out is required")
+	}
+
+	params := templateParams{
+		PackageName: *packageFlag,
+	}
+
+	types, err := ioutil.ReadFile(*typesFlag)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	scanner := bufio.NewScanner(bytes.NewReader(types))
+	for scanner.Scan() {
+		txt := scanner.Text()
+		if strings.HasPrefix(txt, "#") || len(txt) == 0 {
+			continue
+		}
+		parts := strings.Fields(txt)
+		if len(parts) == 1 {
+			continue
+		}
+		params.Mimes = append(params.Mimes, mime{parts[0], parts[1:]})
+	}
+
+	if err = scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+
+	result := bytes.NewBuffer([]byte{})
+
+	if err := template.Must(template.New("_").Parse(tpl)).Execute(result, params); err != nil {
+		log.Fatal(err)
+	}
+
+	if err := ioutil.WriteFile(*outFlag, result.Bytes(), 0600); err != nil {
+		log.Fatal(err)
+	}
+}
+
+var tpl = `// Code generated by github.com/ethereum/go-ethereum/cmd/swarm/mimegen. DO NOT EDIT.
+
+package {{ .PackageName }}
+
+import "mime"
+func init() {
+	var mimeTypes = map[string]string{
+{{- range .Mimes -}}
+	{{ $name := .Name -}}
+	{{- range .Exts }}
+		".{{ . }}": "{{ $name | html }}",
+	{{- end }}
+{{- end }}
+	}
+	for ext, name := range mimeTypes {
+		if err := mime.AddExtensionType(ext, name); err != nil {
+			panic(err)
+		}
+	}
+}
+`

cmd/swarm/mru.go

@@ -1,169 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-// Command resource allows the user to create and update signed mutable resource updates
-package main
-
-import (
-	"fmt"
-	"strconv"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/common/hexutil"
-
-	"github.com/ethereum/go-ethereum/cmd/utils"
-	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
-	"github.com/ethereum/go-ethereum/swarm/storage/mru"
-	"gopkg.in/urfave/cli.v1"
-)
-
-func NewGenericSigner(ctx *cli.Context) mru.Signer {
-	return mru.NewGenericSigner(getPrivKey(ctx))
-}
-
-// swarm resource create <frequency> [--name <name>] [--data <0x Hexdata> [--multihash=false]]
-// swarm resource update <Manifest Address or ENS domain> <0x Hexdata> [--multihash=false]
-// swarm resource info <Manifest Address or ENS domain>
-
-func resourceCreate(ctx *cli.Context) {
-	args := ctx.Args()
-
-	var (
-		bzzapi      = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client      = swarm.NewClient(bzzapi)
-		multihash   = ctx.Bool(SwarmResourceMultihashFlag.Name)
-		initialData = ctx.String(SwarmResourceDataOnCreateFlag.Name)
-		name        = ctx.String(SwarmResourceNameFlag.Name)
-	)
-
-	if len(args) < 1 {
-		fmt.Println("Incorrect number of arguments")
-		cli.ShowCommandHelpAndExit(ctx, "create", 1)
-		return
-	}
-	signer := NewGenericSigner(ctx)
-	frequency, err := strconv.ParseUint(args[0], 10, 64)
-	if err != nil {
-		fmt.Printf("Frequency formatting error: %s\n", err.Error())
-		cli.ShowCommandHelpAndExit(ctx, "create", 1)
-		return
-	}
-
-	metadata := mru.ResourceMetadata{
-		Name:      name,
-		Frequency: frequency,
-		Owner:     signer.Address(),
-	}
-
-	var newResourceRequest *mru.Request
-	if initialData != "" {
-		initialDataBytes, err := hexutil.Decode(initialData)
-		if err != nil {
-			fmt.Printf("Error parsing data: %s\n", err.Error())
-			cli.ShowCommandHelpAndExit(ctx, "create", 1)
-			return
-		}
-		newResourceRequest, err = mru.NewCreateUpdateRequest(&metadata)
-		if err != nil {
-			utils.Fatalf("Error creating new resource request: %s", err)
-		}
-		newResourceRequest.SetData(initialDataBytes, multihash)
-		if err = newResourceRequest.Sign(signer); err != nil {
-			utils.Fatalf("Error signing resource update: %s", err.Error())
-		}
-	} else {
-		newResourceRequest, err = mru.NewCreateRequest(&metadata)
-		if err != nil {
-			utils.Fatalf("Error creating new resource request: %s", err)
-		}
-	}
-
-	manifestAddress, err := client.CreateResource(newResourceRequest)
-	if err != nil {
-		utils.Fatalf("Error creating resource: %s", err.Error())
-		return
-	}
-	fmt.Println(manifestAddress) // output manifest address to the user in a single line (useful for other commands to pick up)
-
-}
-
-func resourceUpdate(ctx *cli.Context) {
-	args := ctx.Args()
-
-	var (
-		bzzapi    = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client    = swarm.NewClient(bzzapi)
-		multihash = ctx.Bool(SwarmResourceMultihashFlag.Name)
-	)
-
-	if len(args) < 2 {
-		fmt.Println("Incorrect number of arguments")
-		cli.ShowCommandHelpAndExit(ctx, "update", 1)
-		return
-	}
-	signer := NewGenericSigner(ctx)
-	manifestAddressOrDomain := args[0]
-	data, err := hexutil.Decode(args[1])
-	if err != nil {
-		utils.Fatalf("Error parsing data: %s", err.Error())
-		return
-	}
-
-	// Retrieve resource status and metadata out of the manifest
-	updateRequest, err := client.GetResourceMetadata(manifestAddressOrDomain)
-	if err != nil {
-		utils.Fatalf("Error retrieving resource status: %s", err.Error())
-	}
-
-	// set the new data
-	updateRequest.SetData(data, multihash)
-
-	// sign update
-	if err = updateRequest.Sign(signer); err != nil {
-		utils.Fatalf("Error signing resource update: %s", err.Error())
-	}
-
-	// post update
-	err = client.UpdateResource(updateRequest)
-	if err != nil {
-		utils.Fatalf("Error updating resource: %s", err.Error())
-		return
-	}
-}
-
-func resourceInfo(ctx *cli.Context) {
-	var (
-		bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client = swarm.NewClient(bzzapi)
-	)
-	args := ctx.Args()
-	if len(args) < 1 {
-		fmt.Println("Incorrect number of arguments.")
-		cli.ShowCommandHelpAndExit(ctx, "info", 1)
-		return
-	}
-	manifestAddressOrDomain := args[0]
-	metadata, err := client.GetResourceMetadata(manifestAddressOrDomain)
-	if err != nil {
-		utils.Fatalf("Error retrieving resource metadata: %s", err.Error())
-		return
-	}
-	encodedMetadata, err := metadata.MarshalJSON()
-	if err != nil {
-		utils.Fatalf("Error encoding metadata to JSON for display:%s", err)
-	}
-	fmt.Println(string(encodedMetadata))
-}

cmd/swarm/run_test.go

@@ -19,6 +19,7 @@ package main
 import (
 	"context"
 	"crypto/ecdsa"
+	"flag"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -41,6 +42,8 @@ import (
 	"github.com/ethereum/go-ethereum/swarm"
 )
 
+var loglevel = flag.Int("loglevel", 3, "verbosity of logs")
+
 func init() {
 	// Run the app if we've been exec'd as "swarm-test" in runSwarm.
 	reexec.Register("swarm-test", func() {
@@ -234,6 +237,7 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
 	// start the node
 	node.Cmd = runSwarm(t,
 		"--port", p2pPort,
+		"--nat", "extip:127.0.0.1",
 		"--nodiscover",
 		"--datadir", dir,
 		"--ipcpath", conf.IPCPath,
@@ -241,7 +245,7 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
 		"--bzzaccount", bzzaccount,
 		"--bzznetworkid", "321",
 		"--bzzport", httpPort,
-		"--verbosity", "6",
+		"--verbosity", fmt.Sprint(*loglevel),
 	)
 	node.Cmd.InputLine(testPassphrase)
 	defer func() {
@@ -284,8 +288,8 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
 	if err := node.Client.Call(&nodeInfo, "admin_nodeInfo"); err != nil {
 		t.Fatal(err)
 	}
-	node.Enode = fmt.Sprintf("enode://%s@127.0.0.1:%s", nodeInfo.ID, p2pPort)
-
+	node.Enode = nodeInfo.Enode
+	node.IpcPath = conf.IPCPath
 	return node
 }
 
@@ -309,6 +313,7 @@ func newTestNode(t *testing.T, dir string) *testNode {
 	// start the node
 	node.Cmd = runSwarm(t,
 		"--port", p2pPort,
+		"--nat", "extip:127.0.0.1",
 		"--nodiscover",
 		"--datadir", dir,
 		"--ipcpath", conf.IPCPath,
@@ -316,7 +321,7 @@ func newTestNode(t *testing.T, dir string) *testNode {
 		"--bzzaccount", account.Address.String(),
 		"--bzznetworkid", "321",
 		"--bzzport", httpPort,
-		"--verbosity", "6",
+		"--verbosity", fmt.Sprint(*loglevel),
 	)
 	node.Cmd.InputLine(testPassphrase)
 	defer func() {
@@ -359,9 +364,8 @@ func newTestNode(t *testing.T, dir string) *testNode {
 	if err := node.Client.Call(&nodeInfo, "admin_nodeInfo"); err != nil {
 		t.Fatal(err)
 	}
-	node.Enode = fmt.Sprintf("enode://%s@127.0.0.1:%s", nodeInfo.ID, p2pPort)
+	node.Enode = nodeInfo.Enode
 	node.IpcPath = conf.IPCPath
-
 	return node
 }
 

cmd/swarm/upload.go

@@ -22,16 +22,15 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"mime"
-	"net/http"
 	"os"
 	"os/user"
 	"path"
 	"path/filepath"
 	"strings"
 
-	"github.com/ethereum/go-ethereum/cmd/utils"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -118,10 +117,9 @@ func upload(ctx *cli.Context) {
 				return "", fmt.Errorf("error opening file: %s", err)
 			}
 			defer f.Close()
-			if mimeType == "" {
-				mimeType = detectMimeType(file)
+			if mimeType != "" {
+				f.ContentType = mimeType
 			}
-			f.ContentType = mimeType
 			return client.Upload(f, "", toEncrypt)
 		}
 	}
@@ -138,6 +136,12 @@ func upload(ctx *cli.Context) {
 // 3. cleans the path, e.g. /a/b/../c -> /a/c
 // Note, it has limitations, e.g. ~someuser/tmp will not be expanded
 func expandPath(p string) string {
+	if i := strings.Index(p, ":"); i > 0 {
+		return p
+	}
+	if i := strings.Index(p, "@"); i > 0 {
+		return p
+	}
 	if strings.HasPrefix(p, "~/") || strings.HasPrefix(p, "~\\") {
 		if home := homeDir(); home != "" {
 			p = home + p[1:]
@@ -155,19 +159,3 @@ func homeDir() string {
 	}
 	return ""
 }
-
-func detectMimeType(file string) string {
-	if ext := filepath.Ext(file); ext != "" {
-		return mime.TypeByExtension(ext)
-	}
-	f, err := os.Open(file)
-	if err != nil {
-		return ""
-	}
-	defer f.Close()
-	buf := make([]byte, 512)
-	if n, _ := f.Read(buf); n > 0 {
-		return http.DetectContentType(buf)
-	}
-	return ""
-}

cmd/swarm/upload_test.go

@@ -18,7 +18,6 @@ package main
 
 import (
 	"bytes"
-	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -26,17 +25,16 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
-	colorable "github.com/mattn/go-colorable"
+	"github.com/mattn/go-colorable"
 )
 
-var loglevel = flag.Int("loglevel", 3, "verbosity of logs")
-
 func init() {
 	log.PrintOrigins(true)
 	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
@@ -45,18 +43,31 @@ func init() {
 // TestCLISwarmUp tests that running 'swarm up' makes the resulting file
 // available from all nodes via the HTTP API
 func TestCLISwarmUp(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
 	testCLISwarmUp(false, t)
 }
 func TestCLISwarmUpRecursive(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
 	testCLISwarmUpRecursive(false, t)
 }
 
 // TestCLISwarmUpEncrypted tests that running 'swarm encrypted-up' makes the resulting file
 // available from all nodes via the HTTP API
 func TestCLISwarmUpEncrypted(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
 	testCLISwarmUp(true, t)
 }
 func TestCLISwarmUpEncryptedRecursive(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
 	testCLISwarmUpRecursive(true, t)
 }
 
@@ -277,6 +288,9 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) {
 // TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute
 // default paths and with encryption.
 func TestCLISwarmUpDefaultPath(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
 	testCLISwarmUpDefaultPath(false, false, t)
 	testCLISwarmUpDefaultPath(false, true, t)
 	testCLISwarmUpDefaultPath(true, false, t)

cmd/utils/flags.go

@@ -51,8 +51,8 @@ import (
 	"github.com/ethereum/go-ethereum/metrics/influxdb"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/discv5"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 	"github.com/ethereum/go-ethereum/params"
@@ -692,9 +692,9 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) {
 		return // already set, don't apply defaults.
 	}
 
-	cfg.BootstrapNodes = make([]*discover.Node, 0, len(urls))
+	cfg.BootstrapNodes = make([]*enode.Node, 0, len(urls))
 	for _, url := range urls {
-		node, err := discover.ParseNode(url)
+		node, err := enode.ParseV4(url)
 		if err != nil {
 			log.Crit("Bootstrap URL invalid", "enode", url, "err", err)
 		}
@@ -1085,11 +1085,14 @@ func checkExclusive(ctx *cli.Context, args ...interface{}) {
 		if i+1 < len(args) {
 			switch option := args[i+1].(type) {
 			case string:
-				// Extended flag, expand the name and shift the arguments
+				// Extended flag check, make sure value set doesn't conflict with passed in option
 				if ctx.GlobalString(flag.GetName()) == option {
 					name += "=" + option
+					set = append(set, "--"+name)
 				}
+				// shift arguments and continue
 				i++
+				continue
 
 			case cli.Flag:
 			default:

cmd/wnode/main.go

@@ -41,7 +41,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/whisper/mailserver"
 	whisper "github.com/ethereum/go-ethereum/whisper/whisperv6"
@@ -175,7 +175,7 @@ func initialize() {
 	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*argVerbosity), log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
 
 	done = make(chan struct{})
-	var peers []*discover.Node
+	var peers []*enode.Node
 	var err error
 
 	if *generateKey {
@@ -203,7 +203,7 @@ func initialize() {
 		if len(*argEnode) == 0 {
 			argEnode = scanLineA("Please enter the peer's enode: ")
 		}
-		peer := discover.MustParseNode(*argEnode)
+		peer := enode.MustParseV4(*argEnode)
 		peers = append(peers, peer)
 	}
 
@@ -747,11 +747,11 @@ func requestExpiredMessagesLoop() {
 }
 
 func extractIDFromEnode(s string) []byte {
-	n, err := discover.ParseNode(s)
+	n, err := enode.ParseV4(s)
 	if err != nil {
 		utils.Fatalf("Failed to parse enode: %s", err)
 	}
-	return n.ID[:]
+	return n.ID().Bytes()
 }
 
 // obfuscateBloom adds 16 random bits to the bloom

contracts/ens/ens.go

@@ -151,6 +151,38 @@ func (self *ENS) Resolve(name string) (common.Hash, error) {
 	return common.BytesToHash(ret[:]), nil
 }
 
+// Addr is a non-transactional call that returns the address associated with a name.
+func (self *ENS) Addr(name string) (common.Address, error) {
+	node := EnsNode(name)
+
+	resolver, err := self.getResolver(node)
+	if err != nil {
+		return common.Address{}, err
+	}
+
+	ret, err := resolver.Addr(node)
+	if err != nil {
+		return common.Address{}, err
+	}
+
+	return common.BytesToAddress(ret[:]), nil
+}
+
+// SetAddress sets the address associated with a name. Only works if the caller
+// owns the name, and the associated resolver implements a `setAddress` function.
+func (self *ENS) SetAddr(name string, addr common.Address) (*types.Transaction, error) {
+	node := EnsNode(name)
+
+	resolver, err := self.getResolver(node)
+	if err != nil {
+		return nil, err
+	}
+
+	opts := self.TransactOpts
+	opts.GasLimit = 200000
+	return resolver.Contract.SetAddr(&opts, node, addr)
+}
+
 // Register registers a new domain name for the caller, making them the owner of the new name.
 // Only works if the registrar for the parent domain implements the FIFS registrar protocol.
 func (self *ENS) Register(name string) (*types.Transaction, error) {

contracts/ens/ens_test.go

@@ -22,16 +22,18 @@ import (
 
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/contracts/ens/contract"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/crypto"
 )
 
 var (
-	key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
-	name   = "my name on ENS"
-	hash   = crypto.Keccak256Hash([]byte("my content"))
-	addr   = crypto.PubkeyToAddress(key.PublicKey)
+	key, _   = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+	name     = "my name on ENS"
+	hash     = crypto.Keccak256Hash([]byte("my content"))
+	addr     = crypto.PubkeyToAddress(key.PublicKey)
+	testAddr = common.HexToAddress("0x1234123412341234123412341234123412341234")
 )
 
 func TestENS(t *testing.T) {
@@ -74,4 +76,19 @@ func TestENS(t *testing.T) {
 	if vhost != hash {
 		t.Fatalf("resolve error, expected %v, got %v", hash.Hex(), vhost.Hex())
 	}
+
+	// set the address for the name
+	if _, err = ens.SetAddr(name, testAddr); err != nil {
+		t.Fatalf("can't set address: %v", err)
+	}
+	contractBackend.Commit()
+
+	// Try to resolve the name to an address
+	recoveredAddr, err := ens.Addr(name)
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+	if vhost != hash {
+		t.Fatalf("resolve error, expected %v, got %v", testAddr.Hex(), recoveredAddr.Hex())
+	}
 }

core/asm/asm.go

@@ -109,9 +109,9 @@ func PrintDisassembled(code string) error {
 	it := NewInstructionIterator(script)
 	for it.Next() {
 		if it.Arg() != nil && 0 < len(it.Arg()) {
-			fmt.Printf("%06v: %v 0x%x\n", it.PC(), it.Op(), it.Arg())
+			fmt.Printf("%05x: %v 0x%x\n", it.PC(), it.Op(), it.Arg())
 		} else {
-			fmt.Printf("%06v: %v\n", it.PC(), it.Op())
+			fmt.Printf("%05x: %v\n", it.PC(), it.Op())
 		}
 	}
 	return it.Error()
@@ -124,9 +124,9 @@ func Disassemble(script []byte) ([]string, error) {
 	it := NewInstructionIterator(script)
 	for it.Next() {
 		if it.Arg() != nil && 0 < len(it.Arg()) {
-			instrs = append(instrs, fmt.Sprintf("%06v: %v 0x%x\n", it.PC(), it.Op(), it.Arg()))
+			instrs = append(instrs, fmt.Sprintf("%05x: %v 0x%x\n", it.PC(), it.Op(), it.Arg()))
 		} else {
-			instrs = append(instrs, fmt.Sprintf("%06v: %v\n", it.PC(), it.Op()))
+			instrs = append(instrs, fmt.Sprintf("%05x: %v\n", it.PC(), it.Op()))
 		}
 	}
 	if err := it.Error(); err != nil {

core/blockchain.go

@@ -55,6 +55,7 @@ var (
 const (
 	bodyCacheLimit      = 256
 	blockCacheLimit     = 256
+	receiptsCacheLimit  = 32
 	maxFutureBlocks     = 256
 	maxTimeFutureBlocks = 30
 	badBlockLimit       = 10
@@ -111,11 +112,12 @@ type BlockChain struct {
 	currentBlock     atomic.Value // Current head of the block chain
 	currentFastBlock atomic.Value // Current head of the fast-sync chain (may be above the block chain!)
 
-	stateCache   state.Database // State database to reuse between imports (contains state cache)
-	bodyCache    *lru.Cache     // Cache for the most recent block bodies
-	bodyRLPCache *lru.Cache     // Cache for the most recent block bodies in RLP encoded format
-	blockCache   *lru.Cache     // Cache for the most recent entire blocks
-	futureBlocks *lru.Cache     // future blocks are blocks added for later processing
+	stateCache    state.Database // State database to reuse between imports (contains state cache)
+	bodyCache     *lru.Cache     // Cache for the most recent block bodies
+	bodyRLPCache  *lru.Cache     // Cache for the most recent block bodies in RLP encoded format
+	receiptsCache *lru.Cache     // Cache for the most recent receipts per block
+	blockCache    *lru.Cache     // Cache for the most recent entire blocks
+	futureBlocks  *lru.Cache     // future blocks are blocks added for later processing
 
 	quit    chan struct{} // blockchain quit channel
 	running int32         // running must be called atomically
@@ -144,6 +146,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 	}
 	bodyCache, _ := lru.New(bodyCacheLimit)
 	bodyRLPCache, _ := lru.New(bodyCacheLimit)
+	receiptsCache, _ := lru.New(receiptsCacheLimit)
 	blockCache, _ := lru.New(blockCacheLimit)
 	futureBlocks, _ := lru.New(maxFutureBlocks)
 	badBlocks, _ := lru.New(badBlockLimit)
@@ -158,6 +161,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 		shouldPreserve: shouldPreserve,
 		bodyCache:      bodyCache,
 		bodyRLPCache:   bodyRLPCache,
+		receiptsCache:  receiptsCache,
 		blockCache:     blockCache,
 		futureBlocks:   futureBlocks,
 		engine:         engine,
@@ -280,6 +284,7 @@ func (bc *BlockChain) SetHead(head uint64) error {
 	// Clear out any stale content from the caches
 	bc.bodyCache.Purge()
 	bc.bodyRLPCache.Purge()
+	bc.receiptsCache.Purge()
 	bc.blockCache.Purge()
 	bc.futureBlocks.Purge()
 
@@ -603,11 +608,18 @@ func (bc *BlockChain) GetBlockByNumber(number uint64) *types.Block {
 
 // GetReceiptsByHash retrieves the receipts for all transactions in a given block.
 func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
+	if receipts, ok := bc.receiptsCache.Get(hash); ok {
+		return receipts.(types.Receipts)
+	}
+
 	number := rawdb.ReadHeaderNumber(bc.db, hash)
 	if number == nil {
 		return nil
 	}
-	return rawdb.ReadReceipts(bc.db, hash, *number)
+
+	receipts := rawdb.ReadReceipts(bc.db, hash, *number)
+	bc.receiptsCache.Add(hash, receipts)
+	return receipts
 }
 
 // GetBlocksFromHash returns the block corresponding to hash and up to n-1 ancestors.

core/chain_indexer.go

@@ -53,14 +53,14 @@ type ChainIndexerChain interface {
 	// CurrentHeader retrieves the latest locally known header.
 	CurrentHeader() *types.Header
 
-	// SubscribeChainEvent subscribes to new head header notifications.
-	SubscribeChainEvent(ch chan<- ChainEvent) event.Subscription
+	// SubscribeChainHeadEvent subscribes to new head header notifications.
+	SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription
 }
 
 // ChainIndexer does a post-processing job for equally sized sections of the
 // canonical chain (like BlooomBits and CHT structures). A ChainIndexer is
 // connected to the blockchain through the event system by starting a
-// ChainEventLoop in a goroutine.
+// ChainHeadEventLoop in a goroutine.
 //
 // Further child ChainIndexers can be added which use the output of the parent
 // section indexer. These child indexers receive new head notifications only
@@ -142,8 +142,8 @@ func (c *ChainIndexer) AddCheckpoint(section uint64, shead common.Hash) {
 // cascading background processing. Children do not need to be started, they
 // are notified about new events by their parents.
 func (c *ChainIndexer) Start(chain ChainIndexerChain) {
-	events := make(chan ChainEvent, 10)
-	sub := chain.SubscribeChainEvent(events)
+	events := make(chan ChainHeadEvent, 10)
+	sub := chain.SubscribeChainHeadEvent(events)
 
 	go c.eventLoop(chain.CurrentHeader(), events, sub)
 }
@@ -190,7 +190,7 @@ func (c *ChainIndexer) Close() error {
 // eventLoop is a secondary - optional - event loop of the indexer which is only
 // started for the outermost indexer to push chain head events into a processing
 // queue.
-func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainEvent, sub event.Subscription) {
+func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainHeadEvent, sub event.Subscription) {
 	// Mark the chain indexer as active, requiring an additional teardown
 	atomic.StoreUint32(&c.active, 1)
 
@@ -219,13 +219,13 @@ func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainE
 			}
 			header := ev.Block.Header()
 			if header.ParentHash != prevHash {
-				// Reorg to the common ancestor (might not exist in light sync mode, skip reorg then)
+				// Reorg to the common ancestor if needed (might not exist in light sync mode, skip reorg then)
 				// TODO(karalabe, zsfelfoldi): This seems a bit brittle, can we detect this case explicitly?
 
-				// TODO(karalabe): This operation is expensive and might block, causing the event system to
-				// potentially also lock up. We need to do with on a different thread somehow.
-				if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, header); h != nil {
-					c.newHead(h.Number.Uint64(), true)
+				if rawdb.ReadCanonicalHash(c.chainDb, prevHeader.Number.Uint64()) != prevHash {
+					if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, header); h != nil {
+						c.newHead(h.Number.Uint64(), true)
+					}
 				}
 			}
 			c.newHead(header.Number.Uint64(), false)

core/types/log.go

@@ -47,7 +47,7 @@ type Log struct {
 	TxIndex uint `json:"transactionIndex" gencodec:"required"`
 	// hash of the block in which the transaction was included
 	BlockHash common.Hash `json:"blockHash"`
-	// index of the log in the receipt
+	// index of the log in the block
 	Index uint `json:"logIndex" gencodec:"required"`
 
 	// The Removed field is true if this log was reverted due to a chain reorganisation.

core/types/transaction.go

@@ -153,16 +153,21 @@ func (tx *Transaction) UnmarshalJSON(input []byte) error {
 	if err := dec.UnmarshalJSON(input); err != nil {
 		return err
 	}
-	var V byte
-	if isProtectedV(dec.V) {
-		chainID := deriveChainId(dec.V).Uint64()
-		V = byte(dec.V.Uint64() - 35 - 2*chainID)
-	} else {
-		V = byte(dec.V.Uint64() - 27)
-	}
-	if !crypto.ValidateSignatureValues(V, dec.R, dec.S, false) {
-		return ErrInvalidSig
+
+	withSignature := dec.V.Sign() != 0 || dec.R.Sign() != 0 || dec.S.Sign() != 0
+	if withSignature {
+		var V byte
+		if isProtectedV(dec.V) {
+			chainID := deriveChainId(dec.V).Uint64()
+			V = byte(dec.V.Uint64() - 35 - 2*chainID)
+		} else {
+			V = byte(dec.V.Uint64() - 27)
+		}
+		if !crypto.ValidateSignatureValues(V, dec.R, dec.S, false) {
+			return ErrInvalidSig
+		}
 	}
+
 	*tx = Transaction{data: dec}
 	return nil
 }

core/types/transaction_signing.go

@@ -227,13 +227,13 @@ func recoverPlain(sighash common.Hash, R, S, Vb *big.Int, homestead bool) (commo
 	if !crypto.ValidateSignatureValues(V, R, S, homestead) {
 		return common.Address{}, ErrInvalidSig
 	}
-	// encode the snature in uncompressed format
+	// encode the signature in uncompressed format
 	r, s := R.Bytes(), S.Bytes()
 	sig := make([]byte, 65)
 	copy(sig[32-len(r):32], r)
 	copy(sig[64-len(s):64], s)
 	sig[64] = V
-	// recover the public key from the snature
+	// recover the public key from the signature
 	pub, err := crypto.Ecrecover(sighash[:], sig)
 	if err != nil {
 		return common.Address{}, err

core/types/transaction_test.go

@@ -185,6 +185,7 @@ func TestTransactionJSON(t *testing.T) {
 	}
 	signer := NewEIP155Signer(common.Big1)
 
+	transactions := make([]*Transaction, 0, 50)
 	for i := uint64(0); i < 25; i++ {
 		var tx *Transaction
 		switch i % 2 {
@@ -193,20 +194,25 @@ func TestTransactionJSON(t *testing.T) {
 		case 1:
 			tx = NewContractCreation(i, common.Big0, 1, common.Big2, []byte("abcdef"))
 		}
+		transactions = append(transactions, tx)
 
-		tx, err := SignTx(tx, signer, key)
+		signedTx, err := SignTx(tx, signer, key)
 		if err != nil {
 			t.Fatalf("could not sign transaction: %v", err)
 		}
 
+		transactions = append(transactions, signedTx)
+	}
+
+	for _, tx := range transactions {
 		data, err := json.Marshal(tx)
 		if err != nil {
-			t.Errorf("json.Marshal failed: %v", err)
+			t.Fatalf("json.Marshal failed: %v", err)
 		}
 
 		var parsedTx *Transaction
 		if err := json.Unmarshal(data, &parsedTx); err != nil {
-			t.Errorf("json.Unmarshal failed: %v", err)
+			t.Fatalf("json.Unmarshal failed: %v", err)
 		}
 
 		// compare nonce, price, gaslimit, recipient, amount, payload, V, R, S

core/vm/analysis.go

@@ -16,34 +16,6 @@
 
 package vm
 
-import (
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-)
-
-// destinations stores one map per contract (keyed by hash of code).
-// The maps contain an entry for each location of a JUMPDEST
-// instruction.
-type destinations map[common.Hash]bitvec
-
-// has checks whether code has a JUMPDEST at dest.
-func (d destinations) has(codehash common.Hash, code []byte, dest *big.Int) bool {
-	// PC cannot go beyond len(code) and certainly can't be bigger than 63bits.
-	// Don't bother checking for JUMPDEST in that case.
-	udest := dest.Uint64()
-	if dest.BitLen() >= 63 || udest >= uint64(len(code)) {
-		return false
-	}
-
-	m, analysed := d[codehash]
-	if !analysed {
-		m = codeBitmap(code)
-		d[codehash] = m
-	}
-	return OpCode(code[udest]) == JUMPDEST && m.codeSegment(udest)
-}
-
 // bitvec is a bit vector which maps bytes in a program.
 // An unset bit means the byte is an opcode, a set bit means
 // it's data (i.e. argument of PUSHxx).

core/vm/analysis_test.go

@@ -16,7 +16,11 @@
 
 package vm
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/ethereum/go-ethereum/crypto"
+)
 
 func TestJumpDestAnalysis(t *testing.T) {
 	tests := []struct {
@@ -49,5 +53,23 @@ func TestJumpDestAnalysis(t *testing.T) {
 			t.Fatalf("expected %x, got %02x", test.exp, ret[test.which])
 		}
 	}
-
+}
+
+func BenchmarkJumpdestAnalysis_1200k(bench *testing.B) {
+	// 1.4 ms
+	code := make([]byte, 1200000)
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		codeBitmap(code)
+	}
+	bench.StopTimer()
+}
+func BenchmarkJumpdestHashing_1200k(bench *testing.B) {
+	// 4 ms
+	code := make([]byte, 1200000)
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		crypto.Keccak256Hash(code)
+	}
+	bench.StopTimer()
 }

core/vm/contract.go

@@ -49,7 +49,8 @@ type Contract struct {
 	caller        ContractRef
 	self          ContractRef
 
-	jumpdests destinations // result of JUMPDEST analysis.
+	jumpdests map[common.Hash]bitvec // Aggregated result of JUMPDEST analysis.
+	analysis  bitvec                 // Locally cached result of JUMPDEST analysis
 
 	Code     []byte
 	CodeHash common.Hash
@@ -58,21 +59,17 @@ type Contract struct {
 
 	Gas   uint64
 	value *big.Int
-
-	Args []byte
-
-	DelegateCall bool
 }
 
 // NewContract returns a new contract environment for the execution of EVM.
 func NewContract(caller ContractRef, object ContractRef, value *big.Int, gas uint64) *Contract {
-	c := &Contract{CallerAddress: caller.Address(), caller: caller, self: object, Args: nil}
+	c := &Contract{CallerAddress: caller.Address(), caller: caller, self: object}
 
 	if parent, ok := caller.(*Contract); ok {
 		// Reuse JUMPDEST analysis from parent context if available.
 		c.jumpdests = parent.jumpdests
 	} else {
-		c.jumpdests = make(destinations)
+		c.jumpdests = make(map[common.Hash]bitvec)
 	}
 
 	// Gas should be a pointer so it can safely be reduced through the run
@@ -84,10 +81,42 @@ func NewContract(caller ContractRef, object ContractRef, value *big.Int, gas uin
 	return c
 }
 
+func (c *Contract) validJumpdest(dest *big.Int) bool {
+	udest := dest.Uint64()
+	// PC cannot go beyond len(code) and certainly can't be bigger than 63bits.
+	// Don't bother checking for JUMPDEST in that case.
+	if dest.BitLen() >= 63 || udest >= uint64(len(c.Code)) {
+		return false
+	}
+	// Only JUMPDESTs allowed for destinations
+	if OpCode(c.Code[udest]) != JUMPDEST {
+		return false
+	}
+	// Do we have a contract hash already?
+	if c.CodeHash != (common.Hash{}) {
+		// Does parent context have the analysis?
+		analysis, exist := c.jumpdests[c.CodeHash]
+		if !exist {
+			// Do the analysis and save in parent context
+			// We do not need to store it in c.analysis
+			analysis = codeBitmap(c.Code)
+			c.jumpdests[c.CodeHash] = analysis
+		}
+		return analysis.codeSegment(udest)
+	}
+	// We don't have the code hash, most likely a piece of initcode not already
+	// in state trie. In that case, we do an analysis, and save it locally, so
+	// we don't have to recalculate it for every JUMP instruction in the execution
+	// However, we don't save it within the parent context
+	if c.analysis == nil {
+		c.analysis = codeBitmap(c.Code)
+	}
+	return c.analysis.codeSegment(udest)
+}
+
 // AsDelegate sets the contract to be a delegate call and returns the current
 // contract (for chaining calls)
 func (c *Contract) AsDelegate() *Contract {
-	c.DelegateCall = true
 	// NOTE: caller must, at all times be a contract. It should never happen
 	// that caller is something other than a Contract.
 	parent := c.caller.(*Contract)
@@ -138,12 +167,6 @@ func (c *Contract) Value() *big.Int {
 	return c.value
 }
 
-// SetCode sets the code to the contract
-func (c *Contract) SetCode(hash common.Hash, code []byte) {
-	c.Code = code
-	c.CodeHash = hash
-}
-
 // SetCallCode sets the code of the contract and address of the backing data
 // object
 func (c *Contract) SetCallCode(addr *common.Address, hash common.Hash, code []byte) {
@@ -151,3 +174,11 @@ func (c *Contract) SetCallCode(addr *common.Address, hash common.Hash, code []by
 	c.CodeHash = hash
 	c.CodeAddr = addr
 }
+
+// SetCodeOptionalHash can be used to provide code, but it's optional to provide hash.
+// In case hash is not provided, the jumpdest analysis will not be saved to the parent context
+func (c *Contract) SetCodeOptionalHash(addr *common.Address, codeAndHash *codeAndHash) {
+	c.Code = codeAndHash.code
+	c.CodeHash = codeAndHash.hash
+	c.CodeAddr = addr
+}

core/vm/evm.go

@@ -212,12 +212,12 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas
 		evm.StateDB.CreateAccount(addr)
 	}
 	evm.Transfer(evm.StateDB, caller.Address(), to.Address(), value)
-
 	// Initialise a new contract and set the code that is to be used by the EVM.
 	// The contract is a scoped environment for this execution context only.
 	contract := NewContract(caller, to, value, gas)
 	contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
 
+	// Even if the account has no code, we need to continue because it might be a precompile
 	start := time.Now()
 
 	// Capture the tracer start/end events in debug mode
@@ -352,8 +352,20 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte
 	return ret, contract.Gas, err
 }
 
+type codeAndHash struct {
+	code []byte
+	hash common.Hash
+}
+
+func (c *codeAndHash) Hash() common.Hash {
+	if c.hash == (common.Hash{}) {
+		c.hash = crypto.Keccak256Hash(c.code)
+	}
+	return c.hash
+}
+
 // create creates a new contract using code as deployment code.
-func (evm *EVM) create(caller ContractRef, code []byte, gas uint64, value *big.Int, address common.Address) ([]byte, common.Address, uint64, error) {
+func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, value *big.Int, address common.Address) ([]byte, common.Address, uint64, error) {
 	// Depth check execution. Fail if we're trying to execute above the
 	// limit.
 	if evm.depth > int(params.CallCreateDepth) {
@@ -382,14 +394,14 @@ func (evm *EVM) create(caller ContractRef, code []byte, gas uint64, value *big.I
 	// EVM. The contract is a scoped environment for this execution context
 	// only.
 	contract := NewContract(caller, AccountRef(address), value, gas)
-	contract.SetCallCode(&address, crypto.Keccak256Hash(code), code)
+	contract.SetCodeOptionalHash(&address, codeAndHash)
 
 	if evm.vmConfig.NoRecursion && evm.depth > 0 {
 		return nil, address, gas, nil
 	}
 
 	if evm.vmConfig.Debug && evm.depth == 0 {
-		evm.vmConfig.Tracer.CaptureStart(caller.Address(), address, true, code, gas, value)
+		evm.vmConfig.Tracer.CaptureStart(caller.Address(), address, true, codeAndHash.code, gas, value)
 	}
 	start := time.Now()
 
@@ -433,7 +445,7 @@ func (evm *EVM) create(caller ContractRef, code []byte, gas uint64, value *big.I
 // Create creates a new contract using code as deployment code.
 func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) {
 	contractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address()))
-	return evm.create(caller, code, gas, value, contractAddr)
+	return evm.create(caller, &codeAndHash{code: code}, gas, value, contractAddr)
 }
 
 // Create2 creates a new contract using code as deployment code.
@@ -441,8 +453,9 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I
 // The different between Create2 with Create is Create2 uses sha3(0xff ++ msg.sender ++ salt ++ sha3(init_code))[12:]
 // instead of the usual sender-and-nonce-hash as the address where the contract is initialized at.
 func (evm *EVM) Create2(caller ContractRef, code []byte, gas uint64, endowment *big.Int, salt *big.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) {
-	contractAddr = crypto.CreateAddress2(caller.Address(), common.BigToHash(salt), code)
-	return evm.create(caller, code, gas, endowment, contractAddr)
+	codeAndHash := &codeAndHash{code: code}
+	contractAddr = crypto.CreateAddress2(caller.Address(), common.BigToHash(salt), codeAndHash.Hash().Bytes())
+	return evm.create(caller, codeAndHash, gas, endowment, contractAddr)
 }
 
 // ChainConfig returns the environment's chain configuration

core/vm/gas_table.go

@@ -347,6 +347,17 @@ func gasCreate2(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack,
 	if gas, overflow = math.SafeAdd(gas, params.Create2Gas); overflow {
 		return 0, errGasUintOverflow
 	}
+	wordGas, overflow := bigUint64(stack.Back(2))
+	if overflow {
+		return 0, errGasUintOverflow
+	}
+	if wordGas, overflow = math.SafeMul(toWordSize(wordGas), params.Sha3WordGas); overflow {
+		return 0, errGasUintOverflow
+	}
+	if gas, overflow = math.SafeAdd(gas, wordGas); overflow {
+		return 0, errGasUintOverflow
+	}
+
 	return gas, nil
 }
 

core/vm/instructions.go

@@ -24,7 +24,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/crypto/sha3"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -373,13 +373,20 @@ func opSAR(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *
 func opSha3(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	offset, size := stack.pop(), stack.pop()
 	data := memory.Get(offset.Int64(), size.Int64())
-	hash := crypto.Keccak256(data)
-	evm := interpreter.evm
 
-	if evm.vmConfig.EnablePreimageRecording {
-		evm.StateDB.AddPreimage(common.BytesToHash(hash), data)
+	if interpreter.hasher == nil {
+		interpreter.hasher = sha3.NewKeccak256().(keccakState)
+	} else {
+		interpreter.hasher.Reset()
 	}
-	stack.push(interpreter.intPool.get().SetBytes(hash))
+	interpreter.hasher.Write(data)
+	interpreter.hasher.Read(interpreter.hasherBuf[:])
+
+	evm := interpreter.evm
+	if evm.vmConfig.EnablePreimageRecording {
+		evm.StateDB.AddPreimage(interpreter.hasherBuf, data)
+	}
+	stack.push(interpreter.intPool.get().SetBytes(interpreter.hasherBuf[:]))
 
 	interpreter.intPool.put(offset, size)
 	return nil, nil
@@ -620,7 +627,7 @@ func opSstore(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memor
 
 func opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	pos := stack.pop()
-	if !contract.jumpdests.has(contract.CodeHash, contract.Code, pos) {
+	if !contract.validJumpdest(pos) {
 		nop := contract.GetOp(pos.Uint64())
 		return nil, fmt.Errorf("invalid jump destination (%v) %v", nop, pos)
 	}
@@ -633,7 +640,7 @@ func opJump(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory
 func opJumpi(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	pos, cond := stack.pop(), stack.pop()
 	if cond.Sign() != 0 {
-		if !contract.jumpdests.has(contract.CodeHash, contract.Code, pos) {
+		if !contract.validJumpdest(pos) {
 			nop := contract.GetOp(pos.Uint64())
 			return nil, fmt.Errorf("invalid jump destination (%v) %v", nop, pos)
 		}
@@ -727,7 +734,7 @@ func opCreate2(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memo
 }
 
 func opCall(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
-	// Pop gas. The actual gas in in interpreter.evm.callGasTemp.
+	// Pop gas. The actual gas in interpreter.evm.callGasTemp.
 	interpreter.intPool.put(stack.pop())
 	gas := interpreter.evm.callGasTemp
 	// Pop other call parameters.

core/vm/instructions_test.go

@@ -17,10 +17,12 @@
 package vm
 
 import (
+	"bytes"
 	"math/big"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -489,3 +491,99 @@ func BenchmarkOpMstore(bench *testing.B) {
 	}
 	poolOfIntPools.put(evmInterpreter.intPool)
 }
+
+func BenchmarkOpSHA3(bench *testing.B) {
+	var (
+		env            = NewEVM(Context{}, nil, params.TestChainConfig, Config{})
+		stack          = newstack()
+		mem            = NewMemory()
+		evmInterpreter = NewEVMInterpreter(env, env.vmConfig)
+	)
+	env.interpreter = evmInterpreter
+	evmInterpreter.intPool = poolOfIntPools.get()
+	mem.Resize(32)
+	pc := uint64(0)
+	start := big.NewInt(0)
+
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		stack.pushN(big.NewInt(32), start)
+		opSha3(&pc, evmInterpreter, nil, mem, stack)
+	}
+	poolOfIntPools.put(evmInterpreter.intPool)
+}
+
+func TestCreate2Addreses(t *testing.T) {
+	type testcase struct {
+		origin   string
+		salt     string
+		code     string
+		expected string
+	}
+
+	for i, tt := range []testcase{
+		{
+			origin:   "0x0000000000000000000000000000000000000000",
+			salt:     "0x0000000000000000000000000000000000000000",
+			code:     "0x00",
+			expected: "0x4d1a2e2bb4f88f0250f26ffff098b0b30b26bf38",
+		},
+		{
+			origin:   "0xdeadbeef00000000000000000000000000000000",
+			salt:     "0x0000000000000000000000000000000000000000",
+			code:     "0x00",
+			expected: "0xB928f69Bb1D91Cd65274e3c79d8986362984fDA3",
+		},
+		{
+			origin:   "0xdeadbeef00000000000000000000000000000000",
+			salt:     "0xfeed000000000000000000000000000000000000",
+			code:     "0x00",
+			expected: "0xD04116cDd17beBE565EB2422F2497E06cC1C9833",
+		},
+		{
+			origin:   "0x0000000000000000000000000000000000000000",
+			salt:     "0x0000000000000000000000000000000000000000",
+			code:     "0xdeadbeef",
+			expected: "0x70f2b2914A2a4b783FaEFb75f459A580616Fcb5e",
+		},
+		{
+			origin:   "0x00000000000000000000000000000000deadbeef",
+			salt:     "0xcafebabe",
+			code:     "0xdeadbeef",
+			expected: "0x60f3f640a8508fC6a86d45DF051962668E1e8AC7",
+		},
+		{
+			origin:   "0x00000000000000000000000000000000deadbeef",
+			salt:     "0xcafebabe",
+			code:     "0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef",
+			expected: "0x1d8bfDC5D46DC4f61D6b6115972536eBE6A8854C",
+		},
+		{
+			origin:   "0x0000000000000000000000000000000000000000",
+			salt:     "0x0000000000000000000000000000000000000000",
+			code:     "0x",
+			expected: "0xE33C0C7F7df4809055C3ebA6c09CFe4BaF1BD9e0",
+		},
+	} {
+
+		origin := common.BytesToAddress(common.FromHex(tt.origin))
+		salt := common.BytesToHash(common.FromHex(tt.salt))
+		code := common.FromHex(tt.code)
+		codeHash := crypto.Keccak256(code)
+		address := crypto.CreateAddress2(origin, salt, codeHash)
+		/*
+			stack          := newstack()
+			// salt, but we don't need that for this test
+			stack.push(big.NewInt(int64(len(code)))) //size
+			stack.push(big.NewInt(0)) // memstart
+			stack.push(big.NewInt(0)) // value
+			gas, _ := gasCreate2(params.GasTable{}, nil, nil, stack, nil, 0)
+			fmt.Printf("Example %d\n* address `0x%x`\n* salt `0x%x`\n* init_code `0x%x`\n* gas (assuming no mem expansion): `%v`\n* result: `%s`\n\n", i,origin, salt, code, gas, address.String())
+		*/
+		expected := common.BytesToAddress(common.FromHex(tt.expected))
+		if !bytes.Equal(expected.Bytes(), address.Bytes()) {
+			t.Errorf("test %d: expected %s, got %s", i, expected.String(), address.String())
+		}
+
+	}
+}

core/vm/interpreter.go

@@ -18,8 +18,10 @@ package vm
 
 import (
 	"fmt"
+	"hash"
 	"sync/atomic"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/params"
 )
@@ -68,12 +70,24 @@ type Interpreter interface {
 	CanRun([]byte) bool
 }
 
+// keccakState wraps sha3.state. In addition to the usual hash methods, it also supports
+// Read to get a variable amount of data from the hash state. Read is faster than Sum
+// because it doesn't copy the internal state, but also modifies the internal state.
+type keccakState interface {
+	hash.Hash
+	Read([]byte) (int, error)
+}
+
 // EVMInterpreter represents an EVM interpreter
 type EVMInterpreter struct {
 	evm      *EVM
 	cfg      Config
 	gasTable params.GasTable
-	intPool  *intPool
+
+	intPool *intPool
+
+	hasher    keccakState // Keccak256 hasher instance shared across opcodes
+	hasherBuf common.Hash // Keccak256 hasher result array shared aross opcodes
 
 	readOnly   bool   // Whether to throw on stateful modifications
 	returnData []byte // Last CALL's return data for subsequent reuse

core/vm/memory.go

@@ -29,7 +29,7 @@ type Memory struct {
 	lastGasCost uint64
 }
 
-// NewMemory returns a new memory memory model.
+// NewMemory returns a new memory model.
 func NewMemory() *Memory {
 	return &Memory{}
 }

core/vm/runtime/runtime_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/params"
 )
 
 func TestDefaults(t *testing.T) {
@@ -148,3 +149,57 @@ func BenchmarkCall(b *testing.B) {
 		}
 	}
 }
+func benchmarkEVM_Create(bench *testing.B, code string) {
+	var (
+		statedb, _ = state.New(common.Hash{}, state.NewDatabase(ethdb.NewMemDatabase()))
+		sender     = common.BytesToAddress([]byte("sender"))
+		receiver   = common.BytesToAddress([]byte("receiver"))
+	)
+
+	statedb.CreateAccount(sender)
+	statedb.SetCode(receiver, common.FromHex(code))
+	runtimeConfig := Config{
+		Origin:      sender,
+		State:       statedb,
+		GasLimit:    10000000,
+		Difficulty:  big.NewInt(0x200000),
+		Time:        new(big.Int).SetUint64(0),
+		Coinbase:    common.Address{},
+		BlockNumber: new(big.Int).SetUint64(1),
+		ChainConfig: &params.ChainConfig{
+			ChainID:             big.NewInt(1),
+			HomesteadBlock:      new(big.Int),
+			ByzantiumBlock:      new(big.Int),
+			ConstantinopleBlock: new(big.Int),
+			DAOForkBlock:        new(big.Int),
+			DAOForkSupport:      false,
+			EIP150Block:         new(big.Int),
+			EIP155Block:         new(big.Int),
+			EIP158Block:         new(big.Int),
+		},
+		EVMConfig: vm.Config{},
+	}
+	// Warm up the intpools and stuff
+	bench.ResetTimer()
+	for i := 0; i < bench.N; i++ {
+		Call(receiver, []byte{}, &runtimeConfig)
+	}
+	bench.StopTimer()
+}
+
+func BenchmarkEVM_CREATE_500(bench *testing.B) {
+	// initcode size 500K, repeatedly calls CREATE and then modifies the mem contents
+	benchmarkEVM_Create(bench, "5b6207a120600080f0600152600056")
+}
+func BenchmarkEVM_CREATE2_500(bench *testing.B) {
+	// initcode size 500K, repeatedly calls CREATE2 and then modifies the mem contents
+	benchmarkEVM_Create(bench, "5b586207a120600080f5600152600056")
+}
+func BenchmarkEVM_CREATE_1200(bench *testing.B) {
+	// initcode size 1200K, repeatedly calls CREATE and then modifies the mem contents
+	benchmarkEVM_Create(bench, "5b62124f80600080f0600152600056")
+}
+func BenchmarkEVM_CREATE2_1200(bench *testing.B) {
+	// initcode size 1200K, repeatedly calls CREATE2 and then modifies the mem contents
+	benchmarkEVM_Create(bench, "5b5862124f80600080f5600152600056")
+}

crypto/crypto.go

@@ -77,9 +77,9 @@ func CreateAddress(b common.Address, nonce uint64) common.Address {
 }
 
 // CreateAddress2 creates an ethereum address given the address bytes, initial
-// contract code and a salt.
-func CreateAddress2(b common.Address, salt [32]byte, code []byte) common.Address {
-	return common.BytesToAddress(Keccak256([]byte{0xff}, b.Bytes(), salt[:], Keccak256(code))[12:])
+// contract code hash and a salt.
+func CreateAddress2(b common.Address, salt [32]byte, inithash []byte) common.Address {
+	return common.BytesToAddress(Keccak256([]byte{0xff}, b.Bytes(), salt[:], inithash)[12:])
 }
 
 // ToECDSA creates a private key with the given D value.

crypto/secp256k1/libsecp256k1/src/num.h

@@ -54,7 +54,7 @@ static void secp256k1_num_mul(secp256k1_num *r, const secp256k1_num *a, const se
     even if r was negative. */
 static void secp256k1_num_mod(secp256k1_num *r, const secp256k1_num *m);
 
-/** Right-shift the passed number by bits bits. */
+/** Right-shift the passed number by bits. */
 static void secp256k1_num_shift(secp256k1_num *r, int bits);
 
 /** Check whether a number is zero. */

eth/api.go

@@ -67,6 +67,15 @@ func (api *PublicEthereumAPI) Hashrate() hexutil.Uint64 {
 	return hexutil.Uint64(api.e.Miner().HashRate())
 }
 
+// ChainId is the EIP-155 replay-protection chain id for the current ethereum chain config.
+func (api *PublicEthereumAPI) ChainId() hexutil.Uint64 {
+	chainID := new(big.Int)
+	if config := api.e.chainConfig; config.IsEIP155(api.e.blockchain.CurrentBlock().Number()) {
+		chainID = config.ChainID
+	}
+	return (hexutil.Uint64)(chainID.Uint64())
+}
+
 // PublicMinerAPI provides an API to control the miner.
 // It offers only methods that operate on data that pose no security risk when it is publicly accessible.
 type PublicMinerAPI struct {

eth/api_backend.go

@@ -25,7 +25,6 @@ import (
 	"github.com/ethereum/go-ethereum/common/math"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/bloombits"
-	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
@@ -107,18 +106,11 @@ func (b *EthAPIBackend) GetBlock(ctx context.Context, hash common.Hash) (*types.
 }
 
 func (b *EthAPIBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) {
-	if number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash); number != nil {
-		return rawdb.ReadReceipts(b.eth.chainDb, hash, *number), nil
-	}
-	return nil, nil
+	return b.eth.blockchain.GetReceiptsByHash(hash), nil
 }
 
 func (b *EthAPIBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) {
-	number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash)
-	if number == nil {
-		return nil, nil
-	}
-	receipts := rawdb.ReadReceipts(b.eth.chainDb, hash, *number)
+	receipts := b.eth.blockchain.GetReceiptsByHash(hash)
 	if receipts == nil {
 		return nil, nil
 	}

eth/api_tracer.go

@@ -391,6 +391,15 @@ func (api *PrivateDebugAPI) TraceBlockFromFile(ctx context.Context, file string,
 	return api.TraceBlock(ctx, blob, config)
 }
 
+// TraceBadBlock returns the structured logs created during the execution of a block
+// within the blockchain 'badblocks' cache
+func (api *PrivateDebugAPI) TraceBadBlock(ctx context.Context, index int, config *TraceConfig) ([]*txTraceResult, error) {
+	if blocks := api.eth.blockchain.BadBlocks(); index < len(blocks) {
+		return api.traceBlock(ctx, blocks[index], config)
+	}
+	return nil, fmt.Errorf("index out of range")
+}
+
 // traceBlock configures a new tracer according to the provided configuration, and
 // executes all the transactions contained within. The return value will be one item
 // per transaction, dependent on the requestd tracer.

eth/downloader/downloader.go

@@ -60,6 +60,9 @@ var (
 	maxHeadersProcess = 2048      // Number of header download results to import at once into the chain
 	maxResultsProcess = 2048      // Number of content download results to import at once into the chain
 
+	reorgProtThreshold   = 48 // Threshold number of recent blocks to disable mini reorg protection
+	reorgProtHeaderDelay = 2  // Number of headers to delay delivering to cover mini reorgs
+
 	fsHeaderCheckFrequency = 100             // Verification frequency of the downloaded headers during fast sync
 	fsHeaderSafetyNet      = 2048            // Number of headers to discard in case a chain violation is detected
 	fsHeaderForceVerify    = 24              // Number of headers to verify before and after the pivot to accept it
@@ -658,8 +661,10 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 					continue
 				}
 				// Otherwise check if we already know the header or not
-				if (d.mode == FullSync && d.blockchain.HasBlock(headers[i].Hash(), headers[i].Number.Uint64())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash(), headers[i].Number.Uint64())) {
-					number, hash = headers[i].Number.Uint64(), headers[i].Hash()
+				h := headers[i].Hash()
+				n := headers[i].Number.Uint64()
+				if (d.mode == FullSync && d.blockchain.HasBlock(h, n)) || (d.mode != FullSync && d.lightchain.HasHeader(h, n)) {
+					number, hash = n, h
 
 					// If every header is known, even future ones, the peer straight out lied about its head
 					if number > height && i == limit-1 {
@@ -723,11 +728,13 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err
 				arrived = true
 
 				// Modify the search interval based on the response
-				if (d.mode == FullSync && !d.blockchain.HasBlock(headers[0].Hash(), headers[0].Number.Uint64())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash(), headers[0].Number.Uint64())) {
+				h := headers[0].Hash()
+				n := headers[0].Number.Uint64()
+				if (d.mode == FullSync && !d.blockchain.HasBlock(h, n)) || (d.mode != FullSync && !d.lightchain.HasHeader(h, n)) {
 					end = check
 					break
 				}
-				header := d.lightchain.GetHeaderByHash(headers[0].Hash()) // Independent of sync mode, header surely exists
+				header := d.lightchain.GetHeaderByHash(h) // Independent of sync mode, header surely exists
 				if header.Number.Uint64() != check {
 					p.log.Debug("Received non requested header", "number", header.Number, "hash", header.Hash(), "request", check)
 					return 0, errBadPeer
@@ -843,6 +850,30 @@ func (d *Downloader) fetchHeaders(p *peerConnection, from uint64, pivot uint64)
 				}
 				headers = filled[proced:]
 				from += uint64(proced)
+			} else {
+				// If we're closing in on the chain head, but haven't yet reached it, delay
+				// the last few headers so mini reorgs on the head don't cause invalid hash
+				// chain errors.
+				if n := len(headers); n > 0 {
+					// Retrieve the current head we're at
+					head := uint64(0)
+					if d.mode == LightSync {
+						head = d.lightchain.CurrentHeader().Number.Uint64()
+					} else {
+						head = d.blockchain.CurrentFastBlock().NumberU64()
+						if full := d.blockchain.CurrentBlock().NumberU64(); head < full {
+							head = full
+						}
+					}
+					// If the head is way older than this batch, delay the last few headers
+					if head+uint64(reorgProtThreshold) < headers[n-1].Number.Uint64() {
+						delay := reorgProtHeaderDelay
+						if delay > n {
+							delay = n
+						}
+						headers = headers[:n-delay]
+					}
+				}
 			}
 			// Insert all the new headers and fetch the next batch
 			if len(headers) > 0 {
@@ -853,8 +884,18 @@ func (d *Downloader) fetchHeaders(p *peerConnection, from uint64, pivot uint64)
 					return errCancelHeaderFetch
 				}
 				from += uint64(len(headers))
+				getHeaders(from)
+			} else {
+				// No headers delivered, or all of them being delayed, sleep a bit and retry
+				p.log.Trace("All headers delayed, waiting")
+				select {
+				case <-time.After(fsHeaderContCheck):
+					getHeaders(from)
+					continue
+				case <-d.cancelCh:
+					return errCancelHeaderFetch
+				}
 			}
-			getHeaders(from)
 
 		case <-timeout.C:
 			if d.dropPeer == nil {

eth/downloader/downloader_test.go

@@ -744,7 +744,7 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) {
 			tester.downloader.queue.lock.Unlock()
 			tester.lock.Unlock()
 
-			if cached == blockCacheItems || retrieved+cached+frozen == targetBlocks+1 {
+			if cached == blockCacheItems || cached == blockCacheItems-reorgProtHeaderDelay || retrieved+cached+frozen == targetBlocks+1 || retrieved+cached+frozen == targetBlocks+1-reorgProtHeaderDelay {
 				break
 			}
 		}
@@ -754,7 +754,7 @@ func testThrottling(t *testing.T, protocol int, mode SyncMode) {
 		tester.lock.RLock()
 		retrieved = len(tester.ownBlocks)
 		tester.lock.RUnlock()
-		if cached != blockCacheItems && retrieved+cached+frozen != targetBlocks+1 {
+		if cached != blockCacheItems && cached != blockCacheItems-reorgProtHeaderDelay && retrieved+cached+frozen != targetBlocks+1 && retrieved+cached+frozen != targetBlocks+1-reorgProtHeaderDelay {
 			t.Fatalf("block count mismatch: have %v, want %v (owned %v, blocked %v, target %v)", cached, blockCacheItems, retrieved, frozen, targetBlocks+1)
 		}
 		// Permit the blocked blocks to import

eth/handler.go

@@ -37,7 +37,7 @@ import (
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 )
@@ -49,6 +49,9 @@ const (
 	// txChanSize is the size of channel listening to NewTxsEvent.
 	// The number is referenced from the size of tx pool.
 	txChanSize = 4096
+
+	// minimim number of peers to broadcast new blocks to
+	minBroadcastPeers = 4
 )
 
 var (
@@ -147,7 +150,7 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne
 			NodeInfo: func() interface{} {
 				return manager.NodeInfo()
 			},
-			PeerInfo: func(id discover.NodeID) interface{} {
+			PeerInfo: func(id enode.ID) interface{} {
 				if p := manager.peers.Peer(fmt.Sprintf("%x", id[:8])); p != nil {
 					return p.Info()
 				}
@@ -705,7 +708,14 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) {
 			return
 		}
 		// Send the block to a subset of our peers
-		transfer := peers[:int(math.Sqrt(float64(len(peers))))]
+		transferLen := int(math.Sqrt(float64(len(peers))))
+		if transferLen < minBroadcastPeers {
+			transferLen = minBroadcastPeers
+		}
+		if transferLen > len(peers) {
+			transferLen = len(peers)
+		}
+		transfer := peers[:transferLen]
 		for _, peer := range transfer {
 			peer.AsyncSendNewBlock(block, td)
 		}

eth/handler_test.go

@@ -17,6 +17,7 @@
 package eth
 
 import (
+	"fmt"
 	"math"
 	"math/big"
 	"math/rand"
@@ -466,14 +467,17 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool
 	}
 	// Create a DAO aware protocol manager
 	var (
-		evmux         = new(event.TypeMux)
-		pow           = ethash.NewFaker()
-		db            = ethdb.NewMemDatabase()
-		config        = &params.ChainConfig{DAOForkBlock: big.NewInt(1), DAOForkSupport: localForked}
-		gspec         = &core.Genesis{Config: config}
-		genesis       = gspec.MustCommit(db)
-		blockchain, _ = core.NewBlockChain(db, nil, config, pow, vm.Config{}, nil)
+		evmux   = new(event.TypeMux)
+		pow     = ethash.NewFaker()
+		db      = ethdb.NewMemDatabase()
+		config  = &params.ChainConfig{DAOForkBlock: big.NewInt(1), DAOForkSupport: localForked}
+		gspec   = &core.Genesis{Config: config}
+		genesis = gspec.MustCommit(db)
 	)
+	blockchain, err := core.NewBlockChain(db, nil, config, pow, vm.Config{}, nil)
+	if err != nil {
+		t.Fatalf("failed to create new blockchain: %v", err)
+	}
 	pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db)
 	if err != nil {
 		t.Fatalf("failed to start test protocol manager: %v", err)
@@ -520,3 +524,90 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool
 		}
 	}
 }
+
+func TestBroadcastBlock(t *testing.T) {
+	var tests = []struct {
+		totalPeers        int
+		broadcastExpected int
+	}{
+		{1, 1},
+		{2, 2},
+		{3, 3},
+		{4, 4},
+		{5, 4},
+		{9, 4},
+		{12, 4},
+		{16, 4},
+		{26, 5},
+		{100, 10},
+	}
+	for _, test := range tests {
+		testBroadcastBlock(t, test.totalPeers, test.broadcastExpected)
+	}
+}
+
+func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) {
+	var (
+		evmux   = new(event.TypeMux)
+		pow     = ethash.NewFaker()
+		db      = ethdb.NewMemDatabase()
+		config  = &params.ChainConfig{}
+		gspec   = &core.Genesis{Config: config}
+		genesis = gspec.MustCommit(db)
+	)
+	blockchain, err := core.NewBlockChain(db, nil, config, pow, vm.Config{}, nil)
+	if err != nil {
+		t.Fatalf("failed to create new blockchain: %v", err)
+	}
+	pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db)
+	if err != nil {
+		t.Fatalf("failed to start test protocol manager: %v", err)
+	}
+	pm.Start(1000)
+	defer pm.Stop()
+	var peers []*testPeer
+	for i := 0; i < totalPeers; i++ {
+		peer, _ := newTestPeer(fmt.Sprintf("peer %d", i), eth63, pm, true)
+		defer peer.close()
+		peers = append(peers, peer)
+	}
+	chain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) {})
+	pm.BroadcastBlock(chain[0], true /*propagate*/)
+
+	errCh := make(chan error, totalPeers)
+	doneCh := make(chan struct{}, totalPeers)
+	for _, peer := range peers {
+		go func(p *testPeer) {
+			if err := p2p.ExpectMsg(p.app, NewBlockMsg, &newBlockData{Block: chain[0], TD: big.NewInt(131136)}); err != nil {
+				errCh <- err
+			} else {
+				doneCh <- struct{}{}
+			}
+		}(peer)
+	}
+	timeoutCh := time.NewTimer(time.Millisecond * 100).C
+	var receivedCount int
+outer:
+	for {
+		select {
+		case err = <-errCh:
+			break outer
+		case <-doneCh:
+			receivedCount++
+			if receivedCount == totalPeers {
+				break outer
+			}
+		case <-timeoutCh:
+			break outer
+		}
+	}
+	for _, peer := range peers {
+		peer.app.Close()
+	}
+	if err != nil {
+		t.Errorf("error matching block by peer: %v", err)
+	}
+	if receivedCount != broadcastExpected {
+		t.Errorf("block broadcast to %d peers, expected %d", receivedCount, broadcastExpected)
+	}
+}

eth/helper_test.go

@@ -37,7 +37,7 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -148,7 +148,7 @@ func newTestPeer(name string, version int, pm *ProtocolManager, shake bool) (*te
 	app, net := p2p.MsgPipe()
 
 	// Generate a random id and create the peer
-	var id discover.NodeID
+	var id enode.ID
 	rand.Read(id[:])
 
 	peer := pm.newPeer(version, p2p.NewPeer(id, name, nil), net)

eth/protocol.go

@@ -37,7 +37,7 @@ const (
 // ProtocolName is the official short name of the protocol used during capability negotiation.
 var ProtocolName = "eth"
 
-// ProtocolVersions are the upported versions of the eth protocol (first is primary).
+// ProtocolVersions are the supported versions of the eth protocol (first is primary).
 var ProtocolVersions = []uint{eth63, eth62}
 
 // ProtocolLengths are the number of implemented message corresponding to different protocol versions.

eth/sync.go

@@ -25,7 +25,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 )
 
 const (
@@ -64,7 +64,7 @@ func (pm *ProtocolManager) syncTransactions(p *peer) {
 // the transactions in small packs to one peer at a time.
 func (pm *ProtocolManager) txsyncLoop() {
 	var (
-		pending = make(map[discover.NodeID]*txsync)
+		pending = make(map[enode.ID]*txsync)
 		sending = false               // whether a send is active
 		pack    = new(txsync)         // the pack that is being sent
 		done    = make(chan error, 1) // result of the send

eth/sync_test.go

@@ -23,7 +23,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 )
 
 // Tests that fast sync gets disabled as soon as a real block is successfully
@@ -42,8 +42,8 @@ func TestFastSyncDisabling(t *testing.T) {
 	// Sync up the two peers
 	io1, io2 := p2p.MsgPipe()
 
-	go pmFull.handle(pmFull.newPeer(63, p2p.NewPeer(discover.NodeID{}, "empty", nil), io2))
-	go pmEmpty.handle(pmEmpty.newPeer(63, p2p.NewPeer(discover.NodeID{}, "full", nil), io1))
+	go pmFull.handle(pmFull.newPeer(63, p2p.NewPeer(enode.ID{}, "empty", nil), io2))
+	go pmEmpty.handle(pmEmpty.newPeer(63, p2p.NewPeer(enode.ID{}, "full", nil), io1))
 
 	time.Sleep(250 * time.Millisecond)
 	pmEmpty.synchronise(pmEmpty.peers.BestPeer())

internal/debug/flags.go

@@ -25,11 +25,11 @@ import (
 	"runtime"
 
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/log/term"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/metrics/exp"
 	"github.com/fjl/memsize/memsizeui"
 	colorable "github.com/mattn/go-colorable"
+	"github.com/mattn/go-isatty"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -101,7 +101,7 @@ var (
 )
 
 func init() {
-	usecolor := term.IsTty(os.Stderr.Fd()) && os.Getenv("TERM") != "dumb"
+	usecolor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb"
 	output := io.Writer(os.Stderr)
 	if usecolor {
 		output = colorable.NewColorableStderr()

internal/ethapi/api.go

@@ -449,7 +449,7 @@ func (s *PrivateAccountAPI) Sign(ctx context.Context, data hexutil.Bytes, addr c
 // addr = ecrecover(hash, signature)
 //
 // Note, the signature must conform to the secp256k1 curve R, S and V values, where
-// the V value must be be 27 or 28 for legacy reasons.
+// the V value must be 27 or 28 for legacy reasons.
 //
 // https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_ecRecover
 func (s *PrivateAccountAPI) EcRecover(ctx context.Context, data, sig hexutil.Bytes) (common.Address, error) {

internal/jsre/deps/web3.js

@@ -1021,7 +1021,7 @@ var formatOutputInt = function (param) {
     var value = param.staticPart() || "0";
 
     // check if it's negative number
-    // it it is, return two's complement
+    // it is, return two's complement
     if (signedIsNegative(value)) {
         return new BigNumber(value, 16).minus(new BigNumber('ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', 16)).minus(1);
     }
@@ -2250,7 +2250,7 @@ var isAddress = function (address) {
         // check if it has the basic requirements of an address
         return false;
     } else if (/^(0x)?[0-9a-f]{40}$/.test(address) || /^(0x)?[0-9A-F]{40}$/.test(address)) {
-        // If it's all small caps or all all caps, return true
+        // If it's all small caps or all caps, return true
         return true;
     } else {
         // Otherwise check each case

internal/web3ext/web3ext.go

@@ -378,6 +378,12 @@ web3._extend({
 			params: 2,
 			inputFormatter: [null, null]
 		}),
+		new web3._extend.Method({
+			name: 'traceBadBlock',
+			call: 'debug_traceBadBlock',
+			params: 1,
+			inputFormatter: [null]
+		}),
 		new web3._extend.Method({
 			name: 'traceBlockByNumber',
 			call: 'debug_traceBlockByNumber',
@@ -433,6 +439,11 @@ const Eth_JS = `
 web3._extend({
 	property: 'eth',
 	methods: [
+		new web3._extend.Method({
+			name: 'chainId',
+			call: 'eth_chainId',
+			params: 0
+		}),
 		new web3._extend.Method({
 			name: 'sign',
 			call: 'eth_sign',

les/commons.go

@@ -26,7 +26,7 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/light"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -63,7 +63,7 @@ func (c *lesCommons) makeProtocols(versions []uint) []p2p.Protocol {
 			Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
 				return c.protocolManager.runPeer(version, p, rw)
 			},
-			PeerInfo: func(id discover.NodeID) interface{} {
+			PeerInfo: func(id enode.ID) interface{} {
 				if p := c.protocolManager.peers.Peer(fmt.Sprintf("%x", id[:8])); p != nil {
 					return p.Info()
 				}

les/distributor_test.go

@@ -87,7 +87,7 @@ const (
 	testDistBufLimit       = 10000000
 	testDistMaxCost        = 1000000
 	testDistPeerCount      = 5
-	testDistReqCount       = 50000
+	testDistReqCount       = 5000
 	testDistMaxResendCount = 3
 )
 

les/fetcher.go

@@ -32,8 +32,9 @@ import (
 )
 
 const (
-	blockDelayTimeout = time.Second * 10 // timeout for a peer to announce a head that has already been confirmed by others
-	maxNodeCount      = 20               // maximum number of fetcherTreeNode entries remembered for each peer
+	blockDelayTimeout    = time.Second * 10 // timeout for a peer to announce a head that has already been confirmed by others
+	maxNodeCount         = 20               // maximum number of fetcherTreeNode entries remembered for each peer
+	serverStateAvailable = 100              // number of recent blocks where state availability is assumed
 )
 
 // lightFetcher implements retrieval of newly announced headers. It also provides a peerHasBlock function for the
@@ -215,8 +216,8 @@ func (f *lightFetcher) syncLoop() {
 // registerPeer adds a new peer to the fetcher's peer set
 func (f *lightFetcher) registerPeer(p *peer) {
 	p.lock.Lock()
-	p.hasBlock = func(hash common.Hash, number uint64) bool {
-		return f.peerHasBlock(p, hash, number)
+	p.hasBlock = func(hash common.Hash, number uint64, hasState bool) bool {
+		return f.peerHasBlock(p, hash, number, hasState)
 	}
 	p.lock.Unlock()
 
@@ -344,21 +345,27 @@ func (f *lightFetcher) announce(p *peer, head *announceData) {
 
 // peerHasBlock returns true if we can assume the peer knows the given block
 // based on its announcements
-func (f *lightFetcher) peerHasBlock(p *peer, hash common.Hash, number uint64) bool {
+func (f *lightFetcher) peerHasBlock(p *peer, hash common.Hash, number uint64, hasState bool) bool {
 	f.lock.Lock()
 	defer f.lock.Unlock()
 
+	fp := f.peers[p]
+	if fp == nil || fp.root == nil {
+		return false
+	}
+
+	if hasState {
+		if fp.lastAnnounced == nil || fp.lastAnnounced.number > number+serverStateAvailable {
+			return false
+		}
+	}
+
 	if f.syncing {
 		// always return true when syncing
 		// false positives are acceptable, a more sophisticated condition can be implemented later
 		return true
 	}
 
-	fp := f.peers[p]
-	if fp == nil || fp.root == nil {
-		return false
-	}
-
 	if number >= fp.root.number {
 		// it is recent enough that if it is known, is should be in the peer's block tree
 		return fp.nodeByHash[hash] != nil

les/handler.go

@@ -213,8 +213,7 @@ func (pm *ProtocolManager) runPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWrit
 	var entry *poolEntry
 	peer := pm.newPeer(int(version), pm.networkId, p, rw)
 	if pm.serverPool != nil {
-		addr := p.RemoteAddr().(*net.TCPAddr)
-		entry = pm.serverPool.connect(peer, addr.IP, uint16(addr.Port))
+		entry = pm.serverPool.connect(peer, peer.Node())
 	}
 	peer.poolEntry = entry
 	select {
@@ -382,7 +381,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 		}
 
 		if p.requestAnnounceType == announceTypeSigned {
-			if err := req.checkSignature(p.pubKey); err != nil {
+			if err := req.checkSignature(p.ID()); err != nil {
 				p.Log().Trace("Invalid announcement signature", "err", err)
 				return err
 			}

les/helper_test.go

@@ -38,7 +38,7 @@ import (
 	"github.com/ethereum/go-ethereum/les/flowcontrol"
 	"github.com/ethereum/go-ethereum/light"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -221,7 +221,7 @@ func newTestPeer(t *testing.T, name string, version int, pm *ProtocolManager, sh
 	app, net := p2p.MsgPipe()
 
 	// Generate a random id and create the peer
-	var id discover.NodeID
+	var id enode.ID
 	rand.Read(id[:])
 
 	peer := pm.newPeer(version, NetworkId, p2p.NewPeer(id, name, nil), net)
@@ -258,7 +258,7 @@ func newTestPeerPair(name string, version int, pm, pm2 *ProtocolManager) (*peer,
 	app, net := p2p.MsgPipe()
 
 	// Generate a random id and create the peer
-	var id discover.NodeID
+	var id enode.ID
 	rand.Read(id[:])
 
 	peer := pm.newPeer(version, NetworkId, p2p.NewPeer(id, name, nil), net)

les/odr_requests.go

@@ -84,7 +84,7 @@ func (r *BlockRequest) GetCost(peer *peer) uint64 {
 
 // CanSend tells if a certain peer is suitable for serving the given request
 func (r *BlockRequest) CanSend(peer *peer) bool {
-	return peer.HasBlock(r.Hash, r.Number)
+	return peer.HasBlock(r.Hash, r.Number, false)
 }
 
 // Request sends an ODR request to the LES network (implementation of LesOdrRequest)
@@ -140,7 +140,7 @@ func (r *ReceiptsRequest) GetCost(peer *peer) uint64 {
 
 // CanSend tells if a certain peer is suitable for serving the given request
 func (r *ReceiptsRequest) CanSend(peer *peer) bool {
-	return peer.HasBlock(r.Hash, r.Number)
+	return peer.HasBlock(r.Hash, r.Number, false)
 }
 
 // Request sends an ODR request to the LES network (implementation of LesOdrRequest)
@@ -202,7 +202,7 @@ func (r *TrieRequest) GetCost(peer *peer) uint64 {
 
 // CanSend tells if a certain peer is suitable for serving the given request
 func (r *TrieRequest) CanSend(peer *peer) bool {
-	return peer.HasBlock(r.Id.BlockHash, r.Id.BlockNumber)
+	return peer.HasBlock(r.Id.BlockHash, r.Id.BlockNumber, true)
 }
 
 // Request sends an ODR request to the LES network (implementation of LesOdrRequest)
@@ -272,7 +272,7 @@ func (r *CodeRequest) GetCost(peer *peer) uint64 {
 
 // CanSend tells if a certain peer is suitable for serving the given request
 func (r *CodeRequest) CanSend(peer *peer) bool {
-	return peer.HasBlock(r.Id.BlockHash, r.Id.BlockNumber)
+	return peer.HasBlock(r.Id.BlockHash, r.Id.BlockNumber, true)
 }
 
 // Request sends an ODR request to the LES network (implementation of LesOdrRequest)

les/odr_test.go

@@ -194,7 +194,7 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) {
 	client.peers.Register(client.rPeer)
 	time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed
 	client.peers.lock.Lock()
-	client.rPeer.hasBlock = func(common.Hash, uint64) bool { return true }
+	client.rPeer.hasBlock = func(common.Hash, uint64, bool) bool { return true }
 	client.peers.lock.Unlock()
 	test(5)
 	// still expect all retrievals to pass, now data should be cached locally

les/peer.go

@@ -18,7 +18,6 @@
 package les
 
 import (
-	"crypto/ecdsa"
 	"errors"
 	"fmt"
 	"math/big"
@@ -51,7 +50,6 @@ const (
 
 type peer struct {
 	*p2p.Peer
-	pubKey *ecdsa.PublicKey
 
 	rw p2p.MsgReadWriter
 
@@ -69,7 +67,7 @@ type peer struct {
 	sendQueue   *execQueue
 
 	poolEntry      *poolEntry
-	hasBlock       func(common.Hash, uint64) bool
+	hasBlock       func(common.Hash, uint64, bool) bool
 	responseErrors int
 
 	fcClient       *flowcontrol.ClientNode // nil if the peer is server only
@@ -80,11 +78,9 @@ type peer struct {
 
 func newPeer(version int, network uint64, p *p2p.Peer, rw p2p.MsgReadWriter) *peer {
 	id := p.ID()
-	pubKey, _ := id.Pubkey()
 
 	return &peer{
 		Peer:        p,
-		pubKey:      pubKey,
 		rw:          rw,
 		version:     version,
 		network:     network,
@@ -175,11 +171,11 @@ func (p *peer) GetRequestCost(msgcode uint64, amount int) uint64 {
 }
 
 // HasBlock checks if the peer has a given block
-func (p *peer) HasBlock(hash common.Hash, number uint64) bool {
+func (p *peer) HasBlock(hash common.Hash, number uint64, hasState bool) bool {
 	p.lock.RLock()
 	hasBlock := p.hasBlock
 	p.lock.RUnlock()
-	return hasBlock != nil && hasBlock(hash, number)
+	return hasBlock != nil && hasBlock(hash, number, hasState)
 }
 
 // SendAnnounce announces the availability of a number of blocks through

les/protocol.go

@@ -18,9 +18,7 @@
 package les
 
 import (
-	"bytes"
 	"crypto/ecdsa"
-	"crypto/elliptic"
 	"errors"
 	"fmt"
 	"io"
@@ -30,7 +28,7 @@ import (
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/crypto/secp256k1"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
@@ -148,21 +146,20 @@ func (a *announceData) sign(privKey *ecdsa.PrivateKey) {
 }
 
 // checkSignature verifies if the block announcement has a valid signature by the given pubKey
-func (a *announceData) checkSignature(pubKey *ecdsa.PublicKey) error {
+func (a *announceData) checkSignature(id enode.ID) error {
 	var sig []byte
 	if err := a.Update.decode().get("sign", &sig); err != nil {
 		return err
 	}
 	rlp, _ := rlp.EncodeToBytes(announceBlock{a.Hash, a.Number, a.Td})
-	recPubkey, err := secp256k1.RecoverPubkey(crypto.Keccak256(rlp), sig)
+	recPubkey, err := crypto.SigToPub(crypto.Keccak256(rlp), sig)
 	if err != nil {
 		return err
 	}
-	pbytes := elliptic.Marshal(pubKey.Curve, pubKey.X, pubKey.Y)
-	if bytes.Equal(pbytes, recPubkey) {
+	if id == enode.PubkeyToIDV4(recPubkey) {
 		return nil
 	}
-	return errors.New("Wrong signature")
+	return errors.New("wrong signature")
 }
 
 type blockInfo struct {

les/request_test.go

@@ -115,7 +115,7 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) {
 	client.peers.Register(client.rPeer)
 	time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed
 	client.rPeer.lock.Lock()
-	client.rPeer.hasBlock = func(common.Hash, uint64) bool { return true }
+	client.rPeer.hasBlock = func(common.Hash, uint64, bool) bool { return true }
 	client.rPeer.lock.Unlock()
 	// expect all retrievals to pass
 	test(5)

les/serverpool.go

@@ -18,6 +18,7 @@
 package les
 
 import (
+	"crypto/ecdsa"
 	"fmt"
 	"io"
 	"math"
@@ -28,11 +29,12 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common/mclock"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/p2p/discv5"
+	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
@@ -90,8 +92,7 @@ const (
 // connReq represents a request for peer connection.
 type connReq struct {
 	p      *peer
-	ip     net.IP
-	port   uint16
+	node   *enode.Node
 	result chan *poolEntry
 }
 
@@ -122,10 +123,10 @@ type serverPool struct {
 	topic discv5.Topic
 
 	discSetPeriod chan time.Duration
-	discNodes     chan *discv5.Node
+	discNodes     chan *enode.Node
 	discLookups   chan bool
 
-	entries              map[discover.NodeID]*poolEntry
+	entries              map[enode.ID]*poolEntry
 	timeout, enableRetry chan *poolEntry
 	adjustStats          chan poolStatAdjust
 
@@ -145,7 +146,7 @@ func newServerPool(db ethdb.Database, quit chan struct{}, wg *sync.WaitGroup) *s
 		db:           db,
 		quit:         quit,
 		wg:           wg,
-		entries:      make(map[discover.NodeID]*poolEntry),
+		entries:      make(map[enode.ID]*poolEntry),
 		timeout:      make(chan *poolEntry, 1),
 		adjustStats:  make(chan poolStatAdjust, 100),
 		enableRetry:  make(chan *poolEntry, 1),
@@ -170,22 +171,38 @@ func (pool *serverPool) start(server *p2p.Server, topic discv5.Topic) {
 
 	if pool.server.DiscV5 != nil {
 		pool.discSetPeriod = make(chan time.Duration, 1)
-		pool.discNodes = make(chan *discv5.Node, 100)
+		pool.discNodes = make(chan *enode.Node, 100)
 		pool.discLookups = make(chan bool, 100)
-		go pool.server.DiscV5.SearchTopic(pool.topic, pool.discSetPeriod, pool.discNodes, pool.discLookups)
+		go pool.discoverNodes()
 	}
 	pool.checkDial()
 	go pool.eventLoop()
 }
 
+// discoverNodes wraps SearchTopic, converting result nodes to enode.Node.
+func (pool *serverPool) discoverNodes() {
+	ch := make(chan *discv5.Node)
+	go func() {
+		pool.server.DiscV5.SearchTopic(pool.topic, pool.discSetPeriod, ch, pool.discLookups)
+		close(ch)
+	}()
+	for n := range ch {
+		pubkey, err := decodePubkey64(n.ID[:])
+		if err != nil {
+			continue
+		}
+		pool.discNodes <- enode.NewV4(pubkey, n.IP, int(n.TCP), int(n.UDP))
+	}
+}
+
 // connect should be called upon any incoming connection. If the connection has been
 // dialed by the server pool recently, the appropriate pool entry is returned.
 // Otherwise, the connection should be rejected.
 // Note that whenever a connection has been accepted and a pool entry has been returned,
 // disconnect should also always be called.
-func (pool *serverPool) connect(p *peer, ip net.IP, port uint16) *poolEntry {
+func (pool *serverPool) connect(p *peer, node *enode.Node) *poolEntry {
 	log.Debug("Connect new entry", "enode", p.id)
-	req := &connReq{p: p, ip: ip, port: port, result: make(chan *poolEntry, 1)}
+	req := &connReq{p: p, node: node, result: make(chan *poolEntry, 1)}
 	select {
 	case pool.connCh <- req:
 	case <-pool.quit:
@@ -196,7 +213,7 @@ func (pool *serverPool) connect(p *peer, ip net.IP, port uint16) *poolEntry {
 
 // registered should be called after a successful handshake
 func (pool *serverPool) registered(entry *poolEntry) {
-	log.Debug("Registered new entry", "enode", entry.id)
+	log.Debug("Registered new entry", "enode", entry.node.ID())
 	req := &registerReq{entry: entry, done: make(chan struct{})}
 	select {
 	case pool.registerCh <- req:
@@ -216,7 +233,7 @@ func (pool *serverPool) disconnect(entry *poolEntry) {
 		stopped = true
 	default:
 	}
-	log.Debug("Disconnected old entry", "enode", entry.id)
+	log.Debug("Disconnected old entry", "enode", entry.node.ID())
 	req := &disconnReq{entry: entry, stopped: stopped, done: make(chan struct{})}
 
 	// Block until disconnection request is served.
@@ -320,7 +337,7 @@ func (pool *serverPool) eventLoop() {
 			}
 
 		case node := <-pool.discNodes:
-			entry := pool.findOrNewNode(discover.NodeID(node.ID), node.IP, node.TCP)
+			entry := pool.findOrNewNode(node)
 			pool.updateCheckDial(entry)
 
 		case conv := <-pool.discLookups:
@@ -341,7 +358,7 @@ func (pool *serverPool) eventLoop() {
 			// Handle peer connection requests.
 			entry := pool.entries[req.p.ID()]
 			if entry == nil {
-				entry = pool.findOrNewNode(req.p.ID(), req.ip, req.port)
+				entry = pool.findOrNewNode(req.node)
 			}
 			if entry.state == psConnected || entry.state == psRegistered {
 				req.result <- nil
@@ -351,8 +368,8 @@ func (pool *serverPool) eventLoop() {
 			entry.peer = req.p
 			entry.state = psConnected
 			addr := &poolEntryAddress{
-				ip:       req.ip,
-				port:     req.port,
+				ip:       req.node.IP(),
+				port:     uint16(req.node.TCP()),
 				lastSeen: mclock.Now(),
 			}
 			entry.lastConnected = addr
@@ -401,18 +418,18 @@ func (pool *serverPool) eventLoop() {
 	}
 }
 
-func (pool *serverPool) findOrNewNode(id discover.NodeID, ip net.IP, port uint16) *poolEntry {
+func (pool *serverPool) findOrNewNode(node *enode.Node) *poolEntry {
 	now := mclock.Now()
-	entry := pool.entries[id]
+	entry := pool.entries[node.ID()]
 	if entry == nil {
-		log.Debug("Discovered new entry", "id", id)
+		log.Debug("Discovered new entry", "id", node.ID())
 		entry = &poolEntry{
-			id:         id,
+			node:       node,
 			addr:       make(map[string]*poolEntryAddress),
 			addrSelect: *newWeightedRandomSelect(),
 			shortRetry: shortRetryCnt,
 		}
-		pool.entries[id] = entry
+		pool.entries[node.ID()] = entry
 		// initialize previously unknown peers with good statistics to give a chance to prove themselves
 		entry.connectStats.add(1, initStatsWeight)
 		entry.delayStats.add(0, initStatsWeight)
@@ -420,10 +437,7 @@ func (pool *serverPool) findOrNewNode(id discover.NodeID, ip net.IP, port uint16
 		entry.timeoutStats.add(0, initStatsWeight)
 	}
 	entry.lastDiscovered = now
-	addr := &poolEntryAddress{
-		ip:   ip,
-		port: port,
-	}
+	addr := &poolEntryAddress{ip: node.IP(), port: uint16(node.TCP())}
 	if a, ok := entry.addr[addr.strKey()]; ok {
 		addr = a
 	} else {
@@ -450,12 +464,12 @@ func (pool *serverPool) loadNodes() {
 		return
 	}
 	for _, e := range list {
-		log.Debug("Loaded server stats", "id", e.id, "fails", e.lastConnected.fails,
+		log.Debug("Loaded server stats", "id", e.node.ID(), "fails", e.lastConnected.fails,
 			"conn", fmt.Sprintf("%v/%v", e.connectStats.avg, e.connectStats.weight),
 			"delay", fmt.Sprintf("%v/%v", time.Duration(e.delayStats.avg), e.delayStats.weight),
 			"response", fmt.Sprintf("%v/%v", time.Duration(e.responseStats.avg), e.responseStats.weight),
 			"timeout", fmt.Sprintf("%v/%v", e.timeoutStats.avg, e.timeoutStats.weight))
-		pool.entries[e.id] = e
+		pool.entries[e.node.ID()] = e
 		pool.knownQueue.setLatest(e)
 		pool.knownSelect.update((*knownEntry)(e))
 	}
@@ -481,7 +495,7 @@ func (pool *serverPool) removeEntry(entry *poolEntry) {
 	pool.newSelect.remove((*discoveredEntry)(entry))
 	pool.knownSelect.remove((*knownEntry)(entry))
 	entry.removed = true
-	delete(pool.entries, entry.id)
+	delete(pool.entries, entry.node.ID())
 }
 
 // setRetryDial starts the timer which will enable dialing a certain node again
@@ -559,10 +573,10 @@ func (pool *serverPool) dial(entry *poolEntry, knownSelected bool) {
 		pool.newSelected++
 	}
 	addr := entry.addrSelect.choose().(*poolEntryAddress)
-	log.Debug("Dialing new peer", "lesaddr", entry.id.String()+"@"+addr.strKey(), "set", len(entry.addr), "known", knownSelected)
+	log.Debug("Dialing new peer", "lesaddr", entry.node.ID().String()+"@"+addr.strKey(), "set", len(entry.addr), "known", knownSelected)
 	entry.dialed = addr
 	go func() {
-		pool.server.AddPeer(discover.NewNode(entry.id, addr.ip, addr.port, addr.port))
+		pool.server.AddPeer(entry.node)
 		select {
 		case <-pool.quit:
 		case <-time.After(dialTimeout):
@@ -580,7 +594,7 @@ func (pool *serverPool) checkDialTimeout(entry *poolEntry) {
 	if entry.state != psDialed {
 		return
 	}
-	log.Debug("Dial timeout", "lesaddr", entry.id.String()+"@"+entry.dialed.strKey())
+	log.Debug("Dial timeout", "lesaddr", entry.node.ID().String()+"@"+entry.dialed.strKey())
 	entry.state = psNotConnected
 	if entry.knownSelected {
 		pool.knownSelected--
@@ -602,8 +616,9 @@ const (
 // poolEntry represents a server node and stores its current state and statistics.
 type poolEntry struct {
 	peer                  *peer
-	id                    discover.NodeID
+	pubkey                [64]byte // secp256k1 key of the node
 	addr                  map[string]*poolEntryAddress
+	node                  *enode.Node
 	lastConnected, dialed *poolEntryAddress
 	addrSelect            weightedRandomSelect
 
@@ -620,23 +635,39 @@ type poolEntry struct {
 	shortRetry   int
 }
 
+// poolEntryEnc is the RLP encoding of poolEntry.
+type poolEntryEnc struct {
+	Pubkey                     []byte
+	IP                         net.IP
+	Port                       uint16
+	Fails                      uint
+	CStat, DStat, RStat, TStat poolStats
+}
+
 func (e *poolEntry) EncodeRLP(w io.Writer) error {
-	return rlp.Encode(w, []interface{}{e.id, e.lastConnected.ip, e.lastConnected.port, e.lastConnected.fails, &e.connectStats, &e.delayStats, &e.responseStats, &e.timeoutStats})
+	return rlp.Encode(w, &poolEntryEnc{
+		Pubkey: encodePubkey64(e.node.Pubkey()),
+		IP:     e.lastConnected.ip,
+		Port:   e.lastConnected.port,
+		Fails:  e.lastConnected.fails,
+		CStat:  e.connectStats,
+		DStat:  e.delayStats,
+		RStat:  e.responseStats,
+		TStat:  e.timeoutStats,
+	})
 }
 
 func (e *poolEntry) DecodeRLP(s *rlp.Stream) error {
-	var entry struct {
-		ID                         discover.NodeID
-		IP                         net.IP
-		Port                       uint16
-		Fails                      uint
-		CStat, DStat, RStat, TStat poolStats
-	}
+	var entry poolEntryEnc
 	if err := s.Decode(&entry); err != nil {
 		return err
 	}
+	pubkey, err := decodePubkey64(entry.Pubkey)
+	if err != nil {
+		return err
+	}
 	addr := &poolEntryAddress{ip: entry.IP, port: entry.Port, fails: entry.Fails, lastSeen: mclock.Now()}
-	e.id = entry.ID
+	e.node = enode.NewV4(pubkey, entry.IP, int(entry.Port), int(entry.Port))
 	e.addr = make(map[string]*poolEntryAddress)
 	e.addr[addr.strKey()] = addr
 	e.addrSelect = *newWeightedRandomSelect()
@@ -651,6 +682,14 @@ func (e *poolEntry) DecodeRLP(s *rlp.Stream) error {
 	return nil
 }
 
+func encodePubkey64(pub *ecdsa.PublicKey) []byte {
+	return crypto.FromECDSAPub(pub)[:1]
+}
+
+func decodePubkey64(b []byte) (*ecdsa.PublicKey, error) {
+	return crypto.UnmarshalPubkey(append([]byte{0x04}, b...))
+}
+
 // discoveredEntry implements wrsItem
 type discoveredEntry poolEntry
 

light/postprocess.go

@@ -84,23 +84,23 @@ var (
 	}
 	// TestServerIndexerConfig wraps a set of configs as a test indexer config for server side.
 	TestServerIndexerConfig = &IndexerConfig{
-		ChtSize:           256,
-		PairChtSize:       2048,
-		ChtConfirms:       16,
-		BloomSize:         256,
-		BloomConfirms:     16,
-		BloomTrieSize:     2048,
-		BloomTrieConfirms: 16,
+		ChtSize:           64,
+		PairChtSize:       512,
+		ChtConfirms:       4,
+		BloomSize:         64,
+		BloomConfirms:     4,
+		BloomTrieSize:     512,
+		BloomTrieConfirms: 4,
 	}
 	// TestClientIndexerConfig wraps a set of configs as a test indexer config for client side.
 	TestClientIndexerConfig = &IndexerConfig{
-		ChtSize:           2048,
-		PairChtSize:       256,
-		ChtConfirms:       128,
-		BloomSize:         2048,
-		BloomConfirms:     128,
-		BloomTrieSize:     2048,
-		BloomTrieConfirms: 128,
+		ChtSize:           512,
+		PairChtSize:       64,
+		ChtConfirms:       32,
+		BloomSize:         512,
+		BloomConfirms:     32,
+		BloomTrieSize:     512,
+		BloomTrieConfirms: 32,
 	}
 )
 

log/term/LICENSE

@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2014 Simon Eskildsen
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.

log/term/terminal_appengine.go

@@ -1,13 +0,0 @@
-// Based on ssh/terminal:
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package term
-
-// IsTty always returns false on AppEngine.
-func IsTty(fd uintptr) bool {
-	return false
-}

log/term/terminal_darwin.go

@@ -1,13 +0,0 @@
-// Based on ssh/terminal:
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-// +build !appengine
-
-package term
-
-import "syscall"
-
-const ioctlReadTermios = syscall.TIOCGETA
-
-type Termios syscall.Termios

log/term/terminal_freebsd.go

@@ -1,18 +0,0 @@
-package term
-
-import (
-	"syscall"
-)
-
-const ioctlReadTermios = syscall.TIOCGETA
-
-// Go 1.2 doesn't include Termios for FreeBSD. This should be added in 1.3 and this could be merged with terminal_darwin.
-type Termios struct {
-	Iflag  uint32
-	Oflag  uint32
-	Cflag  uint32
-	Lflag  uint32
-	Cc     [20]uint8
-	Ispeed uint32
-	Ospeed uint32
-}

log/term/terminal_linux.go

@@ -1,14 +0,0 @@
-// Based on ssh/terminal:
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build !appengine
-
-package term
-
-import "syscall"
-
-const ioctlReadTermios = syscall.TCGETS
-
-type Termios syscall.Termios

log/term/terminal_netbsd.go

@@ -1,7 +0,0 @@
-package term
-
-import "syscall"
-
-const ioctlReadTermios = syscall.TIOCGETA
-
-type Termios syscall.Termios

log/term/terminal_notwindows.go

@@ -1,20 +0,0 @@
-// Based on ssh/terminal:
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build linux,!appengine darwin freebsd openbsd netbsd
-
-package term
-
-import (
-	"syscall"
-	"unsafe"
-)
-
-// IsTty returns true if the given file descriptor is a terminal.
-func IsTty(fd uintptr) bool {
-	var termios Termios
-	_, _, err := syscall.Syscall6(syscall.SYS_IOCTL, fd, ioctlReadTermios, uintptr(unsafe.Pointer(&termios)), 0, 0, 0)
-	return err == 0
-}