build: use golangci-lint (#20295)

* build: use golangci-lint

This changes build/ci.go to download and run golangci-lint instead
of gometalinter.

* core/state: fix unnecessary conversion

* p2p/simulations: fix lock copying (found by go vet)

* signer/core: fix unnecessary conversions

* crypto/ecies: remove unused function cmpPublic

* core/rawdb: remove unused function print

* core/state: remove unused function xTestFuzzCutter

* core/vm: disable TestWriteExpectedValues in a different way

* core/forkid: remove unused function checksum

* les: remove unused type proofsData

* cmd/utils: remove unused functions prefixedNames, prefixFor

* crypto/bn256: run goimports

* p2p/nat: fix goimports lint issue

* cmd/clef: avoid using unkeyed struct fields

* les: cancel context in testRequest

* rlp: delete unreachable code

* core: gofmt

* internal/build: simplify DownloadFile for Go 1.11 compatibility

* build: remove go test --short flag

* .travis.yml: disable build cache

* whisper/whisperv6: fix ineffectual assignment in TestWhisperIdentityManagement

* .golangci.yml: enable goconst and ineffassign linters

* build: print message when there are no lint issues

* internal/build: refactor download a bit

.gitignore

@@ -24,6 +24,7 @@ build/_vendor/pkg
 
 # used by the Makefile
 /build/_workspace/
+/build/cache/
 /build/bin/
 /geth*.zip
 

.golangci.yml

@@ -0,0 +1,45 @@
+# This file configures github.com/golangci/golangci-lint.
+
+run:
+  timeout: 2m
+  tests: true
+  # default is true. Enables skipping of directories:
+  #   vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+  skip-dirs-use-default: true
+
+linters:
+  disable-all: true
+  enable:
+    - deadcode
+    - goconst
+    - goimports
+    - gosimple
+    - govet
+    - ineffassign
+    - misspell
+    # - staticcheck
+    - unconvert
+    # - unused
+    - varcheck
+
+linters-settings:
+  gofmt:
+    simplify: true
+  goconst:
+    min-len: 3 # minimum length of string constant
+    min-occurrences: 6 # minimum number of occurrences
+
+issues:
+  exclude-rules:
+    - path: crypto/blake2b/
+      linters:
+        - deadcode
+    - path: crypto/bn256/cloudflare
+      linters:
+        - deadcode
+    - path: p2p/discv5/
+      linters:
+        - deadcode
+    - path: core/vm/instructions_test.go
+      linters:
+        - goconst

.travis.yml

@@ -87,12 +87,9 @@ jobs:
             - fakeroot
             - python-bzrlib
             - python-paramiko
-      cache:
-        directories:
-          - $HOME/.gobundle
       script:
         - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
-        - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -goversion 1.13.4 -gohash 95dbeab442ee2746b9acf0934c8e2fc26414a0565c008631b04addb8c02e7624 -gobundle $HOME/.gobundle/go.tar.gz
+        - go run build/ci.go debsrc -goversion 1.13.4 -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
 
     # This builder does the Linux Azure uploads
     - stage: build

build/checksums.txt

@@ -0,0 +1,19 @@
+# This file contains sha256 checksums of optional build dependencies.
+
+95dbeab442ee2746b9acf0934c8e2fc26414a0565c008631b04addb8c02e7624  go1.13.4.src.tar.gz
+
+1fcbc9e36f4319eeed02beb8cfd1b3d425ffc2f90ddf09a80f18d5064c51e0cb  golangci-lint-1.21.0-linux-386.tar.gz
+267b4066e67139a38d29499331a002d6a29ad5be7aafc83db3b1e88f1b027f90  golangci-lint-1.21.0-linux-armv6.tar.gz
+a602c1f25f90e46e621019cff0a8cb3f4e1837011f3537f15e730d6a9ebf507b  golangci-lint-1.21.0-freebsd-armv7.tar.gz
+2c861f8dc56b560474aa27cab0c075991628cc01af3451e27ac82f5d10d5106b  golangci-lint-1.21.0-linux-amd64.tar.gz
+a1c39e055280e755acaa906e7abfc20b99a5c28be8af541c57fbc44abbb20dde  golangci-lint-1.21.0-linux-arm64.tar.gz
+a8f8bda8c6a4136acf858091077830b1e83ad5612606cb69d5dced869ce00bd8  golangci-lint-1.21.0-linux-ppc64le.tar.gz
+0a8a8c3bc660ccbca668897ab520f7ee9878f16cc8e4dd24fe46236ceec97ba3  golangci-lint-1.21.0-freebsd-armv6.tar.gz
+699b07f45e216571f54002bcbd83b511c4801464a422162158e299587b095b18  golangci-lint-1.21.0-freebsd-amd64.tar.gz
+980fb4993942154bb5c8129ea3b86de09574fe81b24384ebb58cd7a9d2f04483  golangci-lint-1.21.0-linux-armv7.tar.gz
+f15b689088a47f20d5d3c1d945e9ee7c6238f2b84ea468b5f886cf8713dce62e  golangci-lint-1.21.0-windows-386.zip
+2e40ded7adcf11e59013cb15c24438b15a86526ca241edfcfdf1abd73a5280a8  golangci-lint-1.21.0-windows-amd64.zip
+6052c7cfea4d6dc2fc722f6c12792a5ec087420198db495afffbc22052653bf7  golangci-lint-1.21.0-freebsd-386.tar.gz
+ca00b8eacf9af14a71b908b4149606c762aa5c0eac781e74ca0abedfdfdf6c8c  golangci-lint-1.21.0-linux-s390x.tar.gz
+1365455940c342f95718159d89d66ad2eef19f0846c3e87023e915a3527b929f  golangci-lint-1.21.0-darwin-386.tar.gz
+2b2713ec5007e67883aa501eebb81f22abfab0cf0909134ba90f60a066db3760  golangci-lint-1.21.0-darwin-amd64.tar.gz

build/ci.go

@@ -58,7 +58,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/internal/build"
 	"github.com/ethereum/go-ethereum/params"
 )
@@ -331,7 +330,7 @@ func doTest(cmdline []string) {
 	// Test a single package at a time. CI builders are slow
 	// and some tests run into timeouts under load.
 	gotest := goTool("test", buildFlags(env)...)
-	gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m", "--short")
+	gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m")
 	if *coverage {
 		gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover")
 	}
@@ -340,39 +339,38 @@ func doTest(cmdline []string) {
 	build.MustRun(gotest)
 }
 
-// runs gometalinter on requested packages
+// doLint runs golangci-lint on requested packages.
 func doLint(cmdline []string) {
+	var (
+		cachedir = flag.String("cachedir", "./build/cache", "directory for caching golangci-lint binary.")
+	)
 	flag.CommandLine.Parse(cmdline)
-
 	packages := []string{"./..."}
 	if len(flag.CommandLine.Args()) > 0 {
 		packages = flag.CommandLine.Args()
 	}
-	// Get metalinter and install all supported linters
-	build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v2"))
-	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), "--install")
 
-	// Run fast linters batched together
+	linter := downloadLinter(*cachedir)
-	configs := []string{
+	lflags := []string{"run", "--config", ".golangci.yml"}
-		"--vendor",
+	build.MustRunCommand(linter, append(lflags, packages...)...)
-		"--tests",
+	fmt.Println("You have achieved perfection.")
-		"--deadline=2m",
-		"--disable-all",
-		"--enable=goimports",
-		"--enable=varcheck",
-		"--enable=vet",
-		"--enable=gofmt",
-		"--enable=misspell",
-		"--enable=goconst",
-		"--min-occurrences=6", // for goconst
 }
-	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...)
 
-	// Run slow linters one by one
+// downloadLinter downloads and unpacks golangci-lint.
-	for _, linter := range []string{"unconvert", "gosimple"} {
+func downloadLinter(cachedir string) string {
-		configs = []string{"--vendor", "--tests", "--deadline=10m", "--disable-all", "--enable=" + linter}
+	const version = "1.21.0"
-		build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...)
+
+	csdb := build.MustLoadChecksums("build/checksums.txt")
+	base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH)
+	url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s.tar.gz", version, base)
+	archivePath := filepath.Join(cachedir, base+".tar.gz")
+	if err := csdb.DownloadFile(url, archivePath); err != nil {
+		log.Fatal(err)
 	}
+	if err := build.ExtractTarballArchive(archivePath, cachedir); err != nil {
+		log.Fatal(err)
+	}
+	return filepath.Join(cachedir, base, "golangci-lint")
 }
 
 // Release Packaging
@@ -476,8 +474,7 @@ func maybeSkipArchive(env build.Environment) {
 func doDebianSource(cmdline []string) {
 	var (
 		goversion = flag.String("goversion", "", `Go version to build with (will be included in the source package)`)
-		gobundle  = flag.String("gobundle", "/tmp/go.tar.gz", `Filesystem path to cache the downloaded Go bundles at`)
+		cachedir  = flag.String("cachedir", "./build/cache", `Filesystem path to cache the downloaded Go bundles at`)
-		gohash    = flag.String("gohash", "", `SHA256 checksum of the Go sources requested to build with`)
 		signer    = flag.String("signer", "", `Signing key name, also used as package author`)
 		upload    = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`)
 		sshUser   = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
@@ -495,24 +492,25 @@ func doDebianSource(cmdline []string) {
 		gpg.Stdin = bytes.NewReader(key)
 		build.MustRun(gpg)
 	}
-	// Download and verify the Go source package
+
-	if err := build.EnsureGoSources(*goversion, hexutil.MustDecode("0x"+*gohash), *gobundle); err != nil {
+	// Download and verify the Go source package.
-		log.Fatalf("Failed to ensure Go source package: %v", err)
+	gobundle := downloadGoSources(*goversion, *cachedir)
-	}
+
-	// Create Debian packages and upload them
+	// Create Debian packages and upload them.
 	for _, pkg := range debPackages {
 		for distro, goboot := range debDistroGoBoots {
-			// Prepare the debian package with the go-ethereum sources
+			// Prepare the debian package with the go-ethereum sources.
 			meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables)
 			pkgdir := stageDebianSource(*workdir, meta)
 
-			// Ship the Go sources along so we have a proper thing to build with
+			// Add Go source code.
-			if err := build.ExtractTarballArchive(*gobundle, pkgdir); err != nil {
+			if err := build.ExtractTarballArchive(gobundle, pkgdir); err != nil {
 				log.Fatalf("Failed to extract Go sources: %v", err)
 			}
 			if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil {
 				log.Fatalf("Failed to rename Go source folder: %v", err)
 			}
+
 			// Run the packaging and upload to the PPA
 			debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz")
 			debuild.Dir = pkgdir
@@ -534,6 +532,17 @@ func doDebianSource(cmdline []string) {
 	}
 }
 
+func downloadGoSources(version string, cachedir string) string {
+	csdb := build.MustLoadChecksums("build/checksums.txt")
+	file := fmt.Sprintf("go%s.src.tar.gz", version)
+	url := "https://dl.google.com/go/" + file
+	dst := filepath.Join(cachedir, file)
+	if err := csdb.DownloadFile(url, dst); err != nil {
+		log.Fatal(err)
+	}
+	return dst
+}
+
 func ppaUpload(workdir, ppa, sshUser string, files []string) {
 	p := strings.Split(ppa, "/")
 	if len(p) != 2 {

cmd/clef/main.go

@@ -760,21 +760,19 @@ func testExternalUI(api *core.SignerAPI) {
 		api.UI.ShowInfo("Please approve the next request for signing a clique header")
 		time.Sleep(delay)
 		cliqueHeader := types.Header{
-			common.HexToHash("0000H45H"),
+			ParentHash:  common.HexToHash("0000H45H"),
-			common.HexToHash("0000H45H"),
+			UncleHash:   common.HexToHash("0000H45H"),
-			common.HexToAddress("0000H45H"),
+			Coinbase:    common.HexToAddress("0000H45H"),
-			common.HexToHash("0000H00H"),
+			Root:        common.HexToHash("0000H00H"),
-			common.HexToHash("0000H45H"),
+			TxHash:      common.HexToHash("0000H45H"),
-			common.HexToHash("0000H45H"),
+			ReceiptHash: common.HexToHash("0000H45H"),
-			types.Bloom{},
+			Difficulty:  big.NewInt(1337),
-			big.NewInt(1337),
+			Number:      big.NewInt(1337),
-			big.NewInt(1337),
+			GasLimit:    1338,
-			1338,
+			GasUsed:     1338,
-			1338,
+			Time:        1338,
-			1338,
+			Extra:       []byte("Extra data Extra data Extra data  Extra data  Extra data  Extra data  Extra data Extra data"),
-			[]byte("Extra data Extra data Extra data  Extra data  Extra data  Extra data  Extra data Extra data"),
+			MixDigest:   common.HexToHash("0x0000H45H"),
-			common.HexToHash("0x0000H45H"),
-			types.BlockNonce{},
 		}
 		cliqueRlp, err := rlp.EncodeToBytes(cliqueHeader)
 		if err != nil {
@@ -938,7 +936,7 @@ func GenDoc(ctx *cli.Context) {
 			"of the work in canonicalizing and making sense of the data, and it's up to the UI to present" +
 			"the user with the contents of the `message`"
 		sighash, msg := accounts.TextAndHash([]byte("hello world"))
-		messages := []*core.NameValueType{{"message", msg, accounts.MimetypeTextPlain}}
+		messages := []*core.NameValueType{{Name: "message", Value: msg, Typ: accounts.MimetypeTextPlain}}
 
 		add("SignDataRequest", desc, &core.SignDataRequest{
 			Address:     common.NewMixedcaseAddress(a),
@@ -969,8 +967,8 @@ func GenDoc(ctx *cli.Context) {
 		add("SignTxRequest", desc, &core.SignTxRequest{
 			Meta: meta,
 			Callinfo: []core.ValidationInfo{
-				{"Warning", "Something looks odd, show this message as a warning"},
+				{Typ: "Warning", Message: "Something looks odd, show this message as a warning"},
-				{"Info", "User should see this aswell"},
+				{Typ: "Info", Message: "User should see this as well"},
 			},
 			Transaction: core.SendTxArgs{
 				Data:     &data,
@@ -1036,16 +1034,21 @@ func GenDoc(ctx *cli.Context) {
 			&core.ListRequest{
 				Meta: meta,
 				Accounts: []accounts.Account{
-					{a, accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/a"}},
+					{Address: a, URL: accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/a"}},
-					{b, accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/b"}}},
+					{Address: b, URL: accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/b"}}},
 			})
 
 		add("ListResponse", "Response to list request. The response contains a list of all addresses to show to the caller. "+
 			"Note: the UI is free to respond with any address the caller, regardless of whether it exists or not",
 			&core.ListResponse{
 				Accounts: []accounts.Account{
-					{common.HexToAddress("0xcowbeef000000cowbeef00000000000000000c0w"), accounts.URL{Path: ".. ignored .."}},
+					{
-					{common.HexToAddress("0xffffffffffffffffffffffffffffffffffffffff"), accounts.URL{}},
+						Address: common.HexToAddress("0xcowbeef000000cowbeef00000000000000000c0w"),
+						URL:     accounts.URL{Path: ".. ignored .."},
+					},
+					{
+						Address: common.HexToAddress("0xffffffffffffffffffffffffffffffffffffffff"),
+					},
 				}})
 	}
 

cmd/utils/customflags.go

@@ -185,28 +185,6 @@ func GlobalBig(ctx *cli.Context, name string) *big.Int {
 	return (*big.Int)(val.(*bigValue))
 }
 
-func prefixFor(name string) (prefix string) {
-	if len(name) == 1 {
-		prefix = "-"
-	} else {
-		prefix = "--"
-	}
-
-	return
-}
-
-func prefixedNames(fullName string) (prefixed string) {
-	parts := strings.Split(fullName, ",")
-	for i, name := range parts {
-		name = strings.Trim(name, " ")
-		prefixed += prefixFor(name) + name
-		if i < len(parts)-1 {
-			prefixed += ", "
-		}
-	}
-	return
-}
-
 // Expands a file path
 // 1. replace tilde with users home dir
 // 2. expands embedded environment variables

core/blockchain.go

@@ -42,7 +42,7 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
-	"github.com/hashicorp/golang-lru"
+	lru "github.com/hashicorp/golang-lru"
 )
 
 var (

core/forkid/forkid.go

@@ -186,13 +186,6 @@ func newFilter(config *params.ChainConfig, genesis common.Hash, headfn func() ui
 	}
 }
 
-// checksum calculates the IEEE CRC32 checksum of a block number.
-func checksum(fork uint64) uint32 {
-	var blob [8]byte
-	binary.BigEndian.PutUint64(blob[:], fork)
-	return crc32.ChecksumIEEE(blob[:])
-}
-
 // checksumUpdate calculates the next IEEE CRC32 checksum based on the previous
 // one and a fork block number (equivalent to CRC32(original-blob || fork)).
 func checksumUpdate(hash uint32, fork uint64) uint32 {

core/rawdb/freezer_table_test.go

@@ -41,14 +41,6 @@ func getChunk(size int, b int) []byte {
 	return data
 }
 
-func print(t *testing.T, f *freezerTable, item uint64) {
-	a, err := f.Retrieve(item)
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Logf("db[%d] =  %x\n", item, a)
-}
-
 // TestFreezerBasics test initializing a freezertable from scratch, writing to the table,
 // and reading it back.
 func TestFreezerBasics(t *testing.T) {

core/state/state_object_test.go

@@ -18,10 +18,7 @@ package state
 
 import (
 	"bytes"
-	"fmt"
-	"math/rand"
 	"testing"
-	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 )
@@ -35,9 +32,7 @@ func BenchmarkCutOriginal(b *testing.B) {
 
 func BenchmarkCutsetterFn(b *testing.B) {
 	value := common.HexToHash("0x01")
-	cutSetFn := func(r rune) bool {
+	cutSetFn := func(r rune) bool { return r == 0 }
-		return int32(r) == int32(0)
-	}
 	for i := 0; i < b.N; i++ {
 		bytes.TrimLeftFunc(value[:], cutSetFn)
 	}
@@ -49,22 +44,3 @@ func BenchmarkCutCustomTrim(b *testing.B) {
 		common.TrimLeftZeroes(value[:])
 	}
 }
-
-func xTestFuzzCutter(t *testing.T) {
-	rand.Seed(time.Now().Unix())
-	for {
-		v := make([]byte, 20)
-		zeroes := rand.Intn(21)
-		rand.Read(v[zeroes:])
-		exp := bytes.TrimLeft(v[:], "\x00")
-		got := common.TrimLeftZeroes(v)
-		if !bytes.Equal(exp, got) {
-
-			fmt.Printf("Input %x\n", v)
-			fmt.Printf("Exp %x\n", exp)
-			fmt.Printf("Got %x\n", got)
-			t.Fatalf("Error")
-		}
-		//break
-	}
-}

core/vm/instructions_test.go

@@ -232,7 +232,9 @@ func getResult(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcas
 
 // utility function to fill the json-file with testcases
 // Enable this test to generate the 'testcases_xx.json' files
-func xTestWriteExpectedValues(t *testing.T) {
+func TestWriteExpectedValues(t *testing.T) {
+	t.Skip("Enable this test to create json test cases.")
+
 	for name, method := range twoOpMethods {
 		data, err := json.Marshal(getResult(commonParams, method))
 		if err != nil {
@@ -243,7 +245,6 @@ func xTestWriteExpectedValues(t *testing.T) {
 			t.Fatal(err)
 		}
 	}
-	t.Fatal("This test should not be activated")
 }
 
 // TestJsonTestcases runs through all the testcases defined as json-files

crypto/bn256/bn256_fast.go

@@ -7,17 +7,19 @@
 // Package bn256 implements the Optimal Ate pairing over a 256-bit Barreto-Naehrig curve.
 package bn256
 
-import "github.com/ethereum/go-ethereum/crypto/bn256/cloudflare"
+import (
+	bn256cf "github.com/ethereum/go-ethereum/crypto/bn256/cloudflare"
+)
 
 // G1 is an abstract cyclic group. The zero value is suitable for use as the
 // output of an operation, but cannot be used as an input.
-type G1 = bn256.G1
+type G1 = bn256cf.G1
 
 // G2 is an abstract cyclic group. The zero value is suitable for use as the
 // output of an operation, but cannot be used as an input.
-type G2 = bn256.G2
+type G2 = bn256cf.G2
 
 // PairingCheck calculates the Optimal Ate pairing for a set of points.
 func PairingCheck(a []*G1, b []*G2) bool {
-	return bn256.PairingCheck(a, b)
+	return bn256cf.PairingCheck(a, b)
 }

crypto/ecies/ecies_test.go

@@ -66,22 +66,6 @@ func cmpParams(p1, p2 *ECIESParams) bool {
 		p1.BlockSize == p2.BlockSize
 }
 
-// cmpPublic returns true if the two public keys represent the same pojnt.
-func cmpPublic(pub1, pub2 PublicKey) bool {
-	if pub1.X == nil || pub1.Y == nil {
-		fmt.Println(ErrInvalidPublicKey.Error())
-		return false
-	}
-	if pub2.X == nil || pub2.Y == nil {
-		fmt.Println(ErrInvalidPublicKey.Error())
-		return false
-	}
-	pub1Out := elliptic.Marshal(pub1.Curve, pub1.X, pub1.Y)
-	pub2Out := elliptic.Marshal(pub2.Curve, pub2.X, pub2.Y)
-
-	return bytes.Equal(pub1Out, pub2Out)
-}
-
 // Validate the ECDH component.
 func TestSharedKey(t *testing.T) {
 	prv1, err := GenerateKey(rand.Reader, DefaultCurve, nil)

internal/build/archive.go

@@ -213,11 +213,10 @@ func ExtractTarballArchive(archive string, dest string) error {
 		target := filepath.Join(dest, header.Name)
 
 		switch header.Typeflag {
-		case tar.TypeDir:
+		case tar.TypeReg:
-			if err := os.MkdirAll(target, 0755); err != nil {
+			if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
 				return err
 			}
-		case tar.TypeReg:
 			file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
 			if err != nil {
 				return err

internal/build/download.go

@@ -0,0 +1,149 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package build
+
+import (
+	"bufio"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// ChecksumDB keeps file checksums.
+type ChecksumDB struct {
+	allChecksums []string
+}
+
+// MustLoadChecksums loads a file containing checksums.
+func MustLoadChecksums(file string) *ChecksumDB {
+	content, err := ioutil.ReadFile(file)
+	if err != nil {
+		log.Fatal("can't load checksum file: " + err.Error())
+	}
+	return &ChecksumDB{strings.Split(string(content), "\n")}
+}
+
+// Verify checks whether the given file is valid according to the checksum database.
+func (db *ChecksumDB) Verify(path string) error {
+	fd, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer fd.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil {
+		return err
+	}
+	fileHash := hex.EncodeToString(h.Sum(nil))
+	if !db.findHash(filepath.Base(path), fileHash) {
+		return fmt.Errorf("invalid file hash %s", fileHash)
+	}
+	return nil
+}
+
+func (db *ChecksumDB) findHash(basename, hash string) bool {
+	want := hash + "  " + basename
+	for _, line := range db.allChecksums {
+		if strings.TrimSpace(line) == want {
+			return true
+		}
+	}
+	return false
+}
+
+// DownloadFile downloads a file and verifies its checksum.
+func (db *ChecksumDB) DownloadFile(url, dstPath string) error {
+	if err := db.Verify(dstPath); err == nil {
+		fmt.Printf("%s is up-to-date\n", dstPath)
+		return nil
+	}
+	fmt.Printf("%s is stale\n", dstPath)
+	fmt.Printf("downloading from %s\n", url)
+
+	resp, err := http.Get(url)
+	if err != nil || resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("download error: code %d, err %v", resp.StatusCode, err)
+	}
+	defer resp.Body.Close()
+	if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
+		return err
+	}
+	fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	dst := newDownloadWriter(fd, resp.ContentLength)
+	_, err = io.Copy(dst, resp.Body)
+	dst.Close()
+	if err != nil {
+		return err
+	}
+
+	return db.Verify(dstPath)
+}
+
+type downloadWriter struct {
+	file    *os.File
+	dstBuf  *bufio.Writer
+	size    int64
+	written int64
+	lastpct int64
+}
+
+func newDownloadWriter(dst *os.File, size int64) *downloadWriter {
+	return &downloadWriter{
+		file:   dst,
+		dstBuf: bufio.NewWriter(dst),
+		size:   size,
+	}
+}
+
+func (w *downloadWriter) Write(buf []byte) (int, error) {
+	n, err := w.dstBuf.Write(buf)
+
+	// Report progress.
+	w.written += int64(n)
+	pct := w.written * 10 / w.size * 10
+	if pct != w.lastpct {
+		if w.lastpct != 0 {
+			fmt.Print("...")
+		}
+		fmt.Print(pct, "%")
+		w.lastpct = pct
+	}
+	return n, err
+}
+
+func (w *downloadWriter) Close() error {
+	if w.lastpct > 0 {
+		fmt.Println() // Finish the progress line.
+	}
+	flushErr := w.dstBuf.Flush()
+	closeErr := w.file.Close()
+	if flushErr != nil {
+		return flushErr
+	}
+	return closeErr
+}

internal/build/gosrc.go

@@ -1,81 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package build
-
-import (
-	"bytes"
-	"crypto/sha256"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-)
-
-// EnsureGoSources ensures that path contains a file with the given SHA256 hash,
-// and if not, it downloads a fresh Go source package from upstream and replaces
-// path with it (if the hash matches).
-func EnsureGoSources(version string, hash []byte, path string) error {
-	// Sanity check the destination path to ensure we don't do weird things
-	if !strings.HasSuffix(path, ".tar.gz") {
-		return fmt.Errorf("destination path (%s) must end with .tar.gz", path)
-	}
-	// If the file exists, validate it's hash
-	if archive, err := ioutil.ReadFile(path); err == nil { // Go sources are ~20MB, it's fine to read all
-		hasher := sha256.New()
-		hasher.Write(archive)
-		have := hasher.Sum(nil)
-
-		if bytes.Equal(have, hash) {
-			fmt.Printf("Go %s [%x] available at %s\n", version, hash, path)
-			return nil
-		}
-		fmt.Printf("Go %s hash mismatch (have %x, want %x) at %s, deleting old archive\n", version, have, hash, path)
-		if err := os.Remove(path); err != nil {
-			return err
-		}
-	}
-	// Archive missing or bad hash, download a new one
-	fmt.Printf("Downloading Go %s [want %x] into %s\n", version, hash, path)
-
-	res, err := http.Get(fmt.Sprintf("https://dl.google.com/go/go%s.src.tar.gz", version))
-	if err != nil || res.StatusCode != http.StatusOK {
-		return fmt.Errorf("failed to access Go sources: code %d, err %v", res.StatusCode, err)
-	}
-	defer res.Body.Close()
-
-	archive, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return err
-	}
-	// Sanity check the downloaded archive, save if checks out
-	hasher := sha256.New()
-	hasher.Write(archive)
-
-	if have := hasher.Sum(nil); !bytes.Equal(have, hash) {
-		return fmt.Errorf("downloaded Go %s hash mismatch (have %x, want %x)", version, have, hash)
-	}
-	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
-		return err
-	}
-	if err := ioutil.WriteFile(path, archive, 0644); err != nil {
-		return err
-	}
-	fmt.Printf("Downloaded Go %s [%x] into %s\n", version, hash, path)
-	return nil
-}

les/api_test.go

@@ -326,7 +326,8 @@ func testRequest(ctx context.Context, t *testing.T, client *rpc.Client) bool {
 	var res string
 	var addr common.Address
 	rand.Read(addr[:])
-	c, _ := context.WithTimeout(ctx, time.Second*12)
+	c, cancel := context.WithTimeout(ctx, time.Second*12)
+	defer cancel()
 	err := client.CallContext(c, &res, "eth_getBalance", addr, "latest")
 	if err != nil {
 		t.Log("request error:", err)

les/clientpool.go

@@ -32,7 +32,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/hashicorp/golang-lru"
+	lru "github.com/hashicorp/golang-lru"
 )
 
 const (

les/protocol.go

@@ -235,5 +235,3 @@ func (hn *hashOrNumber) DecodeRLP(s *rlp.Stream) error {
 type CodeData []struct {
 	Value []byte
 }
-
-type proofsData [][]rlp.RawValue

p2p/nat/nat.go

@@ -26,7 +26,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/jackpal/go-nat-pmp"
+	natpmp "github.com/jackpal/go-nat-pmp"
 )
 
 // An implementation of nat.Interface can map local ports to ports

p2p/nat/natpmp.go

@@ -22,7 +22,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/jackpal/go-nat-pmp"
+	natpmp "github.com/jackpal/go-nat-pmp"
 )
 
 // natPMPClient adapts the NAT-PMP protocol implementation so it conforms to

p2p/simulations/events.go

@@ -73,8 +73,7 @@ func NewEvent(v interface{}) *Event {
 	switch v := v.(type) {
 	case *Node:
 		event.Type = EventTypeNode
-		node := *v
+		event.Node = v.copy()
-		event.Node = &node
 	case *Conn:
 		event.Type = EventTypeConn
 		conn := *v

p2p/simulations/http_test.go

@@ -420,16 +420,8 @@ type expectEvents struct {
 }
 
 func (t *expectEvents) nodeEvent(id string, up bool) *Event {
-	node := Node{
+	config := &adapters.NodeConfig{ID: enode.HexID(id)}
-		Config: &adapters.NodeConfig{
+	return &Event{Type: EventTypeNode, Node: newNode(nil, config, up)}
-			ID: enode.HexID(id),
-		},
-		up: up,
-	}
-	return &Event{
-		Type: EventTypeNode,
-		Node: &node,
-	}
 }
 
 func (t *expectEvents) connEvent(one, other string, up bool) *Event {
@@ -450,7 +442,7 @@ loop:
 	for {
 		select {
 		case event := <-t.events:
-			t.Logf("received %s event: %s", event.Type, event)
+			t.Logf("received %s event: %v", event.Type, event)
 
 			if event.Type != EventTypeMsg || event.Msg.Received {
 				continue loop
@@ -486,7 +478,7 @@ func (t *expectEvents) expect(events ...*Event) {
 	for {
 		select {
 		case event := <-t.events:
-			t.Logf("received %s event: %s", event.Type, event)
+			t.Logf("received %s event: %v", event.Type, event)
 
 			expected := events[i]
 			if event.Type != expected.Type {

p2p/simulations/network.go

@@ -119,10 +119,7 @@ func (net *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error)
 	if err != nil {
 		return nil, err
 	}
-	node := &Node{
+	node := newNode(adapterNode, conf, false)
-		Node:   adapterNode,
-		Config: conf,
-	}
 	log.Trace("Node created", "id", conf.ID)
 
 	nodeIndex := len(net.Nodes)
@@ -448,7 +445,7 @@ func (net *Network) GetNodeIDs(excludeIDs ...enode.ID) []enode.ID {
 }
 
 func (net *Network) getNodeIDs(excludeIDs []enode.ID) []enode.ID {
-	// Get all curent nodeIDs
+	// Get all current nodeIDs
 	nodeIDs := make([]enode.ID, 0, len(net.nodeMap))
 	for id := range net.nodeMap {
 		nodeIDs = append(nodeIDs, id)
@@ -735,7 +732,16 @@ type Node struct {
 
 	// up tracks whether or not the node is running
 	up   bool
-	upMu sync.RWMutex
+	upMu *sync.RWMutex
+}
+
+func newNode(an adapters.Node, ac *adapters.NodeConfig, up bool) *Node {
+	return &Node{Node: an, Config: ac, up: up, upMu: new(sync.RWMutex)}
+}
+
+func (n *Node) copy() *Node {
+	configCpy := *n.Config
+	return newNode(n.Node, &configCpy, n.Up())
 }
 
 // Up returns whether the node is currently up (online)
@@ -787,22 +793,19 @@ func (n *Node) MarshalJSON() ([]byte, error) {
 	})
 }
 
-// UnmarshalJSON implements json.Unmarshaler interface so that we don't lose
+// UnmarshalJSON implements json.Unmarshaler interface so that we don't lose Node.up
-// Node.up status. IMPORTANT: The implementation is incomplete; we lose
+// status. IMPORTANT: The implementation is incomplete; we lose p2p.NodeInfo.
-// p2p.NodeInfo.
 func (n *Node) UnmarshalJSON(raw []byte) error {
 	// TODO: How should we turn back NodeInfo into n.Node?
 	// Ticket: https://github.com/ethersphere/go-ethereum/issues/1177
-	node := struct {
+	var node struct {
 		Config *adapters.NodeConfig `json:"config,omitempty"`
 		Up     bool                 `json:"up"`
-	}{}
+	}
 	if err := json.Unmarshal(raw, &node); err != nil {
 		return err
 	}
-
+	*n = *newNode(nil, node.Config, node.Up)
-	n.SetUp(node.Up)
-	n.Config = node.Config
 	return nil
 }
 
@@ -899,7 +902,7 @@ func (net *Network) snapshot(addServices []string, removeServices []string) (*Sn
 		Nodes: make([]NodeSnapshot, len(net.Nodes)),
 	}
 	for i, node := range net.Nodes {
-		snap.Nodes[i] = NodeSnapshot{Node: *node}
+		snap.Nodes[i] = NodeSnapshot{Node: *node.copy()}
 		if !node.Up() {
 			continue
 		}

p2p/simulations/network_test.go

@@ -758,27 +758,22 @@ func benchmarkMinimalServiceTmp(b *testing.B) {
 }
 
 func TestNode_UnmarshalJSON(t *testing.T) {
-	t.Run(
+	t.Run("up_field", func(t *testing.T) {
-		"test unmarshal of Node up field",
-		func(t *testing.T) {
 		runNodeUnmarshalJSON(t, casesNodeUnmarshalJSONUpField())
-		},
+	})
-	)
+	t.Run("config_field", func(t *testing.T) {
-	t.Run(
-		"test unmarshal of Node Config field",
-		func(t *testing.T) {
 		runNodeUnmarshalJSON(t, casesNodeUnmarshalJSONConfigField())
-		},
+	})
-	)
 }
 
 func runNodeUnmarshalJSON(t *testing.T, tests []nodeUnmarshalTestCase) {
 	t.Helper()
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			var got Node
+			var got *Node
-			if err := got.UnmarshalJSON([]byte(tt.marshaled)); err != nil {
+			if err := json.Unmarshal([]byte(tt.marshaled), &got); err != nil {
 				expectErrorMessageToContain(t, err, tt.wantErr)
+				got = nil
 			}
 			expectNodeEquality(t, got, tt.want)
 		})
@@ -788,7 +783,7 @@ func runNodeUnmarshalJSON(t *testing.T, tests []nodeUnmarshalTestCase) {
 type nodeUnmarshalTestCase struct {
 	name      string
 	marshaled string
-	want      Node
+	want      *Node
 	wantErr   string
 }
 
@@ -812,7 +807,7 @@ func expectErrorMessageToContain(t *testing.T, got error, want string) {
 	}
 }
 
-func expectNodeEquality(t *testing.T, got Node, want Node) {
+func expectNodeEquality(t *testing.T, got, want *Node) {
 	t.Helper()
 	if !reflect.DeepEqual(got, want) {
 		t.Errorf("Node.UnmarshalJSON() = %v, want %v", got, want)
@@ -824,23 +819,17 @@ func casesNodeUnmarshalJSONUpField() []nodeUnmarshalTestCase {
 		{
 			name:      "empty json",
 			marshaled: "{}",
-			want: Node{
+			want:      newNode(nil, nil, false),
-				up: false,
-			},
 		},
 		{
 			name:      "a stopped node",
 			marshaled: "{\"up\": false}",
-			want: Node{
+			want:      newNode(nil, nil, false),
-				up: false,
-			},
 		},
 		{
 			name:      "a running node",
 			marshaled: "{\"up\": true}",
-			want: Node{
+			want:      newNode(nil, nil, true),
-				up: true,
-			},
 		},
 		{
 			name:      "invalid JSON value on valid key",
@@ -867,26 +856,17 @@ func casesNodeUnmarshalJSONConfigField() []nodeUnmarshalTestCase {
 		{
 			name:      "Config field is omitted",
 			marshaled: "{}",
-			want: Node{
+			want:      newNode(nil, nil, false),
-				Config: nil,
-			},
 		},
 		{
 			name:      "Config field is nil",
-			marshaled: "{\"config\": nil}",
+			marshaled: "{\"config\": null}",
-			want: Node{
+			want:      newNode(nil, nil, false),
-				Config: nil,
-			},
 		},
 		{
 			name:      "a non default Config field",
 			marshaled: "{\"config\":{\"name\":\"node_ecdd0\",\"port\":44665}}",
-			want: Node{
+			want:      newNode(nil, &adapters.NodeConfig{Name: "node_ecdd0", Port: 44665}, false),
-				Config: &adapters.NodeConfig{
-					Name: "node_ecdd0",
-					Port: 44665,
-				},
-			},
 		},
 	}
 }

rlp/decode.go

@@ -152,7 +152,6 @@ func makeDecoder(typ reflect.Type, tags tags) (dec decoder, err error) {
 	switch {
 	case typ == rawValueType:
 		return decodeRawValue, nil
-		return decodeDecoder, nil
 	case typ.AssignableTo(reflect.PtrTo(bigInt)):
 		return decodeBigInt, nil
 	case typ.AssignableTo(bigInt):

signer/core/signed_data_test.go

@@ -358,7 +358,7 @@ func TestJsonFiles(t *testing.T) {
 			continue
 		}
 		var typedData core.TypedData
-		err = json.Unmarshal([]byte(data), &typedData)
+		err = json.Unmarshal(data, &typedData)
 		if err != nil {
 			t.Errorf("Test %d, file %v, json unmarshalling failed: %v", i, fInfo.Name(), err)
 			continue
@@ -390,7 +390,7 @@ func TestFuzzerFiles(t *testing.T) {
 			continue
 		}
 		var typedData core.TypedData
-		err = json.Unmarshal([]byte(data), &typedData)
+		err = json.Unmarshal(data, &typedData)
 		if err != nil {
 			t.Errorf("Test %d, file %v, json unmarshalling failed: %v", i, fInfo.Name(), err)
 			continue

whisper/whisperv6/whisper_test.go

@@ -262,13 +262,12 @@ func TestWhisperIdentityManagement(t *testing.T) {
 func TestWhisperSymKeyManagement(t *testing.T) {
 	InitSingleTest()
 
-	var err error
+	var (
-	var k1, k2 []byte
+		k1, k2 []byte
-	w := New(&DefaultConfig)
+		w      = New(&DefaultConfig)
-	id1 := string("arbitrary-string-1")
+		id2    = string("arbitrary-string-2")
-	id2 := string("arbitrary-string-2")
+	)
-
+	id1, err := w.GenerateSymKey()
-	id1, err = w.GenerateSymKey()
 	if err != nil {
 		t.Fatalf("failed GenerateSymKey with seed %d: %s.", seed, err)
 	}