params: release Geth v1.8.1

* les: fix light fetcher database race

* les: lightFetcher comments

VERSION

@@ -1 +1 @@
-1.8.0
+1.8.1

cmd/geth/chaincmd.go

@@ -67,6 +67,9 @@ It expects the genesis file as argument.`,
 			utils.DataDirFlag,
 			utils.CacheFlag,
 			utils.LightModeFlag,
+			utils.GCModeFlag,
+			utils.CacheDatabaseFlag,
+			utils.CacheGCFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `

cmd/wnode/main.go

@@ -265,7 +265,7 @@ func initialize() {
 		Config: p2p.Config{
 			PrivateKey:     nodeid,
 			MaxPeers:       maxPeers,
-			Name:           common.MakeName("wnode", "5.0"),
+			Name:           common.MakeName("wnode", "6.0"),
 			Protocols:      shh.Protocols(),
 			ListenAddr:     *argIP,
 			NAT:            nat.Any(),
@@ -656,7 +656,7 @@ func requestExpiredMessagesLoop() {
 		params.PoW = *argServerPoW
 		params.Payload = data
 		params.KeySym = key
-		params.Src = nodeid
+		params.Src = asymKey
 		params.WorkTime = 5
 
 		msg, err := whisper.NewSentMessage(&params)

core/state_transition.go

@@ -215,6 +215,9 @@ func (st *StateTransition) TransitionDb() (ret []byte, usedGas uint64, failed bo
 
 	// Pay intrinsic gas
 	gas, err := IntrinsicGas(st.data, contractCreation, homestead)
+	if err != nil {
+		return nil, 0, false, err
+	}
 	if err = st.useGas(gas); err != nil {
 		return nil, 0, false, err
 	}

core/vm/interpreter.go

@@ -20,9 +20,7 @@ import (
 	"fmt"
 	"sync/atomic"
 
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/math"
-	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/params"
 )
 
@@ -123,11 +121,6 @@ func (in *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err er
 		return nil, nil
 	}
 
-	codehash := contract.CodeHash // codehash is used when doing jump dest caching
-	if codehash == (common.Hash{}) {
-		codehash = crypto.Keccak256Hash(contract.Code)
-	}
-
 	var (
 		op    OpCode        // current opcode
 		mem   = NewMemory() // bound memory

les/fetcher.go

@@ -36,24 +36,26 @@ const (
 	maxNodeCount      = 20               // maximum number of fetcherTreeNode entries remembered for each peer
 )
 
-// lightFetcher
+// lightFetcher implements retrieval of newly announced headers. It also provides a peerHasBlock function for the
+// ODR system to ensure that we only request data related to a certain block from peers who have already processed
+// and announced that block.
 type lightFetcher struct {
 	pm    *ProtocolManager
 	odr   *LesOdr
 	chain *light.LightChain
 
+	lock            sync.Mutex // lock protects access to the fetcher's internal state variables except sent requests
 	maxConfirmedTd  *big.Int
 	peers           map[*peer]*fetcherPeerInfo
 	lastUpdateStats *updateStatsEntry
+	syncing         bool
+	syncDone        chan *peer
 
-	lock       sync.Mutex // qwerqwerqwe
+	reqMu      sync.RWMutex // reqMu protects access to sent header fetch requests
-	deliverChn chan fetchResponse
-	reqMu      sync.RWMutex
 	requested  map[uint64]fetchRequest
+	deliverChn chan fetchResponse
 	timeoutChn chan uint64
 	requestChn chan bool // true if initiated from outside
-	syncing    bool
-	syncDone   chan *peer
 }
 
 // fetcherPeerInfo holds fetcher-specific information about each active peer
@@ -425,6 +427,9 @@ func (f *lightFetcher) nextRequest() (*distReq, uint64) {
 			},
 			canSend: func(dp distPeer) bool {
 				p := dp.(*peer)
+				f.lock.Lock()
+				defer f.lock.Unlock()
+
 				fp := f.peers[p]
 				return fp != nil && fp.nodeByHash[bestHash] != nil
 			},
@@ -557,8 +562,13 @@ func (f *lightFetcher) checkAnnouncedHeaders(fp *fetcherPeerInfo, headers []*typ
 				return true
 			}
 			// we ran out of recently delivered headers but have not reached a node known by this peer yet, continue matching
-			td = f.chain.GetTd(header.ParentHash, header.Number.Uint64()-1)
+			hash, number := header.ParentHash, header.Number.Uint64()-1
-			header = f.chain.GetHeader(header.ParentHash, header.Number.Uint64()-1)
+			td = f.chain.GetTd(hash, number)
+			header = f.chain.GetHeader(hash, number)
+			if header == nil || td == nil {
+				log.Error("Missing parent of validated header", "hash", hash, "number", number)
+				return false
+			}
 		} else {
 			header = headers[i]
 			td = tds[i]
@@ -642,13 +652,18 @@ func (f *lightFetcher) checkKnownNode(p *peer, n *fetcherTreeNode) bool {
 	if td == nil {
 		return false
 	}
+	header := f.chain.GetHeader(n.hash, n.number)
+	// check the availability of both header and td because reads are not protected by chain db mutex
+	// Note: returning false is always safe here
+	if header == nil {
+		return false
+	}
 
 	fp := f.peers[p]
 	if fp == nil {
 		p.Log().Debug("Unknown peer to check known nodes")
 		return false
 	}
-	header := f.chain.GetHeader(n.hash, n.number)
 	if !f.checkAnnouncedHeaders(fp, []*types.Header{header}, []*big.Int{td}) {
 		p.Log().Debug("Inconsistent announcement")
 		go f.pm.removePeer(p.id)

node/node.go

@@ -394,7 +394,7 @@ func (n *Node) startHTTP(endpoint string, apis []rpc.API, modules []string, cors
 		return err
 	}
 	go rpc.NewHTTPServer(cors, vhosts, handler).Serve(listener)
-	n.log.Info("HTTP endpoint opened", "url", fmt.Sprintf("http://%s", endpoint), "cors", strings.Join(cors, ","), "hvosts", strings.Join(vhosts, ","))
+	n.log.Info("HTTP endpoint opened", "url", fmt.Sprintf("http://%s", endpoint), "cors", strings.Join(cors, ","), "vhosts", strings.Join(vhosts, ","))
 	// All listeners booted successfully
 	n.httpEndpoint = endpoint
 	n.httpListener = listener

p2p/discover/database.go

@@ -257,7 +257,7 @@ func (db *nodeDB) expireNodes() error {
 		}
 		// Skip the node if not expired yet (and not self)
 		if !bytes.Equal(id[:], db.self[:]) {
-			if seen := db.lastPong(id); seen.After(threshold) {
+			if seen := db.bondTime(id); seen.After(threshold) {
 				continue
 			}
 		}
@@ -278,13 +278,18 @@ func (db *nodeDB) updateLastPing(id NodeID, instance time.Time) error {
 	return db.storeInt64(makeKey(id, nodeDBDiscoverPing), instance.Unix())
 }
 
-// lastPong retrieves the time of the last successful contact from remote node.
+// bondTime retrieves the time of the last successful pong from remote node.
-func (db *nodeDB) lastPong(id NodeID) time.Time {
+func (db *nodeDB) bondTime(id NodeID) time.Time {
 	return time.Unix(db.fetchInt64(makeKey(id, nodeDBDiscoverPong)), 0)
 }
 
-// updateLastPong updates the last time a remote node successfully contacted.
+// hasBond reports whether the given node is considered bonded.
-func (db *nodeDB) updateLastPong(id NodeID, instance time.Time) error {
+func (db *nodeDB) hasBond(id NodeID) bool {
+	return time.Since(db.bondTime(id)) < nodeDBNodeExpiration
+}
+
+// updateBondTime updates the last pong time of a node.
+func (db *nodeDB) updateBondTime(id NodeID, instance time.Time) error {
 	return db.storeInt64(makeKey(id, nodeDBDiscoverPong), instance.Unix())
 }
 
@@ -327,7 +332,7 @@ seek:
 		if n.ID == db.self {
 			continue seek
 		}
-		if now.Sub(db.lastPong(n.ID)) > maxAge {
+		if now.Sub(db.bondTime(n.ID)) > maxAge {
 			continue seek
 		}
 		for i := range nodes {

p2p/discover/database_test.go

@@ -125,13 +125,13 @@ func TestNodeDBFetchStore(t *testing.T) {
 		t.Errorf("ping: value mismatch: have %v, want %v", stored, inst)
 	}
 	// Check fetch/store operations on a node pong object
-	if stored := db.lastPong(node.ID); stored.Unix() != 0 {
+	if stored := db.bondTime(node.ID); stored.Unix() != 0 {
 		t.Errorf("pong: non-existing object: %v", stored)
 	}
-	if err := db.updateLastPong(node.ID, inst); err != nil {
+	if err := db.updateBondTime(node.ID, inst); err != nil {
 		t.Errorf("pong: failed to update: %v", err)
 	}
-	if stored := db.lastPong(node.ID); stored.Unix() != inst.Unix() {
+	if stored := db.bondTime(node.ID); stored.Unix() != inst.Unix() {
 		t.Errorf("pong: value mismatch: have %v, want %v", stored, inst)
 	}
 	// Check fetch/store operations on a node findnode-failure object
@@ -224,8 +224,8 @@ func TestNodeDBSeedQuery(t *testing.T) {
 		if err := db.updateNode(seed.node); err != nil {
 			t.Fatalf("node %d: failed to insert: %v", i, err)
 		}
-		if err := db.updateLastPong(seed.node.ID, seed.pong); err != nil {
+		if err := db.updateBondTime(seed.node.ID, seed.pong); err != nil {
-			t.Fatalf("node %d: failed to insert lastPong: %v", i, err)
+			t.Fatalf("node %d: failed to insert bondTime: %v", i, err)
 		}
 	}
 
@@ -332,8 +332,8 @@ func TestNodeDBExpiration(t *testing.T) {
 		if err := db.updateNode(seed.node); err != nil {
 			t.Fatalf("node %d: failed to insert: %v", i, err)
 		}
-		if err := db.updateLastPong(seed.node.ID, seed.pong); err != nil {
+		if err := db.updateBondTime(seed.node.ID, seed.pong); err != nil {
-			t.Fatalf("node %d: failed to update pong: %v", i, err)
+			t.Fatalf("node %d: failed to update bondTime: %v", i, err)
 		}
 	}
 	// Expire some of them, and check the rest
@@ -365,8 +365,8 @@ func TestNodeDBSelfExpiration(t *testing.T) {
 		if err := db.updateNode(seed.node); err != nil {
 			t.Fatalf("node %d: failed to insert: %v", i, err)
 		}
-		if err := db.updateLastPong(seed.node.ID, seed.pong); err != nil {
+		if err := db.updateBondTime(seed.node.ID, seed.pong); err != nil {
-			t.Fatalf("node %d: failed to update pong: %v", i, err)
+			t.Fatalf("node %d: failed to update bondTime: %v", i, err)
 		}
 	}
 	// Expire the nodes and make sure self has been evacuated too

p2p/discover/table.go

@@ -455,7 +455,7 @@ func (tab *Table) loadSeedNodes(bond bool) {
 	}
 	for i := range seeds {
 		seed := seeds[i]
-		age := log.Lazy{Fn: func() interface{} { return time.Since(tab.db.lastPong(seed.ID)) }}
+		age := log.Lazy{Fn: func() interface{} { return time.Since(tab.db.bondTime(seed.ID)) }}
 		log.Debug("Found seed node in database", "id", seed.ID, "addr", seed.addr(), "age", age)
 		tab.add(seed)
 	}
@@ -596,7 +596,7 @@ func (tab *Table) bond(pinged bool, id NodeID, addr *net.UDPAddr, tcpPort uint16
 	}
 	// Start bonding if we haven't seen this node for a while or if it failed findnode too often.
 	node, fails := tab.db.node(id), tab.db.findFails(id)
-	age := time.Since(tab.db.lastPong(id))
+	age := time.Since(tab.db.bondTime(id))
 	var result error
 	if fails > 0 || age > nodeDBNodeExpiration {
 		log.Trace("Starting bonding ping/pong", "id", id, "known", node != nil, "failcount", fails, "age", age)
@@ -663,7 +663,7 @@ func (tab *Table) ping(id NodeID, addr *net.UDPAddr) error {
 	if err := tab.net.ping(id, addr); err != nil {
 		return err
 	}
-	tab.db.updateLastPong(id, time.Now())
+	tab.db.updateBondTime(id, time.Now())
 	return nil
 }
 

p2p/discover/udp.go

@@ -613,7 +613,7 @@ func (req *findnode) handle(t *udp, from *net.UDPAddr, fromID NodeID, mac []byte
 	if expired(req.Expiration) {
 		return errExpired
 	}
-	if t.db.node(fromID) == nil {
+	if !t.db.hasBond(fromID) {
 		// No bond exists, we don't process the packet. This prevents
 		// an attack vector where the discovery protocol could be used
 		// to amplify traffic in a DDOS attack. A malicious actor

p2p/discover/udp_test.go

@@ -247,12 +247,8 @@ func TestUDP_findnode(t *testing.T) {
 
 	// ensure there's a bond with the test node,
 	// findnode won't be accepted otherwise.
-	test.table.db.updateNode(NewNode(
+	test.table.db.updateBondTime(PubkeyID(&test.remotekey.PublicKey), time.Now())
-		PubkeyID(&test.remotekey.PublicKey),
+
-		test.remoteaddr.IP,
-		uint16(test.remoteaddr.Port),
-		99,
-	))
 	// check that closest neighbors are returned.
 	test.packetIn(nil, findnodePacket, &findnode{Target: testTarget, Expiration: futureExp})
 	expected := test.table.closest(targetHash, bucketSize)

params/version.go

@@ -23,7 +23,7 @@ import (
 const (
 	VersionMajor = 1        // Major version component of the current release
 	VersionMinor = 8        // Minor version component of the current release
-	VersionPatch = 0        // Patch version component of the current release
+	VersionPatch = 1        // Patch version component of the current release
 	VersionMeta  = "stable" // Version metadata to append to the version string
 )
 

vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go

@@ -48,7 +48,7 @@ var wg sync.WaitGroup      // used to wait until the runloop starts
 // started and is ready via the wg. It also serves purpose of a dummy source,
 // thanks to it the runloop does not return as it also has at least one source
 // registered.
-var source = C.CFRunLoopSourceCreate(refZero, 0, &C.CFRunLoopSourceContext{
+var source = C.CFRunLoopSourceCreate(nil, 0, &C.CFRunLoopSourceContext{
 	perform: (C.CFRunLoopPerformCallBack)(C.gosource),
 })
 
@@ -162,8 +162,8 @@ func (s *stream) Start() error {
 		return nil
 	}
 	wg.Wait()
-	p := C.CFStringCreateWithCStringNoCopy(refZero, C.CString(s.path), C.kCFStringEncodingUTF8, refZero)
+	p := C.CFStringCreateWithCStringNoCopy(nil, C.CString(s.path), C.kCFStringEncodingUTF8, nil)
-	path := C.CFArrayCreate(refZero, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil)
+	path := C.CFArrayCreate(nil, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil)
 	ctx := C.FSEventStreamContext{}
 	ref := C.EventStreamCreate(&ctx, C.uintptr_t(s.info), path, C.FSEventStreamEventId(atomic.LoadUint64(&since)), latency, flags)
 	if ref == nilstream {

vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go

@@ -1,9 +0,0 @@
-// Copyright (c) 2017 The Notify Authors. All rights reserved.
-// Use of this source code is governed by the MIT license that can be
-// found in the LICENSE file.
-
-// +build darwin,!kqueue,go1.10
-
-package notify
-
-const refZero = 0

vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.9.go

@@ -1,14 +0,0 @@
-// Copyright (c) 2017 The Notify Authors. All rights reserved.
-// Use of this source code is governed by the MIT license that can be
-// found in the LICENSE file.
-
-// +build darwin,!kqueue,cgo,!go1.10
-
-package notify
-
-/*
-#include <CoreServices/CoreServices.h>
-*/
-import "C"
-
-var refZero = (*C.struct___CFAllocator)(nil)

vendor/vendor.json

@@ -322,10 +322,10 @@
 			"revisionTime": "2016-11-28T21:05:44Z"
 		},
 		{
-			"checksumSHA1": "1ESHllhZOIBg7MnlGHUdhz047bI=",
+			"checksumSHA1": "28UVHMmHx0iqO0XiJsjx+fwILyI=",
 			"path": "github.com/rjeczalik/notify",
-			"revision": "27b537f07230b3f917421af6dcf044038dbe57e2",
+			"revision": "c31e5f2cb22b3e4ef3f882f413847669bf2652b9",
-			"revisionTime": "2018-01-03T13:19:05Z"
+			"revisionTime": "2018-02-03T14:01:15Z"
 		},
 		{
 			"checksumSHA1": "5uqO4ITTDMklKi3uNaE/D9LQ5nM=",

whisper/whisperv6/peer_test.go

@@ -70,9 +70,8 @@ var keys = []string{
 	"7184c1701569e3a4c4d2ddce691edd983b81e42e09196d332e1ae2f1e062cff4",
 }
 
-const NumNodes = 16 // must not exceed the number of keys (32)
-
 type TestData struct {
+	started int
 	counter [NumNodes]int
 	mutex   sync.RWMutex
 }
@@ -84,21 +83,29 @@ type TestNode struct {
 	filerID string
 }
 
+const NumNodes = 8 // must not exceed the number of keys (32)
+
 var result TestData
 var nodes [NumNodes]*TestNode
 var sharedKey = hexutil.MustDecode("0x03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd31")
+var wrongKey = hexutil.MustDecode("0xf91156714d7ec88d3edc1c652c2181dbb3044e8771c683f3b30d33c12b986b11")
 var sharedTopic = TopicType{0xF, 0x1, 0x2, 0}
-var expectedMessage = []byte("per rectum ad astra")
+var wrongTopic = TopicType{0, 0, 0, 0}
+var expectedMessage = []byte("per aspera ad astra")
+var unexpectedMessage = []byte("per rectum ad astra")
 var masterBloomFilter []byte
 var masterPow = 0.00000001
 var round = 1
+var debugMode = false
+var prevTime time.Time
+var cntPrev int
 
 func TestSimulation(t *testing.T) {
 	// create a chain of whisper nodes,
 	// installs the filters with shared (predefined) parameters
 	initialize(t)
 
-	// each node sends a number of random (undecryptable) messages
+	// each node sends one random (not decryptable) message
 	for i := 0; i < NumNodes; i++ {
 		sendMsg(t, false, i)
 	}
@@ -115,7 +122,6 @@ func TestSimulation(t *testing.T) {
 
 	// send new pow and bloom exchange messages
 	resetParams(t)
-	round++
 
 	// node #1 sends one expected (decryptable) message
 	sendMsg(t, true, 1)
@@ -140,6 +146,8 @@ func resetParams(t *testing.T) {
 	for i := 0; i < NumNodes; i++ {
 		nodes[i].shh.SetBloomFilter(masterBloomFilter)
 	}
+
+	round++
 }
 
 func initBloom(t *testing.T) {
@@ -219,15 +227,22 @@ func initialize(t *testing.T) {
 		nodes[i] = &node
 	}
 
-	for i := 1; i < NumNodes; i++ {
+	for i := 0; i < NumNodes; i++ {
-		go nodes[i].server.Start()
+		go startServer(t, nodes[i].server)
 	}
 
-	// we need to wait until the first node actually starts
+	waitForServersToStart(t)
-	err = nodes[0].server.Start()
+}
+
+func startServer(t *testing.T, s *p2p.Server) {
+	err := s.Start()
 	if err != nil {
 		t.Fatalf("failed to start the fisrt server.")
 	}
+
+	result.mutex.Lock()
+	defer result.mutex.Unlock()
+	result.started++
 }
 
 func stopServers() {
@@ -246,8 +261,10 @@ func checkPropagation(t *testing.T, includingNodeZero bool) {
 		return
 	}
 
-	const cycle = 50
+	prevTime = time.Now()
-	const iterations = 200
+	// (cycle * iterations) should not exceed 50 seconds, since TTL=50
+	const cycle = 200 // time in milliseconds
+	const iterations = 250
 
 	first := 0
 	if !includingNodeZero {
@@ -262,29 +279,29 @@ func checkPropagation(t *testing.T, includingNodeZero bool) {
 			}
 
 			mail := f.Retrieve()
-			if !validateMail(t, i, mail) {
+			validateMail(t, i, mail)
-				return
-			}
 
 			if isTestComplete() {
+				checkTestStatus()
 				return
 			}
 		}
 
+		checkTestStatus()
 		time.Sleep(cycle * time.Millisecond)
 	}
 
-	t.Fatalf("Test was not complete: timeout %d seconds. nodes=%v", iterations*cycle/1000, nodes)
-
 	if !includingNodeZero {
 		f := nodes[0].shh.GetFilter(nodes[0].filerID)
 		if f != nil {
 			t.Fatalf("node zero received a message with low PoW.")
 		}
 	}
+
+	t.Fatalf("Test was not complete (%d round): timeout %d seconds. nodes=%v", round, iterations*cycle/1000, nodes)
 }
 
-func validateMail(t *testing.T, index int, mail []*ReceivedMessage) bool {
+func validateMail(t *testing.T, index int, mail []*ReceivedMessage) {
 	var cnt int
 	for _, m := range mail {
 		if bytes.Equal(m.Payload, expectedMessage) {
@@ -294,14 +311,13 @@ func validateMail(t *testing.T, index int, mail []*ReceivedMessage) bool {
 
 	if cnt == 0 {
 		// no messages received yet: nothing is wrong
-		return true
+		return
 	}
 	if cnt > 1 {
 		t.Fatalf("node %d received %d.", index, cnt)
-		return false
 	}
 
-	if cnt > 0 {
+	if cnt == 1 {
 		result.mutex.Lock()
 		defer result.mutex.Unlock()
 		result.counter[index] += cnt
@@ -309,7 +325,28 @@ func validateMail(t *testing.T, index int, mail []*ReceivedMessage) bool {
 			t.Fatalf("node %d accumulated %d.", index, result.counter[index])
 		}
 	}
-	return true
+}
+
+func checkTestStatus() {
+	var cnt int
+	var arr [NumNodes]int
+
+	for i := 0; i < NumNodes; i++ {
+		arr[i] = nodes[i].server.PeerCount()
+		envelopes := nodes[i].shh.Envelopes()
+		if len(envelopes) >= NumNodes {
+			cnt++
+		}
+	}
+
+	if debugMode {
+		if cntPrev != cnt {
+			fmt.Printf(" %v \t number of nodes that have received all msgs: %d, number of peers per node: %v \n",
+				time.Since(prevTime), cnt, arr)
+			prevTime = time.Now()
+			cntPrev = cnt
+		}
+	}
 }
 
 func isTestComplete() bool {
@@ -324,7 +361,7 @@ func isTestComplete() bool {
 
 	for i := 0; i < NumNodes; i++ {
 		envelopes := nodes[i].shh.Envelopes()
-		if len(envelopes) < 2 {
+		if len(envelopes) < NumNodes+1 {
 			return false
 		}
 	}
@@ -339,9 +376,10 @@ func sendMsg(t *testing.T, expected bool, id int) {
 
 	opt := MessageParams{KeySym: sharedKey, Topic: sharedTopic, Payload: expectedMessage, PoW: 0.00000001, WorkTime: 1}
 	if !expected {
-		opt.KeySym[0]++
+		opt.KeySym = wrongKey
-		opt.Topic[0]++
+		opt.Topic = wrongTopic
-		opt.Payload = opt.Payload[1:]
+		opt.Payload = unexpectedMessage
+		opt.Payload[0] = byte(id)
 	}
 
 	msg, err := NewSentMessage(&opt)
@@ -459,3 +497,14 @@ func checkBloomFilterExchange(t *testing.T) {
 		time.Sleep(50 * time.Millisecond)
 	}
 }
+
+func waitForServersToStart(t *testing.T) {
+	const iterations = 200
+	for j := 0; j < iterations; j++ {
+		time.Sleep(50 * time.Millisecond)
+		if result.started == NumNodes {
+			return
+		}
+	}
+	t.Fatalf("Failed to start all the servers, running: %d", result.started)
+}