From 049e17116e5e00a052384905a433fed5245ea5c4 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 28 Oct 2019 19:59:07 +0800 Subject: [PATCH 1/2] core, eth: implement eth/65 transaction fetcher --- core/tx_pool.go | 6 + eth/downloader/peer.go | 8 +- eth/fetcher/{fetcher.go => block_fetcher.go} | 129 ++++--- ...{fetcher_test.go => block_fetcher_test.go} | 4 +- eth/fetcher/metrics.go | 27 +- eth/fetcher/tx_fetcher.go | 319 +++++++++++++++ eth/fetcher/tx_fetcher_test.go | 318 +++++++++++++++ eth/handler.go | 141 ++++++- eth/handler_test.go | 4 +- eth/helper_test.go | 33 +- eth/peer.go | 363 +++++++++++++++--- eth/protocol.go | 20 +- eth/protocol_test.go | 114 +++++- eth/sync.go | 59 ++- eth/sync_test.go | 10 +- 15 files changed, 1344 insertions(+), 211 deletions(-) rename eth/fetcher/{fetcher.go => block_fetcher.go} (84%) rename eth/fetcher/{fetcher_test.go => block_fetcher_test.go} (99%) create mode 100644 eth/fetcher/tx_fetcher.go create mode 100644 eth/fetcher/tx_fetcher_test.go diff --git a/core/tx_pool.go b/core/tx_pool.go index ae6962c5d9..16d80d644f 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -864,6 +864,12 @@ func (pool *TxPool) Get(hash common.Hash) *types.Transaction { return pool.all.Get(hash) } +// Has returns an indicator whether txpool has a transaction cached with the +// given hash. +func (pool *TxPool) Has(hash common.Hash) bool { + return pool.all.Get(hash) != nil +} + // removeTx removes a single transaction from the queue, moving all subsequent // transactions back to the future queue. func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) { diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 60f86d0e14..5c2020d7d8 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -470,7 +470,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.headerThroughput } - return ps.idlePeers(62, 64, idle, throughput) + return ps.idlePeers(62, 65, idle, throughput) } // BodyIdlePeers retrieves a flat list of all the currently body-idle peers within @@ -484,7 +484,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.blockThroughput } - return ps.idlePeers(62, 64, idle, throughput) + return ps.idlePeers(62, 65, idle, throughput) } // ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers @@ -498,7 +498,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.receiptThroughput } - return ps.idlePeers(63, 64, idle, throughput) + return ps.idlePeers(63, 65, idle, throughput) } // NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle @@ -512,7 +512,7 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.stateThroughput } - return ps.idlePeers(63, 64, idle, throughput) + return ps.idlePeers(63, 65, idle, throughput) } // idlePeers retrieves a flat list of all currently idle peers satisfying the diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/block_fetcher.go similarity index 84% rename from eth/fetcher/fetcher.go rename to eth/fetcher/block_fetcher.go index 28c532d9bd..7395ec83de 100644 --- a/eth/fetcher/fetcher.go +++ b/eth/fetcher/block_fetcher.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package fetcher contains the block announcement based synchronisation. +// Package fetcher contains the announcement based blocks or transaction synchronisation. package fetcher import ( @@ -30,13 +30,16 @@ import ( ) const ( - arriveTimeout = 500 * time.Millisecond // Time allowance before an announced block is explicitly requested + arriveTimeout = 500 * time.Millisecond // Time allowance before an announced block/transaction is explicitly requested gatherSlack = 100 * time.Millisecond // Interval used to collate almost-expired announces with fetches - fetchTimeout = 5 * time.Second // Maximum allotted time to return an explicitly requested block - maxUncleDist = 7 // Maximum allowed backward distance from the chain head - maxQueueDist = 32 // Maximum allowed distance from the chain head to queue - hashLimit = 256 // Maximum number of unique blocks a peer may have announced - blockLimit = 64 // Maximum number of unique blocks a peer may have delivered + fetchTimeout = 5 * time.Second // Maximum allotted time to return an explicitly requested block/transaction +) + +const ( + maxUncleDist = 7 // Maximum allowed backward distance from the chain head + maxQueueDist = 32 // Maximum allowed distance from the chain head to queue + hashLimit = 256 // Maximum number of unique blocks a peer may have announced + blockLimit = 64 // Maximum number of unique blocks a peer may have delivered ) var ( @@ -67,9 +70,9 @@ type chainInsertFn func(types.Blocks) (int, error) // peerDropFn is a callback type for dropping a peer detected as malicious. type peerDropFn func(id string) -// announce is the hash notification of the availability of a new block in the +// blockAnnounce is the hash notification of the availability of a new block in the // network. -type announce struct { +type blockAnnounce struct { hash common.Hash // Hash of the block being announced number uint64 // Number of the block being announced (0 = unknown | old protocol) header *types.Header // Header of the block partially reassembled (new protocol) @@ -97,18 +100,18 @@ type bodyFilterTask struct { time time.Time // Arrival time of the blocks' contents } -// inject represents a schedules import operation. -type inject struct { +// blockInject represents a schedules import operation. +type blockInject struct { origin string block *types.Block } -// Fetcher is responsible for accumulating block announcements from various peers +// BlockFetcher is responsible for accumulating block announcements from various peers // and scheduling them for retrieval. -type Fetcher struct { +type BlockFetcher struct { // Various event channels - notify chan *announce - inject chan *inject + notify chan *blockAnnounce + inject chan *blockInject headerFilter chan chan *headerFilterTask bodyFilter chan chan *bodyFilterTask @@ -117,16 +120,16 @@ type Fetcher struct { quit chan struct{} // Announce states - announces map[string]int // Per peer announce counts to prevent memory exhaustion - announced map[common.Hash][]*announce // Announced blocks, scheduled for fetching - fetching map[common.Hash]*announce // Announced blocks, currently fetching - fetched map[common.Hash][]*announce // Blocks with headers fetched, scheduled for body retrieval - completing map[common.Hash]*announce // Blocks with headers, currently body-completing + announces map[string]int // Per peer blockAnnounce counts to prevent memory exhaustion + announced map[common.Hash][]*blockAnnounce // Announced blocks, scheduled for fetching + fetching map[common.Hash]*blockAnnounce // Announced blocks, currently fetching + fetched map[common.Hash][]*blockAnnounce // Blocks with headers fetched, scheduled for body retrieval + completing map[common.Hash]*blockAnnounce // Blocks with headers, currently body-completing // Block cache - queue *prque.Prque // Queue containing the import operations (block number sorted) - queues map[string]int // Per peer block counts to prevent memory exhaustion - queued map[common.Hash]*inject // Set of already queued blocks (to dedupe imports) + queue *prque.Prque // Queue containing the import operations (block number sorted) + queues map[string]int // Per peer block counts to prevent memory exhaustion + queued map[common.Hash]*blockInject // Set of already queued blocks (to dedupe imports) // Callbacks getBlock blockRetrievalFn // Retrieves a block from the local chain @@ -137,30 +140,30 @@ type Fetcher struct { dropPeer peerDropFn // Drops a peer for misbehaving // Testing hooks - announceChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a hash from the announce list + announceChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a hash from the blockAnnounce list queueChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a block from the import queue fetchingHook func([]common.Hash) // Method to call upon starting a block (eth/61) or header (eth/62) fetch completingHook func([]common.Hash) // Method to call upon starting a block body fetch (eth/62) importedHook func(*types.Block) // Method to call upon successful block import (both eth/61 and eth/62) } -// New creates a block fetcher to retrieve blocks based on hash announcements. -func New(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBlock blockBroadcasterFn, chainHeight chainHeightFn, insertChain chainInsertFn, dropPeer peerDropFn) *Fetcher { - return &Fetcher{ - notify: make(chan *announce), - inject: make(chan *inject), +// NewBlockFetcher creates a block fetcher to retrieve blocks based on hash announcements. +func NewBlockFetcher(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBlock blockBroadcasterFn, chainHeight chainHeightFn, insertChain chainInsertFn, dropPeer peerDropFn) *BlockFetcher { + return &BlockFetcher{ + notify: make(chan *blockAnnounce), + inject: make(chan *blockInject), headerFilter: make(chan chan *headerFilterTask), bodyFilter: make(chan chan *bodyFilterTask), done: make(chan common.Hash), quit: make(chan struct{}), announces: make(map[string]int), - announced: make(map[common.Hash][]*announce), - fetching: make(map[common.Hash]*announce), - fetched: make(map[common.Hash][]*announce), - completing: make(map[common.Hash]*announce), + announced: make(map[common.Hash][]*blockAnnounce), + fetching: make(map[common.Hash]*blockAnnounce), + fetched: make(map[common.Hash][]*blockAnnounce), + completing: make(map[common.Hash]*blockAnnounce), queue: prque.New(nil), queues: make(map[string]int), - queued: make(map[common.Hash]*inject), + queued: make(map[common.Hash]*blockInject), getBlock: getBlock, verifyHeader: verifyHeader, broadcastBlock: broadcastBlock, @@ -172,21 +175,21 @@ func New(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBloc // Start boots up the announcement based synchroniser, accepting and processing // hash notifications and block fetches until termination requested. -func (f *Fetcher) Start() { +func (f *BlockFetcher) Start() { go f.loop() } // Stop terminates the announcement based synchroniser, canceling all pending // operations. -func (f *Fetcher) Stop() { +func (f *BlockFetcher) Stop() { close(f.quit) } // Notify announces the fetcher of the potential availability of a new block in // the network. -func (f *Fetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time, +func (f *BlockFetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time, headerFetcher headerRequesterFn, bodyFetcher bodyRequesterFn) error { - block := &announce{ + block := &blockAnnounce{ hash: hash, number: number, time: time, @@ -203,8 +206,8 @@ func (f *Fetcher) Notify(peer string, hash common.Hash, number uint64, time time } // Enqueue tries to fill gaps the fetcher's future import queue. -func (f *Fetcher) Enqueue(peer string, block *types.Block) error { - op := &inject{ +func (f *BlockFetcher) Enqueue(peer string, block *types.Block) error { + op := &blockInject{ origin: peer, block: block, } @@ -218,7 +221,7 @@ func (f *Fetcher) Enqueue(peer string, block *types.Block) error { // FilterHeaders extracts all the headers that were explicitly requested by the fetcher, // returning those that should be handled differently. -func (f *Fetcher) FilterHeaders(peer string, headers []*types.Header, time time.Time) []*types.Header { +func (f *BlockFetcher) FilterHeaders(peer string, headers []*types.Header, time time.Time) []*types.Header { log.Trace("Filtering headers", "peer", peer, "headers", len(headers)) // Send the filter channel to the fetcher @@ -246,7 +249,7 @@ func (f *Fetcher) FilterHeaders(peer string, headers []*types.Header, time time. // FilterBodies extracts all the block bodies that were explicitly requested by // the fetcher, returning those that should be handled differently. -func (f *Fetcher) FilterBodies(peer string, transactions [][]*types.Transaction, uncles [][]*types.Header, time time.Time) ([][]*types.Transaction, [][]*types.Header) { +func (f *BlockFetcher) FilterBodies(peer string, transactions [][]*types.Transaction, uncles [][]*types.Header, time time.Time) ([][]*types.Transaction, [][]*types.Header) { log.Trace("Filtering bodies", "peer", peer, "txs", len(transactions), "uncles", len(uncles)) // Send the filter channel to the fetcher @@ -274,7 +277,7 @@ func (f *Fetcher) FilterBodies(peer string, transactions [][]*types.Transaction, // Loop is the main fetcher loop, checking and processing various notification // events. -func (f *Fetcher) loop() { +func (f *BlockFetcher) loop() { // Iterate the block fetching until a quit is requested fetchTimer := time.NewTimer(0) completeTimer := time.NewTimer(0) @@ -289,7 +292,7 @@ func (f *Fetcher) loop() { // Import any queued blocks that could potentially fit height := f.chainHeight() for !f.queue.Empty() { - op := f.queue.PopItem().(*inject) + op := f.queue.PopItem().(*blockInject) hash := op.block.Hash() if f.queueChangeHook != nil { f.queueChangeHook(hash, false) @@ -313,24 +316,24 @@ func (f *Fetcher) loop() { // Wait for an outside event to occur select { case <-f.quit: - // Fetcher terminating, abort all operations + // BlockFetcher terminating, abort all operations return case notification := <-f.notify: // A block was announced, make sure the peer isn't DOSing us - propAnnounceInMeter.Mark(1) + blockAnnounceInMeter.Mark(1) count := f.announces[notification.origin] + 1 if count > hashLimit { log.Debug("Peer exceeded outstanding announces", "peer", notification.origin, "limit", hashLimit) - propAnnounceDOSMeter.Mark(1) + blockAnnounceDOSMeter.Mark(1) break } // If we have a valid block number, check that it's potentially useful if notification.number > 0 { if dist := int64(notification.number) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { log.Debug("Peer discarded announcement", "peer", notification.origin, "number", notification.number, "hash", notification.hash, "distance", dist) - propAnnounceDropMeter.Mark(1) + blockAnnounceDropMeter.Mark(1) break } } @@ -352,7 +355,7 @@ func (f *Fetcher) loop() { case op := <-f.inject: // A direct block insertion was requested, try and fill any pending gaps - propBroadcastInMeter.Mark(1) + blockBroadcastInMeter.Mark(1) f.enqueue(op.origin, op.block) case hash := <-f.done: @@ -439,7 +442,7 @@ func (f *Fetcher) loop() { // Split the batch of headers into unknown ones (to return to the caller), // known incomplete ones (requiring body retrievals) and completed blocks. - unknown, incomplete, complete := []*types.Header{}, []*announce{}, []*types.Block{} + unknown, incomplete, complete := []*types.Header{}, []*blockAnnounce{}, []*types.Block{} for _, header := range task.headers { hash := header.Hash() @@ -475,7 +478,7 @@ func (f *Fetcher) loop() { f.forgetHash(hash) } } else { - // Fetcher doesn't know about it, add to the return list + // BlockFetcher doesn't know about it, add to the return list unknown = append(unknown, header) } } @@ -562,8 +565,8 @@ func (f *Fetcher) loop() { } } -// rescheduleFetch resets the specified fetch timer to the next announce timeout. -func (f *Fetcher) rescheduleFetch(fetch *time.Timer) { +// rescheduleFetch resets the specified fetch timer to the next blockAnnounce timeout. +func (f *BlockFetcher) rescheduleFetch(fetch *time.Timer) { // Short circuit if no blocks are announced if len(f.announced) == 0 { return @@ -579,7 +582,7 @@ func (f *Fetcher) rescheduleFetch(fetch *time.Timer) { } // rescheduleComplete resets the specified completion timer to the next fetch timeout. -func (f *Fetcher) rescheduleComplete(complete *time.Timer) { +func (f *BlockFetcher) rescheduleComplete(complete *time.Timer) { // Short circuit if no headers are fetched if len(f.fetched) == 0 { return @@ -596,27 +599,27 @@ func (f *Fetcher) rescheduleComplete(complete *time.Timer) { // enqueue schedules a new future import operation, if the block to be imported // has not yet been seen. -func (f *Fetcher) enqueue(peer string, block *types.Block) { +func (f *BlockFetcher) enqueue(peer string, block *types.Block) { hash := block.Hash() // Ensure the peer isn't DOSing us count := f.queues[peer] + 1 if count > blockLimit { log.Debug("Discarded propagated block, exceeded allowance", "peer", peer, "number", block.Number(), "hash", hash, "limit", blockLimit) - propBroadcastDOSMeter.Mark(1) + blockBroadcastDOSMeter.Mark(1) f.forgetHash(hash) return } // Discard any past or too distant blocks if dist := int64(block.NumberU64()) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { log.Debug("Discarded propagated block, too far away", "peer", peer, "number", block.Number(), "hash", hash, "distance", dist) - propBroadcastDropMeter.Mark(1) + blockBroadcastDropMeter.Mark(1) f.forgetHash(hash) return } // Schedule the block for future importing if _, ok := f.queued[hash]; !ok { - op := &inject{ + op := &blockInject{ origin: peer, block: block, } @@ -633,7 +636,7 @@ func (f *Fetcher) enqueue(peer string, block *types.Block) { // insert spawns a new goroutine to run a block insertion into the chain. If the // block's number is at the same height as the current import phase, it updates // the phase states accordingly. -func (f *Fetcher) insert(peer string, block *types.Block) { +func (f *BlockFetcher) insert(peer string, block *types.Block) { hash := block.Hash() // Run the import on a new thread @@ -651,7 +654,7 @@ func (f *Fetcher) insert(peer string, block *types.Block) { switch err := f.verifyHeader(block.Header()); err { case nil: // All ok, quickly propagate to our peers - propBroadcastOutTimer.UpdateSince(block.ReceivedAt) + blockBroadcastOutTimer.UpdateSince(block.ReceivedAt) go f.broadcastBlock(block, true) case consensus.ErrFutureBlock: @@ -669,7 +672,7 @@ func (f *Fetcher) insert(peer string, block *types.Block) { return } // If import succeeded, broadcast the block - propAnnounceOutTimer.UpdateSince(block.ReceivedAt) + blockAnnounceOutTimer.UpdateSince(block.ReceivedAt) go f.broadcastBlock(block, false) // Invoke the testing hook if needed @@ -681,7 +684,7 @@ func (f *Fetcher) insert(peer string, block *types.Block) { // forgetHash removes all traces of a block announcement from the fetcher's // internal state. -func (f *Fetcher) forgetHash(hash common.Hash) { +func (f *BlockFetcher) forgetHash(hash common.Hash) { // Remove all pending announces and decrement DOS counters for _, announce := range f.announced[hash] { f.announces[announce.origin]-- @@ -723,7 +726,7 @@ func (f *Fetcher) forgetHash(hash common.Hash) { // forgetBlock removes all traces of a queued block from the fetcher's internal // state. -func (f *Fetcher) forgetBlock(hash common.Hash) { +func (f *BlockFetcher) forgetBlock(hash common.Hash) { if insert := f.queued[hash]; insert != nil { f.queues[insert.origin]-- if f.queues[insert.origin] == 0 { diff --git a/eth/fetcher/fetcher_test.go b/eth/fetcher/block_fetcher_test.go similarity index 99% rename from eth/fetcher/fetcher_test.go rename to eth/fetcher/block_fetcher_test.go index 83172c5348..038ead12e7 100644 --- a/eth/fetcher/fetcher_test.go +++ b/eth/fetcher/block_fetcher_test.go @@ -76,7 +76,7 @@ func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common // fetcherTester is a test simulator for mocking out local block chain. type fetcherTester struct { - fetcher *Fetcher + fetcher *BlockFetcher hashes []common.Hash // Hash chain belonging to the tester blocks map[common.Hash]*types.Block // Blocks belonging to the tester @@ -92,7 +92,7 @@ func newTester() *fetcherTester { blocks: map[common.Hash]*types.Block{genesis.Hash(): genesis}, drops: make(map[string]bool), } - tester.fetcher = New(tester.getBlock, tester.verifyHeader, tester.broadcastBlock, tester.chainHeight, tester.insertChain, tester.dropPeer) + tester.fetcher = NewBlockFetcher(tester.getBlock, tester.verifyHeader, tester.broadcastBlock, tester.chainHeight, tester.insertChain, tester.dropPeer) tester.fetcher.Start() return tester diff --git a/eth/fetcher/metrics.go b/eth/fetcher/metrics.go index d68d12f000..b75889938d 100644 --- a/eth/fetcher/metrics.go +++ b/eth/fetcher/metrics.go @@ -23,15 +23,15 @@ import ( ) var ( - propAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/in", nil) - propAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/announces/out", nil) - propAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/drop", nil) - propAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/dos", nil) + blockAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/announces/in", nil) + blockAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/block/announces/out", nil) + blockAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/announces/drop", nil) + blockAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/announces/dos", nil) - propBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/in", nil) - propBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/broadcasts/out", nil) - propBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/drop", nil) - propBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/dos", nil) + blockBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/broadcasts/in", nil) + blockBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/block/broadcasts/out", nil) + blockBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/broadcasts/drop", nil) + blockBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/broadcasts/dos", nil) headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/headers", nil) bodyFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/bodies", nil) @@ -40,4 +40,15 @@ var ( headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/out", nil) bodyFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/in", nil) bodyFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/out", nil) + + txAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/in", nil) + txAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/dos", nil) + txAnnounceSkipMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/skip", nil) + txAnnounceUnderpriceMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/underprice", nil) + txBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/broadcasts/in", nil) + txFetchOutMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/out", nil) + txFetchSuccessMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/success", nil) + txFetchTimeoutMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/timeout", nil) + txFetchInvalidMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/invalid", nil) + txFetchDurationTimer = metrics.NewRegisteredTimer("eth/fetcher/fetch/transaction/duration", nil) ) diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go new file mode 100644 index 0000000000..1dabb0819f --- /dev/null +++ b/eth/fetcher/tx_fetcher.go @@ -0,0 +1,319 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package fetcher + +import ( + "math/rand" + "time" + + mapset "github.com/deckarep/golang-set" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" +) + +var ( + // txAnnounceLimit is the maximum number of unique transaction a peer + // can announce in a short time. + txAnnounceLimit = 4096 + + // txFetchTimeout is the maximum allotted time to return an explicitly + // requested transaction. + txFetchTimeout = 5 * time.Second + + // MaxTransactionFetch is the maximum transaction number can be fetched + // in one request. The rationale to pick this value is: + // In eth protocol, the softResponseLimit is 2MB. Nowdays according to + // Etherscan the average transaction size is around 200B, so in theory + // we can include lots of transaction in a single protocol packet. However + // the maximum size of a single transaction is raised to 128KB, so pick + // a middle value here to ensure we can maximize the efficiency of the + // retrieval and response size overflow won't happen in most cases. + MaxTransactionFetch = 256 + + // underpriceSetSize is the size of underprice set which used for maintaining + // the set of underprice transactions. + underpriceSetSize = 4096 +) + +// txAnnounce is the notification of the availability of a single +// new transaction in the network. +type txAnnounce struct { + origin string // Identifier of the peer originating the notification + time time.Time // Timestamp of the announcement + fetchTxs func([]common.Hash) // Callback for retrieving transaction from specified peer +} + +// txsAnnounce is the notification of the availability of a batch +// of new transactions in the network. +type txsAnnounce struct { + hashes []common.Hash // Batch of transaction hashes being announced + origin string // Identifier of the peer originating the notification + time time.Time // Timestamp of the announcement + fetchTxs func([]common.Hash) // Callback for retrieving transaction from specified peer +} + +// TxFetcher is responsible for retrieving new transaction based +// on the announcement. +type TxFetcher struct { + notify chan *txsAnnounce + cleanup chan []common.Hash + quit chan struct{} + + // Announce states + announces map[string]int // Per peer transaction announce counts to prevent memory exhaustion + announced map[common.Hash][]*txAnnounce // Announced transactions, scheduled for fetching + fetching map[common.Hash]*txAnnounce // Announced transactions, currently fetching + underpriced mapset.Set // Transaction set whose price is too low for accepting + + // Callbacks + hasTx func(common.Hash) bool // Retrieves a tx from the local txpool + addTxs func([]*types.Transaction) []error // Insert a batch of transactions into local txpool + dropPeer func(string) // Drop the specified peer + + // Hooks + announceHook func([]common.Hash) // Hook which is called when a batch transactions are announced + importTxsHook func([]*types.Transaction) // Hook which is called when a batch of transactions are imported. + dropHook func(string) // Hook which is called when a peer is dropped + cleanupHook func([]common.Hash) // Hook which is called when internal status is cleaned + rejectUnderprice func(common.Hash) // Hook which is called when underprice transaction is rejected +} + +// NewTxFetcher creates a transaction fetcher to retrieve transaction +// based on hash announcements. +func NewTxFetcher(hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, dropPeer func(string)) *TxFetcher { + return &TxFetcher{ + notify: make(chan *txsAnnounce), + cleanup: make(chan []common.Hash), + quit: make(chan struct{}), + announces: make(map[string]int), + announced: make(map[common.Hash][]*txAnnounce), + fetching: make(map[common.Hash]*txAnnounce), + underpriced: mapset.NewSet(), + hasTx: hasTx, + addTxs: addTxs, + dropPeer: dropPeer, + } +} + +// Notify announces the fetcher of the potential availability of a +// new transaction in the network. +func (f *TxFetcher) Notify(peer string, hashes []common.Hash, time time.Time, fetchTxs func([]common.Hash)) error { + announce := &txsAnnounce{ + hashes: hashes, + time: time, + origin: peer, + fetchTxs: fetchTxs, + } + select { + case f.notify <- announce: + return nil + case <-f.quit: + return errTerminated + } +} + +// EnqueueTxs imports a batch of received transaction into fetcher. +func (f *TxFetcher) EnqueueTxs(peer string, txs []*types.Transaction) error { + var ( + drop bool + hashes []common.Hash + ) + errs := f.addTxs(txs) + for i, err := range errs { + if err != nil { + // Drop peer if the received transaction isn't signed properly. + drop = (drop || err == core.ErrInvalidSender) + txFetchInvalidMeter.Mark(1) + + // Track the transaction hash if the price is too low for us. + // Avoid re-request this transaction when we receive another + // announcement. + if err == core.ErrUnderpriced { + for f.underpriced.Cardinality() >= underpriceSetSize { + f.underpriced.Pop() + } + f.underpriced.Add(txs[i].Hash()) + } + } + hashes = append(hashes, txs[i].Hash()) + } + if f.importTxsHook != nil { + f.importTxsHook(txs) + } + // Drop the peer if some transaction failed signature verification. + // We can regard this peer is trying to DOS us by feeding lots of + // random hashes. + if drop { + f.dropPeer(peer) + if f.dropHook != nil { + f.dropHook(peer) + } + } + select { + case f.cleanup <- hashes: + return nil + case <-f.quit: + return errTerminated + } +} + +// Start boots up the announcement based synchroniser, accepting and processing +// hash notifications and block fetches until termination requested. +func (f *TxFetcher) Start() { + go f.loop() +} + +// Stop terminates the announcement based synchroniser, canceling all pending +// operations. +func (f *TxFetcher) Stop() { + close(f.quit) +} + +func (f *TxFetcher) loop() { + fetchTimer := time.NewTimer(0) + + for { + // Clean up any expired transaction fetches. + // There are many cases can lead to it: + // * We send the request to busy peer which can reply immediately + // * We send the request to malicious peer which doesn't reply deliberately + // * We send the request to normal peer for a batch of transaction, but some + // transactions have been included into blocks. According to EIP these txs + // won't be included. + // But it's fine to delete the fetching record and reschedule fetching iff we + // receive the annoucement again. + for hash, announce := range f.fetching { + if time.Since(announce.time) > txFetchTimeout { + delete(f.fetching, hash) + txFetchTimeoutMeter.Mark(1) + } + } + select { + case anno := <-f.notify: + txAnnounceInMeter.Mark(int64(len(anno.hashes))) + + // Drop the new announce if there are too many accumulated. + count := f.announces[anno.origin] + len(anno.hashes) + if count > txAnnounceLimit { + txAnnounceDOSMeter.Mark(int64(count - txAnnounceLimit)) + break + } + f.announces[anno.origin] = count + + // All is well, schedule the announce if transaction is not yet downloading + empty := len(f.announced) == 0 + for _, hash := range anno.hashes { + if _, ok := f.fetching[hash]; ok { + continue + } + if f.underpriced.Contains(hash) { + txAnnounceUnderpriceMeter.Mark(1) + if f.rejectUnderprice != nil { + f.rejectUnderprice(hash) + } + continue + } + f.announced[hash] = append(f.announced[hash], &txAnnounce{ + origin: anno.origin, + time: anno.time, + fetchTxs: anno.fetchTxs, + }) + } + if empty && len(f.announced) > 0 { + f.reschedule(fetchTimer) + } + if f.announceHook != nil { + f.announceHook(anno.hashes) + } + case <-fetchTimer.C: + // At least one tx's timer ran out, check for needing retrieval + request := make(map[string][]common.Hash) + + for hash, announces := range f.announced { + if time.Since(announces[0].time) > arriveTimeout-gatherSlack { + // Pick a random peer to retrieve from, reset all others + announce := announces[rand.Intn(len(announces))] + f.forgetHash(hash) + + // Skip fetching if we already receive the transaction. + if f.hasTx(hash) { + txAnnounceSkipMeter.Mark(1) + continue + } + // If the transaction still didn't arrive, queue for fetching + request[announce.origin] = append(request[announce.origin], hash) + f.fetching[hash] = announce + } + } + // Send out all block header requests + for peer, hashes := range request { + log.Trace("Fetching scheduled transactions", "peer", peer, "txs", hashes) + fetchTxs := f.fetching[hashes[0]].fetchTxs + fetchTxs(hashes) + txFetchOutMeter.Mark(int64(len(hashes))) + } + // Schedule the next fetch if blocks are still pending + f.reschedule(fetchTimer) + case hashes := <-f.cleanup: + for _, hash := range hashes { + f.forgetHash(hash) + anno, exist := f.fetching[hash] + if !exist { + txBroadcastInMeter.Mark(1) // Directly transaction propagation + continue + } + txFetchDurationTimer.UpdateSince(anno.time) + txFetchSuccessMeter.Mark(1) + delete(f.fetching, hash) + } + if f.cleanupHook != nil { + f.cleanupHook(hashes) + } + case <-f.quit: + return + } + } +} + +// rescheduleFetch resets the specified fetch timer to the next blockAnnounce timeout. +func (f *TxFetcher) reschedule(fetch *time.Timer) { + // Short circuit if no transactions are announced + if len(f.announced) == 0 { + return + } + // Otherwise find the earliest expiring announcement + earliest := time.Now() + for _, announces := range f.announced { + if earliest.After(announces[0].time) { + earliest = announces[0].time + } + } + fetch.Reset(arriveTimeout - time.Since(earliest)) +} + +func (f *TxFetcher) forgetHash(hash common.Hash) { + // Remove all pending announces and decrement DOS counters + for _, announce := range f.announced[hash] { + f.announces[announce.origin]-- + if f.announces[announce.origin] <= 0 { + delete(f.announces, announce.origin) + } + } + delete(f.announced, hash) +} diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go new file mode 100644 index 0000000000..26f24f3f38 --- /dev/null +++ b/eth/fetcher/tx_fetcher_test.go @@ -0,0 +1,318 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package fetcher + +import ( + "crypto/ecdsa" + "math/big" + "math/rand" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" +) + +func init() { + rand.Seed(int64(time.Now().Nanosecond())) + + txAnnounceLimit = 64 + MaxTransactionFetch = 16 +} + +func makeTransactions(key *ecdsa.PrivateKey, target int) []*types.Transaction { + var txs []*types.Transaction + + for i := 0; i < target; i++ { + random := rand.Uint32() + tx := types.NewTransaction(uint64(random), common.Address{0x1, 0x2, 0x3}, big.NewInt(int64(random)), 100, big.NewInt(int64(random)), nil) + tx, _ = types.SignTx(tx, types.NewEIP155Signer(big.NewInt(1)), key) + txs = append(txs, tx) + } + return txs +} + +func makeUnsignedTransactions(key *ecdsa.PrivateKey, target int) []*types.Transaction { + var txs []*types.Transaction + + for i := 0; i < target; i++ { + random := rand.Uint32() + tx := types.NewTransaction(uint64(random), common.Address{0x1, 0x2, 0x3}, big.NewInt(int64(random)), 100, big.NewInt(int64(random)), nil) + txs = append(txs, tx) + } + return txs +} + +type txfetcherTester struct { + fetcher *TxFetcher + + priceLimit *big.Int + sender *ecdsa.PrivateKey + senderAddr common.Address + signer types.Signer + txs map[common.Hash]*types.Transaction + dropped map[string]struct{} + lock sync.RWMutex +} + +func newTxFetcherTester() *txfetcherTester { + key, _ := crypto.GenerateKey() + addr := crypto.PubkeyToAddress(key.PublicKey) + t := &txfetcherTester{ + sender: key, + senderAddr: addr, + signer: types.NewEIP155Signer(big.NewInt(1)), + txs: make(map[common.Hash]*types.Transaction), + dropped: make(map[string]struct{}), + } + t.fetcher = NewTxFetcher(t.hasTx, t.addTxs, t.dropPeer) + t.fetcher.Start() + return t +} + +func (t *txfetcherTester) hasTx(hash common.Hash) bool { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.txs[hash] != nil +} + +func (t *txfetcherTester) addTxs(txs []*types.Transaction) []error { + t.lock.Lock() + defer t.lock.Unlock() + + var errors []error + for _, tx := range txs { + // Make sure the transaction is signed properly + _, err := types.Sender(t.signer, tx) + if err != nil { + errors = append(errors, core.ErrInvalidSender) + continue + } + // Make sure the price is high enough to accpet + if t.priceLimit != nil && tx.GasPrice().Cmp(t.priceLimit) < 0 { + errors = append(errors, core.ErrUnderpriced) + continue + } + t.txs[tx.Hash()] = tx + errors = append(errors, nil) + } + return errors +} + +func (t *txfetcherTester) dropPeer(id string) { + t.lock.Lock() + defer t.lock.Unlock() + + t.dropped[id] = struct{}{} +} + +// makeTxFetcher retrieves a batch of transaction associated with a simulated peer. +func (t *txfetcherTester) makeTxFetcher(peer string, txs []*types.Transaction) func(hashes []common.Hash) { + closure := make(map[common.Hash]*types.Transaction) + for _, tx := range txs { + closure[tx.Hash()] = tx + } + return func(hashes []common.Hash) { + var txs []*types.Transaction + for _, hash := range hashes { + tx := closure[hash] + if tx == nil { + continue + } + txs = append(txs, tx) + } + // Return on a new thread + go t.fetcher.EnqueueTxs(peer, txs) + } +} + +func TestSequentialTxAnnouncements(t *testing.T) { + tester := newTxFetcherTester() + txs := makeTransactions(tester.sender, txAnnounceLimit) + + retrieveTxs := tester.makeTxFetcher("peer", txs) + + newTxsCh := make(chan struct{}) + tester.fetcher.importTxsHook = func(transactions []*types.Transaction) { + newTxsCh <- struct{}{} + } + for _, tx := range txs { + tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), retrieveTxs) + select { + case <-newTxsCh: + case <-time.NewTimer(time.Second).C: + t.Fatalf("timeout") + } + } + if len(tester.txs) != len(txs) { + t.Fatalf("Imported transaction number mismatch, want %d, got %d", len(txs), len(tester.txs)) + } +} + +func TestConcurrentAnnouncements(t *testing.T) { + tester := newTxFetcherTester() + txs := makeTransactions(tester.sender, txAnnounceLimit) + + txFetcherFn1 := tester.makeTxFetcher("peer1", txs) + txFetcherFn2 := tester.makeTxFetcher("peer2", txs) + + var ( + count uint32 + done = make(chan struct{}) + ) + tester.fetcher.importTxsHook = func(transactions []*types.Transaction) { + atomic.AddUint32(&count, uint32(len(transactions))) + if atomic.LoadUint32(&count) >= uint32(txAnnounceLimit) { + done <- struct{}{} + } + } + for _, tx := range txs { + tester.fetcher.Notify("peer1", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), txFetcherFn1) + tester.fetcher.Notify("peer2", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout+time.Millisecond), txFetcherFn2) + tester.fetcher.Notify("peer2", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout-time.Millisecond), txFetcherFn2) + } + select { + case <-done: + case <-time.NewTimer(time.Second).C: + t.Fatalf("timeout") + } +} + +func TestBatchAnnouncements(t *testing.T) { + tester := newTxFetcherTester() + txs := makeTransactions(tester.sender, txAnnounceLimit) + + retrieveTxs := tester.makeTxFetcher("peer", txs) + + var count uint32 + var done = make(chan struct{}) + tester.fetcher.importTxsHook = func(txs []*types.Transaction) { + atomic.AddUint32(&count, uint32(len(txs))) + + if atomic.LoadUint32(&count) >= uint32(txAnnounceLimit) { + done <- struct{}{} + } + } + // Send all announces which exceeds the limit. + var hashes []common.Hash + for _, tx := range txs { + hashes = append(hashes, tx.Hash()) + } + tester.fetcher.Notify("peer", hashes, time.Now(), retrieveTxs) + + select { + case <-done: + case <-time.NewTimer(time.Second).C: + t.Fatalf("timeout") + } +} + +func TestPropagationAfterAnnounce(t *testing.T) { + tester := newTxFetcherTester() + txs := makeTransactions(tester.sender, txAnnounceLimit) + + var cleaned = make(chan struct{}) + tester.fetcher.cleanupHook = func(hashes []common.Hash) { + cleaned <- struct{}{} + } + retrieveTxs := tester.makeTxFetcher("peer", txs) + for _, tx := range txs { + tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now(), retrieveTxs) + tester.fetcher.EnqueueTxs("peer", []*types.Transaction{tx}) + + // It's ok to read the map directly since no write + // will happen in the same time. + <-cleaned + if len(tester.fetcher.announced) != 0 { + t.Fatalf("Announcement should be cleaned, got %d", len(tester.fetcher.announced)) + } + } +} + +func TestEnqueueTransactions(t *testing.T) { + tester := newTxFetcherTester() + txs := makeTransactions(tester.sender, txAnnounceLimit) + + done := make(chan struct{}) + tester.fetcher.importTxsHook = func(transactions []*types.Transaction) { + if len(transactions) == txAnnounceLimit { + done <- struct{}{} + } + } + go tester.fetcher.EnqueueTxs("peer", txs) + select { + case <-done: + case <-time.NewTimer(time.Second).C: + t.Fatalf("timeout") + } +} + +func TestInvalidTxAnnounces(t *testing.T) { + tester := newTxFetcherTester() + + var txs []*types.Transaction + txs = append(txs, makeUnsignedTransactions(tester.sender, 1)...) + txs = append(txs, makeTransactions(tester.sender, 1)...) + + txFetcherFn := tester.makeTxFetcher("peer", txs) + + dropped := make(chan string, 1) + tester.fetcher.dropHook = func(s string) { dropped <- s } + + for _, tx := range txs { + tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now(), txFetcherFn) + } + select { + case s := <-dropped: + if s != "peer" { + t.Fatalf("invalid dropped peer") + } + case <-time.NewTimer(time.Second).C: + t.Fatalf("timeout") + } +} + +func TestRejectUnderpriced(t *testing.T) { + tester := newTxFetcherTester() + tester.priceLimit = big.NewInt(10000) + + done := make(chan struct{}) + tester.fetcher.importTxsHook = func([]*types.Transaction) { done <- struct{}{} } + reject := make(chan struct{}) + tester.fetcher.rejectUnderprice = func(common.Hash) { reject <- struct{}{} } + + tx := types.NewTransaction(0, common.Address{0x1, 0x2, 0x3}, big.NewInt(int64(100)), 100, big.NewInt(int64(100)), nil) + tx, _ = types.SignTx(tx, types.NewEIP155Signer(big.NewInt(1)), tester.sender) + txFetcherFn := tester.makeTxFetcher("peer", []*types.Transaction{tx}) + + // Send the announcement first time + tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), txFetcherFn) + <-done + + // Resend the announcement, shouldn't schedule fetching this time + tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), txFetcherFn) + select { + case <-reject: + case <-time.NewTimer(time.Second).C: + t.Fatalf("timeout") + } +} diff --git a/eth/handler.go b/eth/handler.go index ae2b764cf6..d527b15d13 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -77,9 +77,10 @@ type ProtocolManager struct { blockchain *core.BlockChain maxPeers int - downloader *downloader.Downloader - fetcher *fetcher.Fetcher - peers *peerSet + downloader *downloader.Downloader + blockFetcher *fetcher.BlockFetcher + txFetcher *fetcher.TxFetcher + peers *peerSet eventMux *event.TypeMux txsCh chan core.NewTxsEvent @@ -97,6 +98,9 @@ type ProtocolManager struct { // wait group is used for graceful shutdowns during downloading // and processing wg sync.WaitGroup + + // Test fields or hooks + broadcastTxAnnouncesOnly bool // Testing field, disable transaction propagation } // NewProtocolManager returns a new Ethereum sub protocol manager. The Ethereum sub protocol manages peers capable @@ -187,7 +191,8 @@ func NewProtocolManager(config *params.ChainConfig, checkpoint *params.TrustedCh } return n, err } - manager.fetcher = fetcher.New(blockchain.GetBlockByHash, validator, manager.BroadcastBlock, heighter, inserter, manager.removePeer) + manager.blockFetcher = fetcher.NewBlockFetcher(blockchain.GetBlockByHash, validator, manager.BroadcastBlock, heighter, inserter, manager.removePeer) + manager.txFetcher = fetcher.NewTxFetcher(txpool.Has, txpool.AddRemotes, manager.removePeer) return manager, nil } @@ -203,7 +208,7 @@ func (pm *ProtocolManager) makeProtocol(version uint) p2p.Protocol { Version: version, Length: length, Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - peer := pm.newPeer(int(version), p, rw) + peer := pm.newPeer(int(version), p, rw, pm.txpool.Get) select { case pm.newPeerCh <- peer: pm.wg.Add(1) @@ -286,8 +291,8 @@ func (pm *ProtocolManager) Stop() { log.Info("Ethereum protocol stopped") } -func (pm *ProtocolManager) newPeer(pv int, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { - return newPeer(pv, p, newMeteredMsgWriter(rw)) +func (pm *ProtocolManager) newPeer(pv int, p *p2p.Peer, rw p2p.MsgReadWriter, getPooledTx func(hash common.Hash) *types.Transaction) *peer { + return newPeer(pv, p, newMeteredMsgWriter(rw), getPooledTx) } // handle is the callback invoked to manage the life cycle of an eth peer. When @@ -514,7 +519,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { p.Log().Debug("Whitelist block verified", "number", headers[0].Number.Uint64(), "hash", want) } // Irrelevant of the fork checks, send the header to the fetcher just in case - headers = pm.fetcher.FilterHeaders(p.id, headers, time.Now()) + headers = pm.blockFetcher.FilterHeaders(p.id, headers, time.Now()) } if len(headers) > 0 || !filter { err := pm.downloader.DeliverHeaders(p.id, headers) @@ -567,7 +572,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Filter out any explicitly requested bodies, deliver the rest to the downloader filter := len(transactions) > 0 || len(uncles) > 0 if filter { - transactions, uncles = pm.fetcher.FilterBodies(p.id, transactions, uncles, time.Now()) + transactions, uncles = pm.blockFetcher.FilterBodies(p.id, transactions, uncles, time.Now()) } if len(transactions) > 0 || len(uncles) > 0 || !filter { err := pm.downloader.DeliverBodies(p.id, transactions, uncles) @@ -678,7 +683,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } } for _, block := range unknown { - pm.fetcher.Notify(p.id, block.Hash, block.Number, time.Now(), p.RequestOneHeader, p.RequestBodies) + pm.blockFetcher.Notify(p.id, block.Hash, block.Number, time.Now(), p.RequestOneHeader, p.RequestBodies) } case msg.Code == NewBlockMsg: @@ -703,7 +708,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Mark the peer as owning the block and schedule it for import p.MarkBlock(request.Block.Hash()) - pm.fetcher.Enqueue(p.id, request.Block) + pm.blockFetcher.Enqueue(p.id, request.Block) // Assuming the block is importable by the peer, but possibly not yet done so, // calculate the head hash and TD that the peer truly must have. @@ -724,6 +729,66 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } } + case msg.Code == NewPooledTransactionHashesMsg && p.version >= eth65: + // New transaction announcement arrived, make sure we have + // a valid and fresh chain to handle them + if atomic.LoadUint32(&pm.acceptTxs) == 0 { + break + } + var hashes []common.Hash + if err := msg.Decode(&hashes); err != nil { + return errResp(ErrDecode, "msg %v: %v", msg, err) + } + // Schedule all the unknown hashes for retrieval + var unknown []common.Hash + for _, hash := range hashes { + // Mark the hashes as present at the remote node + p.MarkTransaction(hash) + + // Filter duplicated transaction announcement. + // Notably we only dedupliate announcement in txpool, check the rationale + // behind in EIP https://github.com/ethereum/EIPs/pull/2464. + if pm.txpool.Has(hash) { + continue + } + unknown = append(unknown, hash) + } + pm.txFetcher.Notify(p.id, unknown, time.Now(), p.AsyncRequestTxs) + + case msg.Code == GetPooledTransactionsMsg && p.version >= eth65: + // Decode the retrieval message + msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size)) + if _, err := msgStream.List(); err != nil { + return err + } + // Gather transactions until the fetch or network limits is reached + var ( + hash common.Hash + bytes int + txs []rlp.RawValue + ) + for bytes < softResponseLimit { + // Retrieve the hash of the next block + if err := msgStream.Decode(&hash); err == rlp.EOL { + break + } else if err != nil { + return errResp(ErrDecode, "msg %v: %v", msg, err) + } + // Retrieve the requested transaction, skipping if unknown to us + tx := pm.txpool.Get(hash) + if tx == nil { + continue + } + // If known, encode and queue for response packet + if encoded, err := rlp.EncodeToBytes(tx); err != nil { + log.Error("Failed to encode transaction", "err", err) + } else { + txs = append(txs, encoded) + bytes += len(encoded) + } + } + return p.SendTransactionRLP(txs) + case msg.Code == TxMsg: // Transactions arrived, make sure we have a valid and fresh chain to handle them if atomic.LoadUint32(&pm.acceptTxs) == 0 { @@ -741,7 +806,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } p.MarkTransaction(tx.Hash()) } - pm.txpool.AddRemotes(txs) + pm.txFetcher.EnqueueTxs(p.id, txs) default: return errResp(ErrInvalidMsgCode, "%v", msg.Code) @@ -791,20 +856,48 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { // BroadcastTxs will propagate a batch of transactions to all peers which are not known to // already have the given transaction. -func (pm *ProtocolManager) BroadcastTxs(txs types.Transactions) { - var txset = make(map[*peer]types.Transactions) - +func (pm *ProtocolManager) BroadcastTxs(txs types.Transactions, propagate bool) { + var ( + txset = make(map[*peer][]common.Hash) + annos = make(map[*peer][]common.Hash) + ) // Broadcast transactions to a batch of peers not knowing about it + if propagate { + for _, tx := range txs { + peers := pm.peers.PeersWithoutTx(tx.Hash()) + + // Send the block to a subset of our peers + transferLen := int(math.Sqrt(float64(len(peers)))) + if transferLen < minBroadcastPeers { + transferLen = minBroadcastPeers + } + if transferLen > len(peers) { + transferLen = len(peers) + } + transfer := peers[:transferLen] + for _, peer := range transfer { + txset[peer] = append(txset[peer], tx.Hash()) + } + log.Trace("Broadcast transaction", "hash", tx.Hash(), "recipients", len(peers)) + } + for peer, hashes := range txset { + peer.AsyncSendTransactions(hashes) + } + return + } + // Otherwise only broadcast the announcement to peers for _, tx := range txs { peers := pm.peers.PeersWithoutTx(tx.Hash()) for _, peer := range peers { - txset[peer] = append(txset[peer], tx) + annos[peer] = append(annos[peer], tx.Hash()) } - log.Trace("Broadcast transaction", "hash", tx.Hash(), "recipients", len(peers)) } - // FIXME include this again: peers = peers[:int(math.Sqrt(float64(len(peers))))] - for peer, txs := range txset { - peer.AsyncSendTransactions(txs) + for peer, hashes := range annos { + if peer.version >= eth65 { + peer.AsyncSendTransactionHashes(hashes) + } else { + peer.AsyncSendTransactions(hashes) + } } } @@ -823,7 +916,13 @@ func (pm *ProtocolManager) txBroadcastLoop() { for { select { case event := <-pm.txsCh: - pm.BroadcastTxs(event.Txs) + // For testing purpose only, disable propagation + if pm.broadcastTxAnnouncesOnly { + pm.BroadcastTxs(event.Txs, false) + continue + } + pm.BroadcastTxs(event.Txs, true) // First propagate transactions to peers + pm.BroadcastTxs(event.Txs, false) // Only then announce to the rest // Err() channel will be closed when unsubscribing. case <-pm.txsSub.Err(): diff --git a/eth/handler_test.go b/eth/handler_test.go index 354cbc068c..97613a9834 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -495,7 +495,7 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo if err != nil { t.Fatalf("failed to create new blockchain: %v", err) } - pm, err := NewProtocolManager(config, cht, syncmode, DefaultConfig.NetworkId, new(event.TypeMux), new(testTxPool), ethash.NewFaker(), blockchain, db, 1, nil) + pm, err := NewProtocolManager(config, cht, syncmode, DefaultConfig.NetworkId, new(event.TypeMux), &testTxPool{pool: make(map[common.Hash]*types.Transaction)}, ethash.NewFaker(), blockchain, db, 1, nil) if err != nil { t.Fatalf("failed to start test protocol manager: %v", err) } @@ -582,7 +582,7 @@ func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) { if err != nil { t.Fatalf("failed to create new blockchain: %v", err) } - pm, err := NewProtocolManager(config, nil, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db, 1, nil) + pm, err := NewProtocolManager(config, nil, downloader.FullSync, DefaultConfig.NetworkId, evmux, &testTxPool{pool: make(map[common.Hash]*types.Transaction)}, pow, blockchain, db, 1, nil) if err != nil { t.Fatalf("failed to start test protocol manager: %v", err) } diff --git a/eth/helper_test.go b/eth/helper_test.go index e66910334f..bec37e16cb 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -68,7 +68,7 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func if _, err := blockchain.InsertChain(chain); err != nil { panic(err) } - pm, err := NewProtocolManager(gspec.Config, nil, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx}, engine, blockchain, db, 1, nil) + pm, err := NewProtocolManager(gspec.Config, nil, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx, pool: make(map[common.Hash]*types.Transaction)}, engine, blockchain, db, 1, nil) if err != nil { return nil, nil, err } @@ -91,22 +91,43 @@ func newTestProtocolManagerMust(t *testing.T, mode downloader.SyncMode, blocks i // testTxPool is a fake, helper transaction pool for testing purposes type testTxPool struct { txFeed event.Feed - pool []*types.Transaction // Collection of all transactions - added chan<- []*types.Transaction // Notification channel for new transactions + pool map[common.Hash]*types.Transaction // Hash map of collected transactions + added chan<- []*types.Transaction // Notification channel for new transactions lock sync.RWMutex // Protects the transaction pool } +// Has returns an indicator whether txpool has a transaction +// cached with the given hash. +func (p *testTxPool) Has(hash common.Hash) bool { + p.lock.Lock() + defer p.lock.Unlock() + + return p.pool[hash] != nil +} + +// Get retrieves the transaction from local txpool with given +// tx hash. +func (p *testTxPool) Get(hash common.Hash) *types.Transaction { + p.lock.Lock() + defer p.lock.Unlock() + + return p.pool[hash] +} + // AddRemotes appends a batch of transactions to the pool, and notifies any // listeners if the addition channel is non nil func (p *testTxPool) AddRemotes(txs []*types.Transaction) []error { p.lock.Lock() defer p.lock.Unlock() - p.pool = append(p.pool, txs...) + for _, tx := range txs { + p.pool[tx.Hash()] = tx + } if p.added != nil { p.added <- txs } + p.txFeed.Send(core.NewTxsEvent{Txs: txs}) return make([]error, len(txs)) } @@ -153,7 +174,7 @@ func newTestPeer(name string, version int, pm *ProtocolManager, shake bool) (*te var id enode.ID rand.Read(id[:]) - peer := pm.newPeer(version, p2p.NewPeer(id, name, nil), net) + peer := pm.newPeer(version, p2p.NewPeer(id, name, nil), net, pm.txpool.Get) // Start the peer on a new thread errc := make(chan error, 1) @@ -191,7 +212,7 @@ func (p *testPeer) handshake(t *testing.T, td *big.Int, head common.Hash, genesi CurrentBlock: head, GenesisBlock: genesis, } - case p.version == eth64: + case p.version >= eth64: msg = &statusData{ ProtocolVersion: uint32(p.version), NetworkID: DefaultConfig.NetworkId, diff --git a/eth/peer.go b/eth/peer.go index 0beec1d844..f4b939b71c 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/fetcher" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rlp" ) @@ -41,24 +42,39 @@ const ( maxKnownTxs = 32768 // Maximum transactions hashes to keep in the known list (prevent DOS) maxKnownBlocks = 1024 // Maximum block hashes to keep in the known list (prevent DOS) - // maxQueuedTxs is the maximum number of transaction lists to queue up before - // dropping broadcasts. This is a sensitive number as a transaction list might - // contain a single transaction, or thousands. - maxQueuedTxs = 128 + // maxQueuedTxs is the maximum number of transactions to queue up before dropping + // broadcasts. + maxQueuedTxs = 4096 - // maxQueuedProps is the maximum number of block propagations to queue up before + // maxQueuedTxAnns is the maximum number of transaction announcements to queue up + // before dropping broadcasts. + maxQueuedTxAnns = 4096 + + // maxQueuedTxRetrieval is the maximum number of tx retrieval requests to queue up + // before dropping requests. + maxQueuedTxRetrieval = 4096 + + // maxQueuedBlocks is the maximum number of block propagations to queue up before // dropping broadcasts. There's not much point in queueing stale blocks, so a few // that might cover uncles should be enough. - maxQueuedProps = 4 + maxQueuedBlocks = 4 - // maxQueuedAnns is the maximum number of block announcements to queue up before + // maxQueuedBlockAnns is the maximum number of block announcements to queue up before // dropping broadcasts. Similarly to block propagations, there's no point to queue // above some healthy uncle limit, so use that. - maxQueuedAnns = 4 + maxQueuedBlockAnns = 4 handshakeTimeout = 5 * time.Second ) +// max is a helper function which returns the larger of the two given integers. +func max(a, b int) int { + if a > b { + return a + } + return b +} + // PeerInfo represents a short summary of the Ethereum sub-protocol metadata known // about a connected peer. type PeerInfo struct { @@ -86,48 +102,48 @@ type peer struct { td *big.Int lock sync.RWMutex - knownTxs mapset.Set // Set of transaction hashes known to be known by this peer - knownBlocks mapset.Set // Set of block hashes known to be known by this peer - queuedTxs chan []*types.Transaction // Queue of transactions to broadcast to the peer - queuedProps chan *propEvent // Queue of blocks to broadcast to the peer - queuedAnns chan *types.Block // Queue of blocks to announce to the peer - term chan struct{} // Termination channel to stop the broadcaster + knownTxs mapset.Set // Set of transaction hashes known to be known by this peer + knownBlocks mapset.Set // Set of block hashes known to be known by this peer + queuedBlocks chan *propEvent // Queue of blocks to broadcast to the peer + queuedBlockAnns chan *types.Block // Queue of blocks to announce to the peer + txPropagation chan []common.Hash // Channel used to queue transaction propagation requests + txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests + txRetrieval chan []common.Hash // Channel used to queue transaction retrieval requests + getPooledTx func(common.Hash) *types.Transaction // Callback used to retrieve transaction from txpool + term chan struct{} // Termination channel to stop the broadcaster } -func newPeer(version int, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { +func newPeer(version int, p *p2p.Peer, rw p2p.MsgReadWriter, getPooledTx func(hash common.Hash) *types.Transaction) *peer { return &peer{ - Peer: p, - rw: rw, - version: version, - id: fmt.Sprintf("%x", p.ID().Bytes()[:8]), - knownTxs: mapset.NewSet(), - knownBlocks: mapset.NewSet(), - queuedTxs: make(chan []*types.Transaction, maxQueuedTxs), - queuedProps: make(chan *propEvent, maxQueuedProps), - queuedAnns: make(chan *types.Block, maxQueuedAnns), - term: make(chan struct{}), + Peer: p, + rw: rw, + version: version, + id: fmt.Sprintf("%x", p.ID().Bytes()[:8]), + knownTxs: mapset.NewSet(), + knownBlocks: mapset.NewSet(), + queuedBlocks: make(chan *propEvent, maxQueuedBlocks), + queuedBlockAnns: make(chan *types.Block, maxQueuedBlockAnns), + txPropagation: make(chan []common.Hash), + txAnnounce: make(chan []common.Hash), + txRetrieval: make(chan []common.Hash), + getPooledTx: getPooledTx, + term: make(chan struct{}), } } -// broadcast is a write loop that multiplexes block propagations, announcements -// and transaction broadcasts into the remote peer. The goal is to have an async -// writer that does not lock up node internals. -func (p *peer) broadcast() { +// broadcastBlocks is a write loop that multiplexes block propagations, +// announcements into the remote peer. The goal is to have an async writer +// that does not lock up node internals. +func (p *peer) broadcastBlocks() { for { select { - case txs := <-p.queuedTxs: - if err := p.SendTransactions(txs); err != nil { - return - } - p.Log().Trace("Broadcast transactions", "count", len(txs)) - - case prop := <-p.queuedProps: + case prop := <-p.queuedBlocks: if err := p.SendNewBlock(prop.block, prop.td); err != nil { return } p.Log().Trace("Propagated block", "number", prop.block.Number(), "hash", prop.block.Hash(), "td", prop.td) - case block := <-p.queuedAnns: + case block := <-p.queuedBlockAnns: if err := p.SendNewBlockHashes([]common.Hash{block.Hash()}, []uint64{block.NumberU64()}); err != nil { return } @@ -139,6 +155,175 @@ func (p *peer) broadcast() { } } +// broadcastTxs is a write loop that multiplexes transaction propagations, +// announcements into the remote peer. The goal is to have an async writer +// that does not lock up node internals. +func (p *peer) broadcastTxs() { + var ( + txProps []common.Hash // Queue of transaction propagations to the peer + txAnnos []common.Hash // Queue of transaction announcements to the peer + done chan struct{} // Non-nil if background network sender routine is active. + errch = make(chan error) // Channel used to receive network error + ) + scheduleTask := func() { + // Short circuit if there already has a inflight task. + if done != nil { + return + } + // Spin up transaction propagation task if there is any + // queued hashes. + if len(txProps) > 0 { + var ( + hashes []common.Hash + txs []*types.Transaction + size common.StorageSize + ) + for i := 0; i < len(txProps) && size < txsyncPackSize; i++ { + if tx := p.getPooledTx(txProps[i]); tx != nil { + txs = append(txs, tx) + size += tx.Size() + } + hashes = append(hashes, txProps[i]) + } + txProps = txProps[:copy(txProps, txProps[len(hashes):])] + if len(txs) > 0 { + done = make(chan struct{}) + go func() { + if err := p.SendNewTransactions(txs); err != nil { + errch <- err + return + } + close(done) + p.Log().Trace("Sent transactions", "count", len(txs)) + }() + return + } + } + // Spin up transaction announcement task if there is any + // queued hashes. + if len(txAnnos) > 0 { + var ( + hashes []common.Hash + pending []common.Hash + size common.StorageSize + ) + for i := 0; i < len(txAnnos) && size < txsyncPackSize; i++ { + if tx := p.getPooledTx(txAnnos[i]); tx != nil { + pending = append(pending, txAnnos[i]) + size += common.HashLength + } + hashes = append(hashes, txAnnos[i]) + } + txAnnos = txAnnos[:copy(txAnnos, txAnnos[len(hashes):])] + if len(pending) > 0 { + done = make(chan struct{}) + go func() { + if err := p.SendNewTransactionHashes(pending); err != nil { + errch <- err + return + } + close(done) + p.Log().Trace("Sent transaction announcements", "count", len(pending)) + }() + } + } + } + + for { + scheduleTask() + select { + case hashes := <-p.txPropagation: + if len(txProps) == maxQueuedTxs { + continue + } + if len(txProps)+len(hashes) > maxQueuedTxs { + hashes = hashes[:maxQueuedTxs-len(txProps)] + } + txProps = append(txProps, hashes...) + + case hashes := <-p.txAnnounce: + if len(txAnnos) == maxQueuedTxAnns { + continue + } + if len(txAnnos)+len(hashes) > maxQueuedTxAnns { + hashes = hashes[:maxQueuedTxAnns-len(txAnnos)] + } + txAnnos = append(txAnnos, hashes...) + + case <-done: + done = nil + + case <-errch: + return + + case <-p.term: + return + } + } +} + +// retrievalTxs is a write loop which is responsible for retrieving transaction +// from the remote peer. The goal is to have an async writer that does not lock +// up node internals. If there are too many requests queued, then new arrival +// requests will be dropped silently so that we can ensure the memory assumption +// is fixed for each peer. +func (p *peer) retrievalTxs() { + var ( + requests []common.Hash // Queue of transaction requests to the peer + done chan struct{} // Non-nil if background network sender routine is active. + errch = make(chan error) // Channel used to receive network error + ) + // pick chooses a reasonble number of transaction hashes for retrieval. + pick := func() []common.Hash { + var ret []common.Hash + if len(requests) > fetcher.MaxTransactionFetch { + ret = requests[:fetcher.MaxTransactionFetch] + } else { + ret = requests[:] + } + requests = requests[:copy(requests, requests[len(ret):])] + return ret + } + // send sends transactions retrieval request. + send := func(hashes []common.Hash, done chan struct{}) { + if err := p.RequestTxs(hashes); err != nil { + errch <- err + return + } + close(done) + p.Log().Trace("Sent transaction retrieval request", "count", len(hashes)) + } + for { + select { + case hashes := <-p.txRetrieval: + if len(requests) == maxQueuedTxRetrieval { + continue + } + if len(requests)+len(hashes) > maxQueuedTxRetrieval { + hashes = hashes[:maxQueuedTxRetrieval-len(requests)] + } + requests = append(requests, hashes...) + if done == nil { + done = make(chan struct{}) + go send(pick(), done) + } + + case <-done: + done = nil + if pending := pick(); len(pending) > 0 { + done = make(chan struct{}) + go send(pending, done) + } + + case <- errch: + return + + case <-p.term: + return + } + } +} + // close signals the broadcast goroutine to terminate. func (p *peer) close() { close(p.term) @@ -194,33 +379,67 @@ func (p *peer) MarkTransaction(hash common.Hash) { p.knownTxs.Add(hash) } -// SendTransactions sends transactions to the peer and includes the hashes -// in its transaction hash set for future reference. -func (p *peer) SendTransactions(txs types.Transactions) error { +// SendNewTransactionHashes sends a batch of transaction hashes to the peer and +// includes the hashes in its transaction hash set for future reference. +func (p *peer) SendNewTransactionHashes(hashes []common.Hash) error { // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { + p.knownTxs.Pop() + } + for _, hash := range hashes { + p.knownTxs.Add(hash) + } + return p2p.Send(p.rw, NewPooledTransactionHashesMsg, hashes) +} + +// SendNewTransactions sends transactions to the peer and includes the hashes +// in its transaction hash set for future reference. +func (p *peer) SendNewTransactions(txs types.Transactions) error { + // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(txs)) { + p.knownTxs.Pop() + } for _, tx := range txs { p.knownTxs.Add(tx.Hash()) } - for p.knownTxs.Cardinality() >= maxKnownTxs { - p.knownTxs.Pop() - } + return p2p.Send(p.rw, TxMsg, txs) +} + +func (p *peer) SendTransactionRLP(txs []rlp.RawValue) error { return p2p.Send(p.rw, TxMsg, txs) } // AsyncSendTransactions queues list of transactions propagation to a remote // peer. If the peer's broadcast queue is full, the event is silently dropped. -func (p *peer) AsyncSendTransactions(txs []*types.Transaction) { +func (p *peer) AsyncSendTransactions(hashes []common.Hash) { select { - case p.queuedTxs <- txs: + case p.txPropagation <- hashes: // Mark all the transactions as known, but ensure we don't overflow our limits - for _, tx := range txs { - p.knownTxs.Add(tx.Hash()) - } - for p.knownTxs.Cardinality() >= maxKnownTxs { + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { p.knownTxs.Pop() } - default: - p.Log().Debug("Dropping transaction propagation", "count", len(txs)) + for _, hash := range hashes { + p.knownTxs.Add(hash) + } + case <-p.term: + p.Log().Debug("Dropping transaction propagation", "count", len(hashes)) + } +} + +// AsyncSendTransactions queues list of transactions propagation to a remote +// peer. If the peer's broadcast queue is full, the event is silently dropped. +func (p *peer) AsyncSendTransactionHashes(hashes []common.Hash) { + select { + case p.txAnnounce <- hashes: + // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { + p.knownTxs.Pop() + } + for _, hash := range hashes { + p.knownTxs.Add(hash) + } + case <-p.term: + p.Log().Debug("Dropping transaction announcement", "count", len(hashes)) } } @@ -228,12 +447,12 @@ func (p *peer) AsyncSendTransactions(txs []*types.Transaction) { // a hash notification. func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error { // Mark all the block hashes as known, but ensure we don't overflow our limits + for p.knownBlocks.Cardinality() > max(0, maxKnownBlocks-len(hashes)) { + p.knownBlocks.Pop() + } for _, hash := range hashes { p.knownBlocks.Add(hash) } - for p.knownBlocks.Cardinality() >= maxKnownBlocks { - p.knownBlocks.Pop() - } request := make(newBlockHashesData, len(hashes)) for i := 0; i < len(hashes); i++ { request[i].Hash = hashes[i] @@ -247,12 +466,12 @@ func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error // dropped. func (p *peer) AsyncSendNewBlockHash(block *types.Block) { select { - case p.queuedAnns <- block: + case p.queuedBlockAnns <- block: // Mark all the block hash as known, but ensure we don't overflow our limits - p.knownBlocks.Add(block.Hash()) for p.knownBlocks.Cardinality() >= maxKnownBlocks { p.knownBlocks.Pop() } + p.knownBlocks.Add(block.Hash()) default: p.Log().Debug("Dropping block announcement", "number", block.NumberU64(), "hash", block.Hash()) } @@ -261,10 +480,10 @@ func (p *peer) AsyncSendNewBlockHash(block *types.Block) { // SendNewBlock propagates an entire block to a remote peer. func (p *peer) SendNewBlock(block *types.Block, td *big.Int) error { // Mark all the block hash as known, but ensure we don't overflow our limits - p.knownBlocks.Add(block.Hash()) for p.knownBlocks.Cardinality() >= maxKnownBlocks { p.knownBlocks.Pop() } + p.knownBlocks.Add(block.Hash()) return p2p.Send(p.rw, NewBlockMsg, []interface{}{block, td}) } @@ -272,12 +491,12 @@ func (p *peer) SendNewBlock(block *types.Block, td *big.Int) error { // the peer's broadcast queue is full, the event is silently dropped. func (p *peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { select { - case p.queuedProps <- &propEvent{block: block, td: td}: + case p.queuedBlocks <- &propEvent{block: block, td: td}: // Mark all the block hash as known, but ensure we don't overflow our limits - p.knownBlocks.Add(block.Hash()) for p.knownBlocks.Cardinality() >= maxKnownBlocks { p.knownBlocks.Pop() } + p.knownBlocks.Add(block.Hash()) default: p.Log().Debug("Dropping block propagation", "number", block.NumberU64(), "hash", block.Hash()) } @@ -352,6 +571,22 @@ func (p *peer) RequestReceipts(hashes []common.Hash) error { return p2p.Send(p.rw, GetReceiptsMsg, hashes) } +// RequestTxs fetches a batch of transactions from a remote node. +func (p *peer) RequestTxs(hashes []common.Hash) error { + p.Log().Debug("Fetching batch of transactions", "count", len(hashes)) + return p2p.Send(p.rw, GetPooledTransactionsMsg, hashes) +} + +// AsyncRequestTxs queues a tx retrieval request to a remote peer. If +// the peer's retrieval queue is full, the event is silently dropped. +func (p *peer) AsyncRequestTxs(hashes []common.Hash) { + select { + case p.txRetrieval <- hashes: + case <-p.term: + p.Log().Debug("Dropping transaction retrieval request", "count", len(hashes)) + } +} + // Handshake executes the eth protocol handshake, negotiating version number, // network IDs, difficulties, head and genesis blocks. func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis common.Hash, forkID forkid.ID, forkFilter forkid.Filter) error { @@ -372,7 +607,7 @@ func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis CurrentBlock: head, GenesisBlock: genesis, }) - case p.version == eth64: + case p.version >= eth64: errc <- p2p.Send(p.rw, StatusMsg, &statusData{ ProtocolVersion: uint32(p.version), NetworkID: network, @@ -389,7 +624,7 @@ func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis switch { case p.version == eth63: errc <- p.readStatusLegacy(network, &status63, genesis) - case p.version == eth64: + case p.version >= eth64: errc <- p.readStatus(network, &status, genesis, forkFilter) default: panic(fmt.Sprintf("unsupported eth protocol version: %d", p.version)) @@ -410,7 +645,7 @@ func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis switch { case p.version == eth63: p.td, p.head = status63.TD, status63.CurrentBlock - case p.version == eth64: + case p.version >= eth64: p.td, p.head = status.TD, status.Head default: panic(fmt.Sprintf("unsupported eth protocol version: %d", p.version)) @@ -511,7 +746,9 @@ func (ps *peerSet) Register(p *peer) error { return errAlreadyRegistered } ps.peers[p.id] = p - go p.broadcast() + go p.broadcastBlocks() + go p.broadcastTxs() + go p.retrievalTxs() return nil } diff --git a/eth/protocol.go b/eth/protocol.go index 62e4d13d14..1cef66adb2 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -33,16 +33,17 @@ import ( const ( eth63 = 63 eth64 = 64 + eth65 = 65 ) // protocolName is the official short name of the protocol used during capability negotiation. const protocolName = "eth" // ProtocolVersions are the supported versions of the eth protocol (first is primary). -var ProtocolVersions = []uint{eth64, eth63} +var ProtocolVersions = []uint{eth65, eth64, eth63} // protocolLengths are the number of implemented message corresponding to different protocol versions. -var protocolLengths = map[uint]uint64{eth64: 17, eth63: 17} +var protocolLengths = map[uint]uint64{eth65: 17, eth64: 17, eth63: 17} const protocolMaxMsgSize = 10 * 1024 * 1024 // Maximum cap on the size of a protocol message @@ -60,6 +61,13 @@ const ( NodeDataMsg = 0x0e GetReceiptsMsg = 0x0f ReceiptsMsg = 0x10 + + // New protocol message codes introduced in eth65 + // + // Previously these message ids(0x08, 0x09) were used by some + // legacy and unsupported eth protocols, reown them here. + NewPooledTransactionHashesMsg = 0x08 + GetPooledTransactionsMsg = 0x09 ) type errCode int @@ -94,6 +102,14 @@ var errorToString = map[int]string{ } type txPool interface { + // Has returns an indicator whether txpool has a transaction + // cached with the given hash. + Has(hash common.Hash) bool + + // Get retrieves the transaction from local txpool with given + // tx hash. + Get(hash common.Hash) *types.Transaction + // AddRemotes should add the given transactions to the pool. AddRemotes([]*types.Transaction) []error diff --git a/eth/protocol_test.go b/eth/protocol_test.go index ca418942bb..e9a1a511ef 100644 --- a/eth/protocol_test.go +++ b/eth/protocol_test.go @@ -20,6 +20,7 @@ import ( "fmt" "math/big" "sync" + "sync/atomic" "testing" "time" @@ -180,16 +181,16 @@ func TestForkIDSplit(t *testing.T) { blocksNoFork, _ = core.GenerateChain(configNoFork, genesisNoFork, engine, dbNoFork, 2, nil) blocksProFork, _ = core.GenerateChain(configProFork, genesisProFork, engine, dbProFork, 2, nil) - ethNoFork, _ = NewProtocolManager(configNoFork, nil, downloader.FullSync, 1, new(event.TypeMux), new(testTxPool), engine, chainNoFork, dbNoFork, 1, nil) - ethProFork, _ = NewProtocolManager(configProFork, nil, downloader.FullSync, 1, new(event.TypeMux), new(testTxPool), engine, chainProFork, dbProFork, 1, nil) + ethNoFork, _ = NewProtocolManager(configNoFork, nil, downloader.FullSync, 1, new(event.TypeMux), &testTxPool{pool: make(map[common.Hash]*types.Transaction)}, engine, chainNoFork, dbNoFork, 1, nil) + ethProFork, _ = NewProtocolManager(configProFork, nil, downloader.FullSync, 1, new(event.TypeMux), &testTxPool{pool: make(map[common.Hash]*types.Transaction)}, engine, chainProFork, dbProFork, 1, nil) ) ethNoFork.Start(1000) ethProFork.Start(1000) // Both nodes should allow the other to connect (same genesis, next fork is the same) p2pNoFork, p2pProFork := p2p.MsgPipe() - peerNoFork := newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork) - peerProFork := newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork) + peerNoFork := newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil) + peerProFork := newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil) errc := make(chan error, 2) go func() { errc <- ethNoFork.handle(peerProFork) }() @@ -207,8 +208,8 @@ func TestForkIDSplit(t *testing.T) { chainProFork.InsertChain(blocksProFork[:1]) p2pNoFork, p2pProFork = p2p.MsgPipe() - peerNoFork = newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork) - peerProFork = newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork) + peerNoFork = newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil) + peerProFork = newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil) errc = make(chan error, 2) go func() { errc <- ethNoFork.handle(peerProFork) }() @@ -226,8 +227,8 @@ func TestForkIDSplit(t *testing.T) { chainProFork.InsertChain(blocksProFork[1:2]) p2pNoFork, p2pProFork = p2p.MsgPipe() - peerNoFork = newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork) - peerProFork = newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork) + peerNoFork = newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil) + peerProFork = newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil) errc = make(chan error, 2) go func() { errc <- ethNoFork.handle(peerProFork) }() @@ -246,6 +247,7 @@ func TestForkIDSplit(t *testing.T) { // This test checks that received transactions are added to the local pool. func TestRecvTransactions63(t *testing.T) { testRecvTransactions(t, 63) } func TestRecvTransactions64(t *testing.T) { testRecvTransactions(t, 64) } +func TestRecvTransactions65(t *testing.T) { testRecvTransactions(t, 65) } func testRecvTransactions(t *testing.T, protocol int) { txAdded := make(chan []*types.Transaction) @@ -274,6 +276,7 @@ func testRecvTransactions(t *testing.T, protocol int) { // This test checks that pending transactions are sent. func TestSendTransactions63(t *testing.T) { testSendTransactions(t, 63) } func TestSendTransactions64(t *testing.T) { testSendTransactions(t, 64) } +func TestSendTransactions65(t *testing.T) { testSendTransactions(t, 65) } func testSendTransactions(t *testing.T, protocol int) { pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, 0, nil, nil) @@ -298,17 +301,43 @@ func testSendTransactions(t *testing.T, protocol int) { } for n := 0; n < len(alltxs) && !t.Failed(); { var txs []*types.Transaction - msg, err := p.app.ReadMsg() - if err != nil { - t.Errorf("%v: read error: %v", p.Peer, err) - } else if msg.Code != TxMsg { - t.Errorf("%v: got code %d, want TxMsg", p.Peer, msg.Code) + var hashes []common.Hash + var forAllHashes func(callback func(hash common.Hash)) + switch protocol { + case 63: + fallthrough + case 64: + msg, err := p.app.ReadMsg() + if err != nil { + t.Errorf("%v: read error: %v", p.Peer, err) + } else if msg.Code != TxMsg { + t.Errorf("%v: got code %d, want TxMsg", p.Peer, msg.Code) + } + if err := msg.Decode(&txs); err != nil { + t.Errorf("%v: %v", p.Peer, err) + } + forAllHashes = func(callback func(hash common.Hash)) { + for _, tx := range txs { + callback(tx.Hash()) + } + } + case 65: + msg, err := p.app.ReadMsg() + if err != nil { + t.Errorf("%v: read error: %v", p.Peer, err) + } else if msg.Code != NewPooledTransactionHashesMsg { + t.Errorf("%v: got code %d, want TxMsg", p.Peer, msg.Code) + } + if err := msg.Decode(&hashes); err != nil { + t.Errorf("%v: %v", p.Peer, err) + } + forAllHashes = func(callback func(hash common.Hash)) { + for _, h := range hashes { + callback(h) + } + } } - if err := msg.Decode(&txs); err != nil { - t.Errorf("%v: %v", p.Peer, err) - } - for _, tx := range txs { - hash := tx.Hash() + forAllHashes(func(hash common.Hash) { seentx, want := seen[hash] if seentx { t.Errorf("%v: got tx more than once: %x", p.Peer, hash) @@ -318,7 +347,7 @@ func testSendTransactions(t *testing.T, protocol int) { } seen[hash] = true n++ - } + }) } } for i := 0; i < 3; i++ { @@ -329,6 +358,53 @@ func testSendTransactions(t *testing.T, protocol int) { wg.Wait() } +func TestTransactionPropagation(t *testing.T) { testSyncTransaction(t, true) } +func TestTransactionAnnouncement(t *testing.T) { testSyncTransaction(t, false) } + +func testSyncTransaction(t *testing.T, propagtion bool) { + // Create a protocol manager for transaction fetcher and sender + pmFetcher, _ := newTestProtocolManagerMust(t, downloader.FastSync, 0, nil, nil) + defer pmFetcher.Stop() + pmSender, _ := newTestProtocolManagerMust(t, downloader.FastSync, 1024, nil, nil) + pmSender.broadcastTxAnnouncesOnly = !propagtion + defer pmSender.Stop() + + // Sync up the two peers + io1, io2 := p2p.MsgPipe() + + go pmSender.handle(pmSender.newPeer(65, p2p.NewPeer(enode.ID{}, "sender", nil), io2, pmSender.txpool.Get)) + go pmFetcher.handle(pmFetcher.newPeer(65, p2p.NewPeer(enode.ID{}, "fetcher", nil), io1, pmFetcher.txpool.Get)) + + time.Sleep(250 * time.Millisecond) + pmFetcher.synchronise(pmFetcher.peers.BestPeer()) + atomic.StoreUint32(&pmFetcher.acceptTxs, 1) + + newTxs := make(chan core.NewTxsEvent, 1024) + sub := pmFetcher.txpool.SubscribeNewTxsEvent(newTxs) + defer sub.Unsubscribe() + + // Fill the pool with new transactions + alltxs := make([]*types.Transaction, 1024) + for nonce := range alltxs { + alltxs[nonce] = newTestTransaction(testAccount, uint64(nonce), 0) + } + pmSender.txpool.AddRemotes(alltxs) + + var got int +loop: + for { + select { + case ev := <-newTxs: + got += len(ev.Txs) + if got == 1024 { + break loop + } + case <-time.NewTimer(time.Second).C: + t.Fatal("Failed to retrieve all transaction") + } + } +} + // Tests that the custom union field encoder and decoder works correctly. func TestGetBlockHeadersDataEncodeDecode(t *testing.T) { // Create a "random" hash for testing diff --git a/eth/sync.go b/eth/sync.go index 9e180ee200..93b2dd2ecf 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -38,8 +38,9 @@ const ( ) type txsync struct { - p *peer - txs []*types.Transaction + p *peer + hashes []common.Hash + txs []*types.Transaction } // syncTransactions starts sending all currently pending transactions to the given peer. @@ -53,7 +54,7 @@ func (pm *ProtocolManager) syncTransactions(p *peer) { return } select { - case pm.txsyncCh <- &txsync{p, txs}: + case pm.txsyncCh <- &txsync{p: p, txs: txs}: case <-pm.quitSync: } } @@ -69,26 +70,46 @@ func (pm *ProtocolManager) txsyncLoop() { pack = new(txsync) // the pack that is being sent done = make(chan error, 1) // result of the send ) - // send starts a sending a pack of transactions from the sync. send := func(s *txsync) { // Fill pack with transactions up to the target size. size := common.StorageSize(0) pack.p = s.p + pack.hashes = pack.hashes[:0] pack.txs = pack.txs[:0] - for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { - pack.txs = append(pack.txs, s.txs[i]) - size += s.txs[i].Size() + if s.p.version >= eth65 { + // Eth65 introduces transaction announcement https://github.com/ethereum/EIPs/pull/2464, + // only txhashes are transferred here. + for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { + pack.hashes = append(pack.hashes, s.txs[i].Hash()) + size += common.HashLength + } + // Remove the transactions that will be sent. + s.txs = s.txs[:copy(s.txs, s.txs[len(pack.hashes):])] + if len(s.txs) == 0 { + delete(pending, s.p.ID()) + } + // Send the pack in the background. + s.p.Log().Trace("Sending batch of transaction announcements", "count", len(pack.hashes), "bytes", size) + sending = true + go func() { done <- pack.p.SendNewTransactionHashes(pack.hashes) }() + } else { + // Legacy eth protocol doesn't have transaction announcement protocol + // message, transfer the whole pending transaction slice. + for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { + pack.txs = append(pack.txs, s.txs[i]) + size += s.txs[i].Size() + } + // Remove the transactions that will be sent. + s.txs = s.txs[:copy(s.txs, s.txs[len(pack.txs):])] + if len(s.txs) == 0 { + delete(pending, s.p.ID()) + } + // Send the pack in the background. + s.p.Log().Trace("Sending batch of transactions", "count", len(pack.txs), "bytes", size) + sending = true + go func() { done <- pack.p.SendNewTransactions(pack.txs) }() } - // Remove the transactions that will be sent. - s.txs = s.txs[:copy(s.txs, s.txs[len(pack.txs):])] - if len(s.txs) == 0 { - delete(pending, s.p.ID()) - } - // Send the pack in the background. - s.p.Log().Trace("Sending batch of transactions", "count", len(pack.txs), "bytes", size) - sending = true - go func() { done <- pack.p.SendTransactions(pack.txs) }() } // pick chooses the next pending sync. @@ -133,8 +154,10 @@ func (pm *ProtocolManager) txsyncLoop() { // downloading hashes and blocks as well as handling the announcement handler. func (pm *ProtocolManager) syncer() { // Start and ensure cleanup of sync mechanisms - pm.fetcher.Start() - defer pm.fetcher.Stop() + pm.blockFetcher.Start() + pm.txFetcher.Start() + defer pm.blockFetcher.Stop() + defer pm.txFetcher.Stop() defer pm.downloader.Terminate() // Wait for different events to fire synchronisation operations diff --git a/eth/sync_test.go b/eth/sync_test.go index e4c99ff587..d02bc57108 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -26,9 +26,13 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" ) +func TestFastSyncDisabling63(t *testing.T) { testFastSyncDisabling(t, 63) } +func TestFastSyncDisabling64(t *testing.T) { testFastSyncDisabling(t, 64) } +func TestFastSyncDisabling65(t *testing.T) { testFastSyncDisabling(t, 65) } + // Tests that fast sync gets disabled as soon as a real block is successfully // imported into the blockchain. -func TestFastSyncDisabling(t *testing.T) { +func testFastSyncDisabling(t *testing.T, protocol int) { // Create a pristine protocol manager, check that fast sync is left enabled pmEmpty, _ := newTestProtocolManagerMust(t, downloader.FastSync, 0, nil, nil) if atomic.LoadUint32(&pmEmpty.fastSync) == 0 { @@ -42,8 +46,8 @@ func TestFastSyncDisabling(t *testing.T) { // Sync up the two peers io1, io2 := p2p.MsgPipe() - go pmFull.handle(pmFull.newPeer(63, p2p.NewPeer(enode.ID{}, "empty", nil), io2)) - go pmEmpty.handle(pmEmpty.newPeer(63, p2p.NewPeer(enode.ID{}, "full", nil), io1)) + go pmFull.handle(pmFull.newPeer(protocol, p2p.NewPeer(enode.ID{}, "empty", nil), io2, pmFull.txpool.Get)) + go pmEmpty.handle(pmEmpty.newPeer(protocol, p2p.NewPeer(enode.ID{}, "full", nil), io1, pmEmpty.txpool.Get)) time.Sleep(250 * time.Millisecond) pmEmpty.synchronise(pmEmpty.peers.BestPeer()) From 9938d954c8391682947682543cf9b52196507a88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 22 Jan 2020 16:39:43 +0200 Subject: [PATCH 2/2] eth: rework tx fetcher to use O(1) ops + manage network requests --- core/tx_pool.go | 9 +- eth/fetcher/block_fetcher.go | 21 + eth/fetcher/metrics.go | 54 - eth/fetcher/tx_fetcher.go | 955 ++++++++-- eth/fetcher/tx_fetcher_test.go | 1676 ++++++++++++++--- eth/handler.go | 59 +- eth/metrics.go | 139 -- eth/peer.go | 318 ++-- eth/protocol.go | 7 +- eth/protocol_test.go | 25 +- eth/sync.go | 75 +- fuzzbuzz.yaml | 8 + ...0151ee1d0db4c74d3bcdfa4f7396a4c8538748c9-2 | 1 + ...020dd7b492a6eb34ff0b7d8ee46189422c37e4a7-6 | Bin 0 -> 14 bytes ...021d1144e359233c496e22c3250609b11b213e9f-4 | 12 + ...0d28327b1fb52c1ba02a6eb96675c31633921bb2-2 | 15 + ...0fcd827b57ded58e91f7ba2ac2b7ea4d25ebedca-7 | 1 + ...109bc9b8fd4fef63493e104c703c79bc4a5e8d34-6 | Bin 0 -> 470 bytes ...163785ab002746452619f31e8dfcb4549e6f8b6e-6 | Bin 0 -> 30 bytes ...1adfa6b9ddf5766220c8ff7ede2926ca241bb947-3 | 11 + ...1b9a02e9a48fea1d2fc3fb77946ada278e152079-4 | Bin 0 -> 17 bytes ...1e14c7ea1faef92890988061b5abe96db7190f98-7 | 1 + ...1e7d05f00e99cbf3ff0ef1cd7ea8dd07ad6dff23-6 | Bin 0 -> 19 bytes ...1ec95e347fd522e6385b5091aa81aa2485be4891-4 | Bin 0 -> 833 bytes ...1fbfa5d214060d2a0905846a589fd6f78d411451-4 | Bin 0 -> 22 bytes ...1fd84ee194e791783a7f18f0a6deab8efe05fc04-2 | 1 + ...21e76b9fca21d94d97f860c1c82f40697a83471b-8 | 3 + ...220a87fed0c92474923054094eb7aff14289cf5e-4 | Bin 0 -> 4 bytes ...23ddcd66aa92fe3d78b7f5b6e7cddb1b55c5f5df-3 | 12 + ...2441d249faf9a859e38c49f6e305b394280c6ea5-1 | Bin 0 -> 55 bytes ...2da1f0635e11283b1927974f418aadd8837ad31e-7 | Bin 0 -> 15 bytes ...2e1853fbf8efe40098b1583224fe3b5f335e7037-6 | Bin 0 -> 26 bytes ...2f25490dc49c103d653843ed47324b310ee7105e-7 | Bin 0 -> 23 bytes ...30494b85bb60ad7f099fa49d427007a761620d8f-5 | 10 + ...316024ca3aaf09c1de5258733ff5fe3d799648d3-4 | 15 + ...32a089e2c439a91f4c1b67a13d52429bcded0dd9-7 | Bin 0 -> 1665 bytes ...33ec1dc0bfeb93d16edee3c07125fec6ac1aa17d-2 | 1 + ...37a0d207700b52caa005ec8aeb344dcb13150ed2-5 | Bin 0 -> 55 bytes ...382f59c66d0ddb6747d3177263279789ca15c2db-5 | Bin 0 -> 14 bytes ...3a010483a4ad8d7215447ce27e0fac3791235c99-4 | 7 + ...3a3b717fcfe7ffb000b906e5a76f32248a576bf7-6 | Bin 0 -> 1141 bytes ...3c37f6d58b8029971935f127f53e6aaeba558445-6 | 2 + ...3c73b63bafa9f535c882ec17189adaf02b58f432-6 | 1 + ...3d11500c4f66b20c73bbdfb1a7bddd7bbf92b29c-5 | Bin 0 -> 453 bytes ...3d8b5bf36c80d6f65802280039f85421f32b5055-6 | Bin 0 -> 23 bytes ...3f99c546a3962256176d566c19e3fffb62072078-1 | 1 + ...408ec46539af27acd82b3d01e863597030882458-8 | Bin 0 -> 1884 bytes ...436154e5bb6487673f6642e6d2a582c01b083c08-8 | 1 + ...45f565cd14b8de1ba2e925047ce776c2682b4b8d-3 | Bin 0 -> 25 bytes ...4a0a12f5b033c8c160cc3b5133692ea1e92c6cdf-7 | 3 + ...550f15ef65230cc4dcfab7fea67de212d9212ff8-8 | Bin 0 -> 44 bytes ...5552213d659fef900a194c52718ffeffdc72d043-3 | Bin 0 -> 11 bytes ...5570ef82893a9b9b9158572d43a7de7537121d2d-1 | 1 + ...5e10f734f8af4116fbd164d96eec67aa53e6228c-5 | Bin 0 -> 40 bytes ...608200b402488b3989ec8ec5f4190ccb537b8ea4-4 | Bin 0 -> 24 bytes ...61e89c3fbdf9eff74bd250ea73cc2e61f8ca0d97-5 | 1 + ...62817a48c78fbf2c12fcdc5ca58e2ca60c43543a-7 | Bin 0 -> 27 bytes ...6782da8f1a432a77306d60d2ac2470c35b98004f-3 | 1 + ...68fb55290cb9d6da5b259017c34bcecf96c944aa-5 | Bin 0 -> 49 bytes ...6a5059bc86872526241d21ab5dae9f0afd3b9ae1-3 | 1 + ...717928e0e2d478c680c6409b173552ca98469ba5-6 | 1 + ...71d22f25419543e437f249ca437823b87ac926b1-6 | Bin 0 -> 27 bytes ...7312a0f31ae5d773ed4fd74abc7521eb14754683-8 | 2 + ...76e413a50dc8861e3756e556f796f1737bec2675-4 | Bin 0 -> 24 bytes ...78480977d5c07386b06e9b37f5c82f5ed86c2f09-3 | 14 + ...7a113cd3c178934cdb64353af86d51462d7080a4-5 | 10 + ...7ea9f71020f3eb783f743f744eba8d8ca4b2582f-3 | 9 + ...84f8c275f3ffbaf8c32c21782af13de10e7de28b-3 | 1 + ...85dfe7ddee0e52aa19115c0ebb9ed28a14e488c6-5 | Bin 0 -> 11 bytes ...87bba5b1e3da38fed8cb5a9bc5c8baa819e83d05-5 | Bin 0 -> 1137 bytes ...8a9ebedfbfec584d8b22761e6121dc1ca0248548-4 | Bin 0 -> 722 bytes ...8ff3bd49f93079e5e1c7f8f2461ba7ee612900c3-5 | Bin 0 -> 40 bytes ...9034aaf45143996a2b14465c352ab0c6fa26b221-2 | 1 + ...92cefdc6251d04896349a464b29be03d6bb04c3d-2 | 1 + ...9613e580ccb69df7c9074f0e2f6886ac6b34ca55-5 | Bin 0 -> 446 bytes ...98afc8970a680fdc4aee0b5d48784f650c566b75-6 | Bin 0 -> 31 bytes ...9dfc92f4ca2ece0167096fca6751ff314765f08b-8 | Bin 0 -> 23 bytes ...9ebcbbfdaf0e98c87652e57226a4d8a35170c67d-4 | 5 + ...9ff520eb8b8319a5fdafbe4d1cbb02a75058d93b-7 | 2 + ...a0b57a12e25ac5adcedb2a5c45915f0f62aee869-4 | Bin 0 -> 242 bytes ...a2684adccf16e036b051c12f283734fa803746e8-6 | Bin 0 -> 25 bytes ...a37305974cf477ecfe65fa92f37b1f51dea25910-4 | Bin 0 -> 15 bytes ...a7eb43926bd14b1f62a66a33107776e487434d32-7 | Bin 0 -> 516 bytes ...a8f7c254eb64a40fd2a77b79979c7bbdac6a760c-4 | 2 + ...a9a8f287d6af24e47d8db468e8f967aa44fb5a1f-7 | Bin 0 -> 106 bytes ...aa7444d8e326158046862590a0db993c07aef372-7 | 1 + ...ae4593626d8796e079a358c2395a4f6c9ddd6a44-6 | 8 + ...b2942d4413a66939cda7db93020dee79eb17788c-9 | Bin 0 -> 18 bytes ...b4614117cdfd147d38f4e8a4d85f5a2bb99a6a4f-5 | Bin 0 -> 838 bytes ...b631ef3291fa405cd6517d11f4d1b9b6d02912d4-2 | 1 + ...b7a91e338cc11f50ebdb2c414610efc4d5be3137-4 | Bin 0 -> 34 bytes ...b858cb282617fb0956d960215c8e84d1ccf909c6-2 | 1 + ...bc9d570aacf3acd39600feda8e72a293a4667da4-1 | 1 + ...be7eed35b245b5d5d2adcdb4c67f07794eb86b24-3 | 2 + ...c010b0cd70c7edbc5bd332fc9e2e91c6a1cbcdc4-5 | 4 + ...c1690698607eb0f4c4244e9f9629968be4beb6bc-8 | 2 + ...c1f435e4f53a9a17578d9e8c4789860f962a1379-6 | Bin 0 -> 1889 bytes ...c298a75334c3acf04bd129a8867447a25c8bacf8-7 | Bin 0 -> 27 bytes ...c42287c7d225e530e822f23bbbba6819a9e48f38-6 | Bin 0 -> 10 bytes ...4cdbb891f3ee76476b7375d5ed51691fed95421-10 | Bin 0 -> 16 bytes ...cc9572d72dfa2937074b1766dcbcff9cc58d1137-4 | Bin 0 -> 1522 bytes ...cd1d73b4e101bc7b979e3f6f135cb12d4594d348-5 | 1 + ...d0acdc8fca32bbd58d368eeac3bd9eaa46f59d27-5 | Bin 0 -> 9 bytes ...d0e43b715fd00953f7bdd6dfad95811985e81396-4 | Bin 0 -> 10 bytes ...d925fbd22c8bc0de34d6a9d1258ce3d2928d0927-8 | Bin 0 -> 22 bytes ...d9ba78cb7425724185d5fa300cd5c03aec2683bb-7 | Bin 0 -> 510 bytes .../da39a3ee5e6b4b0d3255bfef95601890afd80709 | 0 ...dcdb7758b87648b5d766b1b341a65834420cf621-7 | Bin 0 -> 22 bytes ...dd441bd24581332c9ce19e008260a69287aa3cbc-6 | 2 + ...def879fe0fd637a745c00c8f1da340518db8688c-2 | 1 + ...df6c30a9781b93bd6d2f5e97e5592d5945210003-7 | Bin 0 -> 1881 bytes ...dfc1c3a2e3ccdaf6f88c515fd00e8ad08421e431-6 | Bin 0 -> 10 bytes ...e1dcc4e7ead6dfd1139ece7bf57d776cb9dac72d-7 | Bin 0 -> 20 bytes ...e39c2de2c8937d2cbd4339b13d6a0ce94d94f8d2-8 | Bin 0 -> 1275 bytes ...e72f76b9579c792e545d02fe405d9186f0d6c39b-6 | Bin 0 -> 47 bytes ...eb70814d6355a4498b8f301ba8dbc34f895a9947-5 | Bin 0 -> 57 bytes ...ebdc17efe343e412634dca57cecd5a0e1ce1c1c7-5 | Bin 0 -> 54 bytes ...ec0a25eba8966b8f628d821b3cfbdf2dfd4bbb4c-3 | 13 + ...eebe3b76aeba6deed965d17d2b024f7eae1a43f1-5 | Bin 0 -> 10 bytes ...ef8741a9faf030794d98ff113f556c68a24719a5-6 | Bin 0 -> 31 bytes ...efb7410d02418befeba25a43d676cc6124129125-4 | 1 + ...f6f97d781a5a749903790e07db8619866cb7c3a1-6 | Bin 0 -> 22 bytes ...f7a3cd00fa0e57742e7dbbb8283dcaea067eaf7b-5 | 2 + ...f94d60a6c556ce485ab60088291760b8be25776c-6 | 2 + ...f9e627b2cb82ffa1ea5e0c6d7f2802f3000b18a8-6 | Bin 0 -> 11 bytes ...fb3775aa24e5667e658920c05ba4b7b19ff256fb-5 | 1 + ...fd6386548e119a50db96b2fa406e54924c45a2d5-6 | Bin 0 -> 28 bytes tests/fuzzers/txfetcher/txfetcher_fuzzer.go | 199 ++ 128 files changed, 2867 insertions(+), 871 deletions(-) delete mode 100644 eth/fetcher/metrics.go delete mode 100644 eth/metrics.go create mode 100644 tests/fuzzers/txfetcher/corpus/0151ee1d0db4c74d3bcdfa4f7396a4c8538748c9-2 create mode 100644 tests/fuzzers/txfetcher/corpus/020dd7b492a6eb34ff0b7d8ee46189422c37e4a7-6 create mode 100644 tests/fuzzers/txfetcher/corpus/021d1144e359233c496e22c3250609b11b213e9f-4 create mode 100644 tests/fuzzers/txfetcher/corpus/0d28327b1fb52c1ba02a6eb96675c31633921bb2-2 create mode 100644 tests/fuzzers/txfetcher/corpus/0fcd827b57ded58e91f7ba2ac2b7ea4d25ebedca-7 create mode 100644 tests/fuzzers/txfetcher/corpus/109bc9b8fd4fef63493e104c703c79bc4a5e8d34-6 create mode 100644 tests/fuzzers/txfetcher/corpus/163785ab002746452619f31e8dfcb4549e6f8b6e-6 create mode 100644 tests/fuzzers/txfetcher/corpus/1adfa6b9ddf5766220c8ff7ede2926ca241bb947-3 create mode 100644 tests/fuzzers/txfetcher/corpus/1b9a02e9a48fea1d2fc3fb77946ada278e152079-4 create mode 100644 tests/fuzzers/txfetcher/corpus/1e14c7ea1faef92890988061b5abe96db7190f98-7 create mode 100644 tests/fuzzers/txfetcher/corpus/1e7d05f00e99cbf3ff0ef1cd7ea8dd07ad6dff23-6 create mode 100644 tests/fuzzers/txfetcher/corpus/1ec95e347fd522e6385b5091aa81aa2485be4891-4 create mode 100644 tests/fuzzers/txfetcher/corpus/1fbfa5d214060d2a0905846a589fd6f78d411451-4 create mode 100644 tests/fuzzers/txfetcher/corpus/1fd84ee194e791783a7f18f0a6deab8efe05fc04-2 create mode 100644 tests/fuzzers/txfetcher/corpus/21e76b9fca21d94d97f860c1c82f40697a83471b-8 create mode 100644 tests/fuzzers/txfetcher/corpus/220a87fed0c92474923054094eb7aff14289cf5e-4 create mode 100644 tests/fuzzers/txfetcher/corpus/23ddcd66aa92fe3d78b7f5b6e7cddb1b55c5f5df-3 create mode 100644 tests/fuzzers/txfetcher/corpus/2441d249faf9a859e38c49f6e305b394280c6ea5-1 create mode 100644 tests/fuzzers/txfetcher/corpus/2da1f0635e11283b1927974f418aadd8837ad31e-7 create mode 100644 tests/fuzzers/txfetcher/corpus/2e1853fbf8efe40098b1583224fe3b5f335e7037-6 create mode 100644 tests/fuzzers/txfetcher/corpus/2f25490dc49c103d653843ed47324b310ee7105e-7 create mode 100644 tests/fuzzers/txfetcher/corpus/30494b85bb60ad7f099fa49d427007a761620d8f-5 create mode 100644 tests/fuzzers/txfetcher/corpus/316024ca3aaf09c1de5258733ff5fe3d799648d3-4 create mode 100644 tests/fuzzers/txfetcher/corpus/32a089e2c439a91f4c1b67a13d52429bcded0dd9-7 create mode 100644 tests/fuzzers/txfetcher/corpus/33ec1dc0bfeb93d16edee3c07125fec6ac1aa17d-2 create mode 100644 tests/fuzzers/txfetcher/corpus/37a0d207700b52caa005ec8aeb344dcb13150ed2-5 create mode 100644 tests/fuzzers/txfetcher/corpus/382f59c66d0ddb6747d3177263279789ca15c2db-5 create mode 100644 tests/fuzzers/txfetcher/corpus/3a010483a4ad8d7215447ce27e0fac3791235c99-4 create mode 100644 tests/fuzzers/txfetcher/corpus/3a3b717fcfe7ffb000b906e5a76f32248a576bf7-6 create mode 100644 tests/fuzzers/txfetcher/corpus/3c37f6d58b8029971935f127f53e6aaeba558445-6 create mode 100644 tests/fuzzers/txfetcher/corpus/3c73b63bafa9f535c882ec17189adaf02b58f432-6 create mode 100644 tests/fuzzers/txfetcher/corpus/3d11500c4f66b20c73bbdfb1a7bddd7bbf92b29c-5 create mode 100644 tests/fuzzers/txfetcher/corpus/3d8b5bf36c80d6f65802280039f85421f32b5055-6 create mode 100644 tests/fuzzers/txfetcher/corpus/3f99c546a3962256176d566c19e3fffb62072078-1 create mode 100644 tests/fuzzers/txfetcher/corpus/408ec46539af27acd82b3d01e863597030882458-8 create mode 100644 tests/fuzzers/txfetcher/corpus/436154e5bb6487673f6642e6d2a582c01b083c08-8 create mode 100644 tests/fuzzers/txfetcher/corpus/45f565cd14b8de1ba2e925047ce776c2682b4b8d-3 create mode 100644 tests/fuzzers/txfetcher/corpus/4a0a12f5b033c8c160cc3b5133692ea1e92c6cdf-7 create mode 100644 tests/fuzzers/txfetcher/corpus/550f15ef65230cc4dcfab7fea67de212d9212ff8-8 create mode 100644 tests/fuzzers/txfetcher/corpus/5552213d659fef900a194c52718ffeffdc72d043-3 create mode 100644 tests/fuzzers/txfetcher/corpus/5570ef82893a9b9b9158572d43a7de7537121d2d-1 create mode 100644 tests/fuzzers/txfetcher/corpus/5e10f734f8af4116fbd164d96eec67aa53e6228c-5 create mode 100644 tests/fuzzers/txfetcher/corpus/608200b402488b3989ec8ec5f4190ccb537b8ea4-4 create mode 100644 tests/fuzzers/txfetcher/corpus/61e89c3fbdf9eff74bd250ea73cc2e61f8ca0d97-5 create mode 100644 tests/fuzzers/txfetcher/corpus/62817a48c78fbf2c12fcdc5ca58e2ca60c43543a-7 create mode 100644 tests/fuzzers/txfetcher/corpus/6782da8f1a432a77306d60d2ac2470c35b98004f-3 create mode 100644 tests/fuzzers/txfetcher/corpus/68fb55290cb9d6da5b259017c34bcecf96c944aa-5 create mode 100644 tests/fuzzers/txfetcher/corpus/6a5059bc86872526241d21ab5dae9f0afd3b9ae1-3 create mode 100644 tests/fuzzers/txfetcher/corpus/717928e0e2d478c680c6409b173552ca98469ba5-6 create mode 100644 tests/fuzzers/txfetcher/corpus/71d22f25419543e437f249ca437823b87ac926b1-6 create mode 100644 tests/fuzzers/txfetcher/corpus/7312a0f31ae5d773ed4fd74abc7521eb14754683-8 create mode 100644 tests/fuzzers/txfetcher/corpus/76e413a50dc8861e3756e556f796f1737bec2675-4 create mode 100644 tests/fuzzers/txfetcher/corpus/78480977d5c07386b06e9b37f5c82f5ed86c2f09-3 create mode 100644 tests/fuzzers/txfetcher/corpus/7a113cd3c178934cdb64353af86d51462d7080a4-5 create mode 100644 tests/fuzzers/txfetcher/corpus/7ea9f71020f3eb783f743f744eba8d8ca4b2582f-3 create mode 100644 tests/fuzzers/txfetcher/corpus/84f8c275f3ffbaf8c32c21782af13de10e7de28b-3 create mode 100644 tests/fuzzers/txfetcher/corpus/85dfe7ddee0e52aa19115c0ebb9ed28a14e488c6-5 create mode 100644 tests/fuzzers/txfetcher/corpus/87bba5b1e3da38fed8cb5a9bc5c8baa819e83d05-5 create mode 100644 tests/fuzzers/txfetcher/corpus/8a9ebedfbfec584d8b22761e6121dc1ca0248548-4 create mode 100644 tests/fuzzers/txfetcher/corpus/8ff3bd49f93079e5e1c7f8f2461ba7ee612900c3-5 create mode 100644 tests/fuzzers/txfetcher/corpus/9034aaf45143996a2b14465c352ab0c6fa26b221-2 create mode 100644 tests/fuzzers/txfetcher/corpus/92cefdc6251d04896349a464b29be03d6bb04c3d-2 create mode 100644 tests/fuzzers/txfetcher/corpus/9613e580ccb69df7c9074f0e2f6886ac6b34ca55-5 create mode 100644 tests/fuzzers/txfetcher/corpus/98afc8970a680fdc4aee0b5d48784f650c566b75-6 create mode 100644 tests/fuzzers/txfetcher/corpus/9dfc92f4ca2ece0167096fca6751ff314765f08b-8 create mode 100644 tests/fuzzers/txfetcher/corpus/9ebcbbfdaf0e98c87652e57226a4d8a35170c67d-4 create mode 100644 tests/fuzzers/txfetcher/corpus/9ff520eb8b8319a5fdafbe4d1cbb02a75058d93b-7 create mode 100644 tests/fuzzers/txfetcher/corpus/a0b57a12e25ac5adcedb2a5c45915f0f62aee869-4 create mode 100644 tests/fuzzers/txfetcher/corpus/a2684adccf16e036b051c12f283734fa803746e8-6 create mode 100644 tests/fuzzers/txfetcher/corpus/a37305974cf477ecfe65fa92f37b1f51dea25910-4 create mode 100644 tests/fuzzers/txfetcher/corpus/a7eb43926bd14b1f62a66a33107776e487434d32-7 create mode 100644 tests/fuzzers/txfetcher/corpus/a8f7c254eb64a40fd2a77b79979c7bbdac6a760c-4 create mode 100644 tests/fuzzers/txfetcher/corpus/a9a8f287d6af24e47d8db468e8f967aa44fb5a1f-7 create mode 100644 tests/fuzzers/txfetcher/corpus/aa7444d8e326158046862590a0db993c07aef372-7 create mode 100644 tests/fuzzers/txfetcher/corpus/ae4593626d8796e079a358c2395a4f6c9ddd6a44-6 create mode 100644 tests/fuzzers/txfetcher/corpus/b2942d4413a66939cda7db93020dee79eb17788c-9 create mode 100644 tests/fuzzers/txfetcher/corpus/b4614117cdfd147d38f4e8a4d85f5a2bb99a6a4f-5 create mode 100644 tests/fuzzers/txfetcher/corpus/b631ef3291fa405cd6517d11f4d1b9b6d02912d4-2 create mode 100644 tests/fuzzers/txfetcher/corpus/b7a91e338cc11f50ebdb2c414610efc4d5be3137-4 create mode 100644 tests/fuzzers/txfetcher/corpus/b858cb282617fb0956d960215c8e84d1ccf909c6-2 create mode 100644 tests/fuzzers/txfetcher/corpus/bc9d570aacf3acd39600feda8e72a293a4667da4-1 create mode 100644 tests/fuzzers/txfetcher/corpus/be7eed35b245b5d5d2adcdb4c67f07794eb86b24-3 create mode 100644 tests/fuzzers/txfetcher/corpus/c010b0cd70c7edbc5bd332fc9e2e91c6a1cbcdc4-5 create mode 100644 tests/fuzzers/txfetcher/corpus/c1690698607eb0f4c4244e9f9629968be4beb6bc-8 create mode 100644 tests/fuzzers/txfetcher/corpus/c1f435e4f53a9a17578d9e8c4789860f962a1379-6 create mode 100644 tests/fuzzers/txfetcher/corpus/c298a75334c3acf04bd129a8867447a25c8bacf8-7 create mode 100644 tests/fuzzers/txfetcher/corpus/c42287c7d225e530e822f23bbbba6819a9e48f38-6 create mode 100644 tests/fuzzers/txfetcher/corpus/c4cdbb891f3ee76476b7375d5ed51691fed95421-10 create mode 100644 tests/fuzzers/txfetcher/corpus/cc9572d72dfa2937074b1766dcbcff9cc58d1137-4 create mode 100644 tests/fuzzers/txfetcher/corpus/cd1d73b4e101bc7b979e3f6f135cb12d4594d348-5 create mode 100644 tests/fuzzers/txfetcher/corpus/d0acdc8fca32bbd58d368eeac3bd9eaa46f59d27-5 create mode 100644 tests/fuzzers/txfetcher/corpus/d0e43b715fd00953f7bdd6dfad95811985e81396-4 create mode 100644 tests/fuzzers/txfetcher/corpus/d925fbd22c8bc0de34d6a9d1258ce3d2928d0927-8 create mode 100644 tests/fuzzers/txfetcher/corpus/d9ba78cb7425724185d5fa300cd5c03aec2683bb-7 create mode 100644 tests/fuzzers/txfetcher/corpus/da39a3ee5e6b4b0d3255bfef95601890afd80709 create mode 100644 tests/fuzzers/txfetcher/corpus/dcdb7758b87648b5d766b1b341a65834420cf621-7 create mode 100644 tests/fuzzers/txfetcher/corpus/dd441bd24581332c9ce19e008260a69287aa3cbc-6 create mode 100644 tests/fuzzers/txfetcher/corpus/def879fe0fd637a745c00c8f1da340518db8688c-2 create mode 100644 tests/fuzzers/txfetcher/corpus/df6c30a9781b93bd6d2f5e97e5592d5945210003-7 create mode 100644 tests/fuzzers/txfetcher/corpus/dfc1c3a2e3ccdaf6f88c515fd00e8ad08421e431-6 create mode 100644 tests/fuzzers/txfetcher/corpus/e1dcc4e7ead6dfd1139ece7bf57d776cb9dac72d-7 create mode 100644 tests/fuzzers/txfetcher/corpus/e39c2de2c8937d2cbd4339b13d6a0ce94d94f8d2-8 create mode 100644 tests/fuzzers/txfetcher/corpus/e72f76b9579c792e545d02fe405d9186f0d6c39b-6 create mode 100644 tests/fuzzers/txfetcher/corpus/eb70814d6355a4498b8f301ba8dbc34f895a9947-5 create mode 100644 tests/fuzzers/txfetcher/corpus/ebdc17efe343e412634dca57cecd5a0e1ce1c1c7-5 create mode 100644 tests/fuzzers/txfetcher/corpus/ec0a25eba8966b8f628d821b3cfbdf2dfd4bbb4c-3 create mode 100644 tests/fuzzers/txfetcher/corpus/eebe3b76aeba6deed965d17d2b024f7eae1a43f1-5 create mode 100644 tests/fuzzers/txfetcher/corpus/ef8741a9faf030794d98ff113f556c68a24719a5-6 create mode 100644 tests/fuzzers/txfetcher/corpus/efb7410d02418befeba25a43d676cc6124129125-4 create mode 100644 tests/fuzzers/txfetcher/corpus/f6f97d781a5a749903790e07db8619866cb7c3a1-6 create mode 100644 tests/fuzzers/txfetcher/corpus/f7a3cd00fa0e57742e7dbbb8283dcaea067eaf7b-5 create mode 100644 tests/fuzzers/txfetcher/corpus/f94d60a6c556ce485ab60088291760b8be25776c-6 create mode 100644 tests/fuzzers/txfetcher/corpus/f9e627b2cb82ffa1ea5e0c6d7f2802f3000b18a8-6 create mode 100644 tests/fuzzers/txfetcher/corpus/fb3775aa24e5667e658920c05ba4b7b19ff256fb-5 create mode 100644 tests/fuzzers/txfetcher/corpus/fd6386548e119a50db96b2fa406e54924c45a2d5-6 create mode 100644 tests/fuzzers/txfetcher/txfetcher_fuzzer.go diff --git a/core/tx_pool.go b/core/tx_pool.go index 16d80d644f..e2137ca4f4 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -18,7 +18,6 @@ package core import ( "errors" - "fmt" "math" "math/big" "sort" @@ -53,6 +52,10 @@ const ( ) var ( + // ErrAlreadyKnown is returned if the transactions is already contained + // within the pool. + ErrAlreadyKnown = errors.New("already known") + // ErrInvalidSender is returned if the transaction contains an invalid signature. ErrInvalidSender = errors.New("invalid sender") @@ -579,7 +582,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err e if pool.all.Get(hash) != nil { log.Trace("Discarding already known transaction", "hash", hash) knownTxMeter.Mark(1) - return false, fmt.Errorf("known transaction: %x", hash) + return false, ErrAlreadyKnown } // If the transaction fails basic validation, discard it if err := pool.validateTx(tx, local); err != nil { @@ -786,7 +789,7 @@ func (pool *TxPool) addTxs(txs []*types.Transaction, local, sync bool) []error { for i, tx := range txs { // If the transaction is known, pre-set the error slot if pool.all.Get(tx.Hash()) != nil { - errs[i] = fmt.Errorf("known transaction: %x", tx.Hash()) + errs[i] = ErrAlreadyKnown knownTxMeter.Mark(1) continue } diff --git a/eth/fetcher/block_fetcher.go b/eth/fetcher/block_fetcher.go index 7395ec83de..b6cab05deb 100644 --- a/eth/fetcher/block_fetcher.go +++ b/eth/fetcher/block_fetcher.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" ) const ( @@ -42,6 +43,26 @@ const ( blockLimit = 64 // Maximum number of unique blocks a peer may have delivered ) +var ( + blockAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/in", nil) + blockAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/block/announces/out", nil) + blockAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/drop", nil) + blockAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/dos", nil) + + blockBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/in", nil) + blockBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/block/broadcasts/out", nil) + blockBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/drop", nil) + blockBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/dos", nil) + + headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/block/headers", nil) + bodyFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/block/bodies", nil) + + headerFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/in", nil) + headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/out", nil) + bodyFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/in", nil) + bodyFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/out", nil) +) + var ( errTerminated = errors.New("terminated") ) diff --git a/eth/fetcher/metrics.go b/eth/fetcher/metrics.go deleted file mode 100644 index b75889938d..0000000000 --- a/eth/fetcher/metrics.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains the metrics collected by the fetcher. - -package fetcher - -import ( - "github.com/ethereum/go-ethereum/metrics" -) - -var ( - blockAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/announces/in", nil) - blockAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/block/announces/out", nil) - blockAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/announces/drop", nil) - blockAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/announces/dos", nil) - - blockBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/broadcasts/in", nil) - blockBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/block/broadcasts/out", nil) - blockBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/broadcasts/drop", nil) - blockBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/broadcasts/dos", nil) - - headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/headers", nil) - bodyFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/bodies", nil) - - headerFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/in", nil) - headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/out", nil) - bodyFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/in", nil) - bodyFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/out", nil) - - txAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/in", nil) - txAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/dos", nil) - txAnnounceSkipMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/skip", nil) - txAnnounceUnderpriceMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/underprice", nil) - txBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/broadcasts/in", nil) - txFetchOutMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/out", nil) - txFetchSuccessMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/success", nil) - txFetchTimeoutMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/timeout", nil) - txFetchInvalidMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/invalid", nil) - txFetchDurationTimer = metrics.NewRegisteredTimer("eth/fetcher/fetch/transaction/duration", nil) -) diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index 1dabb0819f..c497cebb45 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -17,108 +17,236 @@ package fetcher import ( - "math/rand" + "bytes" + "fmt" + mrand "math/rand" + "sort" "time" mapset "github.com/deckarep/golang-set" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" +) + +const ( + // maxTxAnnounces is the maximum number of unique transaction a peer + // can announce in a short time. + maxTxAnnounces = 4096 + + // maxTxRetrievals is the maximum transaction number can be fetched in one + // request. The rationale to pick 256 is: + // - In eth protocol, the softResponseLimit is 2MB. Nowadays according to + // Etherscan the average transaction size is around 200B, so in theory + // we can include lots of transaction in a single protocol packet. + // - However the maximum size of a single transaction is raised to 128KB, + // so pick a middle value here to ensure we can maximize the efficiency + // of the retrieval and response size overflow won't happen in most cases. + maxTxRetrievals = 256 + + // maxTxUnderpricedSetSize is the size of the underpriced transaction set that + // is used to track recent transactions that have been dropped so we don't + // re-request them. + maxTxUnderpricedSetSize = 32768 + + // txArriveTimeout is the time allowance before an announced transaction is + // explicitly requested. + txArriveTimeout = 500 * time.Millisecond + + // txGatherSlack is the interval used to collate almost-expired announces + // with network fetches. + txGatherSlack = 100 * time.Millisecond ) var ( - // txAnnounceLimit is the maximum number of unique transaction a peer - // can announce in a short time. - txAnnounceLimit = 4096 - // txFetchTimeout is the maximum allotted time to return an explicitly // requested transaction. txFetchTimeout = 5 * time.Second - - // MaxTransactionFetch is the maximum transaction number can be fetched - // in one request. The rationale to pick this value is: - // In eth protocol, the softResponseLimit is 2MB. Nowdays according to - // Etherscan the average transaction size is around 200B, so in theory - // we can include lots of transaction in a single protocol packet. However - // the maximum size of a single transaction is raised to 128KB, so pick - // a middle value here to ensure we can maximize the efficiency of the - // retrieval and response size overflow won't happen in most cases. - MaxTransactionFetch = 256 - - // underpriceSetSize is the size of underprice set which used for maintaining - // the set of underprice transactions. - underpriceSetSize = 4096 ) -// txAnnounce is the notification of the availability of a single -// new transaction in the network. -type txAnnounce struct { - origin string // Identifier of the peer originating the notification - time time.Time // Timestamp of the announcement - fetchTxs func([]common.Hash) // Callback for retrieving transaction from specified peer -} +var ( + txAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/in", nil) + txAnnounceKnownMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/known", nil) + txAnnounceUnderpricedMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/underpriced", nil) + txAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/dos", nil) -// txsAnnounce is the notification of the availability of a batch + txBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/broadcasts/in", nil) + txBroadcastKnownMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/broadcasts/known", nil) + txBroadcastUnderpricedMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/broadcasts/underpriced", nil) + txBroadcastOtherRejectMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/broadcasts/otherreject", nil) + + txRequestOutMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/request/out", nil) + txRequestFailMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/request/fail", nil) + txRequestDoneMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/request/done", nil) + txRequestTimeoutMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/request/timeout", nil) + + txReplyInMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/replies/in", nil) + txReplyKnownMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/replies/known", nil) + txReplyUnderpricedMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/replies/underpriced", nil) + txReplyOtherRejectMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/replies/otherreject", nil) + + txFetcherWaitingPeers = metrics.NewRegisteredGauge("eth/fetcher/transaction/waiting/peers", nil) + txFetcherWaitingHashes = metrics.NewRegisteredGauge("eth/fetcher/transaction/waiting/hashes", nil) + txFetcherQueueingPeers = metrics.NewRegisteredGauge("eth/fetcher/transaction/queueing/peers", nil) + txFetcherQueueingHashes = metrics.NewRegisteredGauge("eth/fetcher/transaction/queueing/hashes", nil) + txFetcherFetchingPeers = metrics.NewRegisteredGauge("eth/fetcher/transaction/fetching/peers", nil) + txFetcherFetchingHashes = metrics.NewRegisteredGauge("eth/fetcher/transaction/fetching/hashes", nil) +) + +// txAnnounce is the notification of the availability of a batch // of new transactions in the network. -type txsAnnounce struct { - hashes []common.Hash // Batch of transaction hashes being announced - origin string // Identifier of the peer originating the notification - time time.Time // Timestamp of the announcement - fetchTxs func([]common.Hash) // Callback for retrieving transaction from specified peer +type txAnnounce struct { + origin string // Identifier of the peer originating the notification + hashes []common.Hash // Batch of transaction hashes being announced } -// TxFetcher is responsible for retrieving new transaction based -// on the announcement. +// txRequest represents an in-flight transaction retrieval request destined to +// a specific peers. +type txRequest struct { + hashes []common.Hash // Transactions having been requested + stolen map[common.Hash]struct{} // Deliveries by someone else (don't re-request) + time mclock.AbsTime // Timestamp of the request +} + +// txDelivery is the notification that a batch of transactions have been added +// to the pool and should be untracked. +type txDelivery struct { + origin string // Identifier of the peer originating the notification + hashes []common.Hash // Batch of transaction hashes having been delivered + direct bool // Whether this is a direct reply or a broadcast +} + +// txDrop is the notiication that a peer has disconnected. +type txDrop struct { + peer string +} + +// TxFetcher is responsible for retrieving new transaction based on announcements. +// +// The fetcher operates in 3 stages: +// - Transactions that are newly discovered are moved into a wait list. +// - After ~500ms passes, transactions from the wait list that have not been +// broadcast to us in whole are moved into a queueing area. +// - When a connected peer doesn't have in-flight retrieval requests, any +// transaction queued up (and announced by the peer) are allocated to the +// peer and moved into a fetching status until it's fulfilled or fails. +// +// The invariants of the fetcher are: +// - Each tracked transaction (hash) must only be present in one of the +// three stages. This ensures that the fetcher operates akin to a finite +// state automata and there's do data leak. +// - Each peer that announced transactions may be scheduled retrievals, but +// only ever one concurrently. This ensures we can immediately know what is +// missing from a reply and reschedule it. type TxFetcher struct { - notify chan *txsAnnounce - cleanup chan []common.Hash + notify chan *txAnnounce + cleanup chan *txDelivery + drop chan *txDrop quit chan struct{} - // Announce states - announces map[string]int // Per peer transaction announce counts to prevent memory exhaustion - announced map[common.Hash][]*txAnnounce // Announced transactions, scheduled for fetching - fetching map[common.Hash]*txAnnounce // Announced transactions, currently fetching - underpriced mapset.Set // Transaction set whose price is too low for accepting + underpriced mapset.Set // Transactions discarded as too cheap (don't re-fetch) + + // Stage 1: Waiting lists for newly discovered transactions that might be + // broadcast without needing explicit request/reply round trips. + waitlist map[common.Hash]map[string]struct{} // Transactions waiting for an potential broadcast + waittime map[common.Hash]mclock.AbsTime // Timestamps when transactions were added to the waitlist + waitslots map[string]map[common.Hash]struct{} // Waiting announcement sgroupped by peer (DoS protection) + + // Stage 2: Queue of transactions that waiting to be allocated to some peer + // to be retrieved directly. + announces map[string]map[common.Hash]struct{} // Set of announced transactions, grouped by origin peer + announced map[common.Hash]map[string]struct{} // Set of download locations, grouped by transaction hash + + // Stage 3: Set of transactions currently being retrieved, some which may be + // fulfilled and some rescheduled. Note, this step shares 'announces' from the + // previous stage to avoid having to duplicate (need it for DoS checks). + fetching map[common.Hash]string // Transaction set currently being retrieved + requests map[string]*txRequest // In-flight transaction retrievals + alternates map[common.Hash]map[string]struct{} // In-flight transaction alternate origins if retrieval fails // Callbacks hasTx func(common.Hash) bool // Retrieves a tx from the local txpool addTxs func([]*types.Transaction) []error // Insert a batch of transactions into local txpool - dropPeer func(string) // Drop the specified peer + fetchTxs func(string, []common.Hash) error // Retrieves a set of txs from a remote peer - // Hooks - announceHook func([]common.Hash) // Hook which is called when a batch transactions are announced - importTxsHook func([]*types.Transaction) // Hook which is called when a batch of transactions are imported. - dropHook func(string) // Hook which is called when a peer is dropped - cleanupHook func([]common.Hash) // Hook which is called when internal status is cleaned - rejectUnderprice func(common.Hash) // Hook which is called when underprice transaction is rejected + step chan struct{} // Notification channel when the fetcher loop iterates + clock mclock.Clock // Time wrapper to simulate in tests + rand *mrand.Rand // Randomizer to use in tests instead of map range loops (soft-random) } // NewTxFetcher creates a transaction fetcher to retrieve transaction // based on hash announcements. -func NewTxFetcher(hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, dropPeer func(string)) *TxFetcher { +func NewTxFetcher(hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error) *TxFetcher { + return NewTxFetcherForTests(hasTx, addTxs, fetchTxs, mclock.System{}, nil) +} + +// NewTxFetcherForTests is a testing method to mock out the realtime clock with +// a simulated version and the internal randomness with a deterministic one. +func NewTxFetcherForTests( + hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, + clock mclock.Clock, rand *mrand.Rand) *TxFetcher { return &TxFetcher{ - notify: make(chan *txsAnnounce), - cleanup: make(chan []common.Hash), + notify: make(chan *txAnnounce), + cleanup: make(chan *txDelivery), + drop: make(chan *txDrop), quit: make(chan struct{}), - announces: make(map[string]int), - announced: make(map[common.Hash][]*txAnnounce), - fetching: make(map[common.Hash]*txAnnounce), + waitlist: make(map[common.Hash]map[string]struct{}), + waittime: make(map[common.Hash]mclock.AbsTime), + waitslots: make(map[string]map[common.Hash]struct{}), + announces: make(map[string]map[common.Hash]struct{}), + announced: make(map[common.Hash]map[string]struct{}), + fetching: make(map[common.Hash]string), + requests: make(map[string]*txRequest), + alternates: make(map[common.Hash]map[string]struct{}), underpriced: mapset.NewSet(), hasTx: hasTx, addTxs: addTxs, - dropPeer: dropPeer, + fetchTxs: fetchTxs, + clock: clock, + rand: rand, } } -// Notify announces the fetcher of the potential availability of a -// new transaction in the network. -func (f *TxFetcher) Notify(peer string, hashes []common.Hash, time time.Time, fetchTxs func([]common.Hash)) error { - announce := &txsAnnounce{ - hashes: hashes, - time: time, - origin: peer, - fetchTxs: fetchTxs, +// Notify announces the fetcher of the potential availability of a new batch of +// transactions in the network. +func (f *TxFetcher) Notify(peer string, hashes []common.Hash) error { + // Keep track of all the announced transactions + txAnnounceInMeter.Mark(int64(len(hashes))) + + // Skip any transaction announcements that we already know of, or that we've + // previously marked as cheap and discarded. This check is of course racey, + // because multiple concurrent notifies will still manage to pass it, but it's + // still valuable to check here because it runs concurrent to the internal + // loop, so anything caught here is time saved internally. + var ( + unknowns = make([]common.Hash, 0, len(hashes)) + duplicate, underpriced int64 + ) + for _, hash := range hashes { + switch { + case f.hasTx(hash): + duplicate++ + + case f.underpriced.Contains(hash): + underpriced++ + + default: + unknowns = append(unknowns, hash) + } + } + txAnnounceKnownMeter.Mark(duplicate) + txAnnounceUnderpricedMeter.Mark(underpriced) + + // If anything's left to announce, push it into the internal loop + if len(unknowns) == 0 { + return nil + } + announce := &txAnnounce{ + origin: peer, + hashes: unknowns, } select { case f.notify <- announce: @@ -128,45 +256,75 @@ func (f *TxFetcher) Notify(peer string, hashes []common.Hash, time time.Time, fe } } -// EnqueueTxs imports a batch of received transaction into fetcher. -func (f *TxFetcher) EnqueueTxs(peer string, txs []*types.Transaction) error { +// Enqueue imports a batch of received transaction into the transaction pool +// and the fetcher. This method may be called by both transaction broadcasts and +// direct request replies. The differentiation is important so the fetcher can +// re-shedule missing transactions as soon as possible. +func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) error { + // Keep track of all the propagated transactions + if direct { + txReplyInMeter.Mark(int64(len(txs))) + } else { + txBroadcastInMeter.Mark(int64(len(txs))) + } + // Push all the transactions into the pool, tracking underpriced ones to avoid + // re-requesting them and dropping the peer in case of malicious transfers. var ( - drop bool - hashes []common.Hash + added = make([]common.Hash, 0, len(txs)) + duplicate int64 + underpriced int64 + otherreject int64 ) errs := f.addTxs(txs) for i, err := range errs { if err != nil { - // Drop peer if the received transaction isn't signed properly. - drop = (drop || err == core.ErrInvalidSender) - txFetchInvalidMeter.Mark(1) - // Track the transaction hash if the price is too low for us. // Avoid re-request this transaction when we receive another // announcement. - if err == core.ErrUnderpriced { - for f.underpriced.Cardinality() >= underpriceSetSize { + if err == core.ErrUnderpriced || err == core.ErrReplaceUnderpriced { + for f.underpriced.Cardinality() >= maxTxUnderpricedSetSize { f.underpriced.Pop() } f.underpriced.Add(txs[i].Hash()) } + // Track a few interesting failure types + switch err { + case nil: // Noop, but need to handle to not count these + + case core.ErrAlreadyKnown: + duplicate++ + + case core.ErrUnderpriced, core.ErrReplaceUnderpriced: + underpriced++ + + default: + otherreject++ + } } - hashes = append(hashes, txs[i].Hash()) + added = append(added, txs[i].Hash()) } - if f.importTxsHook != nil { - f.importTxsHook(txs) - } - // Drop the peer if some transaction failed signature verification. - // We can regard this peer is trying to DOS us by feeding lots of - // random hashes. - if drop { - f.dropPeer(peer) - if f.dropHook != nil { - f.dropHook(peer) - } + if direct { + txReplyKnownMeter.Mark(duplicate) + txReplyUnderpricedMeter.Mark(underpriced) + txReplyOtherRejectMeter.Mark(otherreject) + } else { + txBroadcastKnownMeter.Mark(duplicate) + txBroadcastUnderpricedMeter.Mark(underpriced) + txBroadcastOtherRejectMeter.Mark(otherreject) } select { - case f.cleanup <- hashes: + case f.cleanup <- &txDelivery{origin: peer, hashes: added, direct: direct}: + return nil + case <-f.quit: + return errTerminated + } +} + +// Drop should be called when a peer disconnects. It cleans up all the internal +// data structures of the given node. +func (f *TxFetcher) Drop(peer string) error { + select { + case f.drop <- &txDrop{peer: peer}: return nil case <-f.quit: return errTerminated @@ -186,134 +344,551 @@ func (f *TxFetcher) Stop() { } func (f *TxFetcher) loop() { - fetchTimer := time.NewTimer(0) + var ( + waitTimer = new(mclock.Timer) + timeoutTimer = new(mclock.Timer) + waitTrigger = make(chan struct{}, 1) + timeoutTrigger = make(chan struct{}, 1) + ) for { - // Clean up any expired transaction fetches. - // There are many cases can lead to it: - // * We send the request to busy peer which can reply immediately - // * We send the request to malicious peer which doesn't reply deliberately - // * We send the request to normal peer for a batch of transaction, but some - // transactions have been included into blocks. According to EIP these txs - // won't be included. - // But it's fine to delete the fetching record and reschedule fetching iff we - // receive the annoucement again. - for hash, announce := range f.fetching { - if time.Since(announce.time) > txFetchTimeout { - delete(f.fetching, hash) - txFetchTimeoutMeter.Mark(1) - } - } select { - case anno := <-f.notify: - txAnnounceInMeter.Mark(int64(len(anno.hashes))) - - // Drop the new announce if there are too many accumulated. - count := f.announces[anno.origin] + len(anno.hashes) - if count > txAnnounceLimit { - txAnnounceDOSMeter.Mark(int64(count - txAnnounceLimit)) + case ann := <-f.notify: + // Drop part of the new announcements if there are too many accumulated. + // Note, we could but do not filter already known transactions here as + // the probability of something arriving between this call and the pre- + // filter outside is essentially zero. + used := len(f.waitslots[ann.origin]) + len(f.announces[ann.origin]) + if used >= maxTxAnnounces { + // This can happen if a set of transactions are requested but not + // all fulfilled, so the remainder are rescheduled without the cap + // check. Should be fine as the limit is in the thousands and the + // request size in the hundreds. + txAnnounceDOSMeter.Mark(int64(len(ann.hashes))) break } - f.announces[anno.origin] = count + want := used + len(ann.hashes) + if want > maxTxAnnounces { + txAnnounceDOSMeter.Mark(int64(want - maxTxAnnounces)) + ann.hashes = ann.hashes[:want-maxTxAnnounces] + } + // All is well, schedule the remainder of the transactions + idleWait := len(f.waittime) == 0 + _, oldPeer := f.announces[ann.origin] - // All is well, schedule the announce if transaction is not yet downloading - empty := len(f.announced) == 0 - for _, hash := range anno.hashes { - if _, ok := f.fetching[hash]; ok { - continue - } - if f.underpriced.Contains(hash) { - txAnnounceUnderpriceMeter.Mark(1) - if f.rejectUnderprice != nil { - f.rejectUnderprice(hash) + for _, hash := range ann.hashes { + // If the transaction is already downloading, add it to the list + // of possible alternates (in case the current retrieval fails) and + // also account it for the peer. + if f.alternates[hash] != nil { + f.alternates[hash][ann.origin] = struct{}{} + + // Stage 2 and 3 share the set of origins per tx + if announces := f.announces[ann.origin]; announces != nil { + announces[hash] = struct{}{} + } else { + f.announces[ann.origin] = map[common.Hash]struct{}{hash: struct{}{}} } continue } - f.announced[hash] = append(f.announced[hash], &txAnnounce{ - origin: anno.origin, - time: anno.time, - fetchTxs: anno.fetchTxs, - }) - } - if empty && len(f.announced) > 0 { - f.reschedule(fetchTimer) - } - if f.announceHook != nil { - f.announceHook(anno.hashes) - } - case <-fetchTimer.C: - // At least one tx's timer ran out, check for needing retrieval - request := make(map[string][]common.Hash) + // If the transaction is not downloading, but is already queued + // from a different peer, track it for the new peer too. + if f.announced[hash] != nil { + f.announced[hash][ann.origin] = struct{}{} - for hash, announces := range f.announced { - if time.Since(announces[0].time) > arriveTimeout-gatherSlack { - // Pick a random peer to retrieve from, reset all others - announce := announces[rand.Intn(len(announces))] - f.forgetHash(hash) - - // Skip fetching if we already receive the transaction. - if f.hasTx(hash) { - txAnnounceSkipMeter.Mark(1) - continue + // Stage 2 and 3 share the set of origins per tx + if announces := f.announces[ann.origin]; announces != nil { + announces[hash] = struct{}{} + } else { + f.announces[ann.origin] = map[common.Hash]struct{}{hash: struct{}{}} } - // If the transaction still didn't arrive, queue for fetching - request[announce.origin] = append(request[announce.origin], hash) - f.fetching[hash] = announce - } - } - // Send out all block header requests - for peer, hashes := range request { - log.Trace("Fetching scheduled transactions", "peer", peer, "txs", hashes) - fetchTxs := f.fetching[hashes[0]].fetchTxs - fetchTxs(hashes) - txFetchOutMeter.Mark(int64(len(hashes))) - } - // Schedule the next fetch if blocks are still pending - f.reschedule(fetchTimer) - case hashes := <-f.cleanup: - for _, hash := range hashes { - f.forgetHash(hash) - anno, exist := f.fetching[hash] - if !exist { - txBroadcastInMeter.Mark(1) // Directly transaction propagation continue } - txFetchDurationTimer.UpdateSince(anno.time) - txFetchSuccessMeter.Mark(1) - delete(f.fetching, hash) + // If the transaction is already known to the fetcher, but not + // yet downloading, add the peer as an alternate origin in the + // waiting list. + if f.waitlist[hash] != nil { + f.waitlist[hash][ann.origin] = struct{}{} + + if waitslots := f.waitslots[ann.origin]; waitslots != nil { + waitslots[hash] = struct{}{} + } else { + f.waitslots[ann.origin] = map[common.Hash]struct{}{hash: struct{}{}} + } + continue + } + // Transaction unknown to the fetcher, insert it into the waiting list + f.waitlist[hash] = map[string]struct{}{ann.origin: struct{}{}} + f.waittime[hash] = f.clock.Now() + + if waitslots := f.waitslots[ann.origin]; waitslots != nil { + waitslots[hash] = struct{}{} + } else { + f.waitslots[ann.origin] = map[common.Hash]struct{}{hash: struct{}{}} + } } - if f.cleanupHook != nil { - f.cleanupHook(hashes) + // If a new item was added to the waitlist, schedule it into the fetcher + if idleWait && len(f.waittime) > 0 { + f.rescheduleWait(waitTimer, waitTrigger) } + // If this peer is new and announced something already queued, maybe + // request transactions from them + if !oldPeer && len(f.announces[ann.origin]) > 0 { + f.scheduleFetches(timeoutTimer, timeoutTrigger, map[string]struct{}{ann.origin: struct{}{}}) + } + + case <-waitTrigger: + // At least one transaction's waiting time ran out, push all expired + // ones into the retrieval queues + actives := make(map[string]struct{}) + for hash, instance := range f.waittime { + if time.Duration(f.clock.Now()-instance)+txGatherSlack > txArriveTimeout { + // Transaction expired without propagation, schedule for retrieval + if f.announced[hash] != nil { + panic("announce tracker already contains waitlist item") + } + f.announced[hash] = f.waitlist[hash] + for peer := range f.waitlist[hash] { + if announces := f.announces[peer]; announces != nil { + announces[hash] = struct{}{} + } else { + f.announces[peer] = map[common.Hash]struct{}{hash: struct{}{}} + } + delete(f.waitslots[peer], hash) + if len(f.waitslots[peer]) == 0 { + delete(f.waitslots, peer) + } + actives[peer] = struct{}{} + } + delete(f.waittime, hash) + delete(f.waitlist, hash) + } + } + // If transactions are still waiting for propagation, reschedule the wait timer + if len(f.waittime) > 0 { + f.rescheduleWait(waitTimer, waitTrigger) + } + // If any peers became active and are idle, request transactions from them + if len(actives) > 0 { + f.scheduleFetches(timeoutTimer, timeoutTrigger, actives) + } + + case <-timeoutTrigger: + // Clean up any expired retrievals and avoid re-requesting them from the + // same peer (either overloaded or malicious, useless in both cases). We + // could also penalize (Drop), but there's nothing to gain, and if could + // possibly further increase the load on it. + for peer, req := range f.requests { + if time.Duration(f.clock.Now()-req.time)+txGatherSlack > txFetchTimeout { + txRequestTimeoutMeter.Mark(int64(len(req.hashes))) + + // Reschedule all the not-yet-delivered fetches to alternate peers + for _, hash := range req.hashes { + // Skip rescheduling hashes already delivered by someone else + if req.stolen != nil { + if _, ok := req.stolen[hash]; ok { + continue + } + } + // Move the delivery back from fetching to queued + if _, ok := f.announced[hash]; ok { + panic("announced tracker already contains alternate item") + } + if f.alternates[hash] != nil { // nil if tx was broadcast during fetch + f.announced[hash] = f.alternates[hash] + } + delete(f.announced[hash], peer) + if len(f.announced[hash]) == 0 { + delete(f.announced, hash) + } + delete(f.announces[peer], hash) + delete(f.alternates, hash) + delete(f.fetching, hash) + } + if len(f.announces[peer]) == 0 { + delete(f.announces, peer) + } + // Keep track of the request as dangling, but never expire + f.requests[peer].hashes = nil + } + } + // Schedule a new transaction retrieval + f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) + + // No idea if we sheduled something or not, trigger the timer if needed + // TODO(karalabe): this is kind of lame, can't we dump it into scheduleFetches somehow? + f.rescheduleTimeout(timeoutTimer, timeoutTrigger) + + case delivery := <-f.cleanup: + // Independent if the delivery was direct or broadcast, remove all + // traces of the hash from internal trackers + for _, hash := range delivery.hashes { + if _, ok := f.waitlist[hash]; ok { + for peer, txset := range f.waitslots { + delete(txset, hash) + if len(txset) == 0 { + delete(f.waitslots, peer) + } + } + delete(f.waitlist, hash) + delete(f.waittime, hash) + } else { + for peer, txset := range f.announces { + delete(txset, hash) + if len(txset) == 0 { + delete(f.announces, peer) + } + } + delete(f.announced, hash) + delete(f.alternates, hash) + + // If a transaction currently being fetched from a different + // origin was delivered (delivery stolen), mark it so the + // actual delivery won't double schedule it. + if origin, ok := f.fetching[hash]; ok && (origin != delivery.origin || !delivery.direct) { + stolen := f.requests[origin].stolen + if stolen == nil { + f.requests[origin].stolen = make(map[common.Hash]struct{}) + stolen = f.requests[origin].stolen + } + stolen[hash] = struct{}{} + } + delete(f.fetching, hash) + } + } + // In case of a direct delivery, also reschedule anything missing + // from the original query + if delivery.direct { + // Mark the reqesting successful (independent of individual status) + txRequestDoneMeter.Mark(int64(len(delivery.hashes))) + + // Make sure something was pending, nuke it + req := f.requests[delivery.origin] + if req == nil { + log.Warn("Unexpected transaction delivery", "peer", delivery.origin) + break + } + delete(f.requests, delivery.origin) + + // Anything not delivered should be re-scheduled (with or without + // this peer, depending on the response cutoff) + delivered := make(map[common.Hash]struct{}) + for _, hash := range delivery.hashes { + delivered[hash] = struct{}{} + } + cutoff := len(req.hashes) // If nothing is delivered, assume everything is missing, don't retry!!! + for i, hash := range req.hashes { + if _, ok := delivered[hash]; ok { + cutoff = i + } + } + // Reschedule missing hashes from alternates, not-fulfilled from alt+self + for i, hash := range req.hashes { + // Skip rescheduling hashes already delivered by someone else + if req.stolen != nil { + if _, ok := req.stolen[hash]; ok { + continue + } + } + if _, ok := delivered[hash]; !ok { + if i < cutoff { + delete(f.alternates[hash], delivery.origin) + delete(f.announces[delivery.origin], hash) + if len(f.announces[delivery.origin]) == 0 { + delete(f.announces, delivery.origin) + } + } + if len(f.alternates[hash]) > 0 { + if _, ok := f.announced[hash]; ok { + panic(fmt.Sprintf("announced tracker already contains alternate item: %v", f.announced[hash])) + } + f.announced[hash] = f.alternates[hash] + } + } + delete(f.alternates, hash) + delete(f.fetching, hash) + } + // Something was delivered, try to rechedule requests + f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) // Partial delivery may enable others to deliver too + } + + case drop := <-f.drop: + // A peer was dropped, remove all traces of it + if _, ok := f.waitslots[drop.peer]; ok { + for hash := range f.waitslots[drop.peer] { + delete(f.waitlist[hash], drop.peer) + if len(f.waitlist[hash]) == 0 { + delete(f.waitlist, hash) + delete(f.waittime, hash) + } + } + delete(f.waitslots, drop.peer) + if len(f.waitlist) > 0 { + f.rescheduleWait(waitTimer, waitTrigger) + } + } + // Clean up any active requests + var request *txRequest + if request = f.requests[drop.peer]; request != nil { + for _, hash := range request.hashes { + // Skip rescheduling hashes already delivered by someone else + if request.stolen != nil { + if _, ok := request.stolen[hash]; ok { + continue + } + } + // Undelivered hash, reschedule if there's an alternative origin available + delete(f.alternates[hash], drop.peer) + if len(f.alternates[hash]) == 0 { + delete(f.alternates, hash) + } else { + f.announced[hash] = f.alternates[hash] + delete(f.alternates, hash) + } + delete(f.fetching, hash) + } + delete(f.requests, drop.peer) + } + // Clean up general announcement tracking + if _, ok := f.announces[drop.peer]; ok { + for hash := range f.announces[drop.peer] { + delete(f.announced[hash], drop.peer) + if len(f.announced[hash]) == 0 { + delete(f.announced, hash) + } + } + delete(f.announces, drop.peer) + } + // If a request was cancelled, check if anything needs to be rescheduled + if request != nil { + f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) + f.rescheduleTimeout(timeoutTimer, timeoutTrigger) + } + case <-f.quit: return } + // No idea what happened, but bump some sanity metrics + txFetcherWaitingPeers.Update(int64(len(f.waitslots))) + txFetcherWaitingHashes.Update(int64(len(f.waitlist))) + txFetcherQueueingPeers.Update(int64(len(f.announces) - len(f.requests))) + txFetcherQueueingHashes.Update(int64(len(f.announced))) + txFetcherFetchingPeers.Update(int64(len(f.requests))) + txFetcherFetchingHashes.Update(int64(len(f.fetching))) + + // Loop did something, ping the step notifier if needed (tests) + if f.step != nil { + f.step <- struct{}{} + } + } +} + +// rescheduleWait iterates over all the transactions currently in the waitlist +// and schedules the movement into the fetcher for the earliest. +// +// The method has a granularity of 'gatherSlack', since there's not much point in +// spinning over all the transactions just to maybe find one that should trigger +// a few ms earlier. +func (f *TxFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) { + if *timer != nil { + (*timer).Stop() + } + now := f.clock.Now() + + earliest := now + for _, instance := range f.waittime { + if earliest > instance { + earliest = instance + if txArriveTimeout-time.Duration(now-earliest) < gatherSlack { + break + } + } + } + *timer = f.clock.AfterFunc(txArriveTimeout-time.Duration(now-earliest), func() { + trigger <- struct{}{} + }) +} + +// rescheduleTimeout iterates over all the transactions currently in flight and +// schedules a cleanup run when the first would trigger. +// +// The method has a granularity of 'gatherSlack', since there's not much point in +// spinning over all the transactions just to maybe find one that should trigger +// a few ms earlier. +// +// This method is a bit "flaky" "by design". In theory the timeout timer only ever +// should be rescheduled if some request is pending. In practice, a timeout will +// cause the timer to be rescheduled every 5 secs (until the peer comes through or +// disconnects). This is a limitation of the fetcher code because we don't trac +// pending requests and timed out requests separatey. Without double tracking, if +// we simply didn't reschedule the timer on all-timeout then the timer would never +// be set again since len(request) > 0 => something's running. +func (f *TxFetcher) rescheduleTimeout(timer *mclock.Timer, trigger chan struct{}) { + if *timer != nil { + (*timer).Stop() + } + now := f.clock.Now() + + earliest := now + for _, req := range f.requests { + // If this request already timed out, skip it altogether + if req.hashes == nil { + continue + } + if earliest > req.time { + earliest = req.time + if txFetchTimeout-time.Duration(now-earliest) < gatherSlack { + break + } + } + } + *timer = f.clock.AfterFunc(txFetchTimeout-time.Duration(now-earliest), func() { + trigger <- struct{}{} + }) +} + +// scheduleFetches starts a batch of retrievals for all available idle peers. +func (f *TxFetcher) scheduleFetches(timer *mclock.Timer, timeout chan struct{}, whitelist map[string]struct{}) { + // Gather the set of peers we want to retrieve from (default to all) + actives := whitelist + if actives == nil { + actives = make(map[string]struct{}) + for peer := range f.announces { + actives[peer] = struct{}{} + } + } + if len(actives) == 0 { + return + } + // For each active peer, try to schedule some transaction fetches + idle := len(f.requests) == 0 + + f.forEachPeer(actives, func(peer string) { + if f.requests[peer] != nil { + return // continue in the for-each + } + if len(f.announces[peer]) == 0 { + return // continue in the for-each + } + hashes := make([]common.Hash, 0, maxTxRetrievals) + f.forEachHash(f.announces[peer], func(hash common.Hash) bool { + if _, ok := f.fetching[hash]; !ok { + // Mark the hash as fetching and stash away possible alternates + f.fetching[hash] = peer + + if _, ok := f.alternates[hash]; ok { + panic(fmt.Sprintf("alternate tracker already contains fetching item: %v", f.alternates[hash])) + } + f.alternates[hash] = f.announced[hash] + delete(f.announced, hash) + + // Accumulate the hash and stop if the limit was reached + hashes = append(hashes, hash) + if len(hashes) >= maxTxRetrievals { + return false // break in the for-each + } + } + return true // continue in the for-each + }) + // If any hashes were allocated, request them from the peer + if len(hashes) > 0 { + f.requests[peer] = &txRequest{hashes: hashes, time: f.clock.Now()} + txRequestOutMeter.Mark(int64(len(hashes))) + + go func(peer string, hashes []common.Hash) { + // Try to fetch the transactions, but in case of a request + // failure (e.g. peer disconnected), reschedule the hashes. + if err := f.fetchTxs(peer, hashes); err != nil { + txRequestFailMeter.Mark(int64(len(hashes))) + f.Drop(peer) + } + }(peer, hashes) + } + }) + // If a new request was fired, schedule a timeout timer + if idle && len(f.requests) > 0 { + f.rescheduleTimeout(timer, timeout) + } +} + +// forEachPeer does a range loop over a map of peers in production, but during +// testing it does a deterministic sorted random to allow reproducing issues. +func (f *TxFetcher) forEachPeer(peers map[string]struct{}, do func(peer string)) { + // If we're running production, use whatever Go's map gives us + if f.rand == nil { + for peer := range peers { + do(peer) + } + return + } + // We're running the test suite, make iteration deterministic + list := make([]string, 0, len(peers)) + for peer := range peers { + list = append(list, peer) + } + sort.Strings(list) + rotateStrings(list, f.rand.Intn(len(list))) + for _, peer := range list { + do(peer) + } +} + +// forEachHash does a range loop over a map of hashes in production, but during +// testing it does a deterministic sorted random to allow reproducing issues. +func (f *TxFetcher) forEachHash(hashes map[common.Hash]struct{}, do func(hash common.Hash) bool) { + // If we're running production, use whatever Go's map gives us + if f.rand == nil { + for hash := range hashes { + if !do(hash) { + return + } + } + return + } + // We're running the test suite, make iteration deterministic + list := make([]common.Hash, 0, len(hashes)) + for hash := range hashes { + list = append(list, hash) + } + sortHashes(list) + rotateHashes(list, f.rand.Intn(len(list))) + for _, hash := range list { + if !do(hash) { + return + } } } -// rescheduleFetch resets the specified fetch timer to the next blockAnnounce timeout. -func (f *TxFetcher) reschedule(fetch *time.Timer) { - // Short circuit if no transactions are announced - if len(f.announced) == 0 { - return +// rotateStrings rotates the contents of a slice by n steps. This method is only +// used in tests to simulate random map iteration but keep it deterministic. +func rotateStrings(slice []string, n int) { + orig := make([]string, len(slice)) + copy(orig, slice) + + for i := 0; i < len(orig); i++ { + slice[i] = orig[(i+n)%len(orig)] } - // Otherwise find the earliest expiring announcement - earliest := time.Now() - for _, announces := range f.announced { - if earliest.After(announces[0].time) { - earliest = announces[0].time - } - } - fetch.Reset(arriveTimeout - time.Since(earliest)) } -func (f *TxFetcher) forgetHash(hash common.Hash) { - // Remove all pending announces and decrement DOS counters - for _, announce := range f.announced[hash] { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) +// sortHashes sorts a slice of hashes. This method is only used in tests in order +// to simulate random map iteration but keep it deterministic. +func sortHashes(slice []common.Hash) { + for i := 0; i < len(slice); i++ { + for j := i + 1; j < len(slice); j++ { + if bytes.Compare(slice[i][:], slice[j][:]) > 0 { + slice[i], slice[j] = slice[j], slice[i] + } } } - delete(f.announced, hash) +} + +// rotateHashes rotates the contents of a slice by n steps. This method is only +// used in tests to simulate random map iteration but keep it deterministic. +func rotateHashes(slice []common.Hash, n int) { + orig := make([]common.Hash, len(slice)) + copy(orig, slice) + + for i := 0; i < len(orig); i++ { + slice[i] = orig[(i+n)%len(orig)] + } } diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index 26f24f3f38..c5c198da88 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -17,302 +17,1512 @@ package fetcher import ( - "crypto/ecdsa" + "errors" "math/big" "math/rand" - "sync" - "sync/atomic" "testing" "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" ) -func init() { - rand.Seed(int64(time.Now().Nanosecond())) +var ( + // testTxs is a set of transactions to use during testing that have meaninful hashes. + testTxs = []*types.Transaction{ + types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil), + types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil), + types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil), + types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil), + } + // testTxsHashes is the hashes of the test transactions above + testTxsHashes = []common.Hash{testTxs[0].Hash(), testTxs[1].Hash(), testTxs[2].Hash(), testTxs[3].Hash()} +) - txAnnounceLimit = 64 - MaxTransactionFetch = 16 +type doTxNotify struct { + peer string + hashes []common.Hash +} +type doTxEnqueue struct { + peer string + txs []*types.Transaction + direct bool +} +type doWait struct { + time time.Duration + step bool +} +type doDrop string +type doFunc func() + +type isWaiting map[string][]common.Hash +type isScheduled struct { + tracking map[string][]common.Hash + fetching map[string][]common.Hash + dangling map[string][]common.Hash +} +type isUnderpriced int + +// txFetcherTest represents a test scenario that can be executed by the test +// runner. +type txFetcherTest struct { + init func() *TxFetcher + steps []interface{} } -func makeTransactions(key *ecdsa.PrivateKey, target int) []*types.Transaction { - var txs []*types.Transaction +// Tests that transaction announcements are added to a waitlist, and none +// of them are scheduled for retrieval until the wait expires. +func TestTransactionFetcherWaiting(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Initial announcement to get something into the waitlist + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }), + // Announce from a new peer to check that no overwrite happens + doTxNotify{peer: "B", hashes: []common.Hash{{0x03}, {0x04}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + "B": {{0x03}, {0x04}}, + }), + // Announce clashing hashes but unique new peer + doTxNotify{peer: "C", hashes: []common.Hash{{0x01}, {0x04}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + "B": {{0x03}, {0x04}}, + "C": {{0x01}, {0x04}}, + }), + // Announce existing and clashing hashes from existing peer + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x03}, {0x05}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}, {0x02}, {0x03}, {0x05}}, + "B": {{0x03}, {0x04}}, + "C": {{0x01}, {0x04}}, + }), + isScheduled{tracking: nil, fetching: nil}, - for i := 0; i < target; i++ { - random := rand.Uint32() - tx := types.NewTransaction(uint64(random), common.Address{0x1, 0x2, 0x3}, big.NewInt(int64(random)), 100, big.NewInt(int64(random)), nil) - tx, _ = types.SignTx(tx, types.NewEIP155Signer(big.NewInt(1)), key) - txs = append(txs, tx) - } - return txs + // Wait for the arrival timeout which should move all expired items + // from the wait list to the scheduler + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}, {0x03}, {0x05}}, + "B": {{0x03}, {0x04}}, + "C": {{0x01}, {0x04}}, + }, + fetching: map[string][]common.Hash{ // Depends on deterministic test randomizer + "A": {{0x02}, {0x03}, {0x05}}, + "C": {{0x01}, {0x04}}, + }, + }, + // Queue up a non-fetchable transaction and then trigger it with a new + // peer (weird case to test 1 line in the fetcher) + doTxNotify{peer: "C", hashes: []common.Hash{{0x06}, {0x07}}}, + isWaiting(map[string][]common.Hash{ + "C": {{0x06}, {0x07}}, + }), + doWait{time: txArriveTimeout, step: true}, + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}, {0x03}, {0x05}}, + "B": {{0x03}, {0x04}}, + "C": {{0x01}, {0x04}, {0x06}, {0x07}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x02}, {0x03}, {0x05}}, + "C": {{0x01}, {0x04}}, + }, + }, + doTxNotify{peer: "D", hashes: []common.Hash{{0x06}, {0x07}}}, + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}, {0x03}, {0x05}}, + "B": {{0x03}, {0x04}}, + "C": {{0x01}, {0x04}, {0x06}, {0x07}}, + "D": {{0x06}, {0x07}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x02}, {0x03}, {0x05}}, + "C": {{0x01}, {0x04}}, + "D": {{0x06}, {0x07}}, + }, + }, + }, + }) } -func makeUnsignedTransactions(key *ecdsa.PrivateKey, target int) []*types.Transaction { - var txs []*types.Transaction +// Tests that transaction announcements skip the waiting list if they are +// already scheduled. +func TestTransactionFetcherSkipWaiting(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Push an initial announcement through to the scheduled stage + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }), + isScheduled{tracking: nil, fetching: nil}, - for i := 0; i < target; i++ { - random := rand.Uint32() - tx := types.NewTransaction(uint64(random), common.Address{0x1, 0x2, 0x3}, big.NewInt(int64(random)), 100, big.NewInt(int64(random)), nil) - txs = append(txs, tx) - } - return txs + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + }, + // Announce overlaps from the same peer, ensure the new ones end up + // in stage one, and clashing ones don't get double tracked + doTxNotify{peer: "A", hashes: []common.Hash{{0x02}, {0x03}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x03}}, + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + }, + // Announce overlaps from a new peer, ensure new transactions end up + // in stage one and clashing ones get tracked for the new peer + doTxNotify{peer: "B", hashes: []common.Hash{{0x02}, {0x03}, {0x04}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x03}}, + "B": {{0x03}, {0x04}}, + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + "B": {{0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + }, + }, + }) } -type txfetcherTester struct { - fetcher *TxFetcher +// Tests that only a single transaction request gets scheduled to a peer +// and subsequent announces block or get allotted to someone else. +func TestTransactionFetcherSingletonRequesting(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Push an initial announcement through to the scheduled stage + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }), + isScheduled{tracking: nil, fetching: nil}, - priceLimit *big.Int - sender *ecdsa.PrivateKey - senderAddr common.Address - signer types.Signer - txs map[common.Hash]*types.Transaction - dropped map[string]struct{} - lock sync.RWMutex + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + }, + // Announce a new set of transactions from the same peer and ensure + // they do not start fetching since the peer is already busy + doTxNotify{peer: "A", hashes: []common.Hash{{0x03}, {0x04}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x03}, {0x04}}, + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + }, + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}, {0x03}, {0x04}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + }, + // Announce a duplicate set of transactions from a new peer and ensure + // uniquely new ones start downloading, even if clashing. + doTxNotify{peer: "B", hashes: []common.Hash{{0x02}, {0x03}, {0x05}, {0x06}}}, + isWaiting(map[string][]common.Hash{ + "B": {{0x05}, {0x06}}, + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}, {0x03}, {0x04}}, + "B": {{0x02}, {0x03}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + "B": {{0x03}}, + }, + }, + }, + }) } -func newTxFetcherTester() *txfetcherTester { - key, _ := crypto.GenerateKey() - addr := crypto.PubkeyToAddress(key.PublicKey) - t := &txfetcherTester{ - sender: key, - senderAddr: addr, - signer: types.NewEIP155Signer(big.NewInt(1)), - txs: make(map[common.Hash]*types.Transaction), - dropped: make(map[string]struct{}), - } - t.fetcher = NewTxFetcher(t.hasTx, t.addTxs, t.dropPeer) - t.fetcher.Start() - return t +// Tests that if a transaction retrieval fails, all the transactions get +// instantly schedule back to someone else or the announcements dropped +// if no alternate source is available. +func TestTransactionFetcherFailedRescheduling(t *testing.T) { + // Create a channel to control when tx requests can fail + proceed := make(chan struct{}) + + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(origin string, hashes []common.Hash) error { + <-proceed + return errors.New("peer disconnected") + }, + ) + }, + steps: []interface{}{ + // Push an initial announcement through to the scheduled stage + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }), + isScheduled{tracking: nil, fetching: nil}, + + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + }, + // While the original peer is stuck in the request, push in an second + // data source. + doTxNotify{peer: "B", hashes: []common.Hash{{0x02}}}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + "B": {{0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + }, + // Wait until the original request fails and check that transactions + // are either rescheduled or dropped + doFunc(func() { + proceed <- struct{}{} // Allow peer A to return the failure + }), + doWait{time: 0, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "B": {{0x02}}, + }, + fetching: map[string][]common.Hash{ + "B": {{0x02}}, + }, + }, + doFunc(func() { + proceed <- struct{}{} // Allow peer B to return the failure + }), + doWait{time: 0, step: true}, + isWaiting(nil), + isScheduled{nil, nil, nil}, + }, + }) } -func (t *txfetcherTester) hasTx(hash common.Hash) bool { - t.lock.RLock() - defer t.lock.RUnlock() +// Tests that if a transaction retrieval succeeds, all alternate origins +// are cleaned up. +func TestTransactionFetcherCleanup(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Push an initial announcement through to the scheduled stage + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + isWaiting(map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }), + isScheduled{tracking: nil, fetching: nil}, - return t.txs[hash] != nil + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + }, + // Request should be delivered + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}, direct: true}, + isScheduled{nil, nil, nil}, + }, + }) } -func (t *txfetcherTester) addTxs(txs []*types.Transaction) []error { - t.lock.Lock() - defer t.lock.Unlock() +// Tests that if a transaction retrieval succeeds, but the response is empty (no +// transactions available, then all are nuked instead of being rescheduled (yes, +// this was a bug)). +func TestTransactionFetcherCleanupEmpty(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Push an initial announcement through to the scheduled stage + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + isWaiting(map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }), + isScheduled{tracking: nil, fetching: nil}, - var errors []error - for _, tx := range txs { - // Make sure the transaction is signed properly - _, err := types.Sender(t.signer, tx) - if err != nil { - errors = append(errors, core.ErrInvalidSender) - continue - } - // Make sure the price is high enough to accpet - if t.priceLimit != nil && tx.GasPrice().Cmp(t.priceLimit) < 0 { - errors = append(errors, core.ErrUnderpriced) - continue - } - t.txs[tx.Hash()] = tx - errors = append(errors, nil) - } - return errors + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + }, + // Deliver an empty response and ensure the transaction is cleared, not rescheduled + doTxEnqueue{peer: "A", txs: []*types.Transaction{}, direct: true}, + isScheduled{nil, nil, nil}, + }, + }) } -func (t *txfetcherTester) dropPeer(id string) { - t.lock.Lock() - defer t.lock.Unlock() +// Tests that non-returned transactions are either re-sheduled from a +// different peer, or self if they are after the cutoff point. +func TestTransactionFetcherMissingRescheduling(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Push an initial announcement through to the scheduled stage + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}}, + isWaiting(map[string][]common.Hash{ + "A": {testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}, + }), + isScheduled{tracking: nil, fetching: nil}, - t.dropped[id] = struct{}{} + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}, + }, + }, + // Deliver the middle transaction requested, the one before which + // should be dropped and the one after re-requested. + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}, direct: true}, // This depends on the deterministic random + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[2]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[2]}, + }, + }, + }, + }) } -// makeTxFetcher retrieves a batch of transaction associated with a simulated peer. -func (t *txfetcherTester) makeTxFetcher(peer string, txs []*types.Transaction) func(hashes []common.Hash) { - closure := make(map[common.Hash]*types.Transaction) - for _, tx := range txs { - closure[tx.Hash()] = tx - } - return func(hashes []common.Hash) { - var txs []*types.Transaction - for _, hash := range hashes { - tx := closure[hash] - if tx == nil { - continue - } - txs = append(txs, tx) - } - // Return on a new thread - go t.fetcher.EnqueueTxs(peer, txs) - } +// Tests that out of two transactions, if one is missing and the last is +// delivered, the peer gets properly cleaned out from the internal state. +func TestTransactionFetcherMissingCleanup(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Push an initial announcement through to the scheduled stage + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}}, + isWaiting(map[string][]common.Hash{ + "A": {testTxsHashes[0], testTxsHashes[1]}, + }), + isScheduled{tracking: nil, fetching: nil}, + + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[0], testTxsHashes[1]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0], testTxsHashes[1]}, + }, + }, + // Deliver the middle transaction requested, the one before which + // should be dropped and the one after re-requested. + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[1]}, direct: true}, // This depends on the deterministic random + isScheduled{nil, nil, nil}, + }, + }) } -func TestSequentialTxAnnouncements(t *testing.T) { - tester := newTxFetcherTester() - txs := makeTransactions(tester.sender, txAnnounceLimit) +// Tests that transaction broadcasts properly clean up announcements. +func TestTransactionFetcherBroadcasts(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Set up three transactions to be in different stats, waiting, queued and fetching + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}}, + doWait{time: txArriveTimeout, step: true}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[2]}}, - retrieveTxs := tester.makeTxFetcher("peer", txs) - - newTxsCh := make(chan struct{}) - tester.fetcher.importTxsHook = func(transactions []*types.Transaction) { - newTxsCh <- struct{}{} - } - for _, tx := range txs { - tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), retrieveTxs) - select { - case <-newTxsCh: - case <-time.NewTimer(time.Second).C: - t.Fatalf("timeout") - } - } - if len(tester.txs) != len(txs) { - t.Fatalf("Imported transaction number mismatch, want %d, got %d", len(txs), len(tester.txs)) - } + isWaiting(map[string][]common.Hash{ + "A": {testTxsHashes[2]}, + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[0], testTxsHashes[1]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + }, + // Broadcast all the transactions and ensure everything gets cleaned + // up, but the dangling request is left alone to avoid doing multiple + // concurrent requests. + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1], testTxs[2]}, direct: false}, + isWaiting(nil), + isScheduled{ + tracking: nil, + fetching: nil, + dangling: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + }, + // Deliver the requested hashes + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1], testTxs[2]}, direct: true}, + isScheduled{nil, nil, nil}, + }, + }) } -func TestConcurrentAnnouncements(t *testing.T) { - tester := newTxFetcherTester() - txs := makeTransactions(tester.sender, txAnnounceLimit) +// Tests that the waiting list timers properly reset and reschedule. +func TestTransactionFetcherWaitTimerResets(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}}, + }), + isScheduled{nil, nil, nil}, + doWait{time: txArriveTimeout / 2, step: false}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}}, + }), + isScheduled{nil, nil, nil}, - txFetcherFn1 := tester.makeTxFetcher("peer1", txs) - txFetcherFn2 := tester.makeTxFetcher("peer2", txs) + doTxNotify{peer: "A", hashes: []common.Hash{{0x02}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }), + isScheduled{nil, nil, nil}, + doWait{time: txArriveTimeout / 2, step: true}, + isWaiting(map[string][]common.Hash{ + "A": {{0x02}}, + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}}, + }, + }, - var ( - count uint32 - done = make(chan struct{}) - ) - tester.fetcher.importTxsHook = func(transactions []*types.Transaction) { - atomic.AddUint32(&count, uint32(len(transactions))) - if atomic.LoadUint32(&count) >= uint32(txAnnounceLimit) { - done <- struct{}{} - } - } - for _, tx := range txs { - tester.fetcher.Notify("peer1", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), txFetcherFn1) - tester.fetcher.Notify("peer2", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout+time.Millisecond), txFetcherFn2) - tester.fetcher.Notify("peer2", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout-time.Millisecond), txFetcherFn2) - } - select { - case <-done: - case <-time.NewTimer(time.Second).C: - t.Fatalf("timeout") - } + doWait{time: txArriveTimeout / 2, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}}, + }, + }, + }, + }) } -func TestBatchAnnouncements(t *testing.T) { - tester := newTxFetcherTester() - txs := makeTransactions(tester.sender, txAnnounceLimit) +// Tests that if a transaction request is not replied to, it will time +// out and be re-scheduled for someone else. +func TestTransactionFetcherTimeoutRescheduling(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Push an initial announcement through to the scheduled stage + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + isWaiting(map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }), + isScheduled{tracking: nil, fetching: nil}, - retrieveTxs := tester.makeTxFetcher("peer", txs) + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + }, + // Wait until the delivery times out, everything should be cleaned up + doWait{time: txFetchTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: nil, + fetching: nil, + dangling: map[string][]common.Hash{ + "A": {}, + }, + }, + // Ensure that followup announcements don't get scheduled + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}}, + doWait{time: txArriveTimeout, step: true}, + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[1]}, + }, + fetching: nil, + dangling: map[string][]common.Hash{ + "A": {}, + }, + }, + // If the dangling request arrives a bit later, do not choke + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}, direct: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[1]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[1]}, + }, + }, + }, + }) +} - var count uint32 - var done = make(chan struct{}) - tester.fetcher.importTxsHook = func(txs []*types.Transaction) { - atomic.AddUint32(&count, uint32(len(txs))) +// Tests that the fetching timeout timers properly reset and reschedule. +func TestTransactionFetcherTimeoutTimerResets(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}}, + doWait{time: txArriveTimeout, step: true}, + doTxNotify{peer: "B", hashes: []common.Hash{{0x02}}}, + doWait{time: txArriveTimeout, step: true}, - if atomic.LoadUint32(&count) >= uint32(txAnnounceLimit) { - done <- struct{}{} - } - } - // Send all announces which exceeds the limit. + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}}, + "B": {{0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}}, + "B": {{0x02}}, + }, + }, + doWait{time: txFetchTimeout - txArriveTimeout, step: true}, + isScheduled{ + tracking: map[string][]common.Hash{ + "B": {{0x02}}, + }, + fetching: map[string][]common.Hash{ + "B": {{0x02}}, + }, + dangling: map[string][]common.Hash{ + "A": {}, + }, + }, + doWait{time: txArriveTimeout, step: true}, + isScheduled{ + tracking: nil, + fetching: nil, + dangling: map[string][]common.Hash{ + "A": {}, + "B": {}, + }, + }, + }, + }) +} + +// Tests that if thousands of transactions are announces, only a small +// number of them will be requested at a time. +func TestTransactionFetcherRateLimiting(t *testing.T) { + // Create a slew of transactions and to announce them var hashes []common.Hash - for _, tx := range txs { - hashes = append(hashes, tx.Hash()) + for i := 0; i < maxTxAnnounces; i++ { + hashes = append(hashes, common.Hash{byte(i / 256), byte(i % 256)}) } - tester.fetcher.Notify("peer", hashes, time.Now(), retrieveTxs) - select { - case <-done: - case <-time.NewTimer(time.Second).C: - t.Fatalf("timeout") - } + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Announce all the transactions, wait a bit and ensure only a small + // percentage gets requested + doTxNotify{peer: "A", hashes: hashes}, + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": hashes, + }, + fetching: map[string][]common.Hash{ + "A": hashes[1643 : 1643+maxTxRetrievals], + }, + }, + }, + }) } -func TestPropagationAfterAnnounce(t *testing.T) { - tester := newTxFetcherTester() - txs := makeTransactions(tester.sender, txAnnounceLimit) - - var cleaned = make(chan struct{}) - tester.fetcher.cleanupHook = func(hashes []common.Hash) { - cleaned <- struct{}{} +// Tests that then number of transactions a peer is allowed to announce and/or +// request at the same time is hard capped. +func TestTransactionFetcherDoSProtection(t *testing.T) { + // Create a slew of transactions and to announce them + var hashesA []common.Hash + for i := 0; i < maxTxAnnounces+1; i++ { + hashesA = append(hashesA, common.Hash{0x01, byte(i / 256), byte(i % 256)}) } - retrieveTxs := tester.makeTxFetcher("peer", txs) - for _, tx := range txs { - tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now(), retrieveTxs) - tester.fetcher.EnqueueTxs("peer", []*types.Transaction{tx}) - - // It's ok to read the map directly since no write - // will happen in the same time. - <-cleaned - if len(tester.fetcher.announced) != 0 { - t.Fatalf("Announcement should be cleaned, got %d", len(tester.fetcher.announced)) - } + var hashesB []common.Hash + for i := 0; i < maxTxAnnounces+1; i++ { + hashesB = append(hashesB, common.Hash{0x02, byte(i / 256), byte(i % 256)}) } + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Announce half of the transaction and wait for them to be scheduled + doTxNotify{peer: "A", hashes: hashesA[:maxTxAnnounces/2]}, + doTxNotify{peer: "B", hashes: hashesB[:maxTxAnnounces/2-1]}, + doWait{time: txArriveTimeout, step: true}, + + // Announce the second half and keep them in the wait list + doTxNotify{peer: "A", hashes: hashesA[maxTxAnnounces/2 : maxTxAnnounces]}, + doTxNotify{peer: "B", hashes: hashesB[maxTxAnnounces/2-1 : maxTxAnnounces-1]}, + + // Ensure the hashes are split half and half + isWaiting(map[string][]common.Hash{ + "A": hashesA[maxTxAnnounces/2 : maxTxAnnounces], + "B": hashesB[maxTxAnnounces/2-1 : maxTxAnnounces-1], + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": hashesA[:maxTxAnnounces/2], + "B": hashesB[:maxTxAnnounces/2-1], + }, + fetching: map[string][]common.Hash{ + "A": hashesA[1643 : 1643+maxTxRetrievals], + "B": append(append([]common.Hash{}, hashesB[maxTxAnnounces/2-3:maxTxAnnounces/2-1]...), hashesB[:maxTxRetrievals-2]...), + }, + }, + // Ensure that adding even one more hash results in dropping the hash + doTxNotify{peer: "A", hashes: []common.Hash{hashesA[maxTxAnnounces]}}, + doTxNotify{peer: "B", hashes: hashesB[maxTxAnnounces-1 : maxTxAnnounces+1]}, + + isWaiting(map[string][]common.Hash{ + "A": hashesA[maxTxAnnounces/2 : maxTxAnnounces], + "B": hashesB[maxTxAnnounces/2-1 : maxTxAnnounces], + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": hashesA[:maxTxAnnounces/2], + "B": hashesB[:maxTxAnnounces/2-1], + }, + fetching: map[string][]common.Hash{ + "A": hashesA[1643 : 1643+maxTxRetrievals], + "B": append(append([]common.Hash{}, hashesB[maxTxAnnounces/2-3:maxTxAnnounces/2-1]...), hashesB[:maxTxRetrievals-2]...), + }, + }, + }, + }) } -func TestEnqueueTransactions(t *testing.T) { - tester := newTxFetcherTester() - txs := makeTransactions(tester.sender, txAnnounceLimit) +// Tests that underpriced transactions don't get rescheduled after being rejected. +func TestTransactionFetcherUnderpricedDedup(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + errs := make([]error, len(txs)) + for i := 0; i < len(errs); i++ { + if i%2 == 0 { + errs[i] = core.ErrUnderpriced + } else { + errs[i] = core.ErrReplaceUnderpriced + } + } + return errs + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Deliver a transaction through the fetcher, but reject as underpriced + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}}, + doWait{time: txArriveTimeout, step: true}, + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1]}, direct: true}, + isScheduled{nil, nil, nil}, - done := make(chan struct{}) - tester.fetcher.importTxsHook = func(transactions []*types.Transaction) { - if len(transactions) == txAnnounceLimit { - done <- struct{}{} - } - } - go tester.fetcher.EnqueueTxs("peer", txs) - select { - case <-done: - case <-time.NewTimer(time.Second).C: - t.Fatalf("timeout") - } + // Try to announce the transaction again, ensure it's not scheduled back + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}}, // [2] is needed to force a step in the fetcher + isWaiting(map[string][]common.Hash{ + "A": {testTxsHashes[2]}, + }), + isScheduled{nil, nil, nil}, + }, + }) } -func TestInvalidTxAnnounces(t *testing.T) { - tester := newTxFetcherTester() +// Tests that underpriced transactions don't get rescheduled after being rejected, +// but at the same time there's a hard cap on the number of transactions that are +// tracked. +func TestTransactionFetcherUnderpricedDoSProtection(t *testing.T) { + // Temporarily disable fetch timeouts as they massively mess up the simulated clock + defer func(timeout time.Duration) { txFetchTimeout = timeout }(txFetchTimeout) + txFetchTimeout = 24 * time.Hour + // Create a slew of transactions to max out the underpriced set var txs []*types.Transaction - txs = append(txs, makeUnsignedTransactions(tester.sender, 1)...) - txs = append(txs, makeTransactions(tester.sender, 1)...) - - txFetcherFn := tester.makeTxFetcher("peer", txs) - - dropped := make(chan string, 1) - tester.fetcher.dropHook = func(s string) { dropped <- s } - - for _, tx := range txs { - tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now(), txFetcherFn) + for i := 0; i < maxTxUnderpricedSetSize+1; i++ { + txs = append(txs, types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil)) } - select { - case s := <-dropped: - if s != "peer" { - t.Fatalf("invalid dropped peer") + hashes := make([]common.Hash, len(txs)) + for i, tx := range txs { + hashes[i] = tx.Hash() + } + // Generate a set of steps to announce and deliver the entire set of transactions + var steps []interface{} + for i := 0; i < maxTxUnderpricedSetSize/maxTxRetrievals; i++ { + steps = append(steps, doTxNotify{peer: "A", hashes: hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals]}) + steps = append(steps, isWaiting(map[string][]common.Hash{ + "A": hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals], + })) + steps = append(steps, doWait{time: txArriveTimeout, step: true}) + steps = append(steps, isScheduled{ + tracking: map[string][]common.Hash{ + "A": hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals], + }, + fetching: map[string][]common.Hash{ + "A": hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals], + }, + }) + steps = append(steps, doTxEnqueue{peer: "A", txs: txs[i*maxTxRetrievals : (i+1)*maxTxRetrievals], direct: true}) + steps = append(steps, isWaiting(nil)) + steps = append(steps, isScheduled{nil, nil, nil}) + steps = append(steps, isUnderpriced((i+1)*maxTxRetrievals)) + } + testTransactionFetcher(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + errs := make([]error, len(txs)) + for i := 0; i < len(errs); i++ { + errs[i] = core.ErrUnderpriced + } + return errs + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: append(steps, []interface{}{ + // The preparation of the test has already been done in `steps`, add the last check + doTxNotify{peer: "A", hashes: []common.Hash{hashes[maxTxUnderpricedSetSize]}}, + doWait{time: txArriveTimeout, step: true}, + doTxEnqueue{peer: "A", txs: []*types.Transaction{txs[maxTxUnderpricedSetSize]}, direct: true}, + isUnderpriced(maxTxUnderpricedSetSize), + }...), + }) +} + +// Tests that unexpected deliveries don't corrupt the internal state. +func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Deliver something out of the blue + isWaiting(nil), + isScheduled{nil, nil, nil}, + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}, direct: false}, + isWaiting(nil), + isScheduled{nil, nil, nil}, + + // Set up a few hashes into various stages + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}}, + doWait{time: txArriveTimeout, step: true}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[2]}}, + + isWaiting(map[string][]common.Hash{ + "A": {testTxsHashes[2]}, + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[0], testTxsHashes[1]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + }, + // Deliver everything and more out of the blue + doTxEnqueue{peer: "B", txs: []*types.Transaction{testTxs[0], testTxs[1], testTxs[2], testTxs[3]}, direct: true}, + isWaiting(nil), + isScheduled{ + tracking: nil, + fetching: nil, + dangling: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + }, + }, + }) +} + +// Tests that dropping a peer cleans out all internal data structures in all the +// live or danglng stages. +func TestTransactionFetcherDrop(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Set up a few hashes into various stages + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}}, + doWait{time: txArriveTimeout, step: true}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x02}}}, + doWait{time: txArriveTimeout, step: true}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x03}}}, + + isWaiting(map[string][]common.Hash{ + "A": {{0x03}}, + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}}, + }, + }, + // Drop the peer and ensure everything's cleaned out + doDrop("A"), + isWaiting(nil), + isScheduled{nil, nil, nil}, + + // Push the node into a dangling (timeout) state + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + }, + doWait{time: txFetchTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: nil, + fetching: nil, + dangling: map[string][]common.Hash{ + "A": {}, + }, + }, + // Drop the peer and ensure everything's cleaned out + doDrop("A"), + isWaiting(nil), + isScheduled{nil, nil, nil}, + }, + }) +} + +// Tests that dropping a peer instantly reschedules failed announcements to any +// available peer. +func TestTransactionFetcherDropRescheduling(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Set up a few hashes into various stages + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}}, + doWait{time: txArriveTimeout, step: true}, + doTxNotify{peer: "B", hashes: []common.Hash{{0x01}}}, + + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}}, + "B": {{0x01}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}}, + }, + }, + // Drop the peer and ensure everything's cleaned out + doDrop("A"), + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "B": {{0x01}}, + }, + fetching: map[string][]common.Hash{ + "B": {{0x01}}, + }, + }, + }, + }) +} + +// This test reproduces a crash caught by the fuzzer. The root cause was a +// dangling transaction timing out and clashing on readd with a concurrently +// announced one. +func TestTransactionFetcherFuzzCrash01(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Get a transaction into fetching mode and make it dangling with a broadcast + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}}, + + // Notify the dangling transaction once more and crash via a timeout + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txFetchTimeout, step: true}, + }, + }) +} + +// This test reproduces a crash caught by the fuzzer. The root cause was a +// dangling transaction getting peer-dropped and clashing on readd with a +// concurrently announced one. +func TestTransactionFetcherFuzzCrash02(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Get a transaction into fetching mode and make it dangling with a broadcast + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}}, + + // Notify the dangling transaction once more, re-fetch, and crash via a drop and timeout + doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + doDrop("A"), + doWait{time: txFetchTimeout, step: true}, + }, + }) +} + +// This test reproduces a crash caught by the fuzzer. The root cause was a +// dangling transaction getting rescheduled via a partial delivery, clashing +// with a concurrent notify. +func TestTransactionFetcherFuzzCrash03(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Get a transaction into fetching mode and make it dangling with a broadcast + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}}, + doWait{time: txFetchTimeout, step: true}, + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1]}}, + + // Notify the dangling transaction once more, partially deliver, clash&crash with a timeout + doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[1]}, direct: true}, + doWait{time: txFetchTimeout, step: true}, + }, + }) +} + +// This test reproduces a crash caught by the fuzzer. The root cause was a +// dangling transaction getting rescheduled via a disconnect, clashing with +// a concurrent notify. +func TestTransactionFetcherFuzzCrash04(t *testing.T) { + // Create a channel to control when tx requests can fail + proceed := make(chan struct{}) + + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { + <-proceed + return errors.New("peer disconnected") + }, + ) + }, + steps: []interface{}{ + // Get a transaction into fetching mode and make it dangling with a broadcast + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}}, + + // Notify the dangling transaction once more, re-fetch, and crash via an in-flight disconnect + doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + doFunc(func() { + proceed <- struct{}{} // Allow peer A to return the failure + }), + doWait{time: 0, step: true}, + doWait{time: txFetchTimeout, step: true}, + }, + }) +} + +func testTransactionFetcherParallel(t *testing.T, tt txFetcherTest) { + t.Parallel() + testTransactionFetcher(t, tt) +} + +func testTransactionFetcher(t *testing.T, tt txFetcherTest) { + // Create a fetcher and hook into it's simulated fields + clock := new(mclock.Simulated) + wait := make(chan struct{}) + + fetcher := tt.init() + fetcher.clock = clock + fetcher.step = wait + fetcher.rand = rand.New(rand.NewSource(0x3a29)) + + fetcher.Start() + defer fetcher.Stop() + + // Crunch through all the test steps and execute them + for i, step := range tt.steps { + switch step := step.(type) { + case doTxNotify: + if err := fetcher.Notify(step.peer, step.hashes); err != nil { + t.Errorf("step %d: %v", i, err) + } + <-wait // Fetcher needs to process this, wait until it's done + select { + case <-wait: + panic("wtf") + case <-time.After(time.Millisecond): + } + + case doTxEnqueue: + if err := fetcher.Enqueue(step.peer, step.txs, step.direct); err != nil { + t.Errorf("step %d: %v", i, err) + } + <-wait // Fetcher needs to process this, wait until it's done + + case doWait: + clock.Run(step.time) + if step.step { + <-wait // Fetcher supposed to do something, wait until it's done + } + + case doDrop: + if err := fetcher.Drop(string(step)); err != nil { + t.Errorf("step %d: %v", i, err) + } + <-wait // Fetcher needs to process this, wait until it's done + + case doFunc: + step() + + case isWaiting: + // We need to check that the waiting list (stage 1) internals + // match with the expected set. Check the peer->hash mappings + // first. + for peer, hashes := range step { + waiting := fetcher.waitslots[peer] + if waiting == nil { + t.Errorf("step %d: peer %s missing from waitslots", i, peer) + continue + } + for _, hash := range hashes { + if _, ok := waiting[hash]; !ok { + t.Errorf("step %d, peer %s: hash %x missing from waitslots", i, peer, hash) + } + } + for hash := range waiting { + if !containsHash(hashes, hash) { + t.Errorf("step %d, peer %s: hash %x extra in waitslots", i, peer, hash) + } + } + } + for peer := range fetcher.waitslots { + if _, ok := step[peer]; !ok { + t.Errorf("step %d: peer %s extra in waitslots", i, peer) + } + } + // Peer->hash sets correct, check the hash->peer and timeout sets + for peer, hashes := range step { + for _, hash := range hashes { + if _, ok := fetcher.waitlist[hash][peer]; !ok { + t.Errorf("step %d, hash %x: peer %s missing from waitlist", i, hash, peer) + } + if _, ok := fetcher.waittime[hash]; !ok { + t.Errorf("step %d: hash %x missing from waittime", i, hash) + } + } + } + for hash, peers := range fetcher.waitlist { + if len(peers) == 0 { + t.Errorf("step %d, hash %x: empty peerset in waitlist", i, hash) + } + for peer := range peers { + if !containsHash(step[peer], hash) { + t.Errorf("step %d, hash %x: peer %s extra in waitlist", i, hash, peer) + } + } + } + for hash := range fetcher.waittime { + var found bool + for _, hashes := range step { + if containsHash(hashes, hash) { + found = true + break + } + } + if !found { + t.Errorf("step %d,: hash %x extra in waittime", i, hash) + } + } + + case isScheduled: + // Check that all scheduled announces are accounted for and no + // extra ones are present. + for peer, hashes := range step.tracking { + scheduled := fetcher.announces[peer] + if scheduled == nil { + t.Errorf("step %d: peer %s missing from announces", i, peer) + continue + } + for _, hash := range hashes { + if _, ok := scheduled[hash]; !ok { + t.Errorf("step %d, peer %s: hash %x missing from announces", i, peer, hash) + } + } + for hash := range scheduled { + if !containsHash(hashes, hash) { + t.Errorf("step %d, peer %s: hash %x extra in announces", i, peer, hash) + } + } + } + for peer := range fetcher.announces { + if _, ok := step.tracking[peer]; !ok { + t.Errorf("step %d: peer %s extra in announces", i, peer) + } + } + // Check that all announces required to be fetching are in the + // appropriate sets + for peer, hashes := range step.fetching { + request := fetcher.requests[peer] + if request == nil { + t.Errorf("step %d: peer %s missing from requests", i, peer) + continue + } + for _, hash := range hashes { + if !containsHash(request.hashes, hash) { + t.Errorf("step %d, peer %s: hash %x missing from requests", i, peer, hash) + } + } + for _, hash := range request.hashes { + if !containsHash(hashes, hash) { + t.Errorf("step %d, peer %s: hash %x extra in requests", i, peer, hash) + } + } + } + for peer := range fetcher.requests { + if _, ok := step.fetching[peer]; !ok { + if _, ok := step.dangling[peer]; !ok { + t.Errorf("step %d: peer %s extra in requests", i, peer) + } + } + } + for peer, hashes := range step.fetching { + for _, hash := range hashes { + if _, ok := fetcher.fetching[hash]; !ok { + t.Errorf("step %d, peer %s: hash %x missing from fetching", i, peer, hash) + } + } + } + for hash := range fetcher.fetching { + var found bool + for _, req := range fetcher.requests { + if containsHash(req.hashes, hash) { + found = true + break + } + } + if !found { + t.Errorf("step %d: hash %x extra in fetching", i, hash) + } + } + for _, hashes := range step.fetching { + for _, hash := range hashes { + alternates := fetcher.alternates[hash] + if alternates == nil { + t.Errorf("step %d: hash %x missing from alternates", i, hash) + continue + } + for peer := range alternates { + if _, ok := fetcher.announces[peer]; !ok { + t.Errorf("step %d: peer %s extra in alternates", i, peer) + continue + } + if _, ok := fetcher.announces[peer][hash]; !ok { + t.Errorf("step %d, peer %s: hash %x extra in alternates", i, hash, peer) + continue + } + } + for p := range fetcher.announced[hash] { + if _, ok := alternates[p]; !ok { + t.Errorf("step %d, hash %x: peer %s missing from alternates", i, hash, p) + continue + } + } + } + } + for peer, hashes := range step.dangling { + request := fetcher.requests[peer] + if request == nil { + t.Errorf("step %d: peer %s missing from requests", i, peer) + continue + } + for _, hash := range hashes { + if !containsHash(request.hashes, hash) { + t.Errorf("step %d, peer %s: hash %x missing from requests", i, peer, hash) + } + } + for _, hash := range request.hashes { + if !containsHash(hashes, hash) { + t.Errorf("step %d, peer %s: hash %x extra in requests", i, peer, hash) + } + } + } + // Check that all transaction announces that are scheduled for + // retrieval but not actively being downloaded are tracked only + // in the stage 2 `announced` map. + var queued []common.Hash + for _, hashes := range step.tracking { + for _, hash := range hashes { + var found bool + for _, hs := range step.fetching { + if containsHash(hs, hash) { + found = true + break + } + } + if !found { + queued = append(queued, hash) + } + } + } + for _, hash := range queued { + if _, ok := fetcher.announced[hash]; !ok { + t.Errorf("step %d: hash %x missing from announced", i, hash) + } + } + for hash := range fetcher.announced { + if !containsHash(queued, hash) { + t.Errorf("step %d: hash %x extra in announced", i, hash) + } + } + + case isUnderpriced: + if fetcher.underpriced.Cardinality() != int(step) { + t.Errorf("step %d: underpriced set size mismatch: have %d, want %d", i, fetcher.underpriced.Cardinality(), step) + } + + default: + t.Fatalf("step %d: unknown step type %T", i, step) + } + // After every step, cross validate the internal uniqueness invariants + // between stage one and stage two. + for hash := range fetcher.waittime { + if _, ok := fetcher.announced[hash]; ok { + t.Errorf("step %d: hash %s present in both stage 1 and 2", i, hash) + } } - case <-time.NewTimer(time.Second).C: - t.Fatalf("timeout") } } -func TestRejectUnderpriced(t *testing.T) { - tester := newTxFetcherTester() - tester.priceLimit = big.NewInt(10000) - - done := make(chan struct{}) - tester.fetcher.importTxsHook = func([]*types.Transaction) { done <- struct{}{} } - reject := make(chan struct{}) - tester.fetcher.rejectUnderprice = func(common.Hash) { reject <- struct{}{} } - - tx := types.NewTransaction(0, common.Address{0x1, 0x2, 0x3}, big.NewInt(int64(100)), 100, big.NewInt(int64(100)), nil) - tx, _ = types.SignTx(tx, types.NewEIP155Signer(big.NewInt(1)), tester.sender) - txFetcherFn := tester.makeTxFetcher("peer", []*types.Transaction{tx}) - - // Send the announcement first time - tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), txFetcherFn) - <-done - - // Resend the announcement, shouldn't schedule fetching this time - tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), txFetcherFn) - select { - case <-reject: - case <-time.NewTimer(time.Second).C: - t.Fatalf("timeout") +// containsHash returns whether a hash is contained within a hash slice. +func containsHash(slice []common.Hash, hash common.Hash) bool { + for _, have := range slice { + if have == hash { + return true + } } + return false } diff --git a/eth/handler.go b/eth/handler.go index d527b15d13..fc6c74cfe6 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -51,7 +51,7 @@ const ( // The number is referenced from the size of tx pool. txChanSize = 4096 - // minimim number of peers to broadcast new blocks to + // minimim number of peers to broadcast entire blocks and transactions too. minBroadcastPeers = 4 ) @@ -192,7 +192,15 @@ func NewProtocolManager(config *params.ChainConfig, checkpoint *params.TrustedCh return n, err } manager.blockFetcher = fetcher.NewBlockFetcher(blockchain.GetBlockByHash, validator, manager.BroadcastBlock, heighter, inserter, manager.removePeer) - manager.txFetcher = fetcher.NewTxFetcher(txpool.Has, txpool.AddRemotes, manager.removePeer) + + fetchTx := func(peer string, hashes []common.Hash) error { + p := manager.peers.Peer(peer) + if p == nil { + return errors.New("unknown peer") + } + return p.RequestTxs(hashes) + } + manager.txFetcher = fetcher.NewTxFetcher(txpool.Has, txpool.AddRemotes, fetchTx) return manager, nil } @@ -240,6 +248,8 @@ func (pm *ProtocolManager) removePeer(id string) { // Unregister the peer from the downloader and Ethereum peer set pm.downloader.UnregisterPeer(id) + pm.txFetcher.Drop(id) + if err := pm.peers.Unregister(id); err != nil { log.Error("Peer removal failed", "peer", id, "err", err) } @@ -263,7 +273,7 @@ func (pm *ProtocolManager) Start(maxPeers int) { // start sync handlers go pm.syncer() - go pm.txsyncLoop() + go pm.txsyncLoop64() // TODO(karalabe): Legacy initial tx echange, drop with eth/64. } func (pm *ProtocolManager) Stop() { @@ -292,7 +302,7 @@ func (pm *ProtocolManager) Stop() { } func (pm *ProtocolManager) newPeer(pv int, p *p2p.Peer, rw p2p.MsgReadWriter, getPooledTx func(hash common.Hash) *types.Transaction) *peer { - return newPeer(pv, p, newMeteredMsgWriter(rw), getPooledTx) + return newPeer(pv, p, rw, getPooledTx) } // handle is the callback invoked to manage the life cycle of an eth peer. When @@ -316,9 +326,6 @@ func (pm *ProtocolManager) handle(p *peer) error { p.Log().Debug("Ethereum handshake failed", "err", err) return err } - if rw, ok := p.rw.(*meteredMsgReadWriter); ok { - rw.Init(p.version) - } // Register the peer locally if err := pm.peers.Register(p); err != nil { p.Log().Error("Ethereum peer registration failed", "err", err) @@ -740,20 +747,10 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } // Schedule all the unknown hashes for retrieval - var unknown []common.Hash for _, hash := range hashes { - // Mark the hashes as present at the remote node p.MarkTransaction(hash) - - // Filter duplicated transaction announcement. - // Notably we only dedupliate announcement in txpool, check the rationale - // behind in EIP https://github.com/ethereum/EIPs/pull/2464. - if pm.txpool.Has(hash) { - continue - } - unknown = append(unknown, hash) } - pm.txFetcher.Notify(p.id, unknown, time.Now(), p.AsyncRequestTxs) + pm.txFetcher.Notify(p.id, hashes) case msg.Code == GetPooledTransactionsMsg && p.version >= eth65: // Decode the retrieval message @@ -763,9 +760,10 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } // Gather transactions until the fetch or network limits is reached var ( - hash common.Hash - bytes int - txs []rlp.RawValue + hash common.Hash + bytes int + hashes []common.Hash + txs []rlp.RawValue ) for bytes < softResponseLimit { // Retrieve the hash of the next block @@ -783,13 +781,14 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if encoded, err := rlp.EncodeToBytes(tx); err != nil { log.Error("Failed to encode transaction", "err", err) } else { + hashes = append(hashes, hash) txs = append(txs, encoded) bytes += len(encoded) } } - return p.SendTransactionRLP(txs) + return p.SendPooledTransactionsRLP(hashes, txs) - case msg.Code == TxMsg: + case msg.Code == TransactionMsg || (msg.Code == PooledTransactionsMsg && p.version >= eth65): // Transactions arrived, make sure we have a valid and fresh chain to handle them if atomic.LoadUint32(&pm.acceptTxs) == 0 { break @@ -806,7 +805,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } p.MarkTransaction(tx.Hash()) } - pm.txFetcher.EnqueueTxs(p.id, txs) + pm.txFetcher.Enqueue(p.id, txs, msg.Code == PooledTransactionsMsg) default: return errResp(ErrInvalidMsgCode, "%v", msg.Code) @@ -854,9 +853,9 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { } } -// BroadcastTxs will propagate a batch of transactions to all peers which are not known to +// BroadcastTransactions will propagate a batch of transactions to all peers which are not known to // already have the given transaction. -func (pm *ProtocolManager) BroadcastTxs(txs types.Transactions, propagate bool) { +func (pm *ProtocolManager) BroadcastTransactions(txs types.Transactions, propagate bool) { var ( txset = make(map[*peer][]common.Hash) annos = make(map[*peer][]common.Hash) @@ -894,7 +893,7 @@ func (pm *ProtocolManager) BroadcastTxs(txs types.Transactions, propagate bool) } for peer, hashes := range annos { if peer.version >= eth65 { - peer.AsyncSendTransactionHashes(hashes) + peer.AsyncSendPooledTransactionHashes(hashes) } else { peer.AsyncSendTransactions(hashes) } @@ -918,11 +917,11 @@ func (pm *ProtocolManager) txBroadcastLoop() { case event := <-pm.txsCh: // For testing purpose only, disable propagation if pm.broadcastTxAnnouncesOnly { - pm.BroadcastTxs(event.Txs, false) + pm.BroadcastTransactions(event.Txs, false) continue } - pm.BroadcastTxs(event.Txs, true) // First propagate transactions to peers - pm.BroadcastTxs(event.Txs, false) // Only then announce to the rest + pm.BroadcastTransactions(event.Txs, true) // First propagate transactions to peers + pm.BroadcastTransactions(event.Txs, false) // Only then announce to the rest // Err() channel will be closed when unsubscribing. case <-pm.txsSub.Err(): diff --git a/eth/metrics.go b/eth/metrics.go deleted file mode 100644 index 0533a2a875..0000000000 --- a/eth/metrics.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package eth - -import ( - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/p2p" -) - -var ( - propTxnInPacketsMeter = metrics.NewRegisteredMeter("eth/prop/txns/in/packets", nil) - propTxnInTrafficMeter = metrics.NewRegisteredMeter("eth/prop/txns/in/traffic", nil) - propTxnOutPacketsMeter = metrics.NewRegisteredMeter("eth/prop/txns/out/packets", nil) - propTxnOutTrafficMeter = metrics.NewRegisteredMeter("eth/prop/txns/out/traffic", nil) - propHashInPacketsMeter = metrics.NewRegisteredMeter("eth/prop/hashes/in/packets", nil) - propHashInTrafficMeter = metrics.NewRegisteredMeter("eth/prop/hashes/in/traffic", nil) - propHashOutPacketsMeter = metrics.NewRegisteredMeter("eth/prop/hashes/out/packets", nil) - propHashOutTrafficMeter = metrics.NewRegisteredMeter("eth/prop/hashes/out/traffic", nil) - propBlockInPacketsMeter = metrics.NewRegisteredMeter("eth/prop/blocks/in/packets", nil) - propBlockInTrafficMeter = metrics.NewRegisteredMeter("eth/prop/blocks/in/traffic", nil) - propBlockOutPacketsMeter = metrics.NewRegisteredMeter("eth/prop/blocks/out/packets", nil) - propBlockOutTrafficMeter = metrics.NewRegisteredMeter("eth/prop/blocks/out/traffic", nil) - reqHeaderInPacketsMeter = metrics.NewRegisteredMeter("eth/req/headers/in/packets", nil) - reqHeaderInTrafficMeter = metrics.NewRegisteredMeter("eth/req/headers/in/traffic", nil) - reqHeaderOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/headers/out/packets", nil) - reqHeaderOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/headers/out/traffic", nil) - reqBodyInPacketsMeter = metrics.NewRegisteredMeter("eth/req/bodies/in/packets", nil) - reqBodyInTrafficMeter = metrics.NewRegisteredMeter("eth/req/bodies/in/traffic", nil) - reqBodyOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/bodies/out/packets", nil) - reqBodyOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/bodies/out/traffic", nil) - reqStateInPacketsMeter = metrics.NewRegisteredMeter("eth/req/states/in/packets", nil) - reqStateInTrafficMeter = metrics.NewRegisteredMeter("eth/req/states/in/traffic", nil) - reqStateOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/states/out/packets", nil) - reqStateOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/states/out/traffic", nil) - reqReceiptInPacketsMeter = metrics.NewRegisteredMeter("eth/req/receipts/in/packets", nil) - reqReceiptInTrafficMeter = metrics.NewRegisteredMeter("eth/req/receipts/in/traffic", nil) - reqReceiptOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/receipts/out/packets", nil) - reqReceiptOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/receipts/out/traffic", nil) - miscInPacketsMeter = metrics.NewRegisteredMeter("eth/misc/in/packets", nil) - miscInTrafficMeter = metrics.NewRegisteredMeter("eth/misc/in/traffic", nil) - miscOutPacketsMeter = metrics.NewRegisteredMeter("eth/misc/out/packets", nil) - miscOutTrafficMeter = metrics.NewRegisteredMeter("eth/misc/out/traffic", nil) -) - -// meteredMsgReadWriter is a wrapper around a p2p.MsgReadWriter, capable of -// accumulating the above defined metrics based on the data stream contents. -type meteredMsgReadWriter struct { - p2p.MsgReadWriter // Wrapped message stream to meter - version int // Protocol version to select correct meters -} - -// newMeteredMsgWriter wraps a p2p MsgReadWriter with metering support. If the -// metrics system is disabled, this function returns the original object. -func newMeteredMsgWriter(rw p2p.MsgReadWriter) p2p.MsgReadWriter { - if !metrics.Enabled { - return rw - } - return &meteredMsgReadWriter{MsgReadWriter: rw} -} - -// Init sets the protocol version used by the stream to know which meters to -// increment in case of overlapping message ids between protocol versions. -func (rw *meteredMsgReadWriter) Init(version int) { - rw.version = version -} - -func (rw *meteredMsgReadWriter) ReadMsg() (p2p.Msg, error) { - // Read the message and short circuit in case of an error - msg, err := rw.MsgReadWriter.ReadMsg() - if err != nil { - return msg, err - } - // Account for the data traffic - packets, traffic := miscInPacketsMeter, miscInTrafficMeter - switch { - case msg.Code == BlockHeadersMsg: - packets, traffic = reqHeaderInPacketsMeter, reqHeaderInTrafficMeter - case msg.Code == BlockBodiesMsg: - packets, traffic = reqBodyInPacketsMeter, reqBodyInTrafficMeter - - case rw.version >= eth63 && msg.Code == NodeDataMsg: - packets, traffic = reqStateInPacketsMeter, reqStateInTrafficMeter - case rw.version >= eth63 && msg.Code == ReceiptsMsg: - packets, traffic = reqReceiptInPacketsMeter, reqReceiptInTrafficMeter - - case msg.Code == NewBlockHashesMsg: - packets, traffic = propHashInPacketsMeter, propHashInTrafficMeter - case msg.Code == NewBlockMsg: - packets, traffic = propBlockInPacketsMeter, propBlockInTrafficMeter - case msg.Code == TxMsg: - packets, traffic = propTxnInPacketsMeter, propTxnInTrafficMeter - } - packets.Mark(1) - traffic.Mark(int64(msg.Size)) - - return msg, err -} - -func (rw *meteredMsgReadWriter) WriteMsg(msg p2p.Msg) error { - // Account for the data traffic - packets, traffic := miscOutPacketsMeter, miscOutTrafficMeter - switch { - case msg.Code == BlockHeadersMsg: - packets, traffic = reqHeaderOutPacketsMeter, reqHeaderOutTrafficMeter - case msg.Code == BlockBodiesMsg: - packets, traffic = reqBodyOutPacketsMeter, reqBodyOutTrafficMeter - - case rw.version >= eth63 && msg.Code == NodeDataMsg: - packets, traffic = reqStateOutPacketsMeter, reqStateOutTrafficMeter - case rw.version >= eth63 && msg.Code == ReceiptsMsg: - packets, traffic = reqReceiptOutPacketsMeter, reqReceiptOutTrafficMeter - - case msg.Code == NewBlockHashesMsg: - packets, traffic = propHashOutPacketsMeter, propHashOutTrafficMeter - case msg.Code == NewBlockMsg: - packets, traffic = propBlockOutPacketsMeter, propBlockOutTrafficMeter - case msg.Code == TxMsg: - packets, traffic = propTxnOutPacketsMeter, propTxnOutTrafficMeter - } - packets.Mark(1) - traffic.Mark(int64(msg.Size)) - - // Send the packet to the p2p layer - return rw.MsgReadWriter.WriteMsg(msg) -} diff --git a/eth/peer.go b/eth/peer.go index f4b939b71c..2d22603467 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/fetcher" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rlp" ) @@ -43,17 +42,13 @@ const ( maxKnownBlocks = 1024 // Maximum block hashes to keep in the known list (prevent DOS) // maxQueuedTxs is the maximum number of transactions to queue up before dropping - // broadcasts. + // older broadcasts. maxQueuedTxs = 4096 // maxQueuedTxAnns is the maximum number of transaction announcements to queue up - // before dropping broadcasts. + // before dropping older announcements. maxQueuedTxAnns = 4096 - // maxQueuedTxRetrieval is the maximum number of tx retrieval requests to queue up - // before dropping requests. - maxQueuedTxRetrieval = 4096 - // maxQueuedBlocks is the maximum number of block propagations to queue up before // dropping broadcasts. There's not much point in queueing stale blocks, so a few // that might cover uncles should be enough. @@ -102,15 +97,16 @@ type peer struct { td *big.Int lock sync.RWMutex - knownTxs mapset.Set // Set of transaction hashes known to be known by this peer - knownBlocks mapset.Set // Set of block hashes known to be known by this peer - queuedBlocks chan *propEvent // Queue of blocks to broadcast to the peer - queuedBlockAnns chan *types.Block // Queue of blocks to announce to the peer - txPropagation chan []common.Hash // Channel used to queue transaction propagation requests - txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests - txRetrieval chan []common.Hash // Channel used to queue transaction retrieval requests - getPooledTx func(common.Hash) *types.Transaction // Callback used to retrieve transaction from txpool - term chan struct{} // Termination channel to stop the broadcaster + knownBlocks mapset.Set // Set of block hashes known to be known by this peer + queuedBlocks chan *propEvent // Queue of blocks to broadcast to the peer + queuedBlockAnns chan *types.Block // Queue of blocks to announce to the peer + + knownTxs mapset.Set // Set of transaction hashes known to be known by this peer + txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests + txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests + getPooledTx func(common.Hash) *types.Transaction // Callback used to retrieve transaction from txpool + + term chan struct{} // Termination channel to stop the broadcaster } func newPeer(version int, p *p2p.Peer, rw p2p.MsgReadWriter, getPooledTx func(hash common.Hash) *types.Transaction) *peer { @@ -123,17 +119,16 @@ func newPeer(version int, p *p2p.Peer, rw p2p.MsgReadWriter, getPooledTx func(ha knownBlocks: mapset.NewSet(), queuedBlocks: make(chan *propEvent, maxQueuedBlocks), queuedBlockAnns: make(chan *types.Block, maxQueuedBlockAnns), - txPropagation: make(chan []common.Hash), + txBroadcast: make(chan []common.Hash), txAnnounce: make(chan []common.Hash), - txRetrieval: make(chan []common.Hash), getPooledTx: getPooledTx, term: make(chan struct{}), } } -// broadcastBlocks is a write loop that multiplexes block propagations, -// announcements into the remote peer. The goal is to have an async writer -// that does not lock up node internals. +// broadcastBlocks is a write loop that multiplexes blocks and block accouncements +// to the remote peer. The goal is to have an async writer that does not lock up +// node internals and at the same time rate limits queued data. func (p *peer) broadcastBlocks() { for { select { @@ -155,105 +150,60 @@ func (p *peer) broadcastBlocks() { } } -// broadcastTxs is a write loop that multiplexes transaction propagations, -// announcements into the remote peer. The goal is to have an async writer -// that does not lock up node internals. -func (p *peer) broadcastTxs() { +// broadcastTransactions is a write loop that schedules transaction broadcasts +// to the remote peer. The goal is to have an async writer that does not lock up +// node internals and at the same time rate limits queued data. +func (p *peer) broadcastTransactions() { var ( - txProps []common.Hash // Queue of transaction propagations to the peer - txAnnos []common.Hash // Queue of transaction announcements to the peer - done chan struct{} // Non-nil if background network sender routine is active. - errch = make(chan error) // Channel used to receive network error + queue []common.Hash // Queue of hashes to broadcast as full transactions + done chan struct{} // Non-nil if background broadcaster is running + fail = make(chan error) // Channel used to receive network error ) - scheduleTask := func() { - // Short circuit if there already has a inflight task. - if done != nil { - return - } - // Spin up transaction propagation task if there is any - // queued hashes. - if len(txProps) > 0 { + for { + // If there's no in-flight broadcast running, check if a new one is needed + if done == nil && len(queue) > 0 { + // Pile transaction until we reach our allowed network limit var ( hashes []common.Hash txs []*types.Transaction size common.StorageSize ) - for i := 0; i < len(txProps) && size < txsyncPackSize; i++ { - if tx := p.getPooledTx(txProps[i]); tx != nil { + for i := 0; i < len(queue) && size < txsyncPackSize; i++ { + if tx := p.getPooledTx(queue[i]); tx != nil { txs = append(txs, tx) size += tx.Size() } - hashes = append(hashes, txProps[i]) + hashes = append(hashes, queue[i]) } - txProps = txProps[:copy(txProps, txProps[len(hashes):])] + queue = queue[:copy(queue, queue[len(hashes):])] + + // If there's anything available to transfer, fire up an async writer if len(txs) > 0 { done = make(chan struct{}) go func() { - if err := p.SendNewTransactions(txs); err != nil { - errch <- err + if err := p.sendTransactions(txs); err != nil { + fail <- err return } close(done) p.Log().Trace("Sent transactions", "count", len(txs)) }() - return } } - // Spin up transaction announcement task if there is any - // queued hashes. - if len(txAnnos) > 0 { - var ( - hashes []common.Hash - pending []common.Hash - size common.StorageSize - ) - for i := 0; i < len(txAnnos) && size < txsyncPackSize; i++ { - if tx := p.getPooledTx(txAnnos[i]); tx != nil { - pending = append(pending, txAnnos[i]) - size += common.HashLength - } - hashes = append(hashes, txAnnos[i]) - } - txAnnos = txAnnos[:copy(txAnnos, txAnnos[len(hashes):])] - if len(pending) > 0 { - done = make(chan struct{}) - go func() { - if err := p.SendNewTransactionHashes(pending); err != nil { - errch <- err - return - } - close(done) - p.Log().Trace("Sent transaction announcements", "count", len(pending)) - }() - } - } - } - - for { - scheduleTask() + // Transfer goroutine may or may not have been started, listen for events select { - case hashes := <-p.txPropagation: - if len(txProps) == maxQueuedTxs { - continue + case hashes := <-p.txBroadcast: + // New batch of transactions to be broadcast, queue them (with cap) + queue = append(queue, hashes...) + if len(queue) > maxQueuedTxs { + // Fancy copy and resize to ensure buffer doesn't grow indefinitely + queue = queue[:copy(queue, queue[len(queue)-maxQueuedTxs:])] } - if len(txProps)+len(hashes) > maxQueuedTxs { - hashes = hashes[:maxQueuedTxs-len(txProps)] - } - txProps = append(txProps, hashes...) - - case hashes := <-p.txAnnounce: - if len(txAnnos) == maxQueuedTxAnns { - continue - } - if len(txAnnos)+len(hashes) > maxQueuedTxAnns { - hashes = hashes[:maxQueuedTxAnns-len(txAnnos)] - } - txAnnos = append(txAnnos, hashes...) case <-done: done = nil - case <-errch: + case <-fail: return case <-p.term: @@ -262,60 +212,60 @@ func (p *peer) broadcastTxs() { } } -// retrievalTxs is a write loop which is responsible for retrieving transaction -// from the remote peer. The goal is to have an async writer that does not lock -// up node internals. If there are too many requests queued, then new arrival -// requests will be dropped silently so that we can ensure the memory assumption -// is fixed for each peer. -func (p *peer) retrievalTxs() { +// announceTransactions is a write loop that schedules transaction broadcasts +// to the remote peer. The goal is to have an async writer that does not lock up +// node internals and at the same time rate limits queued data. +func (p *peer) announceTransactions() { var ( - requests []common.Hash // Queue of transaction requests to the peer - done chan struct{} // Non-nil if background network sender routine is active. - errch = make(chan error) // Channel used to receive network error + queue []common.Hash // Queue of hashes to announce as transaction stubs + done chan struct{} // Non-nil if background announcer is running + fail = make(chan error) // Channel used to receive network error ) - // pick chooses a reasonble number of transaction hashes for retrieval. - pick := func() []common.Hash { - var ret []common.Hash - if len(requests) > fetcher.MaxTransactionFetch { - ret = requests[:fetcher.MaxTransactionFetch] - } else { - ret = requests[:] - } - requests = requests[:copy(requests, requests[len(ret):])] - return ret - } - // send sends transactions retrieval request. - send := func(hashes []common.Hash, done chan struct{}) { - if err := p.RequestTxs(hashes); err != nil { - errch <- err - return - } - close(done) - p.Log().Trace("Sent transaction retrieval request", "count", len(hashes)) - } for { - select { - case hashes := <-p.txRetrieval: - if len(requests) == maxQueuedTxRetrieval { - continue + // If there's no in-flight announce running, check if a new one is needed + if done == nil && len(queue) > 0 { + // Pile transaction hashes until we reach our allowed network limit + var ( + hashes []common.Hash + pending []common.Hash + size common.StorageSize + ) + for i := 0; i < len(queue) && size < txsyncPackSize; i++ { + if p.getPooledTx(queue[i]) != nil { + pending = append(pending, queue[i]) + size += common.HashLength + } + hashes = append(hashes, queue[i]) } - if len(requests)+len(hashes) > maxQueuedTxRetrieval { - hashes = hashes[:maxQueuedTxRetrieval-len(requests)] - } - requests = append(requests, hashes...) - if done == nil { + queue = queue[:copy(queue, queue[len(hashes):])] + + // If there's anything available to transfer, fire up an async writer + if len(pending) > 0 { done = make(chan struct{}) - go send(pick(), done) + go func() { + if err := p.sendPooledTransactionHashes(pending); err != nil { + fail <- err + return + } + close(done) + p.Log().Trace("Sent transaction announcements", "count", len(pending)) + }() + } + } + // Transfer goroutine may or may not have been started, listen for events + select { + case hashes := <-p.txAnnounce: + // New batch of transactions to be broadcast, queue them (with cap) + queue = append(queue, hashes...) + if len(queue) > maxQueuedTxAnns { + // Fancy copy and resize to ensure buffer doesn't grow indefinitely + queue = queue[:copy(queue, queue[len(queue)-maxQueuedTxs:])] } case <-done: done = nil - if pending := pick(); len(pending) > 0 { - done = make(chan struct{}) - go send(pending, done) - } - case <- errch: + case <-fail: return case <-p.term: @@ -379,22 +329,22 @@ func (p *peer) MarkTransaction(hash common.Hash) { p.knownTxs.Add(hash) } -// SendNewTransactionHashes sends a batch of transaction hashes to the peer and -// includes the hashes in its transaction hash set for future reference. -func (p *peer) SendNewTransactionHashes(hashes []common.Hash) error { - // Mark all the transactions as known, but ensure we don't overflow our limits - for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { - p.knownTxs.Pop() - } - for _, hash := range hashes { - p.knownTxs.Add(hash) - } - return p2p.Send(p.rw, NewPooledTransactionHashesMsg, hashes) +// SendTransactions64 sends transactions to the peer and includes the hashes +// in its transaction hash set for future reference. +// +// This method is legacy support for initial transaction exchange in eth/64 and +// prior. For eth/65 and higher use SendPooledTransactionHashes. +func (p *peer) SendTransactions64(txs types.Transactions) error { + return p.sendTransactions(txs) } -// SendNewTransactions sends transactions to the peer and includes the hashes +// sendTransactions sends transactions to the peer and includes the hashes // in its transaction hash set for future reference. -func (p *peer) SendNewTransactions(txs types.Transactions) error { +// +// This method is a helper used by the async transaction sender. Don't call it +// directly as the queueing (memory) and transmission (bandwidth) costs should +// not be managed directly. +func (p *peer) sendTransactions(txs types.Transactions) error { // Mark all the transactions as known, but ensure we don't overflow our limits for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(txs)) { p.knownTxs.Pop() @@ -402,18 +352,15 @@ func (p *peer) SendNewTransactions(txs types.Transactions) error { for _, tx := range txs { p.knownTxs.Add(tx.Hash()) } - return p2p.Send(p.rw, TxMsg, txs) + return p2p.Send(p.rw, TransactionMsg, txs) } -func (p *peer) SendTransactionRLP(txs []rlp.RawValue) error { - return p2p.Send(p.rw, TxMsg, txs) -} - -// AsyncSendTransactions queues list of transactions propagation to a remote -// peer. If the peer's broadcast queue is full, the event is silently dropped. +// AsyncSendTransactions queues a list of transactions (by hash) to eventually +// propagate to a remote peer. The number of pending sends are capped (new ones +// will force old sends to be dropped) func (p *peer) AsyncSendTransactions(hashes []common.Hash) { select { - case p.txPropagation <- hashes: + case p.txBroadcast <- hashes: // Mark all the transactions as known, but ensure we don't overflow our limits for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { p.knownTxs.Pop() @@ -426,9 +373,27 @@ func (p *peer) AsyncSendTransactions(hashes []common.Hash) { } } -// AsyncSendTransactions queues list of transactions propagation to a remote -// peer. If the peer's broadcast queue is full, the event is silently dropped. -func (p *peer) AsyncSendTransactionHashes(hashes []common.Hash) { +// sendPooledTransactionHashes sends transaction hashes to the peer and includes +// them in its transaction hash set for future reference. +// +// This method is a helper used by the async transaction announcer. Don't call it +// directly as the queueing (memory) and transmission (bandwidth) costs should +// not be managed directly. +func (p *peer) sendPooledTransactionHashes(hashes []common.Hash) error { + // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { + p.knownTxs.Pop() + } + for _, hash := range hashes { + p.knownTxs.Add(hash) + } + return p2p.Send(p.rw, NewPooledTransactionHashesMsg, hashes) +} + +// AsyncSendPooledTransactionHashes queues a list of transactions hashes to eventually +// announce to a remote peer. The number of pending sends are capped (new ones +// will force old sends to be dropped) +func (p *peer) AsyncSendPooledTransactionHashes(hashes []common.Hash) { select { case p.txAnnounce <- hashes: // Mark all the transactions as known, but ensure we don't overflow our limits @@ -443,6 +408,22 @@ func (p *peer) AsyncSendTransactionHashes(hashes []common.Hash) { } } +// SendPooledTransactionsRLP sends requested transactions to the peer and adds the +// hashes in its transaction hash set for future reference. +// +// Note, the method assumes the hashes are correct and correspond to the list of +// transactions being sent. +func (p *peer) SendPooledTransactionsRLP(hashes []common.Hash, txs []rlp.RawValue) error { + // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { + p.knownTxs.Pop() + } + for _, hash := range hashes { + p.knownTxs.Add(hash) + } + return p2p.Send(p.rw, PooledTransactionsMsg, txs) +} + // SendNewBlockHashes announces the availability of a number of blocks through // a hash notification. func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error { @@ -577,16 +558,6 @@ func (p *peer) RequestTxs(hashes []common.Hash) error { return p2p.Send(p.rw, GetPooledTransactionsMsg, hashes) } -// AsyncRequestTxs queues a tx retrieval request to a remote peer. If -// the peer's retrieval queue is full, the event is silently dropped. -func (p *peer) AsyncRequestTxs(hashes []common.Hash) { - select { - case p.txRetrieval <- hashes: - case <-p.term: - p.Log().Debug("Dropping transaction retrieval request", "count", len(hashes)) - } -} - // Handshake executes the eth protocol handshake, negotiating version number, // network IDs, difficulties, head and genesis blocks. func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis common.Hash, forkID forkid.ID, forkFilter forkid.Filter) error { @@ -746,9 +717,10 @@ func (ps *peerSet) Register(p *peer) error { return errAlreadyRegistered } ps.peers[p.id] = p + go p.broadcastBlocks() - go p.broadcastTxs() - go p.retrievalTxs() + go p.broadcastTransactions() + go p.announceTransactions() return nil } diff --git a/eth/protocol.go b/eth/protocol.go index 1cef66adb2..dc75d6b31a 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -51,7 +51,7 @@ const protocolMaxMsgSize = 10 * 1024 * 1024 // Maximum cap on the size of a prot const ( StatusMsg = 0x00 NewBlockHashesMsg = 0x01 - TxMsg = 0x02 + TransactionMsg = 0x02 GetBlockHeadersMsg = 0x03 BlockHeadersMsg = 0x04 GetBlockBodiesMsg = 0x05 @@ -64,10 +64,11 @@ const ( // New protocol message codes introduced in eth65 // - // Previously these message ids(0x08, 0x09) were used by some - // legacy and unsupported eth protocols, reown them here. + // Previously these message ids were used by some legacy and unsupported + // eth protocols, reown them here. NewPooledTransactionHashesMsg = 0x08 GetPooledTransactionsMsg = 0x09 + PooledTransactionsMsg = 0x0a ) type errCode int diff --git a/eth/protocol_test.go b/eth/protocol_test.go index e9a1a511ef..4bbfe9bd3c 100644 --- a/eth/protocol_test.go +++ b/eth/protocol_test.go @@ -62,7 +62,7 @@ func TestStatusMsgErrors63(t *testing.T) { wantError error }{ { - code: TxMsg, data: []interface{}{}, + code: TransactionMsg, data: []interface{}{}, wantError: errResp(ErrNoStatusMsg, "first msg has code 2 (!= 0)"), }, { @@ -114,7 +114,7 @@ func TestStatusMsgErrors64(t *testing.T) { wantError error }{ { - code: TxMsg, data: []interface{}{}, + code: TransactionMsg, data: []interface{}{}, wantError: errResp(ErrNoStatusMsg, "first msg has code 2 (!= 0)"), }, { @@ -258,7 +258,7 @@ func testRecvTransactions(t *testing.T, protocol int) { defer p.close() tx := newTestTransaction(testAccount, 0, 0) - if err := p2p.Send(p.app, TxMsg, []interface{}{tx}); err != nil { + if err := p2p.Send(p.app, TransactionMsg, []interface{}{tx}); err != nil { t.Fatalf("send error: %v", err) } select { @@ -282,13 +282,16 @@ func testSendTransactions(t *testing.T, protocol int) { pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, 0, nil, nil) defer pm.Stop() - // Fill the pool with big transactions. + // Fill the pool with big transactions (use a subscription to wait until all + // the transactions are announced to avoid spurious events causing extra + // broadcasts). const txsize = txsyncPackSize / 10 alltxs := make([]*types.Transaction, 100) for nonce := range alltxs { alltxs[nonce] = newTestTransaction(testAccount, uint64(nonce), txsize) } pm.txpool.AddRemotes(alltxs) + time.Sleep(100 * time.Millisecond) // Wait until new tx even gets out of the system (lame) // Connect several peers. They should all receive the pending transactions. var wg sync.WaitGroup @@ -300,8 +303,6 @@ func testSendTransactions(t *testing.T, protocol int) { seen[tx.Hash()] = false } for n := 0; n < len(alltxs) && !t.Failed(); { - var txs []*types.Transaction - var hashes []common.Hash var forAllHashes func(callback func(hash common.Hash)) switch protocol { case 63: @@ -310,11 +311,15 @@ func testSendTransactions(t *testing.T, protocol int) { msg, err := p.app.ReadMsg() if err != nil { t.Errorf("%v: read error: %v", p.Peer, err) - } else if msg.Code != TxMsg { + continue + } else if msg.Code != TransactionMsg { t.Errorf("%v: got code %d, want TxMsg", p.Peer, msg.Code) + continue } + var txs []*types.Transaction if err := msg.Decode(&txs); err != nil { t.Errorf("%v: %v", p.Peer, err) + continue } forAllHashes = func(callback func(hash common.Hash)) { for _, tx := range txs { @@ -325,11 +330,15 @@ func testSendTransactions(t *testing.T, protocol int) { msg, err := p.app.ReadMsg() if err != nil { t.Errorf("%v: read error: %v", p.Peer, err) + continue } else if msg.Code != NewPooledTransactionHashesMsg { - t.Errorf("%v: got code %d, want TxMsg", p.Peer, msg.Code) + t.Errorf("%v: got code %d, want NewPooledTransactionHashesMsg", p.Peer, msg.Code) + continue } + var hashes []common.Hash if err := msg.Decode(&hashes); err != nil { t.Errorf("%v: %v", p.Peer, err) + continue } forAllHashes = func(callback func(hash common.Hash)) { for _, h := range hashes { diff --git a/eth/sync.go b/eth/sync.go index 93b2dd2ecf..d5c678a74a 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -38,13 +38,18 @@ const ( ) type txsync struct { - p *peer - hashes []common.Hash - txs []*types.Transaction + p *peer + txs []*types.Transaction } // syncTransactions starts sending all currently pending transactions to the given peer. func (pm *ProtocolManager) syncTransactions(p *peer) { + // Assemble the set of transaction to broadcast or announce to the remote + // peer. Fun fact, this is quite an expensive operation as it needs to sort + // the transactions if the sorting is not cached yet. However, with a random + // order, insertions could overflow the non-executable queues and get dropped. + // + // TODO(karalabe): Figure out if we could get away with random order somehow var txs types.Transactions pending, _ := pm.txpool.Pending() for _, batch := range pending { @@ -53,17 +58,29 @@ func (pm *ProtocolManager) syncTransactions(p *peer) { if len(txs) == 0 { return } + // The eth/65 protocol introduces proper transaction announcements, so instead + // of dripping transactions across multiple peers, just send the entire list as + // an announcement and let the remote side decide what they need (likely nothing). + if p.version >= eth65 { + hashes := make([]common.Hash, len(txs)) + for i, tx := range txs { + hashes[i] = tx.Hash() + } + p.AsyncSendPooledTransactionHashes(hashes) + return + } + // Out of luck, peer is running legacy protocols, drop the txs over select { case pm.txsyncCh <- &txsync{p: p, txs: txs}: case <-pm.quitSync: } } -// txsyncLoop takes care of the initial transaction sync for each new +// txsyncLoop64 takes care of the initial transaction sync for each new // connection. When a new peer appears, we relay all currently pending // transactions. In order to minimise egress bandwidth usage, we send // the transactions in small packs to one peer at a time. -func (pm *ProtocolManager) txsyncLoop() { +func (pm *ProtocolManager) txsyncLoop64() { var ( pending = make(map[enode.ID]*txsync) sending = false // whether a send is active @@ -72,44 +89,26 @@ func (pm *ProtocolManager) txsyncLoop() { ) // send starts a sending a pack of transactions from the sync. send := func(s *txsync) { + if s.p.version >= eth65 { + panic("initial transaction syncer running on eth/65+") + } // Fill pack with transactions up to the target size. size := common.StorageSize(0) pack.p = s.p - pack.hashes = pack.hashes[:0] pack.txs = pack.txs[:0] - if s.p.version >= eth65 { - // Eth65 introduces transaction announcement https://github.com/ethereum/EIPs/pull/2464, - // only txhashes are transferred here. - for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { - pack.hashes = append(pack.hashes, s.txs[i].Hash()) - size += common.HashLength - } - // Remove the transactions that will be sent. - s.txs = s.txs[:copy(s.txs, s.txs[len(pack.hashes):])] - if len(s.txs) == 0 { - delete(pending, s.p.ID()) - } - // Send the pack in the background. - s.p.Log().Trace("Sending batch of transaction announcements", "count", len(pack.hashes), "bytes", size) - sending = true - go func() { done <- pack.p.SendNewTransactionHashes(pack.hashes) }() - } else { - // Legacy eth protocol doesn't have transaction announcement protocol - // message, transfer the whole pending transaction slice. - for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { - pack.txs = append(pack.txs, s.txs[i]) - size += s.txs[i].Size() - } - // Remove the transactions that will be sent. - s.txs = s.txs[:copy(s.txs, s.txs[len(pack.txs):])] - if len(s.txs) == 0 { - delete(pending, s.p.ID()) - } - // Send the pack in the background. - s.p.Log().Trace("Sending batch of transactions", "count", len(pack.txs), "bytes", size) - sending = true - go func() { done <- pack.p.SendNewTransactions(pack.txs) }() + for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { + pack.txs = append(pack.txs, s.txs[i]) + size += s.txs[i].Size() } + // Remove the transactions that will be sent. + s.txs = s.txs[:copy(s.txs, s.txs[len(pack.txs):])] + if len(s.txs) == 0 { + delete(pending, s.p.ID()) + } + // Send the pack in the background. + s.p.Log().Trace("Sending batch of transactions", "count", len(pack.txs), "bytes", size) + sending = true + go func() { done <- pack.p.SendTransactions64(pack.txs) }() } // pick chooses the next pending sync. diff --git a/fuzzbuzz.yaml b/fuzzbuzz.yaml index 72c3835ace..2a4f0c296f 100644 --- a/fuzzbuzz.yaml +++ b/fuzzbuzz.yaml @@ -26,6 +26,14 @@ targets: function: Fuzz package: github.com/ethereum/go-ethereum/tests/fuzzers/trie checkout: github.com/ethereum/go-ethereum/ + - name: txfetcher + language: go + version: "1.13" + corpus: ./fuzzers/txfetcher/corpus + harness: + function: Fuzz + package: github.com/ethereum/go-ethereum/tests/fuzzers/txfetcher + checkout: github.com/ethereum/go-ethereum/ - name: whisperv6 language: go version: "1.13" diff --git a/tests/fuzzers/txfetcher/corpus/0151ee1d0db4c74d3bcdfa4f7396a4c8538748c9-2 b/tests/fuzzers/txfetcher/corpus/0151ee1d0db4c74d3bcdfa4f7396a4c8538748c9-2 new file mode 100644 index 0000000000..2c75e9c7a7 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/0151ee1d0db4c74d3bcdfa4f7396a4c8538748c9-2 @@ -0,0 +1 @@ +¿½ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/020dd7b492a6eb34ff0b7d8ee46189422c37e4a7-6 b/tests/fuzzers/txfetcher/corpus/020dd7b492a6eb34ff0b7d8ee46189422c37e4a7-6 new file mode 100644 index 0000000000000000000000000000000000000000..8d3b57789e79a7046916b2c7646f038f443671ad GIT binary patch literal 14 VcmZRuu&`iYU~sUour#;W4*(9%0>A(O literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/021d1144e359233c496e22c3250609b11b213e9f-4 b/tests/fuzzers/txfetcher/corpus/021d1144e359233c496e22c3250609b11b213e9f-4 new file mode 100644 index 0000000000..73731899d5 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/021d1144e359233c496e22c3250609b11b213e9f-4 @@ -0,0 +1,12 @@ + TESTING KEY----- +MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9 +SjY1bIw4iAJm2gsvvZhIrCHS3l6afab4pZB +l2+XsDlrKBxKKtDrGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTtqJQIDAQAB +AoGAGRzwwir7XvBOAy5tuV6ef6anZzus1s1Y1Clb6HbnWWF/wbZGOpet +3m4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKZTXtdZrh+k7hx0nTP8Jcb +uqFk541awmMogY/EfbWd6IOkp+4xqjlFBEDytgbIECQQDvH/6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz84SHEg1Ak/7KCxmD/sfgS5TeuNi8DoUBEmiSJwm7FX +ftxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su43sjXNueLKH8+ph2UfQuU9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xl/DoCzjA0CQQDU +y2pGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj013sovGKUFfYAqVXVlxtIáo‡X +qUn3Xh9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JMhNRcVFMO8dDaFo +f9Oeos0UotgiDktdQHxdNEwLjQlJBz+OtwwA=---E RATTIEY- \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/0d28327b1fb52c1ba02a6eb96675c31633921bb2-2 b/tests/fuzzers/txfetcher/corpus/0d28327b1fb52c1ba02a6eb96675c31633921bb2-2 new file mode 100644 index 0000000000..8cc3039cb8 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/0d28327b1fb52c1ba02a6eb96675c31633921bb2-2 @@ -0,0 +1,15 @@ +¸&^£áo‡È—-----BEGIN RSA TESTING KEY----- +MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9 +SjY1bIw4iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZhIrCHS3l6afab4pZB +l2+XsDulrKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQAB +AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet +3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp +jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY +fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U +fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj013sovGKUFfYAqVXVlxtIX +qyUBnu3X9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFeo +f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA== +-----END RSA TESTING KEY-----Q_ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/0fcd827b57ded58e91f7ba2ac2b7ea4d25ebedca-7 b/tests/fuzzers/txfetcher/corpus/0fcd827b57ded58e91f7ba2ac2b7ea4d25ebedca-7 new file mode 100644 index 0000000000..8ceee16af1 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/0fcd827b57ded58e91f7ba2ac2b7ea4d25ebedca-7 @@ -0,0 +1 @@ +ð½apï¿ïï��ï¿ï¿¿½½½¿¿½½��¿½ï¿ï¿½ï¿ïÓÌV½¿½ïïï¿ï¿½#ï¿ï¿½&�� \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/109bc9b8fd4fef63493e104c703c79bc4a5e8d34-6 b/tests/fuzzers/txfetcher/corpus/109bc9b8fd4fef63493e104c703c79bc4a5e8d34-6 new file mode 100644 index 0000000000000000000000000000000000000000..df9b986af1005f1e472eaee9d42135411f3cad63 GIT binary patch literal 470 zcmXBPNsgjW0D$4DEvGQ)r7@r&!=e@sdB{8nD6kU{kfE3!lXvU%k@^a4rw;Ic$zQfx zYUxCchXTqX8J;qOlCW-BNUrGREK=HOVq~%x}s(2NsgIr{eR1PWA?-k_w>v}L;@%v7V}lB?YOX7S+`=lZ(i)wVud5Y zY!6BZa8jojm77j=WDEG3q(kN$-N$G=S;6CB{dAPMzB8g9V5#V#Wt=p|pNhFKjy%yC zFpBAMMa)X+HjvSN7$b5hd}DlQ5^HbU{NQ1!jYbc0Xo)I!+*2KCFqw4WsZh_wU=>#O zr1r)kwlZ8qgkQCI1Blc6)Wg<8J2*FT4y<|`;v!3_^cGx#XJ`clF*MUzO#{$fuhoKN zAM^Nc(r*X02+fQKC8*rXv7+O1epiA}A{Chb{qy_R=Z6uCy2&Pra2Fp<%9)R5AkasN o5@|^cIBRywREomC%wfKLePO%ZjwD%%fQGPfz{Yi4`Mgp80htP#Qvd(} literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/163785ab002746452619f31e8dfcb4549e6f8b6e-6 b/tests/fuzzers/txfetcher/corpus/163785ab002746452619f31e8dfcb4549e6f8b6e-6 new file mode 100644 index 0000000000000000000000000000000000000000..55467373d46141a3786f3d38a439ab7fa65d9b1c GIT binary patch literal 30 kcmZShPPx# literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/1adfa6b9ddf5766220c8ff7ede2926ca241bb947-3 b/tests/fuzzers/txfetcher/corpus/1adfa6b9ddf5766220c8ff7ede2926ca241bb947-3 new file mode 100644 index 0000000000..4a593aa28d --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/1adfa6b9ddf5766220c8ff7ede2926ca241bb947-3 @@ -0,0 +1,11 @@ +TAKBgDuLnQA3gey3VBznB39JUtxjeE6myuDkM/uGlfjb +S1w4iA5sBzzh8uxEbi4nW91IJm2gsvvZhICHS3l6ab4pZB +l2DulrKBxKKtD1rGxlG4LncabFn9vLZad2bSysqz/qTAUSTvqJQIDAQAB +AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet +3Z4vMXc7jpTLryzTQIvVdfQbRc6+MUVeLKZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk54MogxEcfbWd6IOkp+4xqFLBEDtgbIECnk+hgN4H +qzzxxr397vWrjrIgbJpQvBv8QeeuNi8DoUBEmiSJwa7FXY +FUtxuvL7XvjwjN5B30pEbc6Iuyt7y4MQJBAIt21su4b3sjphy2tuUE9xblTu14qgHZ6+AiZovGKU--FfYAqVXVlxtIX +qyU3X9ps8ZfjLZ45l6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFeo +f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA== +-----END RSA T \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/1b9a02e9a48fea1d2fc3fb77946ada278e152079-4 b/tests/fuzzers/txfetcher/corpus/1b9a02e9a48fea1d2fc3fb77946ada278e152079-4 new file mode 100644 index 0000000000000000000000000000000000000000..4a56f93d3ba9dc043d85983f6938371424a18831 GIT binary patch literal 17 ZcmdmX;%zzm%{7eWMGPsad6Su6002*L2o3-M literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/1e14c7ea1faef92890988061b5abe96db7190f98-7 b/tests/fuzzers/txfetcher/corpus/1e14c7ea1faef92890988061b5abe96db7190f98-7 new file mode 100644 index 0000000000..d2442fc5a6 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/1e14c7ea1faef92890988061b5abe96db7190f98-7 @@ -0,0 +1 @@ +0000000000000000000000000000000000000000000000000000000000000000000000000 \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/1e7d05f00e99cbf3ff0ef1cd7ea8dd07ad6dff23-6 b/tests/fuzzers/txfetcher/corpus/1e7d05f00e99cbf3ff0ef1cd7ea8dd07ad6dff23-6 new file mode 100644 index 0000000000000000000000000000000000000000..1c342ff53a36366885c5a06494aef83308ed3126 GIT binary patch literal 19 acmZShus5-wASW|9G1$Y=m$7*Nxl#a8k_aaN literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/1ec95e347fd522e6385b5091aa81aa2485be4891-4 b/tests/fuzzers/txfetcher/corpus/1ec95e347fd522e6385b5091aa81aa2485be4891-4 new file mode 100644 index 0000000000000000000000000000000000000000..b0c776bd4d996c4b423b42a871caae498a5bb695 GIT binary patch literal 833 zcmXZb$+DtI0EJvY?uiaW5OC3pLs{H8WBf3>{n2Nad>~2{-XpBKtKy*z7 z+VO+jwx&2CUP!XIdknPs5+C#S?OhhhOc6DhAMLg0*+iY_+G)y&nbovF_e?O(ua@TL z53V#@R+WX6$WV_ED&vV!h4+holrDdwKI_IuW%nq6*C?^Dx@qfOx|0Ugox~tD;mehJ}a=??6)q2HCOwz$325 zp*wt=oTNRZ+sCr=sREM0R}W-Hl{c{bhjYd9bd6;`b{<-^D2sHoI(L^|N=6$OvJI<$ z7^OL`7;Jwyd}`UXMDK@NUHasXIoCPN!xUX=I%YJ9w*(qoqh~Bcc*}BqGwPPF6%qIZ zVNlJ)2&vEF%kO~;ABVP=;sV?8S(JY`Dv!VJqe@8iM`FS#KAp3iGgar|UeiEx&Qhi% zTHa8_;hZ=9)^AApR9+;5r#W-g$;Pm#OjBM1dVeNUKOMpUKXIt;*A@IJ=Id`R@Ha}< B56=Jq literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/1fbfa5d214060d2a0905846a589fd6f78d411451-4 b/tests/fuzzers/txfetcher/corpus/1fbfa5d214060d2a0905846a589fd6f78d411451-4 new file mode 100644 index 0000000000000000000000000000000000000000..75de835c98de4015fd4c9874b4a469e18a3b87d3 GIT binary patch literal 22 ecmdn?ww(Ru8piS>hLqI2$;>Zw^z=$MDF6U-W(gGl literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/1fd84ee194e791783a7f18f0a6deab8efe05fc04-2 b/tests/fuzzers/txfetcher/corpus/1fd84ee194e791783a7f18f0a6deab8efe05fc04-2 new file mode 100644 index 0000000000..3b6d2560ae --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/1fd84ee194e791783a7f18f0a6deab8efe05fc04-2 @@ -0,0 +1 @@ +¸& \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/21e76b9fca21d94d97f860c1c82f40697a83471b-8 b/tests/fuzzers/txfetcher/corpus/21e76b9fca21d94d97f860c1c82f40697a83471b-8 new file mode 100644 index 0000000000..1d4620f49f --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/21e76b9fca21d94d97f860c1c82f40697a83471b-8 @@ -0,0 +1,3 @@ +DtQvfQ+MULKZTXk78c +/fWkpxlQQ/+hgNzVtx9vWgJsafG7b0dA4AFjwVbFLmQcj2PprIMmPNQrooX +L \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/220a87fed0c92474923054094eb7aff14289cf5e-4 b/tests/fuzzers/txfetcher/corpus/220a87fed0c92474923054094eb7aff14289cf5e-4 new file mode 100644 index 0000000000000000000000000000000000000000..175f74fd5aa8883bffb6d3d64727d2265f7bee35 GIT binary patch literal 4 LcmZQz*~^ literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/23ddcd66aa92fe3d78b7f5b6e7cddb1b55c5f5df-3 b/tests/fuzzers/txfetcher/corpus/23ddcd66aa92fe3d78b7f5b6e7cddb1b55c5f5df-3 new file mode 100644 index 0000000000..95892c7b00 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/23ddcd66aa92fe3d78b7f5b6e7cddb1b55c5f5df-3 @@ -0,0 +1,12 @@ +4txjeVE6myuDqkM/uGlfjb9 +SjY1bIw4iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZeIrCHS3l6afab4pZB +l2+XsDlrKBxKKtD1rGxlG4jncdabFn9gvLZad2bSysqz/qTAUSTvqJQIDAQAB +AoGAGRzwwXvBOAy5tM/uV6e+Zf6aZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet +3Z4vD6Mc7pLryzTQIVdfQbRc6+MUVeLKZaTXtdZru+Jk70PJJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+gN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQ2PprIMPcQroo8vpjSHg1Ev14KxmQeDydfsgeuN8UBESJwm7F +UtuL7Xvjw50pNEbc6Iuyty4QJA21su4sjXNueLQphy2U +fQtuUE9txblTu14qN7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6ARYiZPYj1oGUFfYAVVxtI +qyBnu3X9pfLZOAkEAlT4R5Yl6cJQYZHOde3JEhNRcVFMO8dJFo +f9Oeos0UUhgiDkQxdEwLjQf7lJJz5OtwC= +-NRSA TESINGKEY-Q_ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/2441d249faf9a859e38c49f6e305b394280c6ea5-1 b/tests/fuzzers/txfetcher/corpus/2441d249faf9a859e38c49f6e305b394280c6ea5-1 new file mode 100644 index 0000000000000000000000000000000000000000..d76207e992a6e50a53966fa06c881f7c9cab5eb4 GIT binary patch literal 55 zcmV~$fenBl3`IeCEThRMsijI7#|fQ)Fo#p<2;O~ZN#mV^=f=b?TvQR_5T|IO)NpG( K!syDY>)L*r_6#ck literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/2da1f0635e11283b1927974f418aadd8837ad31e-7 b/tests/fuzzers/txfetcher/corpus/2da1f0635e11283b1927974f418aadd8837ad31e-7 new file mode 100644 index 0000000000000000000000000000000000000000..73ae7057014ffcb8c913914888047969958df11c GIT binary patch literal 15 WcmZShus5-wAjgAo?|Zhr`%3{g@&`u% literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/2e1853fbf8efe40098b1583224fe3b5f335e7037-6 b/tests/fuzzers/txfetcher/corpus/2e1853fbf8efe40098b1583224fe3b5f335e7037-6 new file mode 100644 index 0000000000000000000000000000000000000000..692981e614155c59f5ba80f40e832734383d8901 GIT binary patch literal 26 icmZShz~Eha=sv^$|L^zjWze;-urM}dU|?{tumAv_9to%b literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/2f25490dc49c103d653843ed47324b310ee7105e-7 b/tests/fuzzers/txfetcher/corpus/2f25490dc49c103d653843ed47324b310ee7105e-7 new file mode 100644 index 0000000000000000000000000000000000000000..5cf7da75df2dc6c74b3353ec1054ae730ab2a4f6 GIT binary patch literal 23 fcmZShz~Eha=sv^$|L^zjWiVziWnf@%u&@9Cesl=a literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/30494b85bb60ad7f099fa49d427007a761620d8f-5 b/tests/fuzzers/txfetcher/corpus/30494b85bb60ad7f099fa49d427007a761620d8f-5 new file mode 100644 index 0000000000..7ff9d39752 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/30494b85bb60ad7f099fa49d427007a761620d8f-5 @@ -0,0 +1,10 @@ +jXbnWWF/wbZGOpet +3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp +jy4SHEg1AkEA/v13/5M47K9vCxb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY +fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U +fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xl/DoCzjA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6Yj013sovGKUFfYAqVXVlxtIX +qyUBnu3Xh9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dDaFeo +f9Oeos0UotgiDktdQHxdNEwLjQfl \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/316024ca3aaf09c1de5258733ff5fe3d799648d3-4 b/tests/fuzzers/txfetcher/corpus/316024ca3aaf09c1de5258733ff5fe3d799648d3-4 new file mode 100644 index 0000000000..61f7d78f34 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/316024ca3aaf09c1de5258733ff5fe3d799648d3-4 @@ -0,0 +1,15 @@ +¸^áo‡È—----BEGIN RA TTING KEY----- +IIXgIBAAKBQDuLnQI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9 +SjY1bIw4iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJmgsvvZhrCHSl6afab4pZB +l2+XsDulrKBxKKtD1rGxlG4LjcdabF9gvLZad2bSysqz/qTAUStTvqJQDAQAB +AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet +3Z4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp +jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY +fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U +fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj043sovGKUFfYAqVXVlxtIX +qyUBnu3X9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFeo +f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA== +-----END RSA TESTING KEY-----Q_ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/32a089e2c439a91f4c1b67a13d52429bcded0dd9-7 b/tests/fuzzers/txfetcher/corpus/32a089e2c439a91f4c1b67a13d52429bcded0dd9-7 new file mode 100644 index 0000000000000000000000000000000000000000..a986a9d8e753a8e307ab9d904781ec872fa2eec0 GIT binary patch literal 1665 zcmeIzM@|DV7>3~lGKAg+QhHf%TnTO?MT0U^PHGqB?TNL+#c z#cNIglKp(YXFNsLXl$@(kIip7o$l-T$>8&HrI*x$7kGmKH1Gi<_<|q!LjVLq5ClUA zghCjELj*)Z6qq0yVjvdcARZFH42h5g7D$E^NQE>=hYZLBD`Y`7js-XsIp$_Vy0UDtRnxO^k&F2J5f^o3I7j;DH_3g+17Z12}{u zIEE8Ag)=yZ3%Jz3UQ7J@zXE@B1?20O){(7pq*Rl_RHH`5I+Y9?jY3qOmpbB7@$&~) zrP5(rkfPl!cw&pRD#;afM)}(f-XEf6f}dH?%sehr%@^(CpEq}szIMSBsdu{6#v!AR h`=_R1$ph)-(xt6h1&?%ISK8aH%b}_?r%Ji{`5V}g$;JQx literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/33ec1dc0bfeb93d16edee3c07125fec6ac1aa17d-2 b/tests/fuzzers/txfetcher/corpus/33ec1dc0bfeb93d16edee3c07125fec6ac1aa17d-2 new file mode 100644 index 0000000000..d41771b86c --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/33ec1dc0bfeb93d16edee3c07125fec6ac1aa17d-2 @@ -0,0 +1 @@ +ï¿ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/37a0d207700b52caa005ec8aeb344dcb13150ed2-5 b/tests/fuzzers/txfetcher/corpus/37a0d207700b52caa005ec8aeb344dcb13150ed2-5 new file mode 100644 index 0000000000000000000000000000000000000000..2f09c6e28f03ccb6b2e40fe56b8d26768544aab6 GIT binary patch literal 55 zcmZShu=hRV-uL_WGO(qVWay`*mLzAS7U`E1CFT_;CYNO9=jkUEu2ZX Lm*?%>|GpFetm7Di literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/382f59c66d0ddb6747d3177263279789ca15c2db-5 b/tests/fuzzers/txfetcher/corpus/382f59c66d0ddb6747d3177263279789ca15c2db-5 new file mode 100644 index 0000000000000000000000000000000000000000..84441ac374622d3d0e6dcb6c31a11adb110de593 GIT binary patch literal 14 VcmZSCtH8S2`8Wdu!~VStr2rs*1dRXy literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/3a010483a4ad8d7215447ce27e0fac3791235c99-4 b/tests/fuzzers/txfetcher/corpus/3a010483a4ad8d7215447ce27e0fac3791235c99-4 new file mode 100644 index 0000000000..28f5d99b98 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/3a010483a4ad8d7215447ce27e0fac3791235c99-4 @@ -0,0 +1,7 @@ + +lGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet +3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp +jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY +fFUtxuvL7XvjwjN5 \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/3a3b717fcfe7ffb000b906e5a76f32248a576bf7-6 b/tests/fuzzers/txfetcher/corpus/3a3b717fcfe7ffb000b906e5a76f32248a576bf7-6 new file mode 100644 index 0000000000000000000000000000000000000000..022de3c61d4bfaa977cfd74616bc29c66c8a8ea3 GIT binary patch literal 1141 zcmc)IS5g&G5CBj|P|OiT6tjXNAZ8I2vx16ZCYuvR5HaiQhj0I;UKR7h7F>iiTd)tV z;gvPFPxnVp_tdTVHS=e(>i1MKHm4|M6%o@vlQ6+tRG<>`FdtP|fQ49u#aM!+Scc_T z5v)|K!fI5b25V4@wWz~7tVcaIU?VnRGqzwWwqZMVpaG58iCt*IZZx9>t=NM$?8QFp z#{nF~AsogLwBsl`a16(B0-ZRCQ#g$?IE!;Qj|;enE_CA(F5?P%a240ki|e?7o4AGB zxP!asLq7&Eh#?GP1ov)j7;^7kVj6m_hh9^u&+GEa^+4;>wl!_zth7@Gy09bpfvi z7cnX2@R_lyU9|*QJ5)>JHWpBo?ymdemqN0iiYN;UD2Hy}!*N}E=E}{_9`$PBK&^G> z+wjUC9HcS<2r^niC>w5yXs7K}vvJZMW2HC?6{oKanP04L9tSt6`O+3S%Gb%@B7`ms z)mkuR)oB|yV7e-Oo&YnTNLPrqq=Vd@f-9{3=?pr?;tIvPfy6B3coR4NuJ;z5It3}u zBNf#@NfF<7BvAhYVsO~#_AyLfO%aSo#8~EaotD^Xh=i1RptFSF#or0*#E3dG1D+px zAm_2rg;7|qqxu*Bi4I-xVWRTd4t$a$BeA)gP)S1V1Md>oZ2Bl)nj0(jo6*eD^3u?> cGZDY>5a6O3Gifo9tL^r<2)|FK(^2sKe=I1K@c;k- literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/3d8b5bf36c80d6f65802280039f85421f32b5055-6 b/tests/fuzzers/txfetcher/corpus/3d8b5bf36c80d6f65802280039f85421f32b5055-6 new file mode 100644 index 0000000000000000000000000000000000000000..eacd269f317b144dc5fdf3eb138007bbd6270192 GIT binary patch literal 23 ecmZShus5-wASW|9G1$Y=l(9H94mgL_|SBMWiSKDl$?WA&`JdB8U{Rp(r+p6h%Oy2ny<$9C-uY&Q4~C zSSU&nq$nJ40r3)e|9NuaHONe!@3-Gha{5E|h6G52J+K#&AQ|?-emDRra8LxTRF*@K2I-IinQ%BfKh>g7Xgk80 zEI10;kOR4JEW&Y?6OacdAs?8hpa2S?2#Vn}l)#yPwo*0+ltDQ-!37mi31{IPoQEod zn&pa-4;vk&^=84Q5=0qFG{Ucex{gdrG)SMVA}U=-fK zTX+Z2JMvyzD8Q*Lx2}Wv)`bFN{!k#KKB;lPFYsA@QD4v literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/608200b402488b3989ec8ec5f4190ccb537b8ea4-4 b/tests/fuzzers/txfetcher/corpus/608200b402488b3989ec8ec5f4190ccb537b8ea4-4 new file mode 100644 index 0000000000000000000000000000000000000000..d37b018515b8c82be6564a0e1f3e772bbefffd89 GIT binary patch literal 24 gcmdmXqMZHa8piS>hLqI2$;>Zw^z^o8mu^x30EF%e5&!@I literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/61e89c3fbdf9eff74bd250ea73cc2e61f8ca0d97-5 b/tests/fuzzers/txfetcher/corpus/61e89c3fbdf9eff74bd250ea73cc2e61f8ca0d97-5 new file mode 100644 index 0000000000..155744bccc --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/61e89c3fbdf9eff74bd250ea73cc2e61f8ca0d97-5 @@ -0,0 +1 @@ +88242871'392752200424491531672177074144720616417147514758635765020556616¿ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/62817a48c78fbf2c12fcdc5ca58e2ca60c43543a-7 b/tests/fuzzers/txfetcher/corpus/62817a48c78fbf2c12fcdc5ca58e2ca60c43543a-7 new file mode 100644 index 0000000000000000000000000000000000000000..795608a789579add52324341abf22fac2b156d73 GIT binary patch literal 27 jcmZShus5;b|Ns9DiH1hHL5#(zDeoEgzGvIJ|6C~m!tV{; literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/6782da8f1a432a77306d60d2ac2470c35b98004f-3 b/tests/fuzzers/txfetcher/corpus/6782da8f1a432a77306d60d2ac2470c35b98004f-3 new file mode 100644 index 0000000000..f44949e6ae --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/6782da8f1a432a77306d60d2ac2470c35b98004f-3 @@ -0,0 +1 @@ +21888242871'392752200424452601091531672177074144720616417147514758635765020556616¿½ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/68fb55290cb9d6da5b259017c34bcecf96c944aa-5 b/tests/fuzzers/txfetcher/corpus/68fb55290cb9d6da5b259017c34bcecf96c944aa-5 new file mode 100644 index 0000000000000000000000000000000000000000..23d905b827e2a5c4a04388da93af9bb5dafca84d GIT binary patch literal 49 zcmeZ>axzIaN)AZQ$xO{JOD!sJFAmQ#_G9pk42aTqvnY9J8kyr4WRc?HmY?a85*U~R E0HKW##sB~S literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/6a5059bc86872526241d21ab5dae9f0afd3b9ae1-3 b/tests/fuzzers/txfetcher/corpus/6a5059bc86872526241d21ab5dae9f0afd3b9ae1-3 new file mode 100644 index 0000000000..b71d5dff51 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/6a5059bc86872526241d21ab5dae9f0afd3b9ae1-3 @@ -0,0 +1 @@ +¿½ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/717928e0e2d478c680c6409b173552ca98469ba5-6 b/tests/fuzzers/txfetcher/corpus/717928e0e2d478c680c6409b173552ca98469ba5-6 new file mode 100644 index 0000000000..dce5106115 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/717928e0e2d478c680c6409b173552ca98469ba5-6 @@ -0,0 +1 @@ +LvhaJcdaFofenogkjQfJB \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/71d22f25419543e437f249ca437823b87ac926b1-6 b/tests/fuzzers/txfetcher/corpus/71d22f25419543e437f249ca437823b87ac926b1-6 new file mode 100644 index 0000000000000000000000000000000000000000..d07a6c2f3244799a3c836c749bfc3fa614936172 GIT binary patch literal 27 jcmZShus5-wAV)X3?EU_|zKq4GDeoEgzGvIJ|6C~mw;2t$ literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/7312a0f31ae5d773ed4fd74abc7521eb14754683-8 b/tests/fuzzers/txfetcher/corpus/7312a0f31ae5d773ed4fd74abc7521eb14754683-8 new file mode 100644 index 0000000000..3593ce2e19 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/7312a0f31ae5d773ed4fd74abc7521eb14754683-8 @@ -0,0 +1,2 @@ +DtQvfQ+MULKZTXk78c +/fWkpxlyEQQ/+hgNzVtx9vWgJsafG7b0dA4AFjwVbFLmQcj2PprIMmPNQg1AkS5TDEmSJwFVlXsjLZ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/76e413a50dc8861e3756e556f796f1737bec2675-4 b/tests/fuzzers/txfetcher/corpus/76e413a50dc8861e3756e556f796f1737bec2675-4 new file mode 100644 index 0000000000000000000000000000000000000000..623fcf9601e516398a6bce4104435dc4556d99a2 GIT binary patch literal 24 gcmdmX;w}5lHH_s&3@NF3lbK)W=;>|GF5RR60FM$1hyVZp literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/78480977d5c07386b06e9b37f5c82f5ed86c2f09-3 b/tests/fuzzers/txfetcher/corpus/78480977d5c07386b06e9b37f5c82f5ed86c2f09-3 new file mode 100644 index 0000000000..e92863a1c7 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/78480977d5c07386b06e9b37f5c82f5ed86c2f09-3 @@ -0,0 +1,14 @@ + TESTING KEY----- +MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9 +SjY1bIw4iAJm2gsvvZhIrCHS3l6afab4pZB +l2+XsDulrKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQAB +AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet +3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp +jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY +fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U +fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xl/DoCzjA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj013sovGKUFfYAqVXVlxtIX +qyUBnu3Xh9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dDaFeo +f9Oeos0UotgiDktdQHxdNEwLjQflJJBzV+5OtwswCA=----EN RATESTI EY-----Q \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/7a113cd3c178934cdb64353af86d51462d7080a4-5 b/tests/fuzzers/txfetcher/corpus/7a113cd3c178934cdb64353af86d51462d7080a4-5 new file mode 100644 index 0000000000..16818128ae --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/7a113cd3c178934cdb64353af86d51462d7080a4-5 @@ -0,0 +1,10 @@ +l6afab4pZB +l2+XsDlrKBxKKtDrGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTtqJQIDAQAB +AoGAGRzwwir7XvBOAy5tuV6ef6anZzus1s1Y1Clb6HbnWWF/wbZGOpet +3m4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKZTXtdZrh+k7hx0nTP8Jcb +uqFk541awmMogY/EfbWd6IOkp+4xqjlFBEDytgbIECQQDvH/6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz84SHEg1Ak/7KCxmD/sfgS5TeuNi8DoUBEmiSJwm7FX +ftxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su43sjXNueLKH8+ph2UfQuU9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xl/DoCzjA0CQQDU +y2pGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj13sovGKUFfYAqVXVlxtIáo‡X +qUn3X9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JMhNRcVFMO8dDaFo +f9Oeos0UotgiDktdQHxdNEwLjQlJBz+OtwwA=---E ATTIEY- \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/7ea9f71020f3eb783f743f744eba8d8ca4b2582f-3 b/tests/fuzzers/txfetcher/corpus/7ea9f71020f3eb783f743f744eba8d8ca4b2582f-3 new file mode 100644 index 0000000000..08f5bb99f5 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/7ea9f71020f3eb783f743f744eba8d8ca4b2582f-3 @@ -0,0 +1,9 @@ + +l2+DulrKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQAB +AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet +3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp +jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY +fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U +fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU diff --git a/tests/fuzzers/txfetcher/corpus/84f8c275f3ffbaf8c32c21782af13de10e7de28b-3 b/tests/fuzzers/txfetcher/corpus/84f8c275f3ffbaf8c32c21782af13de10e7de28b-3 new file mode 100644 index 0000000000..2d6060c406 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/84f8c275f3ffbaf8c32c21782af13de10e7de28b-3 @@ -0,0 +1 @@ +KKtDlbjVeLKwZatTXtdZrhu+Jk7hx0xxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLQcmPcQETT YQ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/85dfe7ddee0e52aa19115c0ebb9ed28a14e488c6-5 b/tests/fuzzers/txfetcher/corpus/85dfe7ddee0e52aa19115c0ebb9ed28a14e488c6-5 new file mode 100644 index 0000000000000000000000000000000000000000..9b6fe78029e7d3b85115abd651f2bbbff34bbca8 GIT binary patch literal 11 TcmZSh@Sbt+`~746aLQ)#y_k0wLP=ZpFp&S*+VI@{!HP)aKYf*)2 ztP8An*no|w!6t0R7Hq{fY)372pbk4xj|S|*ZtTHc?8AN>KqC&K35ReP&1k_99K|sl z#|fOoDV#T*eh##Wh^V4cx>n+(s|@a0mUki+i|_ z2Y84_7{DNgFpLq5VhoS*1W)k{&+!5;@d~f;25<2W?=g-Ee85LcVhYpvgwObbJig)^ zX7C-e_<^68i_$ox;+Uj`&^t|*N5o-@Nm7`%6iF6RmPBzDg%xrkRuWDjma4NtoTQ;D VPO=bj7Pc!8L%xIiR8uYqk6qWf{iHWl(kl6J{wndw<-Ac6{ii@NU(Cfa={Fu^7* zx5~U7WoYoN>G=4rzJ__}!e_>lb`TcUmZW}!^JJyuKV*IPCxbV1}L-4ze?foTaeL+vV(%Lwzs-QMz~ulq=q0<1aKfIzn}OCL)*E zwHhDtagof)VX{X>sq2AHQS+OUd+CF~Npf8|AP#20+Prve72t2gI`(y6w)oMK38h{* zlPAG5ekJ@PMkkt``&BaowT!_SMq#sKJtD><5W9tq>iLqC$V%>l2;K^4eC~OU8$Q$O zIJw>Pf;AMo&O+6`U-DEfdZN7Ei+(-@J+5Bz8|UtA&mCp6kXtr}A$~&ge8=0oN#MU9 Qxr+1k1is?=`qz);e`P)HIRF3v literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/8ff3bd49f93079e5e1c7f8f2461ba7ee612900c3-5 b/tests/fuzzers/txfetcher/corpus/8ff3bd49f93079e5e1c7f8f2461ba7ee612900c3-5 new file mode 100644 index 0000000000000000000000000000000000000000..a86a66593b4647dd930c1f2c9f07b668e0f6ee50 GIT binary patch literal 40 wcmZShus5-wASW|9G1$Y=)X+#bDA-Xqx$OP^y}pdasVVOn_r7P_yZ>A%07&f-YXATM literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/9034aaf45143996a2b14465c352ab0c6fa26b221-2 b/tests/fuzzers/txfetcher/corpus/9034aaf45143996a2b14465c352ab0c6fa26b221-2 new file mode 100644 index 0000000000..9c95a6ba6a --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/9034aaf45143996a2b14465c352ab0c6fa26b221-2 @@ -0,0 +1 @@ +½ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/92cefdc6251d04896349a464b29be03d6bb04c3d-2 b/tests/fuzzers/txfetcher/corpus/92cefdc6251d04896349a464b29be03d6bb04c3d-2 new file mode 100644 index 0000000000..9b78e45707 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/92cefdc6251d04896349a464b29be03d6bb04c3d-2 @@ -0,0 +1 @@ +ï39402006196394479212279040100143613805079739270465446667948293404245721771496870329047266088258938001861606973112319¿½ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/9613e580ccb69df7c9074f0e2f6886ac6b34ca55-5 b/tests/fuzzers/txfetcher/corpus/9613e580ccb69df7c9074f0e2f6886ac6b34ca55-5 new file mode 100644 index 0000000000000000000000000000000000000000..681adc6a9cd91ac14946e3aa426206220289f31b GIT binary patch literal 446 zcmXBPOOm26007YGEvvahFIj*R0hd7%0U>~jf;&M$5r$s^!Mn}f#;Lx*t9q`1Oj@Ki z+7tv>LdUtXE_=I3opv1cOkhs80*fn}YOg60iK8q}&o|ZfU#$puS60~l2=fmShUREV z>f?uczfY9&URi77x&LnlA2b||;hJx?RK7s~aXFn*w+^#1XnWhMgDmWu=BEgs9MBr8L1 z2}(1S%~YF#7ebpZ29_#A854GQEj<+5$1)P~kg^D^DNm~-TYyd>T4C)CaPb~Enax?> z?0T}yfqv$XQQMx+#g#e|{lH zMKOTPW!FgViBwLj=q}ecwxu-XnYBtgkQd+x-I*ha*f2hI~qI;_h1*kx0-P>Kt^Mwrk=>C!>J)7i) K4MF?hIPM>cYfrWS literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/a2684adccf16e036b051c12f283734fa803746e8-6 b/tests/fuzzers/txfetcher/corpus/a2684adccf16e036b051c12f283734fa803746e8-6 new file mode 100644 index 0000000000000000000000000000000000000000..4e12af2da8e9fc8fbb4f49a1b09323698fdaed26 GIT binary patch literal 25 hcmZShz~HTAUV7+0!`{6N3=S3+#-^6$7W?0q0swI!2@n7P literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/a37305974cf477ecfe65fa92f37b1f51dea25910-4 b/tests/fuzzers/txfetcher/corpus/a37305974cf477ecfe65fa92f37b1f51dea25910-4 new file mode 100644 index 0000000000000000000000000000000000000000..75cb14e8d98ea165366cc682e0b9b72d0510bd70 GIT binary patch literal 15 XcmZShu=hRV-uL_WGO+F4|GpFeM3o4v literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/a7eb43926bd14b1f62a66a33107776e487434d32-7 b/tests/fuzzers/txfetcher/corpus/a7eb43926bd14b1f62a66a33107776e487434d32-7 new file mode 100644 index 0000000000000000000000000000000000000000..88e6127355dd8492243d27d231165c9a7694c32f GIT binary patch literal 516 zcmWmBHBtlt6a-KPTio5<-QC^YT`oYfDF-4Zo7yTOuD~hw3d{@s>+bk}|Hfl~UqSpM zC7B4*qWk#_l3{9KCBk%>flS1Zg>2*?7kS7>0SZwRQLL1p6lEw!1u9X6YSf?>b*M)J z8qtJiw4fDjXh#P+(S>gGpcj4U#{dQ~gkg+e6k{021STAGX#{mv;gkzlG6lXZc1uk)gYuw-#ceuv`9`S@{yaeHE^y4@EJsAdx Ee|Mk`cK`qY literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/a8f7c254eb64a40fd2a77b79979c7bbdac6a760c-4 b/tests/fuzzers/txfetcher/corpus/a8f7c254eb64a40fd2a77b79979c7bbdac6a760c-4 new file mode 100644 index 0000000000..da61777c22 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/a8f7c254eb64a40fd2a77b79979c7bbdac6a760c-4 @@ -0,0 +1,2 @@ +lxtIX +qyU3X9ps8ZfjLZ45l6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFe \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/a9a8f287d6af24e47d8db468e8f967aa44fb5a1f-7 b/tests/fuzzers/txfetcher/corpus/a9a8f287d6af24e47d8db468e8f967aa44fb5a1f-7 new file mode 100644 index 0000000000000000000000000000000000000000..7811921b79e98dab88b68f52779d560f6ef77ced GIT binary patch literal 106 zcmZQkU|sE8oX8UPv~4=`)3zndPj>eJ=`}#SmHBBq5O*{)Kb|}vNbhET+Svxg+nFCv znacdMyB&x-nVC5l7+At{Qxy`674q`)bc<5ca#E8^6x7P}(lS$XQjRl(rymq!V2A<$ D%DO9) literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/aa7444d8e326158046862590a0db993c07aef372-7 b/tests/fuzzers/txfetcher/corpus/aa7444d8e326158046862590a0db993c07aef372-7 new file mode 100644 index 0000000000..870e12ffbc --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/aa7444d8e326158046862590a0db993c07aef372-7 @@ -0,0 +1 @@ +00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000@0000000000000 \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/ae4593626d8796e079a358c2395a4f6c9ddd6a44-6 b/tests/fuzzers/txfetcher/corpus/ae4593626d8796e079a358c2395a4f6c9ddd6a44-6 new file mode 100644 index 0000000000..845deedd0e --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/ae4593626d8796e079a358c2395a4f6c9ddd6a44-6 @@ -0,0 +1,8 @@ +9pmM gY/xEcfbWd6IOkp+4xqjlFLBEDytgbparsing /E6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprLANGcQrooz8vp +jy4SHEg1AkEA/v13/@M47K9vCxb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY +fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U +fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xl/DoCz� jA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6Yj013sovGKUFfYAqVXVlxtIX +qyUBnu3Xh9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYFZHOde3JEMhNRcVFMO8dDaFeo +f9Oeos0Uot \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/b2942d4413a66939cda7db93020dee79eb17788c-9 b/tests/fuzzers/txfetcher/corpus/b2942d4413a66939cda7db93020dee79eb17788c-9 new file mode 100644 index 0000000000000000000000000000000000000000..10aca6512180bfd927400162598662c4589b3c52 GIT binary patch literal 18 ZcmZQ**qfYaXrvp&Se%;5$+-7D8vr(g1>OJv literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/b4614117cdfd147d38f4e8a4d85f5a2bb99a6a4f-5 b/tests/fuzzers/txfetcher/corpus/b4614117cdfd147d38f4e8a4d85f5a2bb99a6a4f-5 new file mode 100644 index 0000000000000000000000000000000000000000..af69eef9b08634dc5bf91be2dc262c5e7a52f334 GIT binary patch literal 838 zcmXZbNv^9#0EJ;^7)Duw=jaJzFxZ1gDNJKp8yhe%qv-51Jw-dp#-g{Rw}spr<*eZ7 zm%gndJsj0iwz?Gj_#R=Z>embv=IO$C!uqU)y}_I`*JbgHQcvQ_ z&EK2N$D{_O$Phdc4;0W)o!B?f%6;CCWB5|L^X0)5T$DwSZ9;)4_N3p7L(%ENtm{~$ zJ7q2u)<9L9L{P3y`DlbuvWJhQZ&#{yfcwDC*lM;a0viYfs1=~x*J*lpygv$8upIE$ z1&>QNo6R3l6|O^1_jEv?3y7e^U3^$QpE5PUP`NfKCYZ2fN0BrAm`9Ccbj#}4>gXkz z4ev2Wny!}``V>ak@o^G%uYuG){A=D-&TTnrF(d|~l|KiTj--h!&21h~vlM6WnQ-d$ z(~#`kqCz$VNnCIao2cvGRlcPpuKJL#++r_8yRIId=Z-n?{+A-SSn$7^065Cx}ug9?=(84}0Qci=91gMc_sJ~2>*H7blhDSRUsiabvbVNF{TfQt8Y zKd_MLk#7xOO)kfSO*XG(?-H4f2Uj}b85Yh!vn^xs`Q;u8Y-qmZU=f$&S?{cUe9I|$ zT=32>exto2-7rv7e4tqZY1s5x?NV5Yuyj+u% z(IoR_-D6KSug64;1O8om=DlEq-ufU*^P;CIi`7p=rYEk%udu`sob*{1zfv N+dKG=P5r+a`(NZO69fPN literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/b631ef3291fa405cd6517d11f4d1b9b6d02912d4-2 b/tests/fuzzers/txfetcher/corpus/b631ef3291fa405cd6517d11f4d1b9b6d02912d4-2 new file mode 100644 index 0000000000..a6b8858b40 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/b631ef3291fa405cd6517d11f4d1b9b6d02912d4-2 @@ -0,0 +1 @@ +&áo‡ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/b7a91e338cc11f50ebdb2c414610efc4d5be3137-4 b/tests/fuzzers/txfetcher/corpus/b7a91e338cc11f50ebdb2c414610efc4d5be3137-4 new file mode 100644 index 0000000000000000000000000000000000000000..9709a1fcb82b88b52db14626b9d8855646ebbd96 GIT binary patch literal 34 ncmV~$!2tju2m(O2@f(O}`wv!kyA*ejbvZ;A9Pth;45V^@e8~qu literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/b858cb282617fb0956d960215c8e84d1ccf909c6-2 b/tests/fuzzers/txfetcher/corpus/b858cb282617fb0956d960215c8e84d1ccf909c6-2 new file mode 100644 index 0000000000..0519ecba6e --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/b858cb282617fb0956d960215c8e84d1ccf909c6-2 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/bc9d570aacf3acd39600feda8e72a293a4667da4-1 b/tests/fuzzers/txfetcher/corpus/bc9d570aacf3acd39600feda8e72a293a4667da4-1 new file mode 100644 index 0000000000..aab27c5909 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/bc9d570aacf3acd39600feda8e72a293a4667da4-1 @@ -0,0 +1 @@ +� \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/be7eed35b245b5d5d2adcdb4c67f07794eb86b24-3 b/tests/fuzzers/txfetcher/corpus/be7eed35b245b5d5d2adcdb4c67f07794eb86b24-3 new file mode 100644 index 0000000000..47c996d33f --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/be7eed35b245b5d5d2adcdb4c67f07794eb86b24-3 @@ -0,0 +1,2 @@ +4LZmbRc6+MUVeLKXtdZr+Jk7hhgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLQcmPcQ SN_ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/c010b0cd70c7edbc5bd332fc9e2e91c6a1cbcdc4-5 b/tests/fuzzers/txfetcher/corpus/c010b0cd70c7edbc5bd332fc9e2e91c6a1cbcdc4-5 new file mode 100644 index 0000000000..474f14d89b --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/c010b0cd70c7edbc5bd332fc9e2e91c6a1cbcdc4-5 @@ -0,0 +1,4 @@ + +Xc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nhgN4H +qzzVtxx7vWrjrIgPbJpvfb \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/c1690698607eb0f4c4244e9f9629968be4beb6bc-8 b/tests/fuzzers/txfetcher/corpus/c1690698607eb0f4c4244e9f9629968be4beb6bc-8 new file mode 100644 index 0000000000..d184a2d8a4 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/c1690698607eb0f4c4244e9f9629968be4beb6bc-8 @@ -0,0 +1,2 @@ +&Ƚ�� � +� � � ���ï¿���ÿÿÿ����������� �!�"�#�$�%�&�'�(�)�*�+�,�-�.�/¿½0 \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/c1f435e4f53a9a17578d9e8c4789860f962a1379-6 b/tests/fuzzers/txfetcher/corpus/c1f435e4f53a9a17578d9e8c4789860f962a1379-6 new file mode 100644 index 0000000000000000000000000000000000000000..f2a68ec3de94cc9015080f2c057f1a707aff0269 GIT binary patch literal 1889 zcmeIz>sCx*9LMn)iAW_1MM#BCqMDk?p&1=?PDfE_bRfbcA!*gT>IU4-tk!BG6iG^o zyyybPCG>mx?z^b9_V@o;`+2q3vuB=d`ycT7LX&|oH8pP6w41TZj4(5Hn-Olt9y9iu z5n;wYGxjT8k#GQ_AR1!eAjHBUI1F(R4@clA9D@W%gyV1ml0u7ft%0ovImyZ?I1R~g z22vmu(%>wdgY%FM8ITECkPR0g2QI=T$b~$}2L}{DAvnPWMQ|Cez*V>g#oz`HT!#`U zg)%6I8&Cn2PzBXc1GP{G^>7m!pb?s&8CswfZb2K|hC9#>9dH-!!F}iiFFb%Q=!PEX zg@^D6`rt7bqJ{Kh&c7sg~5T`lVLXZ?&q{)F1U% zt*d`(d+-0o`-y7n@QP12=SZ>792M3l#FLQB_Ly&*b@rEAXMx*hv-$e_y1Y#l9*2Ff Xr#lqLF35Lj)$MfVIb4O#$__5uJO4h75r literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/c4cdbb891f3ee76476b7375d5ed51691fed95421-10 b/tests/fuzzers/txfetcher/corpus/c4cdbb891f3ee76476b7375d5ed51691fed95421-10 new file mode 100644 index 0000000000000000000000000000000000000000..e365cc52623e848cc2ab0c87573b6653e95477e4 GIT binary patch literal 16 XcmZQ**lTE{8^lQPAnG-^4J!*)#!PJ7f|ad}aSP5n z|K1JAnRA}^tiSo5-oO2tj|bWBm404d)ZztiWuXizP(v7mLj-6b5~3g)v=9Ta5C`#) z0Ev(U$)JN2NQE@eLpo$YCS*Z2VFX5D48~ysCSeMu zVFqSl4(4G27GVjNVFgxU4c1`;oZx~@*n(}?fn9LJ9_)h$4&V@uzzfH40zNneKb*li zTtEOW;R>$dMt-{!2^pHeKb?TsesQfzbW%Y1*i1?lwIWu%-6*1x{yq+0@KbWSJooA^ Mv&Hi4_B|SY0k_#*-v9sr literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/cd1d73b4e101bc7b979e3f6f135cb12d4594d348-5 b/tests/fuzzers/txfetcher/corpus/cd1d73b4e101bc7b979e3f6f135cb12d4594d348-5 new file mode 100644 index 0000000000..3079de5557 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/cd1d73b4e101bc7b979e3f6f135cb12d4594d348-5 @@ -0,0 +1 @@ +822452601031714757585602556 \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/d0acdc8fca32bbd58d368eeac3bd9eaa46f59d27-5 b/tests/fuzzers/txfetcher/corpus/d0acdc8fca32bbd58d368eeac3bd9eaa46f59d27-5 new file mode 100644 index 0000000000000000000000000000000000000000..794d5d86c6a1e18b942a16765778be42f4f583fb GIT binary patch literal 9 QcmZShu$O^t@Ba6t02EdPlmGw# literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/d0e43b715fd00953f7bdd6dfad95811985e81396-4 b/tests/fuzzers/txfetcher/corpus/d0e43b715fd00953f7bdd6dfad95811985e81396-4 new file mode 100644 index 0000000000000000000000000000000000000000..742db5fb3ba97f3f55edce2971d6eabd209d96ef GIT binary patch literal 10 PcmZSC`+h$NF_Z!TB1r}f literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/d925fbd22c8bc0de34d6a9d1258ce3d2928d0927-8 b/tests/fuzzers/txfetcher/corpus/d925fbd22c8bc0de34d6a9d1258ce3d2928d0927-8 new file mode 100644 index 0000000000000000000000000000000000000000..5920dfe6012899e440c8de696128fefd9265fe4e GIT binary patch literal 22 ccmZSS*vmFwS0BO(&rvLx| literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/d9ba78cb7425724185d5fa300cd5c03aec2683bb-7 b/tests/fuzzers/txfetcher/corpus/d9ba78cb7425724185d5fa300cd5c03aec2683bb-7 new file mode 100644 index 0000000000000000000000000000000000000000..c4df1cf210ebbfadf9ae2b676c3950c61f099bd7 GIT binary patch literal 510 zcmWO2OM+uS007WlTg?%AZy`P*c2nXLf<%H~rw9r0uR^(8A8D>I{f_WtJ4thnOvA={bOtbP+p@ezDVmiFAbaF6I24~>UtfRHHrepRPHfB`Kd^$Dc#2^A zc<9Um6XVi95ydBsFsq^yWm47q_DDA0Gv_cbJb24Q+6ls^eO{6j`4P&K@r7xtmR{?w z^{?b$Wst)5px1kWO|^w?obNJbRxK;wBNxp}-P6K4R3F_|G?kzd_USmi+wz;y#I~kO zS2$6_wuNY2M_!iZ$5@SKd4*Q*&B~M(h6Hyx5P)7SA=K^0%PQ{Lk2Y|P%p1FIJIp;* z5#fH)N*2@_U}lzQ{yKtmmgnON58hK{(uw57RNnXg6T9z&+69W%yzkc1q>>)RRG|lG z#k2kUS1pyhSNL7;YA_Q`Oprtb$Fme*B95J~0QsqkT>M*dI_y2Ve-SRA3J>8xlhF!4 zK4yi1cct=lOB6j;$&t$#eU2ElJO2CG9Rsd6a2Y&!8OIHC7U|nN-tkix6d;Vk4>F0; Q@{!z6hbp;93aG*7KfceZ2LJ#7 literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/da39a3ee5e6b4b0d3255bfef95601890afd80709 b/tests/fuzzers/txfetcher/corpus/da39a3ee5e6b4b0d3255bfef95601890afd80709 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/fuzzers/txfetcher/corpus/dcdb7758b87648b5d766b1b341a65834420cf621-7 b/tests/fuzzers/txfetcher/corpus/dcdb7758b87648b5d766b1b341a65834420cf621-7 new file mode 100644 index 0000000000000000000000000000000000000000..78cf11ae2170686d1cd83f0d1b68c1a83552f9cb GIT binary patch literal 22 ecmZShz~Eha=sv^$|L^zjWiU2nU|?{tumAvdh6vLD literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/dd441bd24581332c9ce19e008260a69287aa3cbc-6 b/tests/fuzzers/txfetcher/corpus/dd441bd24581332c9ce19e008260a69287aa3cbc-6 new file mode 100644 index 0000000000..4e0c14006e --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/dd441bd24581332c9ce19e008260a69287aa3cbc-6 @@ -0,0 +1,2 @@ +Dtf1nWk78c +/fWklyEQQ/+hgNzVtxxmDgS5TDETgeXVlXsjLZ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/def879fe0fd637a745c00c8f1da340518db8688c-2 b/tests/fuzzers/txfetcher/corpus/def879fe0fd637a745c00c8f1da340518db8688c-2 new file mode 100644 index 0000000000..555752f0ed --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/def879fe0fd637a745c00c8f1da340518db8688c-2 @@ -0,0 +1 @@ +ù ´ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/df6c30a9781b93bd6d2f5e97e5592d5945210003-7 b/tests/fuzzers/txfetcher/corpus/df6c30a9781b93bd6d2f5e97e5592d5945210003-7 new file mode 100644 index 0000000000000000000000000000000000000000..2a7adb093bcf95dd675fd92635bfc1c3c9f4f75f GIT binary patch literal 1881 zcmeIz>sCy07{~D$Nks@HDLOc|5~-$UoSG4dP>O^cMx}ENDHKcdsvB@Sd(B!6IdxJx z=)j9EV0Q`q{_~x?&|33-KKt2g=FQ$SdmQVQKVXNf7&SWV)O4D$#f%s;wwe)Z#x^tJ z%!oH5!Hn%nuN{yGJ7E{>h9uYndto0WLkjE{AuE;T0Hnb|NQVqKWY3P*>l4}zb0!nA zARBVv2po-YjO93-fRm65%se;+`EVM}KmnYELMVb_a6t)_f*U+=4$i{`xCob^%wTz0 zd~g}c;R;-Z3b+Q9a2;+y72Jeda2u+j2JS#D)WKc22la42az`3i9>7C*1dWk*JZ5+;%FX0vR!E5M;0eAy%;T;UZ5WI&E z5WOQGwS|M6+H~_em~UM;I652-hSg{FMUAPi>YMtmeyE>nTurD+HKnH2FEyiP)o(SY z=GB5)RDaZxT2?D+RsB`})c<|`*FKF@MIT%#x#lra7UuJ$H>U8T?E>}zSZt^AT=kIs7C?jo0`)E%<>Hvsh1Ntyrv literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/dfc1c3a2e3ccdaf6f88c515fd00e8ad08421e431-6 b/tests/fuzzers/txfetcher/corpus/dfc1c3a2e3ccdaf6f88c515fd00e8ad08421e431-6 new file mode 100644 index 0000000000000000000000000000000000000000..59f3442c053c1cbb924c8c024d5eb054e4a74e3c GIT binary patch literal 10 RcmZQkU|sEeoPmL1KL89{0-gW> literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/e1dcc4e7ead6dfd1139ece7bf57d776cb9dac72d-7 b/tests/fuzzers/txfetcher/corpus/e1dcc4e7ead6dfd1139ece7bf57d776cb9dac72d-7 new file mode 100644 index 0000000000000000000000000000000000000000..5ba489f99ddd2944bc6ad575d0f6072858f7e209 GIT binary patch literal 20 ccmZShus5+lH;Az~HRV0y-uG;K_n#{T09*nIoB#j- literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/e39c2de2c8937d2cbd4339b13d6a0ce94d94f8d2-8 b/tests/fuzzers/txfetcher/corpus/e39c2de2c8937d2cbd4339b13d6a0ce94d94f8d2-8 new file mode 100644 index 0000000000000000000000000000000000000000..0e9508938e4ffbe154a5365977bf7aa023fa99c9 GIT binary patch literal 1275 zcmeIwNlF7z6ouhDNu_kI5eESWf+9Fj5J5yx2nl5y`)KTqCb2D>*kC(x;tHG!E~5zO zz%@8?0YYuT^Ybt>S76}Y@7(vQrKo6YH1>9H!b+fuDq|*aQ!G3-$4oe`n3${t850Im z>`kj@7-nG(<{=Azh5=lX*1jFXowHkl{;vY<&Z0yy6WtX zXCo4^SvA#@*F2IDeoMF0uPjy%YWV90pRdh2+K>=Vg2%P1Z6$NxkIx$GlOo<|(q82E hv0pF2@Pc;B>OQrq)FFH|udDia@=|tcOWV}dl^?;h0WSam literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/e72f76b9579c792e545d02fe405d9186f0d6c39b-6 b/tests/fuzzers/txfetcher/corpus/e72f76b9579c792e545d02fe405d9186f0d6c39b-6 new file mode 100644 index 0000000000000000000000000000000000000000..c4d34b1732a239d78e8e6f9e7700ed4b90db1110 GIT binary patch literal 47 zcmZShu=hRV-uL_WGO(qVWay`*mLzASCYNO9=jkUEu2ZXm*?%>|GpFe D|5z1S literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/eb70814d6355a4498b8f301ba8dbc34f895a9947-5 b/tests/fuzzers/txfetcher/corpus/eb70814d6355a4498b8f301ba8dbc34f895a9947-5 new file mode 100644 index 0000000000000000000000000000000000000000..bd57a22fb1e19cda19fcc5ebd923af9d758e9ed0 GIT binary patch literal 57 zcmd0(C`)zG_j2?!GE7SK@h-?P;z|oFDGhbCEU8G!2`Mc!_A^iS2#RttiSl)etaPc! MGS@a`WME(b03AaSMF0Q* literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/ebdc17efe343e412634dca57cecd5a0e1ce1c1c7-5 b/tests/fuzzers/txfetcher/corpus/ebdc17efe343e412634dca57cecd5a0e1ce1c1c7-5 new file mode 100644 index 0000000000000000000000000000000000000000..aaa3f695ab36ddb550af56dbb03615218a420336 GIT binary patch literal 54 zcmd0(C`)zG_j2?!GE7SK@h-?P;z|oFDGhbCEU8G!2`Mc!_A^iS2#RttiSl)etaPc! JGS_Be008Lz5fK0Y literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/ec0a25eba8966b8f628d821b3cfbdf2dfd4bbb4c-3 b/tests/fuzzers/txfetcher/corpus/ec0a25eba8966b8f628d821b3cfbdf2dfd4bbb4c-3 new file mode 100644 index 0000000000..65cf0df801 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/ec0a25eba8966b8f628d821b3cfbdf2dfd4bbb4c-3 @@ -0,0 +1,13 @@ +¸&^£áo‡È—-----BEGIN RSA TESTING KEY----- +MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9 +SjY1bIw4iA5sBBZzHi3z0h1YV8PuxEbi4nW91IJm2gsvvZhIrHS3l6afab4pZB +l2+XsDulrKBxKKtD1rGxlG4Ljncdabn9vLZad2bSysqz/qTAUStvqJQIDAQAB +AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1K1ClbjbE6HXbnWWF/wbZGOpet +3Zm4vD6MXc7jpTLryzQIvVdfQbRc6+MUVeLKwZatTXtZru+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbW6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hg4 +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLcj2pIMPQroozvjg1AkEA/v13/5M47K9vCxmb8QeD/aydfsgS5TeuNi8DoUBEmiSJwmaXY +fFUtxv7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4bjeLKH8Q+ph2 +fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+AYiZ6PYj013sovGKFYqVXVlxtIX +qyUBnu3X9s8ZfjZO7BAkl4R5Yl6cGhaJQYZHOe3JEMhVFaFf9Oes0UUothgiDktdQxdNLj7+5CWA== +-----END RSASQ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/eebe3b76aeba6deed965d17d2b024f7eae1a43f1-5 b/tests/fuzzers/txfetcher/corpus/eebe3b76aeba6deed965d17d2b024f7eae1a43f1-5 new file mode 100644 index 0000000000000000000000000000000000000000..20d62e15b32dce93e2c88cc2c140831d5f22e7b0 GIT binary patch literal 10 Scmeyc_dVm@_xtxUumJ!i;04S8 literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/ef8741a9faf030794d98ff113f556c68a24719a5-6 b/tests/fuzzers/txfetcher/corpus/ef8741a9faf030794d98ff113f556c68a24719a5-6 new file mode 100644 index 0000000000000000000000000000000000000000..09fcd86d77c210e105c543a4eefdbd18e2b25612 GIT binary patch literal 31 lcmZShpkiKn=>Grz@AvOzXxpo6VPRox>Hs1v%`Nu72LSSb4Y>dS literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/efb7410d02418befeba25a43d676cc6124129125-4 b/tests/fuzzers/txfetcher/corpus/efb7410d02418befeba25a43d676cc6124129125-4 new file mode 100644 index 0000000000..2191a7324a --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/efb7410d02418befeba25a43d676cc6124129125-4 @@ -0,0 +1 @@ +88242871'392752200424452601091531672177074144720616417147514758635765020556616¿ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/f6f97d781a5a749903790e07db8619866cb7c3a1-6 b/tests/fuzzers/txfetcher/corpus/f6f97d781a5a749903790e07db8619866cb7c3a1-6 new file mode 100644 index 0000000000000000000000000000000000000000..219a8d3682f5e40e4c1406ecc1b95909f680975d GIT binary patch literal 22 dcmZShus5-wATc;7*ikpR?EU_|zKq4G?*VUO3J(AP literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/f7a3cd00fa0e57742e7dbbb8283dcaea067eaf7b-5 b/tests/fuzzers/txfetcher/corpus/f7a3cd00fa0e57742e7dbbb8283dcaea067eaf7b-5 new file mode 100644 index 0000000000..f01ccd89ef --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/f7a3cd00fa0e57742e7dbbb8283dcaea067eaf7b-5 @@ -0,0 +1,2 @@ +Xyt0Xl/DoCzjA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYi \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/f94d60a6c556ce485ab60088291760b8be25776c-6 b/tests/fuzzers/txfetcher/corpus/f94d60a6c556ce485ab60088291760b8be25776c-6 new file mode 100644 index 0000000000..58d841ff03 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/f94d60a6c556ce485ab60088291760b8be25776c-6 @@ -0,0 +1,2 @@ +HZB4cQZde3JMNRcVFMO8dDFo +f9OeosiDdQQl \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/f9e627b2cb82ffa1ea5e0c6d7f2802f3000b18a8-6 b/tests/fuzzers/txfetcher/corpus/f9e627b2cb82ffa1ea5e0c6d7f2802f3000b18a8-6 new file mode 100644 index 0000000000000000000000000000000000000000..b5dfecc1e9d1cf74363f1eb9213f1fecebdc11eb GIT binary patch literal 11 ScmZSC%evb6I0FO2{=EPd*aP?g literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/fb3775aa24e5667e658920c05ba4b7b19ff256fb-5 b/tests/fuzzers/txfetcher/corpus/fb3775aa24e5667e658920c05ba4b7b19ff256fb-5 new file mode 100644 index 0000000000..6f4927d822 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/fb3775aa24e5667e658920c05ba4b7b19ff256fb-5 @@ -0,0 +1 @@ +HZB4c2cPclieoverpGsumgUtWj3NMYPZ/F8tá5YlNR8dDFoiDdQQl \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/fd6386548e119a50db96b2fa406e54924c45a2d5-6 b/tests/fuzzers/txfetcher/corpus/fd6386548e119a50db96b2fa406e54924c45a2d5-6 new file mode 100644 index 0000000000000000000000000000000000000000..6fff60edd4f0d637706fae63e784667804e3d071 GIT binary patch literal 28 kcmZShus7Jl(bUi=DA-Xqx$OP^y}pdasVVOn_r7NX0G`(ir2qf` literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go new file mode 100644 index 0000000000..10c7eb9424 --- /dev/null +++ b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go @@ -0,0 +1,199 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package txfetcher + +import ( + "bytes" + "fmt" + "math/big" + "math/rand" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/fetcher" +) + +var ( + peers []string + txs []*types.Transaction +) + +func init() { + // Random is nice, but we need it deterministic + rand := rand.New(rand.NewSource(0x3a29)) + + peers = make([]string, 10) + for i := 0; i < len(peers); i++ { + peers[i] = fmt.Sprintf("Peer #%d", i) + } + txs = make([]*types.Transaction, 65536) // We need to bump enough to hit all the limits + for i := 0; i < len(txs); i++ { + txs[i] = types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil) + } +} + +func Fuzz(input []byte) int { + // Don't generate insanely large test cases, not much value in them + if len(input) > 16*1024 { + return -1 + } + r := bytes.NewReader(input) + + // Reduce the problem space for certain fuzz runs. Small tx space is better + // for testing clashes and in general the fetcher, but we should still run + // some tests with large spaces to hit potential issues on limits. + limit, err := r.ReadByte() + if err != nil { + return 0 + } + switch limit % 4 { + case 0: + txs = txs[:4] + case 1: + txs = txs[:256] + case 2: + txs = txs[:4096] + case 3: + // Full run + } + // Create a fetcher and hook into it's simulated fields + clock := new(mclock.Simulated) + rand := rand.New(rand.NewSource(0x3a29)) // Same used in package tests!!! + + f := fetcher.NewTxFetcherForTests( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + clock, rand, + ) + f.Start() + defer f.Stop() + + // Try to throw random junk at the fetcher + for { + // Read the next command and abort if we're done + cmd, err := r.ReadByte() + if err != nil { + return 0 + } + switch cmd % 4 { + case 0: + // Notify a new set of transactions: + // Byte 1: Peer index to announce with + // Byte 2: Number of hashes to announce + // Byte 3-4, 5-6, etc: Transaction indices (2 byte) to announce + peerIdx, err := r.ReadByte() + if err != nil { + return 0 + } + peer := peers[int(peerIdx)%len(peers)] + + announceCnt, err := r.ReadByte() + if err != nil { + return 0 + } + announce := int(announceCnt) % (2 * len(txs)) // No point in generating too many duplicates + + var ( + announceIdxs = make([]int, announce) + announces = make([]common.Hash, announce) + ) + for i := 0; i < len(announces); i++ { + annBuf := make([]byte, 2) + if n, err := r.Read(annBuf); err != nil || n != 2 { + return 0 + } + announceIdxs[i] = (int(annBuf[0])*256 + int(annBuf[1])) % len(txs) + announces[i] = txs[announceIdxs[i]].Hash() + } + fmt.Println("Notify", peer, announceIdxs) + if err := f.Notify(peer, announces); err != nil { + panic(err) + } + + case 1: + // Deliver a new set of transactions: + // Byte 1: Peer index to announce with + // Byte 2: Number of hashes to announce + // Byte 3-4, 5-6, etc: Transaction indices (2 byte) to announce + peerIdx, err := r.ReadByte() + if err != nil { + return 0 + } + peer := peers[int(peerIdx)%len(peers)] + + deliverCnt, err := r.ReadByte() + if err != nil { + return 0 + } + deliver := int(deliverCnt) % (2 * len(txs)) // No point in generating too many duplicates + + var ( + deliverIdxs = make([]int, deliver) + deliveries = make([]*types.Transaction, deliver) + ) + for i := 0; i < len(deliveries); i++ { + deliverBuf := make([]byte, 2) + if n, err := r.Read(deliverBuf); err != nil || n != 2 { + return 0 + } + deliverIdxs[i] = (int(deliverBuf[0])*256 + int(deliverBuf[1])) % len(txs) + deliveries[i] = txs[deliverIdxs[i]] + } + directFlag, err := r.ReadByte() + if err != nil { + return 0 + } + direct := (directFlag % 2) == 0 + + fmt.Println("Enqueue", peer, deliverIdxs, direct) + if err := f.Enqueue(peer, deliveries, direct); err != nil { + panic(err) + } + + case 2: + // Drop a peer: + // Byte 1: Peer index to drop + peerIdx, err := r.ReadByte() + if err != nil { + return 0 + } + peer := peers[int(peerIdx)%len(peers)] + + fmt.Println("Drop", peer) + if err := f.Drop(peer); err != nil { + panic(err) + } + + case 3: + // Move the simulated clock forward + // Byte 1: 100ms increment to move forward + tickCnt, err := r.ReadByte() + if err != nil { + return 0 + } + tick := time.Duration(tickCnt) * 100 * time.Millisecond + + fmt.Println("Sleep", tick) + clock.Run(tick) + } + } +}