From 41a0ad9f03ae8e8389fbe40131f4e6930b5beac5 Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 19 Sep 2023 20:18:29 +0800 Subject: [PATCH] cmd/devp2p: use bootnodes as crawl input (#28139) This PR makes the tool use the --bootnodes list as the input to devp2p crawl. The flag will take effect if the input/output.json file is missing or empty. --- cmd/devp2p/crawl.go | 12 ++++++++++-- cmd/devp2p/discv4cmd.go | 29 ++++++++++++++++++----------- cmd/devp2p/discv5cmd.go | 20 ++++++++++++-------- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/cmd/devp2p/crawl.go b/cmd/devp2p/crawl.go index 8c0defff6..4288a5feb 100644 --- a/cmd/devp2p/crawl.go +++ b/cmd/devp2p/crawl.go @@ -17,6 +17,7 @@ package main import ( + "errors" "sync" "sync/atomic" "time" @@ -51,7 +52,14 @@ type resolver interface { RequestENR(*enode.Node) (*enode.Node, error) } -func newCrawler(input nodeSet, disc resolver, iters ...enode.Iterator) *crawler { +func newCrawler(input nodeSet, bootnodes []*enode.Node, disc resolver, iters ...enode.Iterator) (*crawler, error) { + if len(input) == 0 { + input.add(bootnodes...) + } + if len(input) == 0 { + return nil, errors.New("no input nodes to start crawling") + } + c := &crawler{ input: input, output: make(nodeSet, len(input)), @@ -67,7 +75,7 @@ func newCrawler(input nodeSet, disc resolver, iters ...enode.Iterator) *crawler for id, n := range input { c.output[id] = n } - return c + return c, nil } func (c *crawler) run(timeout time.Duration, nthreads int) nodeSet { diff --git a/cmd/devp2p/discv4cmd.go b/cmd/devp2p/discv4cmd.go index 0117c7eb8..37b139dea 100644 --- a/cmd/devp2p/discv4cmd.go +++ b/cmd/devp2p/discv4cmd.go @@ -143,7 +143,7 @@ var discoveryNodeFlags = []cli.Flag{ func discv4Ping(ctx *cli.Context) error { n := getNodeArg(ctx) - disc := startV4(ctx) + disc, _ := startV4(ctx) defer disc.Close() start := time.Now() @@ -156,7 +156,7 @@ func discv4Ping(ctx *cli.Context) error { func discv4RequestRecord(ctx *cli.Context) error { n := getNodeArg(ctx) - disc := startV4(ctx) + disc, _ := startV4(ctx) defer disc.Close() respN, err := disc.RequestENR(n) @@ -169,7 +169,7 @@ func discv4RequestRecord(ctx *cli.Context) error { func discv4Resolve(ctx *cli.Context) error { n := getNodeArg(ctx) - disc := startV4(ctx) + disc, _ := startV4(ctx) defer disc.Close() fmt.Println(disc.Resolve(n).String()) @@ -196,10 +196,13 @@ func discv4ResolveJSON(ctx *cli.Context) error { nodeargs = append(nodeargs, n) } - // Run the crawler. - disc := startV4(ctx) + disc, config := startV4(ctx) defer disc.Close() - c := newCrawler(inputSet, disc, enode.IterNodes(nodeargs)) + + c, err := newCrawler(inputSet, config.Bootnodes, disc, enode.IterNodes(nodeargs)) + if err != nil { + return err + } c.revalidateInterval = 0 output := c.run(0, 1) writeNodesJSON(nodesFile, output) @@ -211,14 +214,18 @@ func discv4Crawl(ctx *cli.Context) error { return errors.New("need nodes file as argument") } nodesFile := ctx.Args().First() - var inputSet nodeSet + inputSet := make(nodeSet) if common.FileExist(nodesFile) { inputSet = loadNodesJSON(nodesFile) } - disc := startV4(ctx) + disc, config := startV4(ctx) defer disc.Close() - c := newCrawler(inputSet, disc, disc.RandomNodes()) + + c, err := newCrawler(inputSet, config.Bootnodes, disc, disc.RandomNodes()) + if err != nil { + return err + } c.revalidateInterval = 10 * time.Minute output := c.run(ctx.Duration(crawlTimeoutFlag.Name), ctx.Int(crawlParallelismFlag.Name)) writeNodesJSON(nodesFile, output) @@ -238,14 +245,14 @@ func discv4Test(ctx *cli.Context) error { } // startV4 starts an ephemeral discovery V4 node. -func startV4(ctx *cli.Context) *discover.UDPv4 { +func startV4(ctx *cli.Context) (*discover.UDPv4, discover.Config) { ln, config := makeDiscoveryConfig(ctx) socket := listen(ctx, ln) disc, err := discover.ListenV4(socket, ln, config) if err != nil { exit(err) } - return disc + return disc, config } func makeDiscoveryConfig(ctx *cli.Context) (*enode.LocalNode, discover.Config) { diff --git a/cmd/devp2p/discv5cmd.go b/cmd/devp2p/discv5cmd.go index c5e226f0d..0dac94526 100644 --- a/cmd/devp2p/discv5cmd.go +++ b/cmd/devp2p/discv5cmd.go @@ -81,7 +81,7 @@ var ( func discv5Ping(ctx *cli.Context) error { n := getNodeArg(ctx) - disc := startV5(ctx) + disc, _ := startV5(ctx) defer disc.Close() fmt.Println(disc.Ping(n)) @@ -90,7 +90,7 @@ func discv5Ping(ctx *cli.Context) error { func discv5Resolve(ctx *cli.Context) error { n := getNodeArg(ctx) - disc := startV5(ctx) + disc, _ := startV5(ctx) defer disc.Close() fmt.Println(disc.Resolve(n)) @@ -102,14 +102,18 @@ func discv5Crawl(ctx *cli.Context) error { return errors.New("need nodes file as argument") } nodesFile := ctx.Args().First() - var inputSet nodeSet + inputSet := make(nodeSet) if common.FileExist(nodesFile) { inputSet = loadNodesJSON(nodesFile) } - disc := startV5(ctx) + disc, config := startV5(ctx) defer disc.Close() - c := newCrawler(inputSet, disc, disc.RandomNodes()) + + c, err := newCrawler(inputSet, config.Bootnodes, disc, disc.RandomNodes()) + if err != nil { + return err + } c.revalidateInterval = 10 * time.Minute output := c.run(ctx.Duration(crawlTimeoutFlag.Name), ctx.Int(crawlParallelismFlag.Name)) writeNodesJSON(nodesFile, output) @@ -127,7 +131,7 @@ func discv5Test(ctx *cli.Context) error { } func discv5Listen(ctx *cli.Context) error { - disc := startV5(ctx) + disc, _ := startV5(ctx) defer disc.Close() fmt.Println(disc.Self()) @@ -135,12 +139,12 @@ func discv5Listen(ctx *cli.Context) error { } // startV5 starts an ephemeral discovery v5 node. -func startV5(ctx *cli.Context) *discover.UDPv5 { +func startV5(ctx *cli.Context) (*discover.UDPv5, discover.Config) { ln, config := makeDiscoveryConfig(ctx) socket := listen(ctx, ln) disc, err := discover.ListenV5(socket, ln, config) if err != nil { exit(err) } - return disc + return disc, config }