cmd/faucet: use Twitter API instead of scraping webpage (#21850)

This PR adds support for using Twitter API to query the tweet and author details. There are two reasons behind this change:

- Twitter will be deprecating the legacy website on 15th December. The current method is expected to stop working then.
- More importantly, the current system uses Twitter handle for spam protection but the Twitter handle can be changed via automated calls. This allows bots to use the same tweet to withdraw funds infinite times as long as they keep changing their handle between every request. The Rinkeby as well as the Goerli faucet are being actively drained via this method. This PR changes the spam protection to be based on Twitter IDs instead of usernames. A user can not change their Twitter ID.

cmd/faucet/faucet.go

@@ -83,6 +83,8 @@ var (
 
 	noauthFlag = flag.Bool("noauth", false, "Enables funding requests without authentication")
 	logFlag    = flag.Int("loglevel", 3, "Log level to use for Ethereum and the faucet")
+
+	twitterBearerToken = flag.String("twitter.token", "", "Twitter bearer token to authenticate with the twitter API")
 )
 
 var (
@@ -443,6 +445,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 		}
 		// Retrieve the Ethereum address to fund, the requesting user and a profile picture
 		var (
+			id       string
 			username string
 			avatar   string
 			address  common.Address
@@ -462,11 +465,13 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 			}
 			continue
 		case strings.HasPrefix(msg.URL, "https://twitter.com/"):
-			username, avatar, address, err = authTwitter(msg.URL)
+			id, username, avatar, address, err = authTwitter(msg.URL, *twitterBearerToken)
 		case strings.HasPrefix(msg.URL, "https://www.facebook.com/"):
 			username, avatar, address, err = authFacebook(msg.URL)
+			id = username
 		case *noauthFlag:
 			username, avatar, address, err = authNoAuth(msg.URL)
+			id = username
 		default:
 			//lint:ignore ST1005 This error is to be displayed in the browser
 			err = errors.New("Something funky happened, please open an issue at https://github.com/ethereum/go-ethereum/issues")
@@ -486,7 +491,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 			fund    bool
 			timeout time.Time
 		)
-		if timeout = f.timeouts[username]; time.Now().After(timeout) {
+		if timeout = f.timeouts[id]; time.Now().After(timeout) {
 			// User wasn't funded recently, create the funding transaction
 			amount := new(big.Int).Mul(big.NewInt(int64(*payoutFlag)), ether)
 			amount = new(big.Int).Mul(amount, new(big.Int).Exp(big.NewInt(5), big.NewInt(int64(msg.Tier)), nil))
@@ -520,7 +525,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) {
 			timeout := time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute
 			grace := timeout / 288 // 24h timeout => 5m grace
 
-			f.timeouts[username] = time.Now().Add(timeout - grace)
+			f.timeouts[id] = time.Now().Add(timeout - grace)
 			fund = true
 		}
 		f.lock.Unlock()
@@ -684,23 +689,32 @@ func sendSuccess(conn *websocket.Conn, msg string) error {
 }
 
 // authTwitter tries to authenticate a faucet request using Twitter posts, returning
-// the username, avatar URL and Ethereum address to fund on success.
+// the uniqueness identifier (user id/username), username, avatar URL and Ethereum address to fund on success.
-func authTwitter(url string) (string, string, common.Address, error) {
+func authTwitter(url string, token string) (string, string, string, common.Address, error) {
 	// Ensure the user specified a meaningful URL, no fancy nonsense
 	parts := strings.Split(url, "/")
 	if len(parts) < 4 || parts[len(parts)-2] != "status" {
 		//lint:ignore ST1005 This error is to be displayed in the browser
-		return "", "", common.Address{}, errors.New("Invalid Twitter status URL")
+		return "", "", "", common.Address{}, errors.New("Invalid Twitter status URL")
 	}
-	// Twitter's API isn't really friendly with direct links. Still, we don't
+
-	// want to do ask read permissions from users, so just load the public posts
+	// Twitter's API isn't really friendly with direct links.
+	// It is restricted to 300 queries / 15 minute with an app api key.
+	// Anything more will require read only authorization from the users and that we want to avoid.
+
+	// If twitter bearer token is provided, use the twitter api
+	if token != "" {
+		return authTwitterWithToken(parts[len(parts)-1], token)
+	}
+
+	// Twiter API token isn't provided so we just load the public posts
 	// and scrape it for the Ethereum address and profile URL. We need to load
 	// the mobile page though since the main page loads tweet contents via JS.
 	url = strings.Replace(url, "https://twitter.com/", "https://mobile.twitter.com/", 1)
 
 	res, err := http.Get(url)
 	if err != nil {
-		return "", "", common.Address{}, err
+		return "", "", "", common.Address{}, err
 	}
 	defer res.Body.Close()
 
@@ -708,24 +722,77 @@ func authTwitter(url string) (string, string, common.Address, error) {
 	parts = strings.Split(res.Request.URL.String(), "/")
 	if len(parts) < 4 || parts[len(parts)-2] != "status" {
 		//lint:ignore ST1005 This error is to be displayed in the browser
-		return "", "", common.Address{}, errors.New("Invalid Twitter status URL")
+		return "", "", "", common.Address{}, errors.New("Invalid Twitter status URL")
 	}
 	username := parts[len(parts)-3]
 
 	body, err := ioutil.ReadAll(res.Body)
 	if err != nil {
-		return "", "", common.Address{}, err
+		return "", "", "", common.Address{}, err
 	}
 	address := common.HexToAddress(string(regexp.MustCompile("0x[0-9a-fA-F]{40}").Find(body)))
 	if address == (common.Address{}) {
 		//lint:ignore ST1005 This error is to be displayed in the browser
-		return "", "", common.Address{}, errors.New("No Ethereum address found to fund")
+		return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund")
 	}
 	var avatar string
 	if parts = regexp.MustCompile("src=\"([^\"]+twimg.com/profile_images[^\"]+)\"").FindStringSubmatch(string(body)); len(parts) == 2 {
 		avatar = parts[1]
 	}
-	return username + "@twitter", avatar, address, nil
+	return username + "@twitter", username, avatar, address, nil
+}
+
+// authTwitterWithToken tries to authenticate a faucet request using Twitter's API, returning
+// the uniqueness identifier (user id/username), username, avatar URL and Ethereum address to fund on success.
+func authTwitterWithToken(tweetID string, token string) (string, string, string, common.Address, error) {
+	// Strip any query parameters from the tweet id
+	sanitizedTweetID := strings.Split(tweetID, "?")[0]
+
+	// Ensure numeric tweetID
+	if !regexp.MustCompile("^[0-9]+$").MatchString(sanitizedTweetID) {
+		return "", "", "", common.Address{}, errors.New("Invalid Tweet URL")
+	}
+
+	// Query the tweet details from Twitter
+	url := fmt.Sprintf("https://api.twitter.com/2/tweets/%s?expansions=author_id&user.fields=profile_image_url", sanitizedTweetID)
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return "", "", "", common.Address{}, err
+	}
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", "", "", common.Address{}, err
+	}
+	defer res.Body.Close()
+
+	var result struct {
+		Data struct {
+			AuthorID string `json:"author_id"`
+			ID       string `json:"id"`
+			Text     string `json:"text"`
+		} `json:"data"`
+		Includes struct {
+			Users []struct {
+				ProfileImageURL string `json:"profile_image_url"`
+				Username        string `json:"username"`
+				ID              string `json:"id"`
+				Name            string `json:"name"`
+			} `json:"users"`
+		} `json:"includes"`
+	}
+
+	err = json.NewDecoder(res.Body).Decode(&result)
+	if err != nil {
+		return "", "", "", common.Address{}, err
+	}
+
+	address := common.HexToAddress(regexp.MustCompile("0x[0-9a-fA-F]{40}").FindString(result.Data.Text))
+	if address == (common.Address{}) {
+		//lint:ignore ST1005 This error is to be displayed in the browser
+		return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund")
+	}
+	return result.Data.AuthorID + "@twitter", result.Includes.Users[0].Username, result.Includes.Users[0].ProfileImageURL, address, nil
 }
 
 // authFacebook tries to authenticate a faucet request using Facebook posts,

cmd/puppeth/module_faucet.go

@@ -46,6 +46,7 @@ ENTRYPOINT [ \
 	"--faucet.name", "{{.FaucetName}}", "--faucet.amount", "{{.FaucetAmount}}", "--faucet.minutes", "{{.FaucetMinutes}}", "--faucet.tiers", "{{.FaucetTiers}}",             \
 	"--account.json", "/account.json", "--account.pass", "/account.pass"                                                                                                    \
 	{{if .CaptchaToken}}, "--captcha.token", "{{.CaptchaToken}}", "--captcha.secret", "{{.CaptchaSecret}}"{{end}}{{if .NoAuth}}, "--noauth"{{end}}                          \
+	{{if .TwitterToken}}, "--twitter.token", "{{.TwitterToken}}",
 ]`
 
 // faucetComposefile is the docker-compose.yml file required to deploy and maintain
@@ -71,6 +72,7 @@ services:
       - FAUCET_TIERS={{.FaucetTiers}}
       - CAPTCHA_TOKEN={{.CaptchaToken}}
       - CAPTCHA_SECRET={{.CaptchaSecret}}
+      - TWITTER_TOKEN={{.TwitterToken}}
       - NO_AUTH={{.NoAuth}}{{if .VHost}}
       - VIRTUAL_HOST={{.VHost}}
       - VIRTUAL_PORT=8080{{end}}
@@ -103,6 +105,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config
 		"FaucetMinutes": config.minutes,
 		"FaucetTiers":   config.tiers,
 		"NoAuth":        config.noauth,
+		"TwitterToken":  config.twitterToken,
 	})
 	files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes()
 
@@ -120,6 +123,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config
 		"FaucetMinutes": config.minutes,
 		"FaucetTiers":   config.tiers,
 		"NoAuth":        config.noauth,
+		"TwitterToken":  config.twitterToken,
 	})
 	files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes()
 
@@ -152,6 +156,7 @@ type faucetInfos struct {
 	noauth        bool
 	captchaToken  string
 	captchaSecret string
+	twitterToken  string
 }
 
 // Report converts the typed struct into a plain string->string map, containing
@@ -165,6 +170,7 @@ func (info *faucetInfos) Report() map[string]string {
 		"Funding cooldown (base tier)": fmt.Sprintf("%d mins", info.minutes),
 		"Funding tiers":                strconv.Itoa(info.tiers),
 		"Captha protection":            fmt.Sprintf("%v", info.captchaToken != ""),
+		"Using Twitter API":            fmt.Sprintf("%v", info.twitterToken != ""),
 		"Ethstats username":            info.node.ethstats,
 	}
 	if info.noauth {
@@ -243,5 +249,6 @@ func checkFaucet(client *sshClient, network string) (*faucetInfos, error) {
 		captchaToken:  infos.envvars["CAPTCHA_TOKEN"],
 		captchaSecret: infos.envvars["CAPTCHA_SECRET"],
 		noauth:        infos.envvars["NO_AUTH"] == "true",
+		twitterToken:  infos.envvars["TWITTER_TOKEN"],
 	}, nil
 }

cmd/puppeth/wizard_faucet.go

@@ -102,6 +102,29 @@ func (w *wizard) deployFaucet() {
 			infos.captchaSecret = w.readPassword()
 		}
 	}
+
+	// Accessing the twitter api requires a bearer token, request it
+	if infos.twitterToken != "" {
+		fmt.Println()
+		fmt.Println("Reuse previous twitter API Bearer token (y/n)? (default = yes)")
+		if !w.readDefaultYesNo(true) {
+			infos.twitterToken = ""
+		}
+	}
+	if infos.twitterToken == "" {
+		// No previous twitter token (or old one discarded)
+		fmt.Println()
+		fmt.Println("Enable twitter API (y/n)? (default = no)")
+		if !w.readDefaultYesNo(false) {
+			log.Warn("The faucet will fallback to using direct calls")
+		} else {
+			// Twitter api explicitly requested, read the bearer token
+			fmt.Println()
+			fmt.Printf("What is the twitter API Bearer token?\n")
+			infos.twitterToken = w.readString()
+		}
+	}
+
 	// Figure out where the user wants to store the persistent data
 	fmt.Println()
 	if infos.node.datadir == "" {