From b47f4ca5cf3adf7c29e9ee00a6056196f295763c Mon Sep 17 00:00:00 2001 From: Mudit Gupta Date: Fri, 11 Dec 2020 15:05:39 +0530 Subject: [PATCH] cmd/faucet: use Twitter API instead of scraping webpage (#21850) This PR adds support for using Twitter API to query the tweet and author details. There are two reasons behind this change: - Twitter will be deprecating the legacy website on 15th December. The current method is expected to stop working then. - More importantly, the current system uses Twitter handle for spam protection but the Twitter handle can be changed via automated calls. This allows bots to use the same tweet to withdraw funds infinite times as long as they keep changing their handle between every request. The Rinkeby as well as the Goerli faucet are being actively drained via this method. This PR changes the spam protection to be based on Twitter IDs instead of usernames. A user can not change their Twitter ID. --- cmd/faucet/faucet.go | 93 +++++++++++++++++++++++++++++++----- cmd/puppeth/module_faucet.go | 7 +++ cmd/puppeth/wizard_faucet.go | 23 +++++++++ 3 files changed, 110 insertions(+), 13 deletions(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index eaf0dc30c1..d7927ac491 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -83,6 +83,8 @@ var ( noauthFlag = flag.Bool("noauth", false, "Enables funding requests without authentication") logFlag = flag.Int("loglevel", 3, "Log level to use for Ethereum and the faucet") + + twitterBearerToken = flag.String("twitter.token", "", "Twitter bearer token to authenticate with the twitter API") ) var ( @@ -443,6 +445,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { } // Retrieve the Ethereum address to fund, the requesting user and a profile picture var ( + id string username string avatar string address common.Address @@ -462,11 +465,13 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { } continue case strings.HasPrefix(msg.URL, "https://twitter.com/"): - username, avatar, address, err = authTwitter(msg.URL) + id, username, avatar, address, err = authTwitter(msg.URL, *twitterBearerToken) case strings.HasPrefix(msg.URL, "https://www.facebook.com/"): username, avatar, address, err = authFacebook(msg.URL) + id = username case *noauthFlag: username, avatar, address, err = authNoAuth(msg.URL) + id = username default: //lint:ignore ST1005 This error is to be displayed in the browser err = errors.New("Something funky happened, please open an issue at https://github.com/ethereum/go-ethereum/issues") @@ -486,7 +491,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { fund bool timeout time.Time ) - if timeout = f.timeouts[username]; time.Now().After(timeout) { + if timeout = f.timeouts[id]; time.Now().After(timeout) { // User wasn't funded recently, create the funding transaction amount := new(big.Int).Mul(big.NewInt(int64(*payoutFlag)), ether) amount = new(big.Int).Mul(amount, new(big.Int).Exp(big.NewInt(5), big.NewInt(int64(msg.Tier)), nil)) @@ -520,7 +525,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { timeout := time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute grace := timeout / 288 // 24h timeout => 5m grace - f.timeouts[username] = time.Now().Add(timeout - grace) + f.timeouts[id] = time.Now().Add(timeout - grace) fund = true } f.lock.Unlock() @@ -684,23 +689,32 @@ func sendSuccess(conn *websocket.Conn, msg string) error { } // authTwitter tries to authenticate a faucet request using Twitter posts, returning -// the username, avatar URL and Ethereum address to fund on success. -func authTwitter(url string) (string, string, common.Address, error) { +// the uniqueness identifier (user id/username), username, avatar URL and Ethereum address to fund on success. +func authTwitter(url string, token string) (string, string, string, common.Address, error) { // Ensure the user specified a meaningful URL, no fancy nonsense parts := strings.Split(url, "/") if len(parts) < 4 || parts[len(parts)-2] != "status" { //lint:ignore ST1005 This error is to be displayed in the browser - return "", "", common.Address{}, errors.New("Invalid Twitter status URL") + return "", "", "", common.Address{}, errors.New("Invalid Twitter status URL") } - // Twitter's API isn't really friendly with direct links. Still, we don't - // want to do ask read permissions from users, so just load the public posts + + // Twitter's API isn't really friendly with direct links. + // It is restricted to 300 queries / 15 minute with an app api key. + // Anything more will require read only authorization from the users and that we want to avoid. + + // If twitter bearer token is provided, use the twitter api + if token != "" { + return authTwitterWithToken(parts[len(parts)-1], token) + } + + // Twiter API token isn't provided so we just load the public posts // and scrape it for the Ethereum address and profile URL. We need to load // the mobile page though since the main page loads tweet contents via JS. url = strings.Replace(url, "https://twitter.com/", "https://mobile.twitter.com/", 1) res, err := http.Get(url) if err != nil { - return "", "", common.Address{}, err + return "", "", "", common.Address{}, err } defer res.Body.Close() @@ -708,24 +722,77 @@ func authTwitter(url string) (string, string, common.Address, error) { parts = strings.Split(res.Request.URL.String(), "/") if len(parts) < 4 || parts[len(parts)-2] != "status" { //lint:ignore ST1005 This error is to be displayed in the browser - return "", "", common.Address{}, errors.New("Invalid Twitter status URL") + return "", "", "", common.Address{}, errors.New("Invalid Twitter status URL") } username := parts[len(parts)-3] body, err := ioutil.ReadAll(res.Body) if err != nil { - return "", "", common.Address{}, err + return "", "", "", common.Address{}, err } address := common.HexToAddress(string(regexp.MustCompile("0x[0-9a-fA-F]{40}").Find(body))) if address == (common.Address{}) { //lint:ignore ST1005 This error is to be displayed in the browser - return "", "", common.Address{}, errors.New("No Ethereum address found to fund") + return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund") } var avatar string if parts = regexp.MustCompile("src=\"([^\"]+twimg.com/profile_images[^\"]+)\"").FindStringSubmatch(string(body)); len(parts) == 2 { avatar = parts[1] } - return username + "@twitter", avatar, address, nil + return username + "@twitter", username, avatar, address, nil +} + +// authTwitterWithToken tries to authenticate a faucet request using Twitter's API, returning +// the uniqueness identifier (user id/username), username, avatar URL and Ethereum address to fund on success. +func authTwitterWithToken(tweetID string, token string) (string, string, string, common.Address, error) { + // Strip any query parameters from the tweet id + sanitizedTweetID := strings.Split(tweetID, "?")[0] + + // Ensure numeric tweetID + if !regexp.MustCompile("^[0-9]+$").MatchString(sanitizedTweetID) { + return "", "", "", common.Address{}, errors.New("Invalid Tweet URL") + } + + // Query the tweet details from Twitter + url := fmt.Sprintf("https://api.twitter.com/2/tweets/%s?expansions=author_id&user.fields=profile_image_url", sanitizedTweetID) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", "", "", common.Address{}, err + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", "", "", common.Address{}, err + } + defer res.Body.Close() + + var result struct { + Data struct { + AuthorID string `json:"author_id"` + ID string `json:"id"` + Text string `json:"text"` + } `json:"data"` + Includes struct { + Users []struct { + ProfileImageURL string `json:"profile_image_url"` + Username string `json:"username"` + ID string `json:"id"` + Name string `json:"name"` + } `json:"users"` + } `json:"includes"` + } + + err = json.NewDecoder(res.Body).Decode(&result) + if err != nil { + return "", "", "", common.Address{}, err + } + + address := common.HexToAddress(regexp.MustCompile("0x[0-9a-fA-F]{40}").FindString(result.Data.Text)) + if address == (common.Address{}) { + //lint:ignore ST1005 This error is to be displayed in the browser + return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund") + } + return result.Data.AuthorID + "@twitter", result.Includes.Users[0].Username, result.Includes.Users[0].ProfileImageURL, address, nil } // authFacebook tries to authenticate a faucet request using Facebook posts, diff --git a/cmd/puppeth/module_faucet.go b/cmd/puppeth/module_faucet.go index 987bed14aa..2527e137f2 100644 --- a/cmd/puppeth/module_faucet.go +++ b/cmd/puppeth/module_faucet.go @@ -46,6 +46,7 @@ ENTRYPOINT [ \ "--faucet.name", "{{.FaucetName}}", "--faucet.amount", "{{.FaucetAmount}}", "--faucet.minutes", "{{.FaucetMinutes}}", "--faucet.tiers", "{{.FaucetTiers}}", \ "--account.json", "/account.json", "--account.pass", "/account.pass" \ {{if .CaptchaToken}}, "--captcha.token", "{{.CaptchaToken}}", "--captcha.secret", "{{.CaptchaSecret}}"{{end}}{{if .NoAuth}}, "--noauth"{{end}} \ + {{if .TwitterToken}}, "--twitter.token", "{{.TwitterToken}}", ]` // faucetComposefile is the docker-compose.yml file required to deploy and maintain @@ -71,6 +72,7 @@ services: - FAUCET_TIERS={{.FaucetTiers}} - CAPTCHA_TOKEN={{.CaptchaToken}} - CAPTCHA_SECRET={{.CaptchaSecret}} + - TWITTER_TOKEN={{.TwitterToken}} - NO_AUTH={{.NoAuth}}{{if .VHost}} - VIRTUAL_HOST={{.VHost}} - VIRTUAL_PORT=8080{{end}} @@ -103,6 +105,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config "FaucetMinutes": config.minutes, "FaucetTiers": config.tiers, "NoAuth": config.noauth, + "TwitterToken": config.twitterToken, }) files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes() @@ -120,6 +123,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config "FaucetMinutes": config.minutes, "FaucetTiers": config.tiers, "NoAuth": config.noauth, + "TwitterToken": config.twitterToken, }) files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes() @@ -152,6 +156,7 @@ type faucetInfos struct { noauth bool captchaToken string captchaSecret string + twitterToken string } // Report converts the typed struct into a plain string->string map, containing @@ -165,6 +170,7 @@ func (info *faucetInfos) Report() map[string]string { "Funding cooldown (base tier)": fmt.Sprintf("%d mins", info.minutes), "Funding tiers": strconv.Itoa(info.tiers), "Captha protection": fmt.Sprintf("%v", info.captchaToken != ""), + "Using Twitter API": fmt.Sprintf("%v", info.twitterToken != ""), "Ethstats username": info.node.ethstats, } if info.noauth { @@ -243,5 +249,6 @@ func checkFaucet(client *sshClient, network string) (*faucetInfos, error) { captchaToken: infos.envvars["CAPTCHA_TOKEN"], captchaSecret: infos.envvars["CAPTCHA_SECRET"], noauth: infos.envvars["NO_AUTH"] == "true", + twitterToken: infos.envvars["TWITTER_TOKEN"], }, nil } diff --git a/cmd/puppeth/wizard_faucet.go b/cmd/puppeth/wizard_faucet.go index 9f753ad68b..47e05cd9c1 100644 --- a/cmd/puppeth/wizard_faucet.go +++ b/cmd/puppeth/wizard_faucet.go @@ -102,6 +102,29 @@ func (w *wizard) deployFaucet() { infos.captchaSecret = w.readPassword() } } + + // Accessing the twitter api requires a bearer token, request it + if infos.twitterToken != "" { + fmt.Println() + fmt.Println("Reuse previous twitter API Bearer token (y/n)? (default = yes)") + if !w.readDefaultYesNo(true) { + infos.twitterToken = "" + } + } + if infos.twitterToken == "" { + // No previous twitter token (or old one discarded) + fmt.Println() + fmt.Println("Enable twitter API (y/n)? (default = no)") + if !w.readDefaultYesNo(false) { + log.Warn("The faucet will fallback to using direct calls") + } else { + // Twitter api explicitly requested, read the bearer token + fmt.Println() + fmt.Printf("What is the twitter API Bearer token?\n") + infos.twitterToken = w.readString() + } + } + // Figure out where the user wants to store the persistent data fmt.Println() if infos.node.datadir == "" {