From 88c7eaed061ae9a6798733a97e4e87011d36b8e7 Mon Sep 17 00:00:00 2001 From: Richard Moore Date: Tue, 14 Jul 2020 02:26:45 -0400 Subject: [PATCH] Added initial throttling support (#139, #904, #926). --- packages/providers/src.ts/alchemy-provider.ts | 15 +- .../providers/src.ts/etherscan-provider.ts | 44 ++++- packages/providers/src.ts/formatter.ts | 20 +++ packages/providers/src.ts/infura-provider.ts | 11 +- .../providers/src.ts/nodesmith-provider.ts | 2 + packages/web/src.ts/index.ts | 166 ++++++++++++------ 6 files changed, 191 insertions(+), 67 deletions(-) diff --git a/packages/providers/src.ts/alchemy-provider.ts b/packages/providers/src.ts/alchemy-provider.ts index 7ba8a28b8..e56c56445 100644 --- a/packages/providers/src.ts/alchemy-provider.ts +++ b/packages/providers/src.ts/alchemy-provider.ts @@ -1,7 +1,9 @@ "use strict"; import { Network, Networkish } from "@ethersproject/networks"; +import { ConnectionInfo } from "@ethersproject/web"; +import { showThrottleMessage } from "./formatter"; import { WebSocketProvider } from "./websocket-provider"; import { Logger } from "@ethersproject/logger"; @@ -18,7 +20,6 @@ import { UrlJsonRpcProvider } from "./url-json-rpc-provider"; const defaultApiKey = "_gg7wSSi0KMBsdKnGVfHDueq6xMB9EkC" export class AlchemyProvider extends UrlJsonRpcProvider { - readonly apiKey: string; static getWebSocketProvider(network?: Networkish, apiKey?: any): WebSocketProvider { const provider = new AlchemyProvider(network, apiKey); @@ -37,7 +38,7 @@ export class AlchemyProvider extends UrlJsonRpcProvider { return apiKey; } - static getUrl(network: Network, apiKey: string): string { + static getUrl(network: Network, apiKey: string): ConnectionInfo { let host = null; switch (network.name) { case "homestead": @@ -59,6 +60,14 @@ export class AlchemyProvider extends UrlJsonRpcProvider { logger.throwArgumentError("unsupported network", "network", arguments[0]); } - return ("https:/" + "/" + host + apiKey); + return { + url: ("https:/" + "/" + host + apiKey), + throttleCallback: (attempt: number, url: string) => { + if (apiKey === defaultApiKey) { + showThrottleMessage(); + } + return Promise.resolve(true); + } + }; } } diff --git a/packages/providers/src.ts/etherscan-provider.ts b/packages/providers/src.ts/etherscan-provider.ts index fdd6e56ba..c4bd68f82 100644 --- a/packages/providers/src.ts/etherscan-provider.ts +++ b/packages/providers/src.ts/etherscan-provider.ts @@ -6,6 +6,8 @@ import { Network, Networkish } from "@ethersproject/networks"; import { deepCopy, defineReadOnly } from "@ethersproject/properties"; import { fetchJson } from "@ethersproject/web"; +import { showThrottleMessage } from "./formatter"; + import { Logger } from "@ethersproject/logger"; import { version } from "./_version"; const logger = new Logger(version); @@ -34,9 +36,11 @@ function getResult(result: { status?: number, message?: string, result?: any }): } if (result.status != 1 || result.message != "OK") { - // @TODO: not any const error: any = new Error("invalid response"); error.result = JSON.stringify(result); + if ((result.result || "").toLowerCase().indexOf("rate limit") >= 0) { + error.throttleRetry = true; + } throw error; } @@ -44,6 +48,14 @@ function getResult(result: { status?: number, message?: string, result?: any }): } function getJsonResult(result: { jsonrpc: string, result?: any, error?: { code?: number, data?: any, message?: string} } ): any { + // This response indicates we are being throttled + if (result && (result).status == 0 && (result).message == "NOTOK" && (result.result || "").toLowerCase().indexOf("rate limit") >= 0) { + const error: any = new Error("throttled response"); + error.result = JSON.stringify(result); + error.throttleRetry = true; + throw error; + } + if (result.jsonrpc != "2.0") { // @TODO: not any const error: any = new Error("invalid response"); @@ -76,6 +88,7 @@ const defaultApiKey = "9D13ZE7XSBTJ94N9BNJ2MA33VMAY2YPIRB"; export class EtherscanProvider extends BaseProvider{ readonly baseUrl: string; readonly apiKey: string; + constructor(network?: Networkish, apiKey?: string) { logger.checkNew(new.target, EtherscanProvider); @@ -126,7 +139,18 @@ export class EtherscanProvider extends BaseProvider{ provider: this }); - const result = await fetchJson(url, null, procFunc || getJsonResult); + + const connection = { + url: url, + throttleCallback: (attempt: number, url: string) => { + if (this.apiKey === defaultApiKey) { + showThrottleMessage(); + } + return Promise.resolve(true); + } + }; + + const result = await fetchJson(connection, null, procFunc || getJsonResult); this.emit("debug", { action: "response", @@ -162,13 +186,13 @@ export class EtherscanProvider extends BaseProvider{ case "getCode": url += "/api?module=proxy&action=eth_getCode&address=" + params.address; url += "&tag=" + params.blockTag + apiKey; - return get(url, getJsonResult); + return get(url); case "getStorageAt": url += "/api?module=proxy&action=eth_getStorageAt&address=" + params.address; url += "&position=" + params.position; url += "&tag=" + params.blockTag + apiKey; - return get(url, getJsonResult); + return get(url); case "sendTransaction": @@ -325,7 +349,17 @@ export class EtherscanProvider extends BaseProvider{ provider: this }); - return fetchJson(url, null, getResult).then((result: Array) => { + const connection = { + url: url, + throttleCallback: (attempt: number, url: string) => { + if (this.apiKey === defaultApiKey) { + showThrottleMessage(); + } + return Promise.resolve(true); + } + } + + return fetchJson(connection, null, getResult).then((result: Array) => { this.emit("debug", { action: "response", request: url, diff --git a/packages/providers/src.ts/formatter.ts b/packages/providers/src.ts/formatter.ts index f57767a3c..a90da339e 100644 --- a/packages/providers/src.ts/formatter.ts +++ b/packages/providers/src.ts/formatter.ts @@ -455,3 +455,23 @@ export class Formatter { } } +// Show the throttle message only once +let throttleMessage = false; +export function showThrottleMessage() { + if (throttleMessage) { return; } + throttleMessage = true; + + console.log("========= NOTICE =========") + console.log("Request-Rate Exceeded (this message will not be repeated)"); + console.log(""); + console.log("The default API keys for each service are provided as a highly-throttled,"); + console.log("community resource for low-traffic projects and early prototyping."); + console.log(""); + console.log("While your application will continue to function, we highly recommended"); + console.log("signing up for your own API keys to improve performance, increase your"); + console.log("request rate/limit and enable other perks, such as metrics and advanced APIs."); + console.log(""); + console.log("For more details: https:/\/docs.ethers.io/api-keys/"); + console.log("=========================="); +} + diff --git a/packages/providers/src.ts/infura-provider.ts b/packages/providers/src.ts/infura-provider.ts index b7de2fa38..2a9f21a04 100644 --- a/packages/providers/src.ts/infura-provider.ts +++ b/packages/providers/src.ts/infura-provider.ts @@ -4,6 +4,7 @@ import { Network, Networkish } from "@ethersproject/networks"; import { ConnectionInfo } from "@ethersproject/web"; import { WebSocketProvider } from "./websocket-provider"; +import { showThrottleMessage } from "./formatter"; import { Logger } from "@ethersproject/logger"; import { version } from "./_version"; @@ -61,7 +62,7 @@ export class InfuraProvider extends UrlJsonRpcProvider { return apiKeyObj; } - static getUrl(network: Network, apiKey: any): string | ConnectionInfo { + static getUrl(network: Network, apiKey: any): ConnectionInfo { let host: string = null; switch(network ? network.name: "unknown") { case "homestead": @@ -87,7 +88,13 @@ export class InfuraProvider extends UrlJsonRpcProvider { } const connection: ConnectionInfo = { - url: ("https:/" + "/" + host + "/v3/" + apiKey.projectId) + url: ("https:/" + "/" + host + "/v3/" + apiKey.projectId), + throttleCallback: (attempt: number, url: string) => { + if (apiKey.projectId === defaultProjectId) { + showThrottleMessage(); + } + return Promise.resolve(true); + } }; if (apiKey.projectSecret != null) { diff --git a/packages/providers/src.ts/nodesmith-provider.ts b/packages/providers/src.ts/nodesmith-provider.ts index 07b4c315f..f712b9b83 100644 --- a/packages/providers/src.ts/nodesmith-provider.ts +++ b/packages/providers/src.ts/nodesmith-provider.ts @@ -1,3 +1,5 @@ +/* istanbul ignore file */ + "use strict"; import { Network } from "@ethersproject/networks"; diff --git a/packages/web/src.ts/index.ts b/packages/web/src.ts/index.ts index 6508952e5..555227eaf 100644 --- a/packages/web/src.ts/index.ts +++ b/packages/web/src.ts/index.ts @@ -10,15 +10,26 @@ const logger = new Logger(version); import { getUrl, GetUrlResponse } from "./geturl"; +function staller(duration: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, duration); + }); +} + // Exported Types export type ConnectionInfo = { url: string, + headers?: { [key: string]: string | number } + user?: string, password?: string, + allowInsecureAuthentication?: boolean, + throttleLimit?: number, + throttleCallback?: (attempt: number, url: string) => Promise, + timeout?: number, - headers?: { [key: string]: string | number } }; export interface OnceBlockable { @@ -48,6 +59,14 @@ export type FetchJsonResponse = { type Header = { key: string, value: string }; export function fetchJson(connection: string | ConnectionInfo, json?: string, processFunc?: (value: any, response: FetchJsonResponse) => any): Promise { + + // How many times to retry in the event of a throttle + const attemptLimit = (typeof(connection) === "object" && connection.throttleLimit != null) ? connection.throttleLimit: 12; + logger.assertArgument((attemptLimit > 0 && (attemptLimit % 1) === 0), + "invalid connection throttle limit", "connection.throttleLimit", attemptLimit); + + const throttleCallback = ((typeof(connection) === "object") ? connection.throttleCallback: null); + const headers: { [key: string]: Header } = { }; let url: string = null; @@ -143,72 +162,105 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr const runningFetch = (async function() { - let response: GetUrlResponse = null; - try { - response = await getUrl(url, options); - } catch (error) { - response = (error).response; - if (response == null) { + for (let attempt = 0; attempt < attemptLimit; attempt++) { + let response: GetUrlResponse = null; + + try { + response = await getUrl(url, options); + + // Exponential back-off throttling (interval = 100ms) + if (response.statusCode === 429 && attempt < attemptLimit) { + let tryAgain = true; + if (throttleCallback) { + tryAgain = await throttleCallback(attempt, url); + } + + if (tryAgain) { + const timeout = 100 * parseInt(String(Math.random() * Math.pow(2, attempt))); + await staller(timeout); + continue; + } + } + + } catch (error) { + response = (error).response; + if (response == null) { + runningTimeout.cancel(); + logger.throwError("missing response", Logger.errors.SERVER_ERROR, { + requestBody: (options.body || null), + requestMethod: options.method, + serverError: error, + url: url + }); + } + } + + + let body = response.body; + + if (allow304 && response.statusCode === 304) { + body = null; + + } else if (response.statusCode < 200 || response.statusCode >= 300) { runningTimeout.cancel(); - logger.throwError("missing response", Logger.errors.SERVER_ERROR, { - requestBody: (options.body || null), - requestMethod: options.method, - serverError: error, - url: url - }); - } - } - - - let body = response.body; - - if (allow304 && response.statusCode === 304) { - body = null; - - } else if (response.statusCode < 200 || response.statusCode >= 300) { - runningTimeout.cancel(); - logger.throwError("bad response", Logger.errors.SERVER_ERROR, { - status: response.statusCode, - headers: response.headers, - body: body, - requestBody: (options.body || null), - requestMethod: options.method, - url: url - }); - } - - runningTimeout.cancel(); - - let json: any = null; - if (body != null) { - try { - json = JSON.parse(body); - } catch (error) { - logger.throwError("invalid JSON", Logger.errors.SERVER_ERROR, { + logger.throwError("bad response", Logger.errors.SERVER_ERROR, { + status: response.statusCode, + headers: response.headers, body: body, - error: error, requestBody: (options.body || null), requestMethod: options.method, url: url }); } - } - if (processFunc) { - try { - json = await processFunc(json, response); - } catch (error) { - logger.throwError("processing response error", Logger.errors.SERVER_ERROR, { - body: json, - error: error, - requestBody: (options.body || null), - requestMethod: options.method, - url: url - }); + let json: any = null; + if (body != null) { + try { + json = JSON.parse(body); + } catch (error) { + runningTimeout.cancel(); + logger.throwError("invalid JSON", Logger.errors.SERVER_ERROR, { + body: body, + error: error, + requestBody: (options.body || null), + requestMethod: options.method, + url: url + }); + } } - } - return json; + if (processFunc) { + try { + json = await processFunc(json, response); + } catch (error) { + // Allow the processFunc to trigger a throttle + if (error.throttleRetry && attempt < attemptLimit) { + let tryAgain = true; + if (throttleCallback) { + tryAgain = await throttleCallback(attempt, url); + } + + if (tryAgain) { + const timeout = 100 * parseInt(String(Math.random() * Math.pow(2, attempt))); + await staller(timeout); + continue; + } + } + + runningTimeout.cancel(); + logger.throwError("processing response error", Logger.errors.SERVER_ERROR, { + body: json, + error: error, + requestBody: (options.body || null), + requestMethod: options.method, + url: url + }); + } + } + + runningTimeout.cancel(); + return json; + } })(); return Promise.race([ runningTimeout.promise, runningFetch ]);