tornado-packages/ethers.js
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added initial throttling support (#139, #904, #926).

Browse Source
This commit is contained in:
Richard Moore 2020-07-14 02:26:45 -04:00
parent c730cbc629
commit 88c7eaed06
No known key found for this signature in database
GPG Key ID: 665176BE8E9DC651
6 changed files with 191 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
packages/providers/src.ts/alchemy-provider.ts
View File

@ -1,7 +1,9 @@
"use strict";
import { Network, Networkish } from "@ethersproject/networks";
import { ConnectionInfo } from "@ethersproject/web";
import { showThrottleMessage } from "./formatter";
import { WebSocketProvider } from "./websocket-provider";
import { Logger } from "@ethersproject/logger";
@ -18,7 +20,6 @@ import { UrlJsonRpcProvider } from "./url-json-rpc-provider";
const defaultApiKey = "_gg7wSSi0KMBsdKnGVfHDueq6xMB9EkC"
export class AlchemyProvider extends UrlJsonRpcProvider {
readonly apiKey: string;
static getWebSocketProvider(network?: Networkish, apiKey?: any): WebSocketProvider {
const provider = new AlchemyProvider(network, apiKey);
@ -37,7 +38,7 @@ export class AlchemyProvider extends UrlJsonRpcProvider {
return apiKey;
}
static getUrl(network: Network, apiKey: string): string {
static getUrl(network: Network, apiKey: string): ConnectionInfo {
let host = null;
switch (network.name) {
case "homestead":
@ -59,6 +60,14 @@ export class AlchemyProvider extends UrlJsonRpcProvider {
logger.throwArgumentError("unsupported network", "network", arguments[0]);
}
return ("https:/" + "/" + host + apiKey);
return {
url: ("https:/" + "/" + host + apiKey),
throttleCallback: (attempt: number, url: string) => {
if (apiKey === defaultApiKey) {
showThrottleMessage();
}
return Promise.resolve(true);
}
};
}
}

44
packages/providers/src.ts/etherscan-provider.ts
View File

@ -6,6 +6,8 @@ import { Network, Networkish } from "@ethersproject/networks";
import { deepCopy, defineReadOnly } from "@ethersproject/properties";
import { fetchJson } from "@ethersproject/web";
import { showThrottleMessage } from "./formatter";
import { Logger } from "@ethersproject/logger";
import { version } from "./_version";
const logger = new Logger(version);
@ -34,9 +36,11 @@ function getResult(result: { status?: number, message?: string, result?: any }):
}
if (result.status != 1 || result.message != "OK") {
// @TODO: not any
const error: any = new Error("invalid response");
error.result = JSON.stringify(result);
if ((result.result || "").toLowerCase().indexOf("rate limit") >= 0) {
error.throttleRetry = true;
}
throw error;
}
@ -44,6 +48,14 @@ function getResult(result: { status?: number, message?: string, result?: any }):
}
function getJsonResult(result: { jsonrpc: string, result?: any, error?: { code?: number, data?: any, message?: string} } ): any {
// This response indicates we are being throttled
if (result && (<any>result).status == 0 && (<any>result).message == "NOTOK" && (result.result || "").toLowerCase().indexOf("rate limit") >= 0) {
const error: any = new Error("throttled response");
error.result = JSON.stringify(result);
error.throttleRetry = true;
throw error;
}
if (result.jsonrpc != "2.0") {
// @TODO: not any
const error: any = new Error("invalid response");
@ -76,6 +88,7 @@ const defaultApiKey = "9D13ZE7XSBTJ94N9BNJ2MA33VMAY2YPIRB";
export class EtherscanProvider extends BaseProvider{
readonly baseUrl: string;
readonly apiKey: string;
constructor(network?: Networkish, apiKey?: string) {
logger.checkNew(new.target, EtherscanProvider);
@ -126,7 +139,18 @@ export class EtherscanProvider extends BaseProvider{
provider: this
});
const result = await fetchJson(url, null, procFunc || getJsonResult);
const connection = {
url: url,
throttleCallback: (attempt: number, url: string) => {
if (this.apiKey === defaultApiKey) {
showThrottleMessage();
}
return Promise.resolve(true);
}
};
const result = await fetchJson(connection, null, procFunc || getJsonResult);
this.emit("debug", {
action: "response",
@ -162,13 +186,13 @@ export class EtherscanProvider extends BaseProvider{
case "getCode":
url += "/api?module=proxy&action=eth_getCode&address=" + params.address;
url += "&tag=" + params.blockTag + apiKey;
return get(url, getJsonResult);
return get(url);
case "getStorageAt":
url += "/api?module=proxy&action=eth_getStorageAt&address=" + params.address;
url += "&position=" + params.position;
url += "&tag=" + params.blockTag + apiKey;
return get(url, getJsonResult);
return get(url);
case "sendTransaction":
@ -325,7 +349,17 @@ export class EtherscanProvider extends BaseProvider{
provider: this
});
return fetchJson(url, null, getResult).then((result: Array<any>) => {
const connection = {
url: url,
throttleCallback: (attempt: number, url: string) => {
if (this.apiKey === defaultApiKey) {
showThrottleMessage();
}
return Promise.resolve(true);
}
}
return fetchJson(connection, null, getResult).then((result: Array<any>) => {
this.emit("debug", {
action: "response",
request: url,

20
packages/providers/src.ts/formatter.ts
View File

@ -455,3 +455,23 @@ export class Formatter {
}
}
// Show the throttle message only once
let throttleMessage = false;
export function showThrottleMessage() {
if (throttleMessage) { return; }
throttleMessage = true;
console.log("========= NOTICE =========")
console.log("Request-Rate Exceeded (this message will not be repeated)");
console.log("");
console.log("The default API keys for each service are provided as a highly-throttled,");
console.log("community resource for low-traffic projects and early prototyping.");
console.log("");
console.log("While your application will continue to function, we highly recommended");
console.log("signing up for your own API keys to improve performance, increase your");
console.log("request rate/limit and enable other perks, such as metrics and advanced APIs.");
console.log("");
console.log("For more details: https:/\/docs.ethers.io/api-keys/");
console.log("==========================");
}

11
packages/providers/src.ts/infura-provider.ts
View File

@ -4,6 +4,7 @@ import { Network, Networkish } from "@ethersproject/networks";
import { ConnectionInfo } from "@ethersproject/web";
import { WebSocketProvider } from "./websocket-provider";
import { showThrottleMessage } from "./formatter";
import { Logger } from "@ethersproject/logger";
import { version } from "./_version";
@ -61,7 +62,7 @@ export class InfuraProvider extends UrlJsonRpcProvider {
return apiKeyObj;
}
static getUrl(network: Network, apiKey: any): string | ConnectionInfo {
static getUrl(network: Network, apiKey: any): ConnectionInfo {
let host: string = null;
switch(network ? network.name: "unknown") {
case "homestead":
@ -87,7 +88,13 @@ export class InfuraProvider extends UrlJsonRpcProvider {
}
const connection: ConnectionInfo = {
url: ("https:/" + "/" + host + "/v3/" + apiKey.projectId)
url: ("https:/" + "/" + host + "/v3/" + apiKey.projectId),
throttleCallback: (attempt: number, url: string) => {
if (apiKey.projectId === defaultProjectId) {
showThrottleMessage();
}
return Promise.resolve(true);
}
};
if (apiKey.projectSecret != null) {

2
packages/providers/src.ts/nodesmith-provider.ts
View File

@ -1,3 +1,5 @@
/* istanbul ignore file */
"use strict";
import { Network } from "@ethersproject/networks";

58
packages/web/src.ts/index.ts
View File

@ -10,15 +10,26 @@ const logger = new Logger(version);
import { getUrl, GetUrlResponse } from "./geturl";
function staller(duration: number): Promise<void> {
return new Promise((resolve) => {
setTimeout(resolve, duration);
});
}
// Exported Types
export type ConnectionInfo = {
url: string,
headers?: { [key: string]: string | number }
user?: string,
password?: string,
allowInsecureAuthentication?: boolean,
throttleLimit?: number,
throttleCallback?: (attempt: number, url: string) => Promise<boolean>,
timeout?: number,
headers?: { [key: string]: string | number }
};
export interface OnceBlockable {
@ -48,6 +59,14 @@ export type FetchJsonResponse = {
type Header = { key: string, value: string };
export function fetchJson(connection: string | ConnectionInfo, json?: string, processFunc?: (value: any, response: FetchJsonResponse) => any): Promise<any> {
// How many times to retry in the event of a throttle
const attemptLimit = (typeof(connection) === "object" && connection.throttleLimit != null) ? connection.throttleLimit: 12;
logger.assertArgument((attemptLimit > 0 && (attemptLimit % 1) === 0),
"invalid connection throttle limit", "connection.throttleLimit", attemptLimit);
const throttleCallback = ((typeof(connection) === "object") ? connection.throttleCallback: null);
const headers: { [key: string]: Header } = { };
let url: string = null;
@ -143,9 +162,26 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr
const runningFetch = (async function() {
for (let attempt = 0; attempt < attemptLimit; attempt++) {
let response: GetUrlResponse = null;
try {
response = await getUrl(url, options);
// Exponential back-off throttling (interval = 100ms)
if (response.statusCode === 429 && attempt < attemptLimit) {
let tryAgain = true;
if (throttleCallback) {
tryAgain = await throttleCallback(attempt, url);
}
if (tryAgain) {
const timeout = 100 * parseInt(String(Math.random() * Math.pow(2, attempt)));
await staller(timeout);
continue;
}
}
} catch (error) {
response = (<any>error).response;
if (response == null) {
@ -177,13 +213,12 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr
});
}
runningTimeout.cancel();
let json: any = null;
if (body != null) {
try {
json = JSON.parse(body);
} catch (error) {
runningTimeout.cancel();
logger.throwError("invalid JSON", Logger.errors.SERVER_ERROR, {
body: body,
error: error,
@ -198,6 +233,21 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr
try {
json = await processFunc(json, response);
} catch (error) {
// Allow the processFunc to trigger a throttle
if (error.throttleRetry && attempt < attemptLimit) {
let tryAgain = true;
if (throttleCallback) {
tryAgain = await throttleCallback(attempt, url);
}
if (tryAgain) {
const timeout = 100 * parseInt(String(Math.random() * Math.pow(2, attempt)));
await staller(timeout);
continue;
}
}
runningTimeout.cancel();
logger.throwError("processing response error", Logger.errors.SERVER_ERROR, {
body: json,
error: error,
@ -208,7 +258,9 @@ export function fetchJson(connection: string | ConnectionInfo, json?: string, pr
}
}
runningTimeout.cancel();
return json;
}
})();
return Promise.race([ runningTimeout.promise, runningFetch ]);