tornado-packages/ethers.js
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More robust configuration options for FetchRequest getUrl functions (#4353).

Browse Source
This commit is contained in:
Richard Moore 2023-09-13 02:54:03 -04:00
parent c7f8abefa4
commit 9541f2f70c
3 changed files with 200 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
src.ts/utils/fetch.ts
View File

@ -23,7 +23,7 @@ import { assert, assertArgument } from "./errors.js";
import { defineProperties } from "./properties.js";
import { toUtf8Bytes, toUtf8String } from "./utf8.js"
import { getUrl } from "./geturl.js";
import { createGetUrl } from "./geturl.js";
/**
* An environments implementation of ``getUrl`` must return this type.
@ -77,7 +77,7 @@ const MAX_ATTEMPTS = 12;
const SLOT_INTERVAL = 250;
// The global FetchGetUrlFunc implementation.
let getUrlFunc: FetchGetUrlFunc = getUrl;
let defaultGetUrlFunc: FetchGetUrlFunc = createGetUrl();
const reData = new RegExp("^data:([^;:]*)?(;base64)?,(.*)$", "i");
const reIpfs = new RegExp("^ipfs:/\/(ipfs/)?(.*)$", "i");
@ -201,6 +201,8 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
#throttle: Required<FetchThrottleParams>;
#getUrlFunc: null | FetchGetUrlFunc;
/**
* The fetch URI to requrest.
*/
@ -429,6 +431,28 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
this.#retry = retry;
}
/**
* This function is called to fetch content from HTTP and
* HTTPS URLs and is platform specific (e.g. nodejs vs
* browsers).
*
* This is by default the currently registered global getUrl
* function, which can be changed using [[registerGetUrl]].
* If this has been set, setting is to ``null`` will cause
* this FetchRequest (and any future clones) to revert back to
* using the currently registered global getUrl function.
*
* Setting this is generally not necessary, but may be useful
* for developers that wish to intercept requests or to
* configurege a proxy or other agent.
*/
get getUrlFunc(): FetchGetUrlFunc {
return this.#getUrlFunc || defaultGetUrlFunc;
}
set getUrlFunc(value: null | FetchGetUrlFunc) {
this.#getUrlFunc = value;
}
/**
* Create a new FetchRequest instance with default values.
*
@ -448,6 +472,8 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
slotInterval: SLOT_INTERVAL,
maxAttempts: MAX_ATTEMPTS
};
this.#getUrlFunc = null;
}
toString(): string {
@ -510,7 +536,7 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
// We have a preflight function; update the request
if (this.preflightFunc) { req = await this.preflightFunc(req); }
const resp = await getUrlFunc(req, checkSignal(_request.#signal));
const resp = await this.getUrlFunc(req, checkSignal(_request.#signal));
let response = new FetchResponse(resp.statusCode, resp.statusMessage, resp.headers, resp.body, _request);
if (response.statusCode === 301 || response.statusCode === 302) {
@ -641,6 +667,8 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
clone.#process = this.#process;
clone.#retry = this.#retry;
clone.#getUrlFunc = this.#getUrlFunc;
return clone;
}
@ -686,7 +714,22 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
*/
static registerGetUrl(getUrl: FetchGetUrlFunc): void {
if (locked) { throw new Error("gateways locked"); }
getUrlFunc = getUrl;
defaultGetUrlFunc = getUrl;
}
/**
* Creates a getUrl function that fetches content from HTTP and
* HTTPS URLs.
*
* The available %%options%% are dependent on the platform
* implementation of the default getUrl function.
*
* This is not generally something that is needed, but is useful
* when trying to customize simple behaviour when fetching HTTP
* content.
*/
static createGetUrlFunc(options?: Record<string, any>): FetchGetUrlFunc {
return createGetUrl(options);
}
/**

88
src.ts/utils/geturl-browser.ts
View File

@ -1,6 +1,8 @@
import { assert } from "./errors.js";
import type { FetchRequest, FetchCancelSignal, GetUrlResponse } from "./fetch.js";
import type {
FetchGetUrlFunc, FetchRequest, FetchCancelSignal, GetUrlResponse
} from "./fetch.js";
declare global {
@ -27,46 +29,58 @@ declare global {
// @TODO: timeout is completely ignored; start a Promise.any with a reject?
export async function getUrl(req: FetchRequest, _signal?: FetchCancelSignal): Promise<GetUrlResponse> {
const protocol = req.url.split(":")[0].toLowerCase();
export function createGetUrl(options?: Record<string, any>): FetchGetUrlFunc {
assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", {
info: { protocol },
operation: "request"
});
async function getUrl(req: FetchRequest, _signal?: FetchCancelSignal): Promise<GetUrlResponse> {
const protocol = req.url.split(":")[0].toLowerCase();
assert(protocol === "https" || !req.credentials || req.allowInsecureAuthentication, "insecure authorized connections unsupported", "UNSUPPORTED_OPERATION", {
operation: "request"
});
assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", {
info: { protocol },
operation: "request"
});
let signal: undefined | AbortSignal = undefined;
if (_signal) {
const controller = new AbortController();
signal = controller.signal;
_signal.addListener(() => { controller.abort(); });
assert(protocol === "https" || !req.credentials || req.allowInsecureAuthentication, "insecure authorized connections unsupported", "UNSUPPORTED_OPERATION", {
operation: "request"
});
let signal: undefined | AbortSignal = undefined;
if (_signal) {
const controller = new AbortController();
signal = controller.signal;
_signal.addListener(() => { controller.abort(); });
}
const init = {
method: req.method,
headers: new Headers(Array.from(req)),
body: req.body || undefined,
signal
};
const resp = await fetch(req.url, init);
const headers: Record<string, string> = { };
resp.headers.forEach((value, key) => {
headers[key.toLowerCase()] = value;
});
const respBody = await resp.arrayBuffer();
const body = (respBody == null) ? null: new Uint8Array(respBody);
return {
statusCode: resp.status,
statusMessage: resp.statusText,
headers, body
};
}
const init = {
method: req.method,
headers: new Headers(Array.from(req)),
body: req.body || undefined,
signal
};
const resp = await fetch(req.url, init);
const headers: Record<string, string> = { };
resp.headers.forEach((value, key) => {
headers[key.toLowerCase()] = value;
});
const respBody = await resp.arrayBuffer();
const body = (respBody == null) ? null: new Uint8Array(respBody);
return {
statusCode: resp.status,
statusMessage: resp.statusText,
headers, body
};
return getUrl;
}
// @TODO: remove in v7; provided for backwards compat
const defaultGetUrl: FetchGetUrlFunc = createGetUrl({ });
export async function getUrl(req: FetchRequest, _signal?: FetchCancelSignal): Promise<GetUrlResponse> {
return defaultGetUrl(req, _signal);
}

184
src.ts/utils/geturl.ts
View File

@ -5,92 +5,112 @@ import { gunzipSync } from "zlib";
import { assert } from "./errors.js";
import { getBytes } from "./data.js";
import type { FetchRequest, FetchCancelSignal, GetUrlResponse } from "./fetch.js";
import type {
FetchGetUrlFunc, FetchRequest, FetchCancelSignal, GetUrlResponse
} from "./fetch.js";
/**
* @_ignore:
*/
export function createGetUrl(options?: Record<string, any>): FetchGetUrlFunc {
async function getUrl(req: FetchRequest, signal?: FetchCancelSignal): Promise<GetUrlResponse> {
const protocol = req.url.split(":")[0].toLowerCase();
assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", {
info: { protocol },
operation: "request"
});
assert(protocol === "https" || !req.credentials || req.allowInsecureAuthentication, "insecure authorized connections unsupported", "UNSUPPORTED_OPERATION", {
operation: "request"
});
const method = req.method;
const headers = Object.assign({ }, req.headers);
const reqOptions: any = { method, headers };
if (options) {
if (options.agent) { reqOptions.agent = options.agent; }
}
const request = ((protocol === "http") ? http: https).request(req.url, reqOptions);
request.setTimeout(req.timeout);
const body = req.body;
if (body) { request.write(Buffer.from(body)); }
request.end();
return new Promise((resolve, reject) => {
// @TODO: Node 15 added AbortSignal; once we drop support for
// Node14, we can add that in here too
request.once("response", (resp: http.IncomingMessage) => {
const statusCode = resp.statusCode || 0;
const statusMessage = resp.statusMessage || "";
const headers = Object.keys(resp.headers || {}).reduce((accum, name) => {
let value = resp.headers[name] || "";
if (Array.isArray(value)) {
value = value.join(", ");
}
accum[name] = value;
return accum;
}, <{ [ name: string ]: string }>{ });
let body: null | Uint8Array = null;
//resp.setEncoding("utf8");
resp.on("data", (chunk: Uint8Array) => {
if (signal) {
try {
signal.checkSignal();
} catch (error) {
return reject(error);
}
}
if (body == null) {
body = chunk;
} else {
const newBody = new Uint8Array(body.length + chunk.length);
newBody.set(body, 0);
newBody.set(chunk, body.length);
body = newBody;
}
});
resp.on("end", () => {
if (headers["content-encoding"] === "gzip" && body) {
body = getBytes(gunzipSync(body));
}
resolve({ statusCode, statusMessage, headers, body });
});
resp.on("error", (error) => {
//@TODO: Should this just return nornal response with a server error?
(<any>error).response = { statusCode, statusMessage, headers, body };
reject(error);
});
});
request.on("error", (error) => { reject(error); });
});
}
return getUrl;
}
// @TODO: remove in v7; provided for backwards compat
const defaultGetUrl: FetchGetUrlFunc = createGetUrl({ });
/**
* @_ignore:
*/
export async function getUrl(req: FetchRequest, signal?: FetchCancelSignal): Promise<GetUrlResponse> {
const protocol = req.url.split(":")[0].toLowerCase();
assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", {
info: { protocol },
operation: "request"
});
assert(protocol === "https" || !req.credentials || req.allowInsecureAuthentication, "insecure authorized connections unsupported", "UNSUPPORTED_OPERATION", {
operation: "request"
});
const method = req.method;
const headers = Object.assign({ }, req.headers);
const options: any = { method, headers };
const request = ((protocol === "http") ? http: https).request(req.url, options);
request.setTimeout(req.timeout);
const body = req.body;
if (body) { request.write(Buffer.from(body)); }
request.end();
return new Promise((resolve, reject) => {
// @TODO: Node 15 added AbortSignal; once we drop support for
// Node14, we can add that in here too
request.once("response", (resp: http.IncomingMessage) => {
const statusCode = resp.statusCode || 0;
const statusMessage = resp.statusMessage || "";
const headers = Object.keys(resp.headers || {}).reduce((accum, name) => {
let value = resp.headers[name] || "";
if (Array.isArray(value)) {
value = value.join(", ");
}
accum[name] = value;
return accum;
}, <{ [ name: string ]: string }>{ });
let body: null | Uint8Array = null;
//resp.setEncoding("utf8");
resp.on("data", (chunk: Uint8Array) => {
if (signal) {
try {
signal.checkSignal();
} catch (error) {
return reject(error);
}
}
if (body == null) {
body = chunk;
} else {
const newBody = new Uint8Array(body.length + chunk.length);
newBody.set(body, 0);
newBody.set(chunk, body.length);
body = newBody;
}
});
resp.on("end", () => {
if (headers["content-encoding"] === "gzip" && body) {
body = getBytes(gunzipSync(body));
}
resolve({ statusCode, statusMessage, headers, body });
});
resp.on("error", (error) => {
//@TODO: Should this just return nornal response with a server error?
(<any>error).response = { statusCode, statusMessage, headers, body };
reject(error);
});
});
request.on("error", (error) => { reject(error); });
});
return defaultGetUrl(req, signal);
}