/** * Fetching content from the web is environment-specific, so Ethers * provides an abstraction that each environment can implement to provide * this service. * * On [Node.js](link-node), the ``http`` and ``https`` libs are used to * create a request object, register event listeners and process data * and populate the [[FetchResponse]]. * * In a browser, the [DOM fetch](link-js-fetch) is used, and the resulting * ``Promise`` is waited on to retrieve the payload. * * The [[FetchRequest]] is responsible for handling many common situations, * such as redirects, server throttling, authentication, etc. * * It also handles common gateways, such as IPFS and data URIs. * * @_section api/utils/fetching:Fetching Web Content [about-fetch] */ import { decodeBase64, encodeBase64 } from "./base64.js"; import { hexlify } from "./data.js"; import { assert, assertArgument } from "./errors.js"; import { defineProperties } from "./properties.js"; import { toUtf8Bytes, toUtf8String } from "./utf8.js"; import { createGetUrl } from "./geturl.js"; const MAX_ATTEMPTS = 12; const SLOT_INTERVAL = 250; // The global FetchGetUrlFunc implementation. let defaultGetUrlFunc = createGetUrl(); const reData = new RegExp("^data:([^;:]*)?(;base64)?,(.*)$", "i"); const reIpfs = new RegExp("^ipfs:/\/(ipfs/)?(.*)$", "i"); // If locked, new Gateways cannot be added let locked = false; // https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URLs async function dataGatewayFunc(url, signal) { try { const match = url.match(reData); if (!match) { throw new Error("invalid data"); } return new FetchResponse(200, "OK", { "content-type": (match[1] || "text/plain"), }, (match[2] ? decodeBase64(match[3]) : unpercent(match[3]))); } catch (error) { return new FetchResponse(599, "BAD REQUEST (invalid data: URI)", {}, null, new FetchRequest(url)); } } /** * Returns a [[FetchGatewayFunc]] for fetching content from a standard * IPFS gateway hosted at %%baseUrl%%. */ function getIpfsGatewayFunc(baseUrl) { async function gatewayIpfs(url, signal) { try { const match = url.match(reIpfs); if (!match) { throw new Error("invalid link"); } return new FetchRequest(`${baseUrl}${match[2]}`); } catch (error) { return new FetchResponse(599, "BAD REQUEST (invalid IPFS URI)", {}, null, new FetchRequest(url)); } } return gatewayIpfs; } const Gateways = { "data": dataGatewayFunc, "ipfs": getIpfsGatewayFunc("https:/\/gateway.ipfs.io/ipfs/") }; const fetchSignals = new WeakMap(); /** * @_ignore */ export class FetchCancelSignal { #listeners; #cancelled; constructor(request) { this.#listeners = []; this.#cancelled = false; fetchSignals.set(request, () => { if (this.#cancelled) { return; } this.#cancelled = true; for (const listener of this.#listeners) { setTimeout(() => { listener(); }, 0); } this.#listeners = []; }); } addListener(listener) { assert(!this.#cancelled, "singal already cancelled", "UNSUPPORTED_OPERATION", { operation: "fetchCancelSignal.addCancelListener" }); this.#listeners.push(listener); } get cancelled() { return this.#cancelled; } checkSignal() { assert(!this.cancelled, "cancelled", "CANCELLED", {}); } } // Check the signal, throwing if it is cancelled function checkSignal(signal) { if (signal == null) { throw new Error("missing signal; should not happen"); } signal.checkSignal(); return signal; } /** * Represents a request for a resource using a URI. * * By default, the supported schemes are ``HTTP``, ``HTTPS``, ``data:``, * and ``IPFS:``. * * Additional schemes can be added globally using [[registerGateway]]. * * @example: * req = new FetchRequest("https://www.ricmoo.com") * resp = await req.send() * resp.body.length * //_result: */ export class FetchRequest { #allowInsecure; #gzip; #headers; #method; #timeout; #url; #body; #bodyType; #creds; // Hooks #preflight; #process; #retry; #signal; #throttle; #getUrlFunc; /** * The fetch URL to request. */ get url() { return this.#url; } set url(url) { this.#url = String(url); } /** * The fetch body, if any, to send as the request body. //(default: null)// * * When setting a body, the intrinsic ``Content-Type`` is automatically * set and will be used if **not overridden** by setting a custom * header. * * If %%body%% is null, the body is cleared (along with the * intrinsic ``Content-Type``). * * If %%body%% is a string, the intrinsic ``Content-Type`` is set to * ``text/plain``. * * If %%body%% is a Uint8Array, the intrinsic ``Content-Type`` is set to * ``application/octet-stream``. * * If %%body%% is any other object, the intrinsic ``Content-Type`` is * set to ``application/json``. */ get body() { if (this.#body == null) { return null; } return new Uint8Array(this.#body); } set body(body) { if (body == null) { this.#body = undefined; this.#bodyType = undefined; } else if (typeof (body) === "string") { this.#body = toUtf8Bytes(body); this.#bodyType = "text/plain"; } else if (body instanceof Uint8Array) { this.#body = body; this.#bodyType = "application/octet-stream"; } else if (typeof (body) === "object") { this.#body = toUtf8Bytes(JSON.stringify(body)); this.#bodyType = "application/json"; } else { throw new Error("invalid body"); } } /** * Returns true if the request has a body. */ hasBody() { return (this.#body != null); } /** * The HTTP method to use when requesting the URI. If no method * has been explicitly set, then ``GET`` is used if the body is * null and ``POST`` otherwise. */ get method() { if (this.#method) { return this.#method; } if (this.hasBody()) { return "POST"; } return "GET"; } set method(method) { if (method == null) { method = ""; } this.#method = String(method).toUpperCase(); } /** * The headers that will be used when requesting the URI. All * keys are lower-case. * * This object is a copy, so any changes will **NOT** be reflected * in the ``FetchRequest``. * * To set a header entry, use the ``setHeader`` method. */ get headers() { const headers = Object.assign({}, this.#headers); if (this.#creds) { headers["authorization"] = `Basic ${encodeBase64(toUtf8Bytes(this.#creds))}`; } ; if (this.allowGzip) { headers["accept-encoding"] = "gzip"; } if (headers["content-type"] == null && this.#bodyType) { headers["content-type"] = this.#bodyType; } if (this.body) { headers["content-length"] = String(this.body.length); } return headers; } /** * Get the header for %%key%%, ignoring case. */ getHeader(key) { return this.headers[key.toLowerCase()]; } /** * Set the header for %%key%% to %%value%%. All values are coerced * to a string. */ setHeader(key, value) { this.#headers[String(key).toLowerCase()] = String(value); } /** * Clear all headers, resetting all intrinsic headers. */ clearHeaders() { this.#headers = {}; } [Symbol.iterator]() { const headers = this.headers; const keys = Object.keys(headers); let index = 0; return { next: () => { if (index < keys.length) { const key = keys[index++]; return { value: [key, headers[key]], done: false }; } return { value: undefined, done: true }; } }; } /** * The value that will be sent for the ``Authorization`` header. * * To set the credentials, use the ``setCredentials`` method. */ get credentials() { return this.#creds || null; } /** * Sets an ``Authorization`` for %%username%% with %%password%%. */ setCredentials(username, password) { assertArgument(!username.match(/:/), "invalid basic authentication username", "username", "[REDACTED]"); this.#creds = `${username}:${password}`; } /** * Enable and request gzip-encoded responses. The response will * automatically be decompressed. //(default: true)// */ get allowGzip() { return this.#gzip; } set allowGzip(value) { this.#gzip = !!value; } /** * Allow ``Authentication`` credentials to be sent over insecure * channels. //(default: false)// */ get allowInsecureAuthentication() { return !!this.#allowInsecure; } set allowInsecureAuthentication(value) { this.#allowInsecure = !!value; } /** * The timeout (in milliseconds) to wait for a complete response. * //(default: 5 minutes)// */ get timeout() { return this.#timeout; } set timeout(timeout) { assertArgument(timeout >= 0, "timeout must be non-zero", "timeout", timeout); this.#timeout = timeout; } /** * This function is called prior to each request, for example * during a redirection or retry in case of server throttling. * * This offers an opportunity to populate headers or update * content before sending a request. */ get preflightFunc() { return this.#preflight || null; } set preflightFunc(preflight) { this.#preflight = preflight; } /** * This function is called after each response, offering an * opportunity to provide client-level throttling or updating * response data. * * Any error thrown in this causes the ``send()`` to throw. * * To schedule a retry attempt (assuming the maximum retry limit * has not been reached), use [[response.throwThrottleError]]. */ get processFunc() { return this.#process || null; } set processFunc(process) { this.#process = process; } /** * This function is called on each retry attempt. */ get retryFunc() { return this.#retry || null; } set retryFunc(retry) { this.#retry = retry; } /** * This function is called to fetch content from HTTP and * HTTPS URLs and is platform specific (e.g. nodejs vs * browsers). * * This is by default the currently registered global getUrl * function, which can be changed using [[registerGetUrl]]. * If this has been set, setting is to ``null`` will cause * this FetchRequest (and any future clones) to revert back to * using the currently registered global getUrl function. * * Setting this is generally not necessary, but may be useful * for developers that wish to intercept requests or to * configurege a proxy or other agent. */ get getUrlFunc() { return this.#getUrlFunc || defaultGetUrlFunc; } set getUrlFunc(value) { this.#getUrlFunc = value; } /** * Create a new FetchRequest instance with default values. * * Once created, each property may be set before issuing a * ``.send()`` to make the request. */ constructor(url) { this.#url = String(url); this.#allowInsecure = false; this.#gzip = true; this.#headers = {}; this.#method = ""; this.#timeout = 300000; this.#throttle = { slotInterval: SLOT_INTERVAL, maxAttempts: MAX_ATTEMPTS }; this.#getUrlFunc = null; } toString() { return ``; } /** * Update the throttle parameters used to determine maximum * attempts and exponential-backoff properties. */ setThrottleParams(params) { if (params.slotInterval != null) { this.#throttle.slotInterval = params.slotInterval; } if (params.maxAttempts != null) { this.#throttle.maxAttempts = params.maxAttempts; } } async #send(attempt, expires, delay, _request, _response) { if (attempt >= this.#throttle.maxAttempts) { return _response.makeServerError("exceeded maximum retry limit"); } assert(getTime() <= expires, "timeout", "TIMEOUT", { operation: "request.send", reason: "timeout", request: _request }); if (delay > 0) { await wait(delay); } let req = this.clone(); const scheme = (req.url.split(":")[0] || "").toLowerCase(); // Process any Gateways if (scheme in Gateways) { const result = await Gateways[scheme](req.url, checkSignal(_request.#signal)); if (result instanceof FetchResponse) { let response = result; if (this.processFunc) { checkSignal(_request.#signal); try { response = await this.processFunc(req, response); } catch (error) { // Something went wrong during processing; throw a 5xx server error if (error.throttle == null || typeof (error.stall) !== "number") { response.makeServerError("error in post-processing function", error).assertOk(); } // Ignore throttling } } return response; } req = result; } // We have a preflight function; update the request if (this.preflightFunc) { req = await this.preflightFunc(req); } const resp = await this.getUrlFunc(req, checkSignal(_request.#signal)); let response = new FetchResponse(resp.statusCode, resp.statusMessage, resp.headers, resp.body, _request); if (response.statusCode === 301 || response.statusCode === 302) { // Redirect try { const location = response.headers.location || ""; return req.redirect(location).#send(attempt + 1, expires, 0, _request, response); } catch (error) { } // Things won't get any better on another attempt; abort return response; } else if (response.statusCode === 429) { // Throttle if (this.retryFunc == null || (await this.retryFunc(req, response, attempt))) { const retryAfter = response.headers["retry-after"]; let delay = this.#throttle.slotInterval * Math.trunc(Math.random() * Math.pow(2, attempt)); if (typeof (retryAfter) === "string" && retryAfter.match(/^[1-9][0-9]*$/)) { delay = parseInt(retryAfter); } return req.clone().#send(attempt + 1, expires, delay, _request, response); } } if (this.processFunc) { checkSignal(_request.#signal); try { response = await this.processFunc(req, response); } catch (error) { // Something went wrong during processing; throw a 5xx server error if (error.throttle == null || typeof (error.stall) !== "number") { response.makeServerError("error in post-processing function", error).assertOk(); } // Throttle let delay = this.#throttle.slotInterval * Math.trunc(Math.random() * Math.pow(2, attempt)); ; if (error.stall >= 0) { delay = error.stall; } return req.clone().#send(attempt + 1, expires, delay, _request, response); } } return response; } /** * Resolves to the response by sending the request. */ send() { assert(this.#signal == null, "request already sent", "UNSUPPORTED_OPERATION", { operation: "fetchRequest.send" }); this.#signal = new FetchCancelSignal(this); return this.#send(0, getTime() + this.timeout, 0, this, new FetchResponse(0, "", {}, null, this)); } /** * Cancels the inflight response, causing a ``CANCELLED`` * error to be rejected from the [[send]]. */ cancel() { assert(this.#signal != null, "request has not been sent", "UNSUPPORTED_OPERATION", { operation: "fetchRequest.cancel" }); const signal = fetchSignals.get(this); if (!signal) { throw new Error("missing signal; should not happen"); } signal(); } /** * Returns a new [[FetchRequest]] that represents the redirection * to %%location%%. */ redirect(location) { // Redirection; for now we only support absolute locations const current = this.url.split(":")[0].toLowerCase(); const target = location.split(":")[0].toLowerCase(); // Don't allow redirecting: // - non-GET requests // - downgrading the security (e.g. https => http) // - to non-HTTP (or non-HTTPS) protocols [this could be relaxed?] assert(this.method === "GET" && (current !== "https" || target !== "http") && location.match(/^https?:/), `unsupported redirect`, "UNSUPPORTED_OPERATION", { operation: `redirect(${this.method} ${JSON.stringify(this.url)} => ${JSON.stringify(location)})` }); // Create a copy of this request, with a new URL const req = new FetchRequest(location); req.method = "GET"; req.allowGzip = this.allowGzip; req.timeout = this.timeout; req.#headers = Object.assign({}, this.#headers); if (this.#body) { req.#body = new Uint8Array(this.#body); } req.#bodyType = this.#bodyType; // Do not forward credentials unless on the same domain; only absolute //req.allowInsecure = false; // paths are currently supported; may want a way to specify to forward? //setStore(req.#props, "creds", getStore(this.#pros, "creds")); return req; } /** * Create a new copy of this request. */ clone() { const clone = new FetchRequest(this.url); // Preserve "default method" (i.e. null) clone.#method = this.#method; // Preserve "default body" with type, copying the Uint8Array is present if (this.#body) { clone.#body = this.#body; } clone.#bodyType = this.#bodyType; // Preserve "default headers" clone.#headers = Object.assign({}, this.#headers); // Credentials is readonly, so we copy internally clone.#creds = this.#creds; if (this.allowGzip) { clone.allowGzip = true; } clone.timeout = this.timeout; if (this.allowInsecureAuthentication) { clone.allowInsecureAuthentication = true; } clone.#preflight = this.#preflight; clone.#process = this.#process; clone.#retry = this.#retry; clone.#throttle = Object.assign({}, this.#throttle); clone.#getUrlFunc = this.#getUrlFunc; return clone; } /** * Locks all static configuration for gateways and FetchGetUrlFunc * registration. */ static lockConfig() { locked = true; } /** * Get the current Gateway function for %%scheme%%. */ static getGateway(scheme) { return Gateways[scheme.toLowerCase()] || null; } /** * Use the %%func%% when fetching URIs using %%scheme%%. * * This method affects all requests globally. * * If [[lockConfig]] has been called, no change is made and this * throws. */ static registerGateway(scheme, func) { scheme = scheme.toLowerCase(); if (scheme === "http" || scheme === "https") { throw new Error(`cannot intercept ${scheme}; use registerGetUrl`); } if (locked) { throw new Error("gateways locked"); } Gateways[scheme] = func; } /** * Use %%getUrl%% when fetching URIs over HTTP and HTTPS requests. * * This method affects all requests globally. * * If [[lockConfig]] has been called, no change is made and this * throws. */ static registerGetUrl(getUrl) { if (locked) { throw new Error("gateways locked"); } defaultGetUrlFunc = getUrl; } /** * Creates a getUrl function that fetches content from HTTP and * HTTPS URLs. * * The available %%options%% are dependent on the platform * implementation of the default getUrl function. * * This is not generally something that is needed, but is useful * when trying to customize simple behaviour when fetching HTTP * content. */ static createGetUrlFunc(options) { return createGetUrl(options); } /** * Creates a function that can "fetch" data URIs. * * Note that this is automatically done internally to support * data URIs, so it is not necessary to register it. * * This is not generally something that is needed, but may * be useful in a wrapper to perfom custom data URI functionality. */ static createDataGateway() { return dataGatewayFunc; } /** * Creates a function that will fetch IPFS (unvalidated) from * a custom gateway baseUrl. * * The default IPFS gateway used internally is * ``"https:/\/gateway.ipfs.io/ipfs/"``. */ static createIpfsGatewayFunc(baseUrl) { return getIpfsGatewayFunc(baseUrl); } } ; /** * The response for a FetchRequest. */ export class FetchResponse { #statusCode; #statusMessage; #headers; #body; #request; #error; toString() { return ``; } /** * The response status code. */ get statusCode() { return this.#statusCode; } /** * The response status message. */ get statusMessage() { return this.#statusMessage; } /** * The response headers. All keys are lower-case. */ get headers() { return Object.assign({}, this.#headers); } /** * The response body, or ``null`` if there was no body. */ get body() { return (this.#body == null) ? null : new Uint8Array(this.#body); } /** * The response body as a UTF-8 encoded string, or the empty * string (i.e. ``""``) if there was no body. * * An error is thrown if the body is invalid UTF-8 data. */ get bodyText() { try { return (this.#body == null) ? "" : toUtf8String(this.#body); } catch (error) { assert(false, "response body is not valid UTF-8 data", "UNSUPPORTED_OPERATION", { operation: "bodyText", info: { response: this } }); } } /** * The response body, decoded as JSON. * * An error is thrown if the body is invalid JSON-encoded data * or if there was no body. */ get bodyJson() { try { return JSON.parse(this.bodyText); } catch (error) { assert(false, "response body is not valid JSON", "UNSUPPORTED_OPERATION", { operation: "bodyJson", info: { response: this } }); } } [Symbol.iterator]() { const headers = this.headers; const keys = Object.keys(headers); let index = 0; return { next: () => { if (index < keys.length) { const key = keys[index++]; return { value: [key, headers[key]], done: false }; } return { value: undefined, done: true }; } }; } constructor(statusCode, statusMessage, headers, body, request) { this.#statusCode = statusCode; this.#statusMessage = statusMessage; this.#headers = Object.keys(headers).reduce((accum, k) => { accum[k.toLowerCase()] = String(headers[k]); return accum; }, {}); this.#body = ((body == null) ? null : new Uint8Array(body)); this.#request = (request || null); this.#error = { message: "" }; } /** * Return a Response with matching headers and body, but with * an error status code (i.e. 599) and %%message%% with an * optional %%error%%. */ makeServerError(message, error) { let statusMessage; if (!message) { message = `${this.statusCode} ${this.statusMessage}`; statusMessage = `CLIENT ESCALATED SERVER ERROR (${message})`; } else { statusMessage = `CLIENT ESCALATED SERVER ERROR (${this.statusCode} ${this.statusMessage}; ${message})`; } const response = new FetchResponse(599, statusMessage, this.headers, this.body, this.#request || undefined); response.#error = { message, error }; return response; } /** * If called within a [request.processFunc](FetchRequest-processFunc) * call, causes the request to retry as if throttled for %%stall%% * milliseconds. */ throwThrottleError(message, stall) { if (stall == null) { stall = -1; } else { assertArgument(Number.isInteger(stall) && stall >= 0, "invalid stall timeout", "stall", stall); } const error = new Error(message || "throttling requests"); defineProperties(error, { stall, throttle: true }); throw error; } /** * Get the header value for %%key%%, ignoring case. */ getHeader(key) { return this.headers[key.toLowerCase()]; } /** * Returns true if the response has a body. */ hasBody() { return (this.#body != null); } /** * The request made for this response. */ get request() { return this.#request; } /** * Returns true if this response was a success statusCode. */ ok() { return (this.#error.message === "" && this.statusCode >= 200 && this.statusCode < 300); } /** * Throws a ``SERVER_ERROR`` if this response is not ok. */ assertOk() { if (this.ok()) { return; } let { message, error } = this.#error; if (message === "") { message = `server response ${this.statusCode} ${this.statusMessage}`; } let requestUrl = null; if (this.request) { requestUrl = this.request.url; } let responseBody = null; try { if (this.#body) { responseBody = toUtf8String(this.#body); } } catch (e) { } assert(false, message, "SERVER_ERROR", { request: (this.request || "unknown request"), response: this, error, info: { requestUrl, responseBody, responseStatus: `${this.statusCode} ${this.statusMessage}` } }); } } function getTime() { return (new Date()).getTime(); } function unpercent(value) { return toUtf8Bytes(value.replace(/%([0-9a-f][0-9a-f])/gi, (all, code) => { return String.fromCharCode(parseInt(code, 16)); })); } function wait(delay) { return new Promise((resolve) => setTimeout(resolve, delay)); } //# sourceMappingURL=fetch.js.map