More robust configuration options for FetchRequest getUrl functions (#4353).

src.ts/utils/fetch.ts

@@ -23,7 +23,7 @@ import { assert, assertArgument } from "./errors.js";
 import { defineProperties } from "./properties.js";
 import { toUtf8Bytes, toUtf8String } from "./utf8.js"
 
-import { getUrl } from "./geturl.js";
+import { createGetUrl } from "./geturl.js";
 
 /**
  *  An environments implementation of ``getUrl`` must return this type.
@@ -77,7 +77,7 @@ const MAX_ATTEMPTS = 12;
 const SLOT_INTERVAL = 250;
 
 // The global FetchGetUrlFunc implementation.
-let getUrlFunc: FetchGetUrlFunc = getUrl;
+let defaultGetUrlFunc: FetchGetUrlFunc = createGetUrl();
 
 const reData = new RegExp("^data:([^;:]*)?(;base64)?,(.*)$", "i");
 const reIpfs = new RegExp("^ipfs:/\/(ipfs/)?(.*)$", "i");
@@ -201,6 +201,8 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
 
     #throttle: Required<FetchThrottleParams>;
 
+    #getUrlFunc: null | FetchGetUrlFunc;
+
     /**
      *  The fetch URI to requrest.
      */
@@ -429,6 +431,28 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
         this.#retry = retry;
     }
 
+    /**
+     *  This function is called to fetch content from HTTP and
+     *  HTTPS URLs and is platform specific (e.g. nodejs vs
+     *  browsers).
+     *
+     *  This is by default the currently registered global getUrl
+     *  function, which can be changed using [[registerGetUrl]].
+     *  If this has been set, setting is to ``null`` will cause
+     *  this FetchRequest (and any future clones) to revert back to
+     *  using the currently registered global getUrl function.
+     *
+     *  Setting this is generally not necessary, but may be useful
+     *  for developers that wish to intercept requests or to
+     *  configurege a proxy or other agent.
+     */
+    get getUrlFunc(): FetchGetUrlFunc {
+        return this.#getUrlFunc || defaultGetUrlFunc;
+    }
+    set getUrlFunc(value: null | FetchGetUrlFunc) {
+        this.#getUrlFunc = value;
+    }
+
     /**
      *  Create a new FetchRequest instance with default values.
      *
@@ -448,6 +472,8 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
             slotInterval: SLOT_INTERVAL,
             maxAttempts: MAX_ATTEMPTS
         };
+
+        this.#getUrlFunc = null;
     }
 
     toString(): string {
@@ -510,7 +536,7 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
         // We have a preflight function; update the request
         if (this.preflightFunc) { req = await this.preflightFunc(req); }
 
-        const resp = await getUrlFunc(req, checkSignal(_request.#signal));
+        const resp = await this.getUrlFunc(req, checkSignal(_request.#signal));
         let response = new FetchResponse(resp.statusCode, resp.statusMessage, resp.headers, resp.body, _request);
 
         if (response.statusCode === 301 || response.statusCode === 302) {
@@ -641,6 +667,8 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
         clone.#process = this.#process;
         clone.#retry = this.#retry;
 
+        clone.#getUrlFunc = this.#getUrlFunc;
+
         return clone;
     }
 
@@ -686,7 +714,22 @@ export class FetchRequest implements Iterable<[ key: string, value: string ]> {
      */
     static registerGetUrl(getUrl: FetchGetUrlFunc): void {
         if (locked) { throw new Error("gateways locked"); }
-        getUrlFunc = getUrl;
+        defaultGetUrlFunc = getUrl;
+    }
+
+    /**
+     *  Creates a getUrl function that fetches content from HTTP and
+     *  HTTPS URLs.
+     *
+     *  The available %%options%% are dependent on the platform
+     *  implementation of the default getUrl function.
+     *
+     *  This is not generally something that is needed, but is useful
+     *  when trying to customize simple behaviour when fetching HTTP
+     *  content.
+     */
+    static createGetUrlFunc(options?: Record<string, any>): FetchGetUrlFunc {
+        return createGetUrl(options);
     }
 
     /**

src.ts/utils/geturl-browser.ts

@@ -1,6 +1,8 @@
 import { assert } from "./errors.js";
 
-import type { FetchRequest, FetchCancelSignal, GetUrlResponse } from "./fetch.js";
+import type {
+    FetchGetUrlFunc, FetchRequest, FetchCancelSignal, GetUrlResponse
+} from "./fetch.js";
 
 
 declare global {
@@ -27,46 +29,58 @@ declare global {
 
 // @TODO: timeout is completely ignored; start a Promise.any with a reject?
 
-export async function getUrl(req: FetchRequest, _signal?: FetchCancelSignal): Promise<GetUrlResponse> {
+export function createGetUrl(options?: Record<string, any>): FetchGetUrlFunc {
-    const protocol = req.url.split(":")[0].toLowerCase();
 
-    assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", {
+    async function getUrl(req: FetchRequest, _signal?: FetchCancelSignal): Promise<GetUrlResponse> {
-        info: { protocol },
+        const protocol = req.url.split(":")[0].toLowerCase();
-        operation: "request"
-    });
 
-    assert(protocol === "https" || !req.credentials || req.allowInsecureAuthentication, "insecure authorized connections unsupported", "UNSUPPORTED_OPERATION", {
+        assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", {
-        operation: "request"
+            info: { protocol },
-    });
+            operation: "request"
+        });
 
-    let signal: undefined | AbortSignal = undefined;
+        assert(protocol === "https" || !req.credentials || req.allowInsecureAuthentication, "insecure authorized connections unsupported", "UNSUPPORTED_OPERATION", {
-    if (_signal) {
+            operation: "request"
-        const controller = new AbortController();
+        });
-        signal = controller.signal;
+
-        _signal.addListener(() => { controller.abort(); });
+        let signal: undefined | AbortSignal = undefined;
+        if (_signal) {
+            const controller = new AbortController();
+            signal = controller.signal;
+            _signal.addListener(() => { controller.abort(); });
+        }
+
+        const init = {
+            method: req.method,
+            headers: new Headers(Array.from(req)),
+            body: req.body || undefined,
+            signal
+        };
+
+        const resp = await fetch(req.url, init);
+
+        const headers: Record<string, string> = { };
+        resp.headers.forEach((value, key) => {
+            headers[key.toLowerCase()] = value;
+        });
+
+        const respBody = await resp.arrayBuffer();
+        const body = (respBody == null) ? null: new Uint8Array(respBody);
+
+        return {
+            statusCode: resp.status,
+            statusMessage: resp.statusText,
+            headers, body
+        };
     }
 
-    const init = {
+    return getUrl;
-        method: req.method,
+}
-        headers: new Headers(Array.from(req)),
+
-        body: req.body || undefined,
+// @TODO: remove in v7; provided for backwards compat
-        signal
+const defaultGetUrl: FetchGetUrlFunc = createGetUrl({ });
-    };
+
-
+export async function getUrl(req: FetchRequest, _signal?: FetchCancelSignal): Promise<GetUrlResponse> {
-    const resp = await fetch(req.url, init);
+    return defaultGetUrl(req, _signal);
-
-    const headers: Record<string, string> = { };
-    resp.headers.forEach((value, key) => {
-        headers[key.toLowerCase()] = value;
-    });
-
-    const respBody = await resp.arrayBuffer();
-    const body = (respBody == null) ? null: new Uint8Array(respBody);
-
-    return {
-        statusCode: resp.status,
-        statusMessage: resp.statusText,
-        headers, body
-    };
 }
 

src.ts/utils/geturl.ts

@@ -5,92 +5,112 @@ import { gunzipSync } from "zlib";
 import { assert } from "./errors.js";
 import { getBytes } from "./data.js";
 
-import type { FetchRequest, FetchCancelSignal, GetUrlResponse } from "./fetch.js";
+import type {
+    FetchGetUrlFunc, FetchRequest, FetchCancelSignal, GetUrlResponse
+} from "./fetch.js";
+
+/**
+ *  @_ignore:
+ */
+export function createGetUrl(options?: Record<string, any>): FetchGetUrlFunc {
+
+    async function getUrl(req: FetchRequest, signal?: FetchCancelSignal): Promise<GetUrlResponse> {
+
+        const protocol = req.url.split(":")[0].toLowerCase();
+
+        assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", {
+            info: { protocol },
+            operation: "request"
+        });
+
+        assert(protocol === "https" || !req.credentials || req.allowInsecureAuthentication, "insecure authorized connections unsupported", "UNSUPPORTED_OPERATION", {
+            operation: "request"
+        });
+
+        const method = req.method;
+        const headers = Object.assign({ }, req.headers);
+
+        const reqOptions: any = { method, headers };
+        if (options) {
+            if (options.agent) { reqOptions.agent = options.agent; }
+        }
+
+        const request = ((protocol === "http") ? http: https).request(req.url, reqOptions);
+
+        request.setTimeout(req.timeout);
+
+        const body = req.body;
+        if (body) { request.write(Buffer.from(body)); }
+
+        request.end();
+
+        return new Promise((resolve, reject) => {
+            // @TODO: Node 15 added AbortSignal; once we drop support for
+            // Node14, we can add that in here too
+
+            request.once("response", (resp: http.IncomingMessage) => {
+                const statusCode = resp.statusCode || 0;
+                const statusMessage = resp.statusMessage || "";
+                const headers = Object.keys(resp.headers || {}).reduce((accum, name) => {
+                    let value = resp.headers[name] || "";
+                    if (Array.isArray(value)) {
+                        value = value.join(", ");
+                    }
+                    accum[name] = value;
+                    return accum;
+                }, <{ [ name: string ]: string }>{ });
+
+                let body: null | Uint8Array = null;
+                //resp.setEncoding("utf8");
+
+                resp.on("data", (chunk: Uint8Array) => {
+                    if (signal) {
+                        try {
+                            signal.checkSignal();
+                        } catch (error) {
+                            return reject(error);
+                        }
+                    }
+
+                    if (body == null) {
+                        body = chunk;
+                    } else {
+                        const newBody = new Uint8Array(body.length + chunk.length);
+                        newBody.set(body, 0);
+                        newBody.set(chunk, body.length);
+                        body = newBody;
+                    }
+                });
+
+                resp.on("end", () => {
+                    if (headers["content-encoding"] === "gzip" && body) {
+                        body = getBytes(gunzipSync(body));
+                    }
+
+                    resolve({ statusCode, statusMessage, headers, body });
+                });
+
+                resp.on("error", (error) => {
+                //@TODO: Should this just return nornal response with a server error?
+                    (<any>error).response = { statusCode, statusMessage, headers, body };
+                    reject(error);
+                });
+            });
+
+            request.on("error", (error) => { reject(error); });
+        });
+    }
+
+    return getUrl;
+}
+
+// @TODO: remove in v7; provided for backwards compat
+const defaultGetUrl: FetchGetUrlFunc = createGetUrl({ });
 
 /**
  *  @_ignore:
  */
 export async function getUrl(req: FetchRequest, signal?: FetchCancelSignal): Promise<GetUrlResponse> {
-
+    return defaultGetUrl(req, signal);
-    const protocol = req.url.split(":")[0].toLowerCase();
-
-    assert(protocol === "http" || protocol === "https", `unsupported protocol ${ protocol }`, "UNSUPPORTED_OPERATION", {
-        info: { protocol },
-        operation: "request"
-    });
-
-    assert(protocol === "https" || !req.credentials || req.allowInsecureAuthentication, "insecure authorized connections unsupported", "UNSUPPORTED_OPERATION", {
-        operation: "request"
-    });
-
-    const method = req.method;
-    const headers = Object.assign({ }, req.headers);
-
-    const options: any = { method, headers };
-
-    const request = ((protocol === "http") ? http: https).request(req.url, options);
-
-    request.setTimeout(req.timeout);
-
-    const body = req.body;
-    if (body) { request.write(Buffer.from(body)); }
-
-    request.end();
-
-    return new Promise((resolve, reject) => {
-        // @TODO: Node 15 added AbortSignal; once we drop support for
-        // Node14, we can add that in here too
-
-        request.once("response", (resp: http.IncomingMessage) => {
-            const statusCode = resp.statusCode || 0;
-            const statusMessage = resp.statusMessage || "";
-            const headers = Object.keys(resp.headers || {}).reduce((accum, name) => {
-                let value = resp.headers[name] || "";
-                if (Array.isArray(value)) {
-                    value = value.join(", ");
-                }
-                accum[name] = value;
-                return accum;
-            }, <{ [ name: string ]: string }>{ });
-
-            let body: null | Uint8Array = null;
-            //resp.setEncoding("utf8");
-
-            resp.on("data", (chunk: Uint8Array) => {
-                if (signal) {
-                    try {
-                        signal.checkSignal();
-                    } catch (error) {
-                        return reject(error);
-                    }
-                }
-
-                if (body == null) {
-                    body = chunk;
-                } else {
-                    const newBody = new Uint8Array(body.length + chunk.length);
-                    newBody.set(body, 0);
-                    newBody.set(chunk, body.length);
-                    body = newBody;
-                }
-            });
-
-            resp.on("end", () => {
-                if (headers["content-encoding"] === "gzip" && body) {
-                    body = getBytes(gunzipSync(body));
-                }
-
-                resolve({ statusCode, statusMessage, headers, body });
-            });
-
-            resp.on("error", (error) => {
-            //@TODO: Should this just return nornal response with a server error?
-                (<any>error).response = { statusCode, statusMessage, headers, body };
-                reject(error);
-            });
-        });
-
-        request.on("error", (error) => { reject(error); });
-    });
 }