IndexedDB

dist/events/db.d.ts

@@ -0,0 +1,30 @@
+import { IndexedDB } from '../idb';
+import { BaseTornadoService, BaseTornadoServiceConstructor } from './base';
+import { BaseEvents, MinimalEvents, DepositsEvents, WithdrawalsEvents, CachedEvents } from './types';
+export declare function saveDBEvents<T extends MinimalEvents>({ idb, instanceName, events, lastBlock, }: {
+    idb: IndexedDB;
+    instanceName: string;
+    events: T[];
+    lastBlock: number;
+}): Promise<void>;
+export declare function loadDBEvents<T extends MinimalEvents>({ idb, instanceName, }: {
+    idb: IndexedDB;
+    instanceName: string;
+}): Promise<BaseEvents<T>>;
+export declare function loadRemoteEvents<T extends MinimalEvents>({ staticUrl, instanceName, deployedBlock, }: {
+    staticUrl: string;
+    instanceName: string;
+    deployedBlock: number;
+}): Promise<CachedEvents<T>>;
+export interface DBTornadoServiceConstructor extends BaseTornadoServiceConstructor {
+    staticUrl: string;
+    idb: IndexedDB;
+}
+export declare class DBTornadoService extends BaseTornadoService {
+    staticUrl: string;
+    idb: IndexedDB;
+    constructor(params: DBTornadoServiceConstructor);
+    getEventsFromDB(): Promise<BaseEvents<DepositsEvents | WithdrawalsEvents>>;
+    getEventsFromCache(): Promise<CachedEvents<DepositsEvents | WithdrawalsEvents>>;
+    saveEvents({ events, lastBlock }: BaseEvents<DepositsEvents | WithdrawalsEvents>): Promise<void>;
+}

dist/events/index.d.ts

@@ -1,2 +1,3 @@
 export * from './types';
 export * from './base';
+export * from './db';

dist/idb.d.ts

@@ -0,0 +1,84 @@
+import { OpenDBCallbacks, IDBPDatabase } from 'idb';
+import { NetIdType } from './networkConfig';
+export declare const INDEX_DB_ERROR = "A mutation operation was attempted on a database that did not allow mutations.";
+export interface IDBIndex {
+    name: string;
+    unique?: boolean;
+}
+export interface IDBStores {
+    name: string;
+    keyPath?: string;
+    indexes?: IDBIndex[];
+}
+export interface IDBConstructor {
+    dbName: string;
+    stores?: IDBStores[];
+}
+export declare class IndexedDB {
+    dbExists: boolean;
+    isBlocked: boolean;
+    options: OpenDBCallbacks<any>;
+    dbName: string;
+    dbVersion: number;
+    db?: IDBPDatabase<any>;
+    constructor({ dbName, stores }: IDBConstructor);
+    initDB(): Promise<void>;
+    _removeExist(): Promise<void>;
+    getFromIndex<T>({ storeName, indexName, key, }: {
+        storeName: string;
+        indexName: string;
+        key?: string;
+    }): Promise<T | undefined>;
+    getAllFromIndex<T>({ storeName, indexName, key, count, }: {
+        storeName: string;
+        indexName: string;
+        key?: string;
+        count?: number;
+    }): Promise<T>;
+    getItem<T>({ storeName, key }: {
+        storeName: string;
+        key: string;
+    }): Promise<T | undefined>;
+    addItem({ storeName, data, key }: {
+        storeName: string;
+        data: any;
+        key: string;
+    }): Promise<void>;
+    putItem({ storeName, data, key }: {
+        storeName: string;
+        data: any;
+        key?: string;
+    }): Promise<void>;
+    deleteItem({ storeName, key }: {
+        storeName: string;
+        key: string;
+    }): Promise<void>;
+    getAll<T>({ storeName }: {
+        storeName: string;
+    }): Promise<T>;
+    /**
+     * Simple key-value store inspired by idb-keyval package
+     */
+    getValue<T>(key: string): Promise<T | undefined>;
+    setValue(key: string, data: any): Promise<void>;
+    delValue(key: string): Promise<void>;
+    clearStore({ storeName, mode }: {
+        storeName: string;
+        mode: IDBTransactionMode;
+    }): Promise<void>;
+    createTransactions({ storeName, data, mode, }: {
+        storeName: string;
+        data: any;
+        mode: IDBTransactionMode;
+    }): Promise<void>;
+    createMultipleTransactions({ storeName, data, index, mode, }: {
+        storeName: string;
+        data: any[];
+        index?: any;
+        mode?: IDBTransactionMode;
+    }): Promise<void>;
+}
+/**
+ * Should check if DB is initialized well
+ */
+export declare function getIndexedDB(netId?: NetIdType): Promise<IndexedDB>;

dist/index.d.ts

@@ -6,6 +6,7 @@ export * from './batch';
 export * from './deposits';
 export * from './encryptedNotes';
 export * from './fees';
+export * from './idb';
 export * from './merkleTree';
 export * from './mimc';
 export * from './multicall';
@@ -18,3 +19,4 @@ export * from './tokens';
 export * from './tovarishClient';
 export * from './utils';
 export * from './websnark';
+export * from './zip';

dist/zip.d.ts

@@ -0,0 +1,9 @@
+import { AsyncZippable, Unzipped } from 'fflate';
+export declare function zipAsync(file: AsyncZippable): Promise<Uint8Array>;
+export declare function unzipAsync(data: Uint8Array): Promise<Unzipped>;
+export declare function downloadZip<T>({ staticUrl, zipName, zipDigest, parseJson, }: {
+    staticUrl?: string;
+    zipName: string;
+    zipDigest?: string;
+    parseJson?: boolean;
+}): Promise<T>;

package.json

@@ -42,7 +42,8 @@
     "cross-fetch": "^4.0.0",
     "ethers": "^6.13.2",
     "ffjavascript": "0.2.48",
-    "fflate": "^0.8.2"
+    "fflate": "^0.8.2",
+    "idb": "^8.0.0"
   },
   "devDependencies": {
     "@rollup/plugin-commonjs": "^28.0.0",

src/events/db.ts

@@ -0,0 +1,150 @@
+import { downloadZip } from '../zip';
+import { IndexedDB } from '../idb';
+
+import { BaseTornadoService, BaseTornadoServiceConstructor } from './base';
+import { BaseEvents, MinimalEvents, DepositsEvents, WithdrawalsEvents, CachedEvents } from './types';
+
+export async function saveDBEvents<T extends MinimalEvents>({
+  idb,
+  instanceName,
+  events,
+  lastBlock,
+}: {
+  idb: IndexedDB;
+  instanceName: string;
+  events: T[];
+  lastBlock: number;
+}) {
+  try {
+    await idb.createMultipleTransactions({
+      data: events,
+      storeName: instanceName,
+    });
+
+    await idb.putItem({
+      data: {
+        blockNumber: lastBlock,
+        name: instanceName,
+      },
+      storeName: 'lastEvents',
+    });
+  } catch (err) {
+    console.log('Method saveDBEvents has error');
+    console.log(err);
+  }
+}
+
+export async function loadDBEvents<T extends MinimalEvents>({
+  idb,
+  instanceName,
+}: {
+  idb: IndexedDB;
+  instanceName: string;
+}): Promise<BaseEvents<T>> {
+  try {
+    const lastBlockStore = await idb.getItem<{ blockNumber: number; name: string }>({
+      storeName: 'lastEvents',
+      key: instanceName,
+    });
+
+    if (!lastBlockStore?.blockNumber) {
+      return {
+        events: [],
+        lastBlock: 0,
+      };
+    }
+
+    return {
+      events: await idb.getAll<T[]>({ storeName: instanceName }),
+      lastBlock: lastBlockStore.blockNumber,
+    };
+  } catch (err) {
+    console.log('Method loadDBEvents has error');
+    console.log(err);
+
+    return {
+      events: [],
+      lastBlock: 0,
+    };
+  }
+}
+
+export async function loadRemoteEvents<T extends MinimalEvents>({
+  staticUrl,
+  instanceName,
+  deployedBlock,
+}: {
+  staticUrl: string;
+  instanceName: string;
+  deployedBlock: number;
+}): Promise<CachedEvents<T>> {
+  try {
+    const zipName = `${instanceName}.json`.toLowerCase();
+
+    const events = await downloadZip<T[]>({
+      staticUrl,
+      zipName,
+    });
+
+    if (!Array.isArray(events)) {
+      const errStr = `Invalid events from ${staticUrl}/${zipName}`;
+      throw new Error(errStr);
+    }
+
+    return {
+      events,
+      lastBlock: events[events.length - 1]?.blockNumber || deployedBlock,
+      fromCache: true,
+    };
+  } catch (err) {
+    console.log('Method loadRemoteEvents has error');
+    console.log(err);
+
+    return {
+      events: [],
+      lastBlock: deployedBlock,
+      fromCache: true,
+    };
+  }
+}
+
+export interface DBTornadoServiceConstructor extends BaseTornadoServiceConstructor {
+  staticUrl: string;
+  idb: IndexedDB;
+}
+
+export class DBTornadoService extends BaseTornadoService {
+  staticUrl: string;
+  idb: IndexedDB;
+
+  constructor(params: DBTornadoServiceConstructor) {
+    super(params);
+
+    this.staticUrl = params.staticUrl;
+    this.idb = params.idb;
+  }
+
+  async getEventsFromDB() {
+    return await loadDBEvents<DepositsEvents | WithdrawalsEvents>({
+      idb: this.idb,
+      instanceName: this.getInstanceName(),
+    });
+  }
+
+  async getEventsFromCache() {
+    return await loadRemoteEvents<DepositsEvents | WithdrawalsEvents>({
+      staticUrl: this.staticUrl,
+      instanceName: this.getInstanceName(),
+      deployedBlock: this.deployedBlock,
+    });
+  }
+
+  async saveEvents({ events, lastBlock }: BaseEvents<DepositsEvents | WithdrawalsEvents>) {
+    await saveDBEvents<DepositsEvents | WithdrawalsEvents>({
+      idb: this.idb,
+      instanceName: this.getInstanceName(),
+      events,
+      lastBlock,
+    });
+  }
+}

src/events/index.ts

@@ -1,2 +1,3 @@
 export * from './types';
 export * from './base';
+export * from './db';

src/idb.ts

@@ -0,0 +1,395 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { openDB, deleteDB, OpenDBCallbacks, IDBPDatabase } from 'idb';
+import { getConfig, getNetworkConfig, NetId, NetIdType } from './networkConfig';
+
+export const INDEX_DB_ERROR = 'A mutation operation was attempted on a database that did not allow mutations.';
+
+export interface IDBIndex {
+  name: string;
+  unique?: boolean;
+}
+
+export interface IDBStores {
+  name: string;
+  keyPath?: string;
+  indexes?: IDBIndex[];
+}
+
+export interface IDBConstructor {
+  dbName: string;
+  stores?: IDBStores[];
+}
+
+export class IndexedDB {
+  dbExists: boolean;
+  isBlocked: boolean;
+  // todo: TestDBSchema on any
+  options: OpenDBCallbacks<any>;
+  dbName: string;
+  dbVersion: number;
+  db?: IDBPDatabase<any>;
+
+  constructor({ dbName, stores }: IDBConstructor) {
+    this.dbExists = false;
+    this.isBlocked = false;
+
+    this.options = {
+      upgrade(db) {
+        Object.values(db.objectStoreNames).forEach((value) => {
+          db.deleteObjectStore(value);
+        });
+
+        [{ name: 'keyval' }, ...(stores || [])].forEach(({ name, keyPath, indexes }) => {
+          const store = db.createObjectStore(name, {
+            keyPath,
+            autoIncrement: true,
+          });
+
+          if (Array.isArray(indexes)) {
+            indexes.forEach(({ name, unique = false }) => {
+              store.createIndex(name, name, { unique });
+            });
+          }
+        });
+      },
+    };
+
+    this.dbName = dbName;
+    this.dbVersion = 34;
+  }
+
+  async initDB() {
+    try {
+      if (this.dbExists || this.isBlocked) {
+        return;
+      }
+
+      this.db = await openDB(this.dbName, this.dbVersion, this.options);
+      this.db.addEventListener('onupgradeneeded', async () => {
+        await this._removeExist();
+      });
+
+      this.dbExists = true;
+    } catch (err: any) {
+      // needed for private mode firefox browser
+      if (err.message.includes(INDEX_DB_ERROR)) {
+        console.log('This browser does not support IndexedDB!');
+        this.isBlocked = true;
+        return;
+      }
+
+      if (err.message.includes('less than the existing version')) {
+        console.log(`Upgrading DB ${this.dbName} to ${this.dbVersion}`);
+        await this._removeExist();
+        return;
+      }
+
+      console.error(`Method initDB has error: ${err.message}`);
+    }
+  }
+
+  async _removeExist() {
+    await deleteDB(this.dbName);
+    this.dbExists = false;
+
+    await this.initDB();
+  }
+
+  async getFromIndex<T>({
+    storeName,
+    indexName,
+    key,
+  }: {
+    storeName: string;
+    indexName: string;
+    key?: string;
+  }): Promise<T | undefined> {
+    await this.initDB();
+
+    if (!this.db) {
+      return;
+    }
+
+    try {
+      return (await this.db.getFromIndex(storeName, indexName, key)) as T;
+    } catch (err: any) {
+      throw new Error(`Method getFromIndex has error: ${err.message}`);
+    }
+  }
+
+  async getAllFromIndex<T>({
+    storeName,
+    indexName,
+    key,
+    count,
+  }: {
+    storeName: string;
+    indexName: string;
+    key?: string;
+    count?: number;
+  }): Promise<T> {
+    await this.initDB();
+
+    if (!this.db) {
+      return [] as T;
+    }
+
+    try {
+      return (await this.db.getAllFromIndex(storeName, indexName, key, count)) as T;
+    } catch (err: any) {
+      throw new Error(`Method getAllFromIndex has error: ${err.message}`);
+    }
+  }
+
+  async getItem<T>({ storeName, key }: { storeName: string; key: string }): Promise<T | undefined> {
+    await this.initDB();
+
+    if (!this.db) {
+      return;
+    }
+
+    try {
+      const store = this.db.transaction(storeName).objectStore(storeName);
+
+      return (await store.get(key)) as T;
+    } catch (err: any) {
+      throw new Error(`Method getItem has error: ${err.message}`);
+    }
+  }
+
+  async addItem({ storeName, data, key = '' }: { storeName: string; data: any; key: string }) {
+    await this.initDB();
+
+    if (!this.db) {
+      return;
+    }
+
+    try {
+      const tx = this.db.transaction(storeName, 'readwrite');
+      const isExist = await tx.objectStore(storeName).get(key);
+
+      if (!isExist) {
+        await tx.objectStore(storeName).add(data);
+      }
+    } catch (err: any) {
+      throw new Error(`Method addItem has error: ${err.message}`);
+    }
+  }
+
+  async putItem({ storeName, data, key }: { storeName: string; data: any; key?: string }) {
+    await this.initDB();
+
+    if (!this.db) {
+      return;
+    }
+
+    try {
+      const tx = this.db.transaction(storeName, 'readwrite');
+
+      await tx.objectStore(storeName).put(data, key);
+    } catch (err: any) {
+      throw new Error(`Method putItem has error: ${err.message}`);
+    }
+  }
+
+  async deleteItem({ storeName, key }: { storeName: string; key: string }) {
+    await this.initDB();
+
+    if (!this.db) {
+      return;
+    }
+
+    try {
+      const tx = this.db.transaction(storeName, 'readwrite');
+
+      await tx.objectStore(storeName).delete(key);
+    } catch (err: any) {
+      throw new Error(`Method deleteItem has error: ${err.message}`);
+    }
+  }
+
+  async getAll<T>({ storeName }: { storeName: string }): Promise<T> {
+    await this.initDB();
+
+    if (!this.db) {
+      return [] as T;
+    }
+
+    try {
+      const tx = this.db.transaction(storeName, 'readonly');
+
+      return (await tx.objectStore(storeName).getAll()) as T;
+    } catch (err: any) {
+      throw new Error(`Method getAll has error: ${err.message}`);
+    }
+  }
+
+  /**
+   * Simple key-value store inspired by idb-keyval package
+   */
+  getValue<T>(key: string) {
+    return this.getItem<T>({ storeName: 'keyval', key });
+  }
+
+  setValue(key: string, data: any) {
+    return this.putItem({ storeName: 'keyval', key, data });
+  }
+
+  delValue(key: string) {
+    return this.deleteItem({ storeName: 'keyval', key });
+  }
+
+  async clearStore({ storeName, mode = 'readwrite' }: { storeName: string; mode: IDBTransactionMode }) {
+    await this.initDB();
+
+    if (!this.db) {
+      return;
+    }
+
+    try {
+      const tx = this.db.transaction(storeName, mode);
+
+      await (tx.objectStore(storeName).clear as () => Promise<void>)();
+    } catch (err: any) {
+      throw new Error(`Method clearStore has error: ${err.message}`);
+    }
+  }
+
+  async createTransactions({
+    storeName,
+    data,
+    mode = 'readwrite',
+  }: {
+    storeName: string;
+    data: any;
+    mode: IDBTransactionMode;
+  }) {
+    await this.initDB();
+
+    if (!this.db) {
+      return;
+    }
+
+    try {
+      const tx = this.db.transaction(storeName, mode);
+
+      await (tx.objectStore(storeName).add as (value: any, key?: any) => Promise<any>)(data);
+      await tx.done;
+    } catch (err: any) {
+      throw new Error(`Method createTransactions has error: ${err.message}`);
+    }
+  }
+
+  async createMultipleTransactions({
+    storeName,
+    data,
+    index,
+    mode = 'readwrite',
+  }: {
+    storeName: string;
+    data: any[];
+    index?: any;
+    mode?: IDBTransactionMode;
+  }) {
+    await this.initDB();
+
+    if (!this.db) {
+      return;
+    }
+
+    try {
+      const tx = this.db.transaction(storeName, mode);
+
+      for (const item of data) {
+        if (item) {
+          await (tx.store.put as (value: any, key?: any) => Promise<any>)({ ...item, ...index });
+        }
+      }
+    } catch (err: any) {
+      throw new Error(`Method createMultipleTransactions has error: ${err.message}`);
+    }
+  }
+}
+
+/**
+ * Should check if DB is initialized well
+ */
+export async function getIndexedDB(netId?: NetIdType) {
+  // key-value db for settings
+  if (!netId) {
+    const idb = new IndexedDB({ dbName: 'tornado-core' });
+    await idb.initDB();
+    return idb;
+  }
+
+  const DEPOSIT_INDEXES = [
+    { name: 'transactionHash', unique: false },
+    { name: 'commitment', unique: true },
+  ];
+  const WITHDRAWAL_INDEXES = [
+    { name: 'nullifierHash', unique: true }, // keys on which the index is created
+  ];
+  const LAST_EVENT_INDEXES = [{ name: 'name', unique: false }];
+
+  const defaultState = [
+    {
+      name: 'encrypted_events',
+      keyPath: 'transactionHash',
+    },
+    {
+      name: 'lastEvents',
+      keyPath: 'name',
+      indexes: LAST_EVENT_INDEXES,
+    },
+  ];
+
+  const config = getConfig(netId);
+
+  const { tokens, nativeCurrency } = config;
+
+  const stores = [...defaultState];
+
+  if (netId === NetId.MAINNET) {
+    stores.push({
+      name: 'register_events',
+      keyPath: 'ensName',
+    });
+  }
+
+  Object.entries(tokens).forEach(([token, { instanceAddress }]) => {
+    Object.keys(instanceAddress).forEach((amount) => {
+      if (nativeCurrency === token) {
+        stores.push({
+          name: `stringify_bloom_${netId}_${token}_${amount}`,
+          keyPath: 'hashBloom',
+        });
+      }
+
+      stores.push(
+        {
+          name: `deposits_${netId}_${token}_${amount}`,
+          keyPath: 'leafIndex', // the key by which it refers to the object must be in all instances of the storage
+          indexes: DEPOSIT_INDEXES,
+        },
+        {
+          name: `withdrawals_${netId}_${token}_${amount}`,
+          keyPath: 'blockNumber',
+          indexes: WITHDRAWAL_INDEXES,
+        },
+        {
+          name: `stringify_tree_${netId}_${token}_${amount}`,
+          keyPath: 'hashTree',
+        },
+      );
+    });
+  });
+
+  const idb = new IndexedDB({
+    dbName: `tornado_core_${netId}`,
+    stores,
+  });
+
+  await idb.initDB();
+
+  return idb;
+}

src/index.ts

@@ -6,6 +6,7 @@ export * from './batch';
 export * from './deposits';
 export * from './encryptedNotes';
 export * from './fees';
+export * from './idb';
 export * from './merkleTree';
 export * from './mimc';
 export * from './multicall';
@@ -18,3 +19,4 @@ export * from './tokens';
 export * from './tovarishClient';
 export * from './utils';
 export * from './websnark';
+export * from './zip';

src/zip.ts

@@ -0,0 +1,66 @@
+import { zip, unzip, AsyncZippable, Unzipped } from 'fflate';
+import { fetchData } from './providers';
+import { bytesToBase64, digest } from './utils';
+
+export function zipAsync(file: AsyncZippable): Promise<Uint8Array> {
+  return new Promise((res, rej) => {
+    zip(file, { mtime: new Date('1/1/1980') }, (err, data) => {
+      if (err) {
+        rej(err);
+        return;
+      }
+      res(data);
+    });
+  });
+}
+
+export function unzipAsync(data: Uint8Array): Promise<Unzipped> {
+  return new Promise((res, rej) => {
+    unzip(data, {}, (err, data) => {
+      if (err) {
+        rej(err);
+        return;
+      }
+      res(data);
+    });
+  });
+}
+
+export async function downloadZip<T>({
+  staticUrl = '',
+  zipName,
+  zipDigest,
+  parseJson = true,
+}: {
+  staticUrl?: string;
+  zipName: string;
+  zipDigest?: string;
+  parseJson?: boolean;
+}): Promise<T> {
+  const url = `${staticUrl}/${zipName}.zip`;
+
+  const resp = (await fetchData(url, {
+    method: 'GET',
+    returnResponse: true,
+  })) as Response;
+
+  const data = new Uint8Array(await resp.arrayBuffer());
+
+  // If the zip has digest value, compare it
+  if (zipDigest) {
+    const hash = 'sha384-' + bytesToBase64(await digest(data));
+
+    if (zipDigest !== hash) {
+      const errMsg = `Invalid digest hash for file ${url}, wants ${zipDigest} has ${hash}`;
+      throw new Error(errMsg);
+    }
+  }
+
+  const { [zipName]: content } = await unzipAsync(data);
+
+  if (parseJson) {
+    return JSON.parse(new TextDecoder().decode(content)) as T;
+  }
+
+  return content as T;
+}

yarn.lock

@@ -3096,6 +3096,11 @@ iconv-lite@^0.4.24:
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 
+idb@^8.0.0:
+  version "8.0.0"
+  resolved "https://registry.yarnpkg.com/idb/-/idb-8.0.0.tgz#33d7ed894ed36e23bcb542fb701ad579bfaad41f"
+  integrity sha512-l//qvlAKGmQO31Qn7xdzagVPPaHTxXx199MhrAFuVBTPqydcPYBWjkrbv4Y0ktB+GmWOiwHl237UUOrLmQxLvw==
+
 ieee754@^1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"