IndexedDB

This commit is contained in:
Tornado Contrib 2024-10-04 12:58:47 +00:00
parent f73b9ecbff
commit 42db44ca3b
Signed by: tornadocontrib
GPG Key ID: 60B4DF1A076C64B1
15 changed files with 5937 additions and 453 deletions

30
dist/events/db.d.ts vendored Normal file

@ -0,0 +1,30 @@
import { IndexedDB } from '../idb';
import { BaseTornadoService, BaseTornadoServiceConstructor } from './base';
import { BaseEvents, MinimalEvents, DepositsEvents, WithdrawalsEvents, CachedEvents } from './types';
export declare function saveDBEvents<T extends MinimalEvents>({ idb, instanceName, events, lastBlock, }: {
idb: IndexedDB;
instanceName: string;
events: T[];
lastBlock: number;
}): Promise<void>;
export declare function loadDBEvents<T extends MinimalEvents>({ idb, instanceName, }: {
idb: IndexedDB;
instanceName: string;
}): Promise<BaseEvents<T>>;
export declare function loadRemoteEvents<T extends MinimalEvents>({ staticUrl, instanceName, deployedBlock, }: {
staticUrl: string;
instanceName: string;
deployedBlock: number;
}): Promise<CachedEvents<T>>;
export interface DBTornadoServiceConstructor extends BaseTornadoServiceConstructor {
staticUrl: string;
idb: IndexedDB;
}
export declare class DBTornadoService extends BaseTornadoService {
staticUrl: string;
idb: IndexedDB;
constructor(params: DBTornadoServiceConstructor);
getEventsFromDB(): Promise<BaseEvents<DepositsEvents | WithdrawalsEvents>>;
getEventsFromCache(): Promise<CachedEvents<DepositsEvents | WithdrawalsEvents>>;
saveEvents({ events, lastBlock }: BaseEvents<DepositsEvents | WithdrawalsEvents>): Promise<void>;
}

@ -1,2 +1,3 @@
export * from './types';
export * from './base';
export * from './db';

84
dist/idb.d.ts vendored Normal file

@ -0,0 +1,84 @@
import { OpenDBCallbacks, IDBPDatabase } from 'idb';
import { NetIdType } from './networkConfig';
export declare const INDEX_DB_ERROR = "A mutation operation was attempted on a database that did not allow mutations.";
export interface IDBIndex {
name: string;
unique?: boolean;
}
export interface IDBStores {
name: string;
keyPath?: string;
indexes?: IDBIndex[];
}
export interface IDBConstructor {
dbName: string;
stores?: IDBStores[];
}
export declare class IndexedDB {
dbExists: boolean;
isBlocked: boolean;
options: OpenDBCallbacks<any>;
dbName: string;
dbVersion: number;
db?: IDBPDatabase<any>;
constructor({ dbName, stores }: IDBConstructor);
initDB(): Promise<void>;
_removeExist(): Promise<void>;
getFromIndex<T>({ storeName, indexName, key, }: {
storeName: string;
indexName: string;
key?: string;
}): Promise<T | undefined>;
getAllFromIndex<T>({ storeName, indexName, key, count, }: {
storeName: string;
indexName: string;
key?: string;
count?: number;
}): Promise<T>;
getItem<T>({ storeName, key }: {
storeName: string;
key: string;
}): Promise<T | undefined>;
addItem({ storeName, data, key }: {
storeName: string;
data: any;
key: string;
}): Promise<void>;
putItem({ storeName, data, key }: {
storeName: string;
data: any;
key?: string;
}): Promise<void>;
deleteItem({ storeName, key }: {
storeName: string;
key: string;
}): Promise<void>;
getAll<T>({ storeName }: {
storeName: string;
}): Promise<T>;
/**
* Simple key-value store inspired by idb-keyval package
*/
getValue<T>(key: string): Promise<T | undefined>;
setValue(key: string, data: any): Promise<void>;
delValue(key: string): Promise<void>;
clearStore({ storeName, mode }: {
storeName: string;
mode: IDBTransactionMode;
}): Promise<void>;
createTransactions({ storeName, data, mode, }: {
storeName: string;
data: any;
mode: IDBTransactionMode;
}): Promise<void>;
createMultipleTransactions({ storeName, data, index, mode, }: {
storeName: string;
data: any[];
index?: any;
mode?: IDBTransactionMode;
}): Promise<void>;
}
/**
* Should check if DB is initialized well
*/
export declare function getIndexedDB(netId?: NetIdType): Promise<IndexedDB>;

2
dist/index.d.ts vendored

@ -6,6 +6,7 @@ export * from './batch';
export * from './deposits';
export * from './encryptedNotes';
export * from './fees';
export * from './idb';
export * from './merkleTree';
export * from './mimc';
export * from './multicall';
@ -18,3 +19,4 @@ export * from './tokens';
export * from './tovarishClient';
export * from './utils';
export * from './websnark';
export * from './zip';

941
dist/index.js vendored

File diff suppressed because it is too large Load Diff

933
dist/index.mjs vendored

File diff suppressed because it is too large Load Diff

3768
dist/tornado.umd.js vendored

File diff suppressed because it is too large Load Diff

9
dist/zip.d.ts vendored Normal file

@ -0,0 +1,9 @@
import { AsyncZippable, Unzipped } from 'fflate';
export declare function zipAsync(file: AsyncZippable): Promise<Uint8Array>;
export declare function unzipAsync(data: Uint8Array): Promise<Unzipped>;
export declare function downloadZip<T>({ staticUrl, zipName, zipDigest, parseJson, }: {
staticUrl?: string;
zipName: string;
zipDigest?: string;
parseJson?: boolean;
}): Promise<T>;

@ -42,7 +42,8 @@
"cross-fetch": "^4.0.0",
"ethers": "^6.13.2",
"ffjavascript": "0.2.48",
"fflate": "^0.8.2"
"fflate": "^0.8.2",
"idb": "^8.0.0"
},
"devDependencies": {
"@rollup/plugin-commonjs": "^28.0.0",

150
src/events/db.ts Normal file

@ -0,0 +1,150 @@
import { downloadZip } from '../zip';
import { IndexedDB } from '../idb';
import { BaseTornadoService, BaseTornadoServiceConstructor } from './base';
import { BaseEvents, MinimalEvents, DepositsEvents, WithdrawalsEvents, CachedEvents } from './types';
export async function saveDBEvents<T extends MinimalEvents>({
idb,
instanceName,
events,
lastBlock,
}: {
idb: IndexedDB;
instanceName: string;
events: T[];
lastBlock: number;
}) {
try {
await idb.createMultipleTransactions({
data: events,
storeName: instanceName,
});
await idb.putItem({
data: {
blockNumber: lastBlock,
name: instanceName,
},
storeName: 'lastEvents',
});
} catch (err) {
console.log('Method saveDBEvents has error');
console.log(err);
}
}
export async function loadDBEvents<T extends MinimalEvents>({
idb,
instanceName,
}: {
idb: IndexedDB;
instanceName: string;
}): Promise<BaseEvents<T>> {
try {
const lastBlockStore = await idb.getItem<{ blockNumber: number; name: string }>({
storeName: 'lastEvents',
key: instanceName,
});
if (!lastBlockStore?.blockNumber) {
return {
events: [],
lastBlock: 0,
};
}
return {
events: await idb.getAll<T[]>({ storeName: instanceName }),
lastBlock: lastBlockStore.blockNumber,
};
} catch (err) {
console.log('Method loadDBEvents has error');
console.log(err);
return {
events: [],
lastBlock: 0,
};
}
}
export async function loadRemoteEvents<T extends MinimalEvents>({
staticUrl,
instanceName,
deployedBlock,
}: {
staticUrl: string;
instanceName: string;
deployedBlock: number;
}): Promise<CachedEvents<T>> {
try {
const zipName = `${instanceName}.json`.toLowerCase();
const events = await downloadZip<T[]>({
staticUrl,
zipName,
});
if (!Array.isArray(events)) {
const errStr = `Invalid events from ${staticUrl}/${zipName}`;
throw new Error(errStr);
}
return {
events,
lastBlock: events[events.length - 1]?.blockNumber || deployedBlock,
fromCache: true,
};
} catch (err) {
console.log('Method loadRemoteEvents has error');
console.log(err);
return {
events: [],
lastBlock: deployedBlock,
fromCache: true,
};
}
}
export interface DBTornadoServiceConstructor extends BaseTornadoServiceConstructor {
staticUrl: string;
idb: IndexedDB;
}
export class DBTornadoService extends BaseTornadoService {
staticUrl: string;
idb: IndexedDB;
constructor(params: DBTornadoServiceConstructor) {
super(params);
this.staticUrl = params.staticUrl;
this.idb = params.idb;
}
async getEventsFromDB() {
return await loadDBEvents<DepositsEvents | WithdrawalsEvents>({
idb: this.idb,
instanceName: this.getInstanceName(),
});
}
async getEventsFromCache() {
return await loadRemoteEvents<DepositsEvents | WithdrawalsEvents>({
staticUrl: this.staticUrl,
instanceName: this.getInstanceName(),
deployedBlock: this.deployedBlock,
});
}
async saveEvents({ events, lastBlock }: BaseEvents<DepositsEvents | WithdrawalsEvents>) {
await saveDBEvents<DepositsEvents | WithdrawalsEvents>({
idb: this.idb,
instanceName: this.getInstanceName(),
events,
lastBlock,
});
}
}

@ -1,2 +1,3 @@
export * from './types';
export * from './base';
export * from './db';

395
src/idb.ts Normal file

@ -0,0 +1,395 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { openDB, deleteDB, OpenDBCallbacks, IDBPDatabase } from 'idb';
import { getConfig, getNetworkConfig, NetId, NetIdType } from './networkConfig';
export const INDEX_DB_ERROR = 'A mutation operation was attempted on a database that did not allow mutations.';
export interface IDBIndex {
name: string;
unique?: boolean;
}
export interface IDBStores {
name: string;
keyPath?: string;
indexes?: IDBIndex[];
}
export interface IDBConstructor {
dbName: string;
stores?: IDBStores[];
}
export class IndexedDB {
dbExists: boolean;
isBlocked: boolean;
// todo: TestDBSchema on any
options: OpenDBCallbacks<any>;
dbName: string;
dbVersion: number;
db?: IDBPDatabase<any>;
constructor({ dbName, stores }: IDBConstructor) {
this.dbExists = false;
this.isBlocked = false;
this.options = {
upgrade(db) {
Object.values(db.objectStoreNames).forEach((value) => {
db.deleteObjectStore(value);
});
[{ name: 'keyval' }, ...(stores || [])].forEach(({ name, keyPath, indexes }) => {
const store = db.createObjectStore(name, {
keyPath,
autoIncrement: true,
});
if (Array.isArray(indexes)) {
indexes.forEach(({ name, unique = false }) => {
store.createIndex(name, name, { unique });
});
}
});
},
};
this.dbName = dbName;
this.dbVersion = 34;
}
async initDB() {
try {
if (this.dbExists || this.isBlocked) {
return;
}
this.db = await openDB(this.dbName, this.dbVersion, this.options);
this.db.addEventListener('onupgradeneeded', async () => {
await this._removeExist();
});
this.dbExists = true;
} catch (err: any) {
// needed for private mode firefox browser
if (err.message.includes(INDEX_DB_ERROR)) {
console.log('This browser does not support IndexedDB!');
this.isBlocked = true;
return;
}
if (err.message.includes('less than the existing version')) {
console.log(`Upgrading DB ${this.dbName} to ${this.dbVersion}`);
await this._removeExist();
return;
}
console.error(`Method initDB has error: ${err.message}`);
}
}
async _removeExist() {
await deleteDB(this.dbName);
this.dbExists = false;
await this.initDB();
}
async getFromIndex<T>({
storeName,
indexName,
key,
}: {
storeName: string;
indexName: string;
key?: string;
}): Promise<T | undefined> {
await this.initDB();
if (!this.db) {
return;
}
try {
return (await this.db.getFromIndex(storeName, indexName, key)) as T;
} catch (err: any) {
throw new Error(`Method getFromIndex has error: ${err.message}`);
}
}
async getAllFromIndex<T>({
storeName,
indexName,
key,
count,
}: {
storeName: string;
indexName: string;
key?: string;
count?: number;
}): Promise<T> {
await this.initDB();
if (!this.db) {
return [] as T;
}
try {
return (await this.db.getAllFromIndex(storeName, indexName, key, count)) as T;
} catch (err: any) {
throw new Error(`Method getAllFromIndex has error: ${err.message}`);
}
}
async getItem<T>({ storeName, key }: { storeName: string; key: string }): Promise<T | undefined> {
await this.initDB();
if (!this.db) {
return;
}
try {
const store = this.db.transaction(storeName).objectStore(storeName);
return (await store.get(key)) as T;
} catch (err: any) {
throw new Error(`Method getItem has error: ${err.message}`);
}
}
async addItem({ storeName, data, key = '' }: { storeName: string; data: any; key: string }) {
await this.initDB();
if (!this.db) {
return;
}
try {
const tx = this.db.transaction(storeName, 'readwrite');
const isExist = await tx.objectStore(storeName).get(key);
if (!isExist) {
await tx.objectStore(storeName).add(data);
}
} catch (err: any) {
throw new Error(`Method addItem has error: ${err.message}`);
}
}
async putItem({ storeName, data, key }: { storeName: string; data: any; key?: string }) {
await this.initDB();
if (!this.db) {
return;
}
try {
const tx = this.db.transaction(storeName, 'readwrite');
await tx.objectStore(storeName).put(data, key);
} catch (err: any) {
throw new Error(`Method putItem has error: ${err.message}`);
}
}
async deleteItem({ storeName, key }: { storeName: string; key: string }) {
await this.initDB();
if (!this.db) {
return;
}
try {
const tx = this.db.transaction(storeName, 'readwrite');
await tx.objectStore(storeName).delete(key);
} catch (err: any) {
throw new Error(`Method deleteItem has error: ${err.message}`);
}
}
async getAll<T>({ storeName }: { storeName: string }): Promise<T> {
await this.initDB();
if (!this.db) {
return [] as T;
}
try {
const tx = this.db.transaction(storeName, 'readonly');
return (await tx.objectStore(storeName).getAll()) as T;
} catch (err: any) {
throw new Error(`Method getAll has error: ${err.message}`);
}
}
/**
* Simple key-value store inspired by idb-keyval package
*/
getValue<T>(key: string) {
return this.getItem<T>({ storeName: 'keyval', key });
}
setValue(key: string, data: any) {
return this.putItem({ storeName: 'keyval', key, data });
}
delValue(key: string) {
return this.deleteItem({ storeName: 'keyval', key });
}
async clearStore({ storeName, mode = 'readwrite' }: { storeName: string; mode: IDBTransactionMode }) {
await this.initDB();
if (!this.db) {
return;
}
try {
const tx = this.db.transaction(storeName, mode);
await (tx.objectStore(storeName).clear as () => Promise<void>)();
} catch (err: any) {
throw new Error(`Method clearStore has error: ${err.message}`);
}
}
async createTransactions({
storeName,
data,
mode = 'readwrite',
}: {
storeName: string;
data: any;
mode: IDBTransactionMode;
}) {
await this.initDB();
if (!this.db) {
return;
}
try {
const tx = this.db.transaction(storeName, mode);
await (tx.objectStore(storeName).add as (value: any, key?: any) => Promise<any>)(data);
await tx.done;
} catch (err: any) {
throw new Error(`Method createTransactions has error: ${err.message}`);
}
}
async createMultipleTransactions({
storeName,
data,
index,
mode = 'readwrite',
}: {
storeName: string;
data: any[];
index?: any;
mode?: IDBTransactionMode;
}) {
await this.initDB();
if (!this.db) {
return;
}
try {
const tx = this.db.transaction(storeName, mode);
for (const item of data) {
if (item) {
await (tx.store.put as (value: any, key?: any) => Promise<any>)({ ...item, ...index });
}
}
} catch (err: any) {
throw new Error(`Method createMultipleTransactions has error: ${err.message}`);
}
}
}
/**
* Should check if DB is initialized well
*/
export async function getIndexedDB(netId?: NetIdType) {
// key-value db for settings
if (!netId) {
const idb = new IndexedDB({ dbName: 'tornado-core' });
await idb.initDB();
return idb;
}
const DEPOSIT_INDEXES = [
{ name: 'transactionHash', unique: false },
{ name: 'commitment', unique: true },
];
const WITHDRAWAL_INDEXES = [
{ name: 'nullifierHash', unique: true }, // keys on which the index is created
];
const LAST_EVENT_INDEXES = [{ name: 'name', unique: false }];
const defaultState = [
{
name: 'encrypted_events',
keyPath: 'transactionHash',
},
{
name: 'lastEvents',
keyPath: 'name',
indexes: LAST_EVENT_INDEXES,
},
];
const config = getConfig(netId);
const { tokens, nativeCurrency } = config;
const stores = [...defaultState];
if (netId === NetId.MAINNET) {
stores.push({
name: 'register_events',
keyPath: 'ensName',
});
}
Object.entries(tokens).forEach(([token, { instanceAddress }]) => {
Object.keys(instanceAddress).forEach((amount) => {
if (nativeCurrency === token) {
stores.push({
name: `stringify_bloom_${netId}_${token}_${amount}`,
keyPath: 'hashBloom',
});
}
stores.push(
{
name: `deposits_${netId}_${token}_${amount}`,
keyPath: 'leafIndex', // the key by which it refers to the object must be in all instances of the storage
indexes: DEPOSIT_INDEXES,
},
{
name: `withdrawals_${netId}_${token}_${amount}`,
keyPath: 'blockNumber',
indexes: WITHDRAWAL_INDEXES,
},
{
name: `stringify_tree_${netId}_${token}_${amount}`,
keyPath: 'hashTree',
},
);
});
});
const idb = new IndexedDB({
dbName: `tornado_core_${netId}`,
stores,
});
await idb.initDB();
return idb;
}

@ -6,6 +6,7 @@ export * from './batch';
export * from './deposits';
export * from './encryptedNotes';
export * from './fees';
export * from './idb';
export * from './merkleTree';
export * from './mimc';
export * from './multicall';
@ -18,3 +19,4 @@ export * from './tokens';
export * from './tovarishClient';
export * from './utils';
export * from './websnark';
export * from './zip';

66
src/zip.ts Normal file

@ -0,0 +1,66 @@
import { zip, unzip, AsyncZippable, Unzipped } from 'fflate';
import { fetchData } from './providers';
import { bytesToBase64, digest } from './utils';
export function zipAsync(file: AsyncZippable): Promise<Uint8Array> {
return new Promise((res, rej) => {
zip(file, { mtime: new Date('1/1/1980') }, (err, data) => {
if (err) {
rej(err);
return;
}
res(data);
});
});
}
export function unzipAsync(data: Uint8Array): Promise<Unzipped> {
return new Promise((res, rej) => {
unzip(data, {}, (err, data) => {
if (err) {
rej(err);
return;
}
res(data);
});
});
}
export async function downloadZip<T>({
staticUrl = '',
zipName,
zipDigest,
parseJson = true,
}: {
staticUrl?: string;
zipName: string;
zipDigest?: string;
parseJson?: boolean;
}): Promise<T> {
const url = `${staticUrl}/${zipName}.zip`;
const resp = (await fetchData(url, {
method: 'GET',
returnResponse: true,
})) as Response;
const data = new Uint8Array(await resp.arrayBuffer());
// If the zip has digest value, compare it
if (zipDigest) {
const hash = 'sha384-' + bytesToBase64(await digest(data));
if (zipDigest !== hash) {
const errMsg = `Invalid digest hash for file ${url}, wants ${zipDigest} has ${hash}`;
throw new Error(errMsg);
}
}
const { [zipName]: content } = await unzipAsync(data);
if (parseJson) {
return JSON.parse(new TextDecoder().decode(content)) as T;
}
return content as T;
}

@ -3096,6 +3096,11 @@ iconv-lite@^0.4.24:
dependencies:
safer-buffer ">= 2.1.2 < 3"
idb@^8.0.0:
version "8.0.0"
resolved "https://registry.yarnpkg.com/idb/-/idb-8.0.0.tgz#33d7ed894ed36e23bcb542fb701ad579bfaad41f"
integrity sha512-l//qvlAKGmQO31Qn7xdzagVPPaHTxXx199MhrAFuVBTPqydcPYBWjkrbv4Y0ktB+GmWOiwHl237UUOrLmQxLvw==
ieee754@^1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"