From 8fcb9ed387063afe5fde17f5c29ef6b7529f5c1a Mon Sep 17 00:00:00 2001 From: Tornado Contrib Date: Wed, 8 May 2024 18:57:13 +0000 Subject: [PATCH] Use cached events from frontend and workers --- README.md | 6 ++++ assets/events.worker.js | 9 ++++++ assets/nullifier.worker.js | 9 ++++++ assets/services/downloadEvents.js | 37 +++++++++++++++++++++++ services/events/index.ts | 40 ++++++++++++++++++++----- services/events/load.ts | 49 +++++++++++++++++++++++++++++++ 6 files changed, 143 insertions(+), 7 deletions(-) create mode 100644 assets/services/downloadEvents.js create mode 100644 services/events/load.ts diff --git a/README.md b/README.md index 45436f7..43f9cca 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,12 @@ ## Build Setup +If you use the latest Node.js version, you should modify your NODE_OPTIONS env + +```bash +export NODE_OPTIONS="--openssl-legacy-provider" +``` + ```bash # install dependencies $ yarn install diff --git a/assets/events.worker.js b/assets/events.worker.js index 1f886eb..6e150bb 100644 --- a/assets/events.worker.js +++ b/assets/events.worker.js @@ -11,6 +11,7 @@ import { ExtendedProvider } from './services/provider' import { POOL_CONTRACT, RPC_LIST, FALLBACK_RPC_LIST, workerEvents, numbers } from './services/constants' import { sleep } from './services/utilities' import { poolAbi } from './services/pool' +import { downloadEvents } from './services/downloadEvents' const getProviderWithSigner = (chainId) => { return new ExtendedProvider(RPC_LIST[chainId], chainId, FALLBACK_RPC_LIST[chainId]) @@ -165,6 +166,14 @@ const getCommitments = async ({ withCache, lastSyncBlock }) => { return { commitmentEvents: cachedEvents } } blockFrom = newBlockFrom > currentBlock ? currentBlock : newBlockFrom + } else { + const downloadedEvents = await downloadEvents(`commitments_${self.chainId}.json`, blockFrom) + + if (downloadedEvents.events.length) { + cachedEvents.push(...downloadedEvents.events) + + blockFrom = downloadedEvents.lastBlock + } } const commitmentEvents = await getCommitmentBatch({ blockFrom, blockTo: currentBlock, cachedEvents, withCache }) diff --git a/assets/nullifier.worker.js b/assets/nullifier.worker.js index 7bcf302..73b7f1d 100644 --- a/assets/nullifier.worker.js +++ b/assets/nullifier.worker.js @@ -8,6 +8,7 @@ import { ExtendedProvider } from './services/provider' import { POOL_CONTRACT, RPC_LIST, FALLBACK_RPC_LIST, workerEvents, numbers } from './services/constants' import { sleep } from './services/utilities' import { poolAbi } from './services/pool' +import { downloadEvents } from './services/downloadEvents' const getProviderWithSigner = (chainId) => { return new ExtendedProvider(RPC_LIST[chainId], chainId, FALLBACK_RPC_LIST[chainId]) @@ -120,6 +121,14 @@ const getCachedEvents = async () => { return { blockFrom, cachedEvents } } blockFrom = newBlockFrom > currentBlock ? currentBlock : newBlockFrom + } else { + const downloadedEvents = await downloadEvents(`nullifiers_${self.chainId}.json`, blockFrom) + + if (downloadedEvents.events.length) { + cachedEvents.push(...downloadedEvents.events) + + blockFrom = downloadedEvents.lastBlock + } } return { blockFrom, cachedEvents } diff --git a/assets/services/downloadEvents.js b/assets/services/downloadEvents.js new file mode 100644 index 0000000..5f312a2 --- /dev/null +++ b/assets/services/downloadEvents.js @@ -0,0 +1,37 @@ +import { unzipAsync } from "./zip" + +export async function downloadEvents(fileName, deployedBlock) { + fileName = fileName.toLowerCase() + + // @ts-ignore + const prefix = __webpack_public_path__.slice(0, -7) + + try { + const resp = await fetch(`${prefix}/${fileName}.zip`, { + method: 'GET', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + } + }) + + const arrayBuffer = await resp.arrayBuffer() + + const { [fileName]: content } = await unzipAsync(new Uint8Array(arrayBuffer)) + + const events = JSON.parse(new TextDecoder().decode(content)) + + const lastBlock = events && Array.isArray(events) && events[events.length - 1] + ? events[events.length - 1].blockNumber + : deployedBlock + + return { + events, + lastBlock + } + } catch { + return { + events: [], + lastBlock: deployedBlock + } + } +} \ No newline at end of file diff --git a/services/events/index.ts b/services/events/index.ts index b0c10cb..40bd0ea 100644 --- a/services/events/index.ts +++ b/services/events/index.ts @@ -7,6 +7,7 @@ import { isEmpty, sleep, toChecksumAddress } from '@/utilities' import { getBridgeHelper, getBridgeProxy, getAmbBridge } from '@/contracts' import { EventsClass, GetAffirmationParams, GetRelayedMessageParams, SaveEventsParams } from './@types' +import { downloadEvents } from './load' export * from './batch' @@ -33,19 +34,31 @@ class EventAggregator implements EventsClass { storeName: `${IndexDBStores.ACCOUNT_EVENTS}_${chainId}`, }) + const newEvents = [] + if (cachedEvents?.length) { const [latestEvent] = cachedEvents.slice(-numbers.ONE) blockFrom = Number(latestEvent.blockNumber) + numbers.ONE + } else { + const downloadedEvents = await downloadEvents(`accounts_${chainId}.json`, blockFrom) + + if (downloadedEvents.events.length) { + newEvents.push(...downloadedEvents.events) + + blockFrom = downloadedEvents.lastBlock + } } const { events: graphEvents, lastSyncBlock } = await getAllAccounts({ fromBlock: blockFrom, chainId }) - const [account] = graphEvents.filter((e: { key: string }) => e.key === publicKey) + newEvents.push(...graphEvents) + + const [account] = newEvents.filter((e: { key: string }) => e.key === publicKey) if (account) { this.saveEvents({ chainId, - events: graphEvents, + events: newEvents, storeName: IndexDBStores.ACCOUNT_EVENTS, }) return account.owner @@ -68,7 +81,7 @@ class EventAggregator implements EventsClass { } }) - const newEvents = graphEvents.concat(accountEvents) + newEvents.push(...accountEvents) this.saveEvents({ chainId, @@ -76,7 +89,7 @@ class EventAggregator implements EventsClass { storeName: IndexDBStores.ACCOUNT_EVENTS, }) - const events = cachedEvents.concat(newEvents).filter((e: { key: string }) => e.key === publicKey) + const events = newEvents.filter((e: { key: string }) => e.key === publicKey) if (isEmpty(events)) { return undefined @@ -87,6 +100,7 @@ class EventAggregator implements EventsClass { return event.owner } catch (err) { + console.log(err) return undefined } } @@ -113,19 +127,30 @@ class EventAggregator implements EventsClass { storeName: `${IndexDBStores.ACCOUNT_EVENTS}_${chainId}`, }) + const newEvents = [] + if (cachedEvents?.length) { const [latestEvent] = cachedEvents.slice(-numbers.ONE) blockFrom = Number(latestEvent.blockNumber) + numbers.ONE + } else { + const downloadedEvents = await downloadEvents(`accounts_${chainId}.json`, blockFrom) + + if (downloadedEvents.events.length) { + newEvents.push(...downloadedEvents.events) + + blockFrom = downloadedEvents.lastBlock + } } const { events: graphEvents, lastSyncBlock } = await getAllAccounts({ fromBlock: blockFrom, chainId }) + newEvents.push(...graphEvents) - const [account] = graphEvents.filter((e: { owner: string }) => toChecksumAddress(e.owner) === toChecksumAddress(address)) + const [account] = newEvents.filter((e: { owner: string }) => toChecksumAddress(e.owner) === toChecksumAddress(address)) if (account) { this.saveEvents({ chainId, - events: graphEvents, + events: newEvents, storeName: IndexDBStores.ACCOUNT_EVENTS, }) return account.key @@ -148,7 +173,7 @@ class EventAggregator implements EventsClass { } }) - const newEvents = graphEvents.concat(accountEvents) + newEvents.push(...accountEvents) this.saveEvents({ chainId, @@ -169,6 +194,7 @@ class EventAggregator implements EventsClass { return event.key } catch (err) { + console.log(err) return undefined } } diff --git a/services/events/load.ts b/services/events/load.ts new file mode 100644 index 0000000..332ce02 --- /dev/null +++ b/services/events/load.ts @@ -0,0 +1,49 @@ +import { unzip } from 'fflate' + +export function unzipAsync(data: Uint8Array) { + return new Promise((res, rej) => { + unzip(data, {}, (err, data) => { + if (err) { + rej(err); + return; + } + res(data); + }); + }); +} + +export async function downloadEvents(fileName: string, deployedBlock: number) { + fileName = fileName.toLowerCase() + + // @ts-ignore + const prefix = __webpack_public_path__.slice(0, -7) + + try { + const resp = await fetch(`${prefix}/${fileName}.zip`, { + method: 'GET', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + } + }) + + const arrayBuffer = await resp.arrayBuffer() + + const { [fileName]: content } = (await unzipAsync(new Uint8Array(arrayBuffer))) as any + + const events = JSON.parse(new TextDecoder().decode(content)) + + const lastBlock = events && Array.isArray(events) && events[events.length - 1] + ? events[events.length - 1].blockNumber + : deployedBlock + + return { + events, + lastBlock + } + } catch { + return { + events: [], + lastBlock: deployedBlock + } + } +} \ No newline at end of file