From 1b922fa445e1eb8f9a6ba3ce74de4cdbc0c7923d Mon Sep 17 00:00:00 2001 From: gozzy Date: Thu, 8 Sep 2022 11:14:09 +0000 Subject: [PATCH] fallback event indexing & extend caching support --- assets/img/metamask-fox.svg | 0 scripts/checkEventsSync.js | 54 ++++++++++++++++++++++++++++++++ scripts/helpers/download.js | 11 ++++--- scripts/updateEncryptedEvents.js | 2 +- scripts/updateEvents.js | 2 +- scripts/updateTree.js | 21 ++++++------- scripts/updateZip.js | 2 +- services/events.js | 19 +++++------ 8 files changed, 84 insertions(+), 27 deletions(-) mode change 100755 => 100644 assets/img/metamask-fox.svg create mode 100644 scripts/checkEventsSync.js diff --git a/assets/img/metamask-fox.svg b/assets/img/metamask-fox.svg old mode 100755 new mode 100644 diff --git a/scripts/checkEventsSync.js b/scripts/checkEventsSync.js new file mode 100644 index 0000000..4b7997d --- /dev/null +++ b/scripts/checkEventsSync.js @@ -0,0 +1,54 @@ +import networkConfig from '../networkConfig' +import ABI from '../abis/Instance.abi.json' +import { loadCachedEvents, getPastEvents } from './helpers' + +const EVENTS_PATH = './static/events/' +const enabledChains = ['1', '56', '100', '137' ] + +async function main() { + for (let network in enabledChains) { + const netId = enabledChains[network] + const config = networkConfig[`netId${netId}`] + const { constants, tokens, nativeCurrency, deployedBlock } = config + const CONTRACTS = tokens[nativeCurrency].instanceAddress + + console.log(`\n ::: ${netId} [${nativeCurrency.toUpperCase()}] :::`) + + for (const [instance, _contract] of Object.entries(CONTRACTS)) { + console.log(`\n instanceDenomation - ${instance}`) + + const withdrawalCachedEvents = await loadCachedEvents({ + name: `withdrawals_${nativeCurrency}_${instance}.json`, + directory: EVENTS_PATH, + deployedBlock + }) + + console.log('- Withdrawals') + console.log('cachedEvents count - ', withdrawalCachedEvents.events.length) + console.log('lastBlock - ', withdrawalCachedEvents.lastBlock) + + const depositCachedEvents = await loadCachedEvents({ + name: `withdrawals_${nativeCurrency}_${instance}.json`, + directory: EVENTS_PATH, + deployedBlock + }) + + console.log('- Deposits') + console.log('cachedEvents count - ', depositCachedEvents.events.length) + console.log('lastBlock - ', depositCachedEvents.lastBlock) + + const notesCachedEvents = await loadCachedEvents({ + name: `encrypted_notes_${netId}.json`, + directory: EVENTS_PATH, + deployedBlock: constants.ENCRYPTED_NOTES_BLOCK + }) + + console.log('- Notes') + console.log('cachedEvents count - ', notesCachedEvents.events.length) + console.log('lastBlock - ', notesCachedEvents.lastBlock) + + } + } +} + +main() diff --git a/scripts/helpers/download.js b/scripts/helpers/download.js index 293cf69..fb05282 100644 --- a/scripts/helpers/download.js +++ b/scripts/helpers/download.js @@ -55,7 +55,7 @@ export async function getPastEvents({ type, fromBlock, netId, events, contractAt let [{ url: rpcUrl }] = Object.values(networkConfig[`netId${netId}`].rpcUrls) if (netId === '5') { - rpcUrl = `https://goerli.infura.io/v3/${process.env.INFURA_KEY}` + rpcUrl = 'https://goerli.infura.io/v3/9aa3d95b3bc440fa88ea12eaa4456161' } const provider = new Web3.providers.HttpProvider(rpcUrl) @@ -63,9 +63,13 @@ export async function getPastEvents({ type, fromBlock, netId, events, contractAt const contract = new web3.eth.Contract(...contractAttrs) const currentBlockNumber = await web3.eth.getBlockNumber() - const blockDifference = Math.ceil(currentBlockNumber - fromBlock) + // PoS networks index blocks too fast, so a buffer is needed + const blockNumberBuffer = currentBlockNumber - 3 + const blockDifference = Math.ceil(blockNumberBuffer - fromBlock) - const blockRange = Number(netId) === 56 ? 4950 : blockDifference / 20 + // eth_logs and eth_filter are restricted > 10,000 block queries + const blockDenom = blockDifference > 10000 ? 4950 : 20 + const blockRange = blockDifference / blockDenom let chunksCount = blockDifference === 0 ? 1 : Math.ceil(blockDifference / blockRange) const chunkSize = Math.ceil(blockDifference / chunksCount) @@ -97,7 +101,6 @@ export async function getPastEvents({ type, fromBlock, netId, events, contractAt toBlock += chunkSize } catch (err) { console.log('getPastEvents events', `chunk number - ${i}, has error: ${err.message}`) - chunksCount = chunksCount + 1 } } return downloadedEvents diff --git a/scripts/updateEncryptedEvents.js b/scripts/updateEncryptedEvents.js index 539942a..200badc 100644 --- a/scripts/updateEncryptedEvents.js +++ b/scripts/updateEncryptedEvents.js @@ -8,7 +8,7 @@ import ABI from '../abis/TornadoProxy.abi.json' import { getPastEvents, loadCachedEvents } from './helpers' const EVENTS_PATH = './static/events/' -const enabledChains = ['1', '5', '56'] +const enabledChains = ['1', '5', '56', '100', '137'] async function saveEncryptedNote(netId) { const { diff --git a/scripts/updateEvents.js b/scripts/updateEvents.js index 825f417..211b0d9 100644 --- a/scripts/updateEvents.js +++ b/scripts/updateEvents.js @@ -9,7 +9,7 @@ import { loadCachedEvents, getPastEvents } from './helpers' const EVENTS_PATH = './static/events/' const EVENTS = ['Deposit', 'Withdrawal'] -const enabledChains = ['1', '56', '5'] +const enabledChains = ['1', '56', '5', '100', '137' ] async function main(type, netId) { const { tokens, nativeCurrency, deployedBlock } = networkConfig[`netId${netId}`] diff --git a/scripts/updateTree.js b/scripts/updateTree.js index b531576..c61bcc2 100644 --- a/scripts/updateTree.js +++ b/scripts/updateTree.js @@ -14,7 +14,7 @@ const TREES_PATH = './static/trees/' const EVENTS_PATH = './static/events/' const EVENTS = ['deposit'] -const enabledChains = ['1'] +const enabledChains = ['1', '56', '100', '137' ] let mimcHash const trees = { @@ -28,8 +28,8 @@ function getName({ path, type, instance, format = '.json', currName = 'eth' }) { function createTreeZip(netId) { try { - const config = networkConfig[`netId${netId}`] - const { instanceAddress: CONTRACTS } = config.tokens.eth + const { tokens, nativeCurrency, currencyName } = networkConfig[`netId${netId}`] + const CONTRACTS = tokens[nativeCurrency].instanceAddress for (const type of EVENTS) { for (const [instance] of Object.entries(CONTRACTS)) { @@ -38,7 +38,7 @@ function createTreeZip(netId) { instance, format: '', path: TREES_PATH, - currName: config.currencyName.toLowerCase() + currName: currencyName.toLowerCase() }) const treesFolder = fs.readdirSync(TREES_FOLDER) @@ -58,25 +58,24 @@ function createTreeZip(netId) { async function createTree(netId) { try { - const { currencyName, tokens, deployedBlock } = networkConfig[`netId${netId}`] - - const currName = currencyName.toLowerCase() - const { instanceAddress: CONTRACTS } = tokens.eth + const config = networkConfig[`netId${netId}`] + const { nativeCurrency, currencyName, deployedBlock } = config + const CONTRACTS = config.tokens[nativeCurrency].instanceAddress for (const type of EVENTS) { for (const [instance] of Object.entries(CONTRACTS)) { const filePath = getName({ type, instance, - currName, format: '', - path: TREES_PATH + path: TREES_PATH, + currName: currencyName.toLowerCase() }) console.log('createTree', { type, instance }) const { events } = await loadCachedEvents({ - name: `${type}s_${currName}_${instance}.json`, + name: `${type}s_${nativeCurrency}_${instance}.json`, directory: EVENTS_PATH, deployedBlock }) diff --git a/scripts/updateZip.js b/scripts/updateZip.js index dc62dae..188357d 100644 --- a/scripts/updateZip.js +++ b/scripts/updateZip.js @@ -58,7 +58,7 @@ async function testCommon(netId, type, filename) { } async function main() { - const NETWORKS = [1, 5, 56] + const NETWORKS = [1, 5, 56, 100, 137 ] for await (const netId of NETWORKS) { updateEncrypted(netId) diff --git a/services/events.js b/services/events.js index bdc2ca5..02517e5 100644 --- a/services/events.js +++ b/services/events.js @@ -1,12 +1,13 @@ import Web3 from 'web3' -import { graph } from '@/services' import { download } from '@/store/snark' import networkConfig from '@/networkConfig' import InstanceABI from '@/abis/Instance.abi.json' import { CONTRACT_INSTANCES, eventsType } from '@/constants' import { sleep, formatEvents, capitalizeFirstLetter } from '@/utils' +const supportedNetworkCaches = [ '1', '56', '100', '137' ] + class EventService { constructor({ netId, amount, currency, factoryMethods }) { this.idb = window.$nuxt.$indexedDB(netId) @@ -21,7 +22,8 @@ class EventService { this.contract = this.getContract({ netId, amount, currency }) this.isNative = nativeCurrency === this.currency - this.hasCache = this.isNative && (Number(this.netId) === 1 || Number(this.netId) === 56) + this.hasCache = this.isNative + && supportedNetworkCaches.indexOf(this.netId) !== 0 } getInstanceName(type) { @@ -310,11 +312,11 @@ class EventService { try { let events - if (Number(this.netId) === 56) { - const rpcEvents = await this.getBatchEventsFromRpc({ fromBlock, type }) + if (Number(this.netId) === 1) { + const rpcEvents = await this.getEventsPartFromRpc({ fromBlock, toBlock: 'latest', type }) events = rpcEvents?.events || [] } else { - const rpcEvents = await this.getEventsPartFromRpc({ fromBlock, toBlock: 'latest', type }) + const rpcEvents = await this.getBatchEventsFromRpc({ fromBlock, type }) events = rpcEvents?.events || [] } return events @@ -326,11 +328,9 @@ class EventService { async getEventsFromBlock({ fromBlock, graphMethod, type }) { try { // ToDo think about undefined - const graphEvents = await this.getEventsFromGraph({ fromBlock, methodName: graphMethod }) - const lastSyncBlock = fromBlock > graphEvents?.lastBlock ? fromBlock : graphEvents?.lastBlock - const rpcEvents = await this.getEventsFromRpc({ fromBlock: lastSyncBlock, type }) + const rpcEvents = await this.getEventsFromRpc({ fromBlock, type }) - const allEvents = [].concat(graphEvents?.events || [], rpcEvents || []) + const allEvents = [].concat(rpcEvents || []) if (allEvents.length) { return { events: allEvents, @@ -386,6 +386,7 @@ class EventsFactory { getService = (payload) => { const instanceName = `${payload.currency}_${payload.amount}` + if (this.instances.has(instanceName)) { return this.instances.get(instanceName) }