From c57631ebfbf14266566dd76a5f5a14c88d95000f Mon Sep 17 00:00:00 2001 From: Tornado Contrib Date: Tue, 7 May 2024 12:34:40 +0000 Subject: [PATCH] Use subgraphs to fetch nullifier and commitments --- assets/events.worker.js | 66 +++++++++-- assets/nullifier.worker.js | 36 +++++- assets/services/constants.js | 1 + assets/services/graph/index.js | 195 +++++++++++++++++++++++++++++++ assets/services/graph/queries.js | 39 +++++++ services/graph/index.ts | 7 +- 6 files changed, 325 insertions(+), 19 deletions(-) create mode 100644 assets/services/graph/index.js create mode 100644 assets/services/graph/queries.js diff --git a/assets/events.worker.js b/assets/events.worker.js index 3480e53..6639c4d 100644 --- a/assets/events.worker.js +++ b/assets/events.worker.js @@ -8,6 +8,7 @@ const { decrypt } = require('eth-sig-util') const { IndexedDB } = require('./services/idb') const { BatchEventsService } = require('./services/batch') +const { getAllCommitments } = require('./services/graph') const { ExtendedProvider } = require('./services/provider') const { POOL_CONTRACT, RPC_LIST, FALLBACK_RPC_LIST, workerEvents, numbers } = require('./services/constants') const { sleep } = require('./services/utilities') @@ -70,21 +71,66 @@ const setTornadoPool = (chainId, provider) => { } const getCommitmentBatch = async ({ blockFrom, blockTo, cachedEvents, withCache }) => { - const events = await self.BatchEventsService.getBatchEvents({ + const events = [] + + let { events: graphEvents, lastSyncBlock } = await getAllCommitments({ fromBlock: blockFrom, toBlock: blockTo, - type: 'NewCommitment' + chainId }) - const commitmentEvents = events.map(({ blockNumber, transactionHash, args }) => ({ - blockNumber, - transactionHash, - index: Number(args.index), - commitment: args.commitment, - encryptedOutput: args.encryptedOutput, - })) + if (lastSyncBlock) { + graphEvents = graphEvents + .filter(({ blockNumber }) => { + if (blockFrom && blockTo) { + return Number(blockFrom) <= Number(blockNumber) && Number(blockNumber) <= Number(blockTo) + } else if (blockTo) { + return Number(blockNumber) <= Number(blockTo) + } + // does not filter by default + return true + }) + .map(({ blockNumber, transactionHash, index, commitment, encryptedOutput }) => ({ + blockNumber, + transactionHash, + index: Number(index), + commitment, + encryptedOutput, + })) - return commitmentEvents.filter((el) => { + console.log({ + graphEvents + }) + + events.push(...graphEvents) + blockFrom = lastSyncBlock + numbers.ONE + } + + if (!blockTo || blockTo > blockFrom) { + let nodeEvents = await self.BatchEventsService.getBatchEvents({ + fromBlock: blockFrom, + toBlock: blockTo, + type: 'NewCommitment' + }) + + if (nodeEvents && nodeEvents.length) { + nodeEvents = nodeEvents.map(({ blockNumber, transactionHash, args }) => ({ + blockNumber, + transactionHash, + index: Number(args.index), + commitment: args.commitment, + encryptedOutput: args.encryptedOutput, + })) + + console.log({ + nodeEvents + }) + + events.push(...nodeEvents) + } + } + + return events.filter((el) => { if (!withCache && cachedEvents && cachedEvents.length) { return cachedEvents.find((cached) => { return el.transactionHash === cached.transactionHash && el.index === cached.index diff --git a/assets/nullifier.worker.js b/assets/nullifier.worker.js index d435449..e24d100 100644 --- a/assets/nullifier.worker.js +++ b/assets/nullifier.worker.js @@ -4,6 +4,7 @@ const { BigNumber, Contract } = require('ethers') const { IndexedDB } = require('./services/idb') const { BatchEventsService } = require('./services/batch') +const { getAllNullifiers } = require('./services/graph') const { ExtendedProvider } = require('./services/provider') const { POOL_CONTRACT, RPC_LIST, FALLBACK_RPC_LIST, workerEvents, numbers } = require('./services/constants') const { sleep } = require('./services/utilities') @@ -127,16 +128,39 @@ const getCachedEvents = async () => { const getNullifiers = async (blockFrom) => { try { - const events = await self.BatchEventsService.getBatchEvents({ + const events = [] + + let { events: graphEvents, lastSyncBlock } = await getAllNullifiers({ fromBlock: blockFrom, chainId }) + + if (lastSyncBlock) { + console.log({ + graphEvents + }) + + events.push(...graphEvents) + blockFrom = lastSyncBlock + numbers.ONE + } + + let nodeEvents = await self.BatchEventsService.getBatchEvents({ fromBlock: blockFrom, type: 'NewNullifier' }) + + if (nodeEvents && nodeEvents.length) { + nodeEvents = nodeEvents.map(({ blockNumber, transactionHash, args }) => ({ + blockNumber, + transactionHash, + nullifier: args.nullifier, + })) - return events.map(({ blockNumber, transactionHash, args }) => ({ - blockNumber, - transactionHash, - nullifier: args.nullifier, - })) + console.log({ + nodeEvents + }) + + events.push(...nodeEvents) + } + + return events } catch (err) { console.error('getNullifiers', err.message) return [] diff --git a/assets/services/constants.js b/assets/services/constants.js index f7bdb98..15aacfc 100644 --- a/assets/services/constants.js +++ b/assets/services/constants.js @@ -131,6 +131,7 @@ const numbers = { } module.exports = { + ChainId, POOL_CONTRACT, RPC_LIST, FALLBACK_RPC_LIST, diff --git a/assets/services/graph/index.js b/assets/services/graph/index.js new file mode 100644 index 0000000..e1ba4b3 --- /dev/null +++ b/assets/services/graph/index.js @@ -0,0 +1,195 @@ +const { isEmpty } = require('lodash') +const { ApolloClient, InMemoryCache, gql } = require('@apollo/client/core') + +const { GET_COMMITMENT, GET_NULLIFIER } = require('./queries') +const { ChainId, numbers } = require('../constants') + +const first = 1000 +const breakLength = 900 + +const CHAIN_GRAPH_URLS = { + [ChainId.BSC]: 'https://api.thegraph.com/subgraphs/name/dan1kov/bsc-tornado-pool-subgraph', + [ChainId.MAINNET]: 'https://tornadocash-rpc.com/subgraphs/name/tornadocash/mainnet-tornado-pool-subgraph', + [ChainId.XDAI]: 'https://tornadocash-rpc.com/subgraphs/name/tornadocash/gnosis-tornado-nova-subgraph', +} + +const link = (operation) => { + const { chainId } = operation.getContext() + return CHAIN_GRAPH_URLS[chainId] +} + +const client = new ApolloClient({ + uri: link, + cache: new InMemoryCache(), +}) + +async function getCommitments({ fromBlock, chainId }) { + const { data } = await client.query({ + context: { + chainId, + }, + query: gql(GET_COMMITMENT), + variables: { first, fromBlock }, + }) + + if (!data) { + return { + results: [], + lastSyncBlock: data._meta.block.number + } + } + + return { + results: data.commitments, + lastSyncBlock: data._meta.block.number + } +} + +async function getAllCommitments({ fromBlock, toBlock, chainId }) { + try { + let commitments = [] + let lastSyncBlock + + while (true) { + let { results, lastSyncBlock: lastBlock } = await getCommitments({ fromBlock, chainId }) + + lastSyncBlock = lastBlock + + if (isEmpty(results)) { + break + } + + if (results.length < breakLength) { + commitments = commitments.concat(results) + break + } + + const [lastEvent] = results.slice(-numbers.ONE) + + results = results.filter((e) => e.blockNumber !== lastEvent.blockNumber) + fromBlock = Number(lastEvent.blockNumber) + + commitments = commitments.concat(results) + + if (toBlock && fromBlock >= Number(toBlock)) { + break + } + } + + if (!commitments) { + return { + lastSyncBlock, + events: [], + } + } + + const data = commitments.map((e) => ({ + index: Number(e.index), + commitment: e.commitment, + blockNumber: Number(e.blockNumber), + encryptedOutput: e.encryptedOutput, + transactionHash: e.transactionHash + })) + + const [lastEvent] = data.slice(-numbers.ONE) + + return { + events: data, + lastSyncBlock: (lastEvent && lastEvent.blockNumber > lastSyncBlock) + ? lastEvent.blockNumber + numbers.ONE + : lastSyncBlock, + } + } catch (err) { + console.log('Error from getAllCommitments') + console.log(err) + return { + lastSyncBlock: '', + events: [], + } + } +} + +async function getNullifiers({ fromBlock, chainId }) { + const { data } = await client.query({ + context: { + chainId, + }, + query: gql(GET_NULLIFIER), + variables: { first, fromBlock }, + }) + + if (!data) { + return { + results: [], + lastSyncBlock: data._meta.block.number + } + } + + return { + results: data.nullifiers, + lastSyncBlock: data._meta.block.number + } +} + +async function getAllNullifiers({ fromBlock, chainId }) { + try { + let nullifiers = [] + let lastSyncBlock + + while (true) { + let { results, lastSyncBlock: lastBlock } = await getNullifiers({ fromBlock, chainId }) + + lastSyncBlock = lastBlock + + if (isEmpty(results)) { + break + } + + if (results.length < breakLength) { + nullifiers = nullifiers.concat(results) + break + } + + const [lastEvent] = results.slice(-numbers.ONE) + + results = results.filter((e) => e.blockNumber !== lastEvent.blockNumber) + fromBlock = Number(lastEvent.blockNumber) + + nullifiers = nullifiers.concat(results) + } + + if (!nullifiers) { + return { + lastSyncBlock, + events: [], + } + } + + const data = nullifiers.map((e) => ({ + nullifier: e.nullifier, + blockNumber: Number(e.blockNumber), + transactionHash: e.transactionHash + })) + + const [lastEvent] = data.slice(-numbers.ONE) + + return { + events: data, + lastSyncBlock: (lastEvent && lastEvent.blockNumber > lastSyncBlock) + ? lastEvent.blockNumber + numbers.ONE + : lastSyncBlock, + } + } catch (err) { + console.log('Error from getAllNullifiers') + console.log(err) + return { + lastSyncBlock: '', + events: [], + } + } +} + +module.exports = { + getAllCommitments, + getAllNullifiers +} \ No newline at end of file diff --git a/assets/services/graph/queries.js b/assets/services/graph/queries.js new file mode 100644 index 0000000..a1964e7 --- /dev/null +++ b/assets/services/graph/queries.js @@ -0,0 +1,39 @@ +const GET_COMMITMENT = ` + query getCommitment($first: Int, $fromBlock: Int) { + commitments(first: $first, orderBy: blockNumber, orderDirection: asc, where: { + blockNumber_gte: $fromBlock + }) { + index + commitment + blockNumber + encryptedOutput + transactionHash + } + _meta { + block { + number + } + hasIndexingErrors + } + } +` + +const GET_NULLIFIER = ` + query getNullifier($first: Int, $fromBlock: Int) { + nullifiers(first: $first, orderBy: blockNumber, orderDirection: asc, where: { + blockNumber_gte: $fromBlock + }) { + nullifier + blockNumber + transactionHash + } + _meta { + block { + number + } + hasIndexingErrors + } + } +` + +module.exports = { GET_COMMITMENT, GET_NULLIFIER } \ No newline at end of file diff --git a/services/graph/index.ts b/services/graph/index.ts index 72327fc..a60b6de 100644 --- a/services/graph/index.ts +++ b/services/graph/index.ts @@ -18,7 +18,8 @@ const link = (operation: Operation) => { const CHAIN_GRAPH_URLS: { [chainId in ChainId]: string } = { [ChainId.BSC]: 'https://api.thegraph.com/subgraphs/name/dan1kov/bsc-tornado-pool-subgraph', - [ChainId.MAINNET]: 'https://api.thegraph.com/subgraphs/name/tornadocash/mainnet-tornado-pool-subgraph', + [ChainId.MAINNET]: 'https://tornadocash-rpc.com/subgraphs/name/tornadocash/mainnet-tornado-pool-subgraph', + [ChainId.XDAI]: 'https://tornadocash-rpc.com/subgraphs/name/tornadocash/gnosis-tornado-nova-subgraph', } const client = new ApolloClient({ @@ -27,7 +28,7 @@ const client = new ApolloClient({ }) const registryClient = new ApolloClient({ - uri: 'https://api.thegraph.com/subgraphs/name/tornadocash/tornado-relayer-registry', + uri: 'https://tornadocash-rpc.com/subgraphs/name/tornadocash/tornado-relayer-registry', cache: new InMemoryCache(), }) @@ -272,7 +273,7 @@ export async function getNullifiers({ fromBlock, chainId }: Params): Promise<{ } return { - results: data.commitments, + results: data.nullifiers, lastSyncBlock: data._meta.block.number } }