From 48774c321e1912a5a83ff4cbba49c60ef6e2021e Mon Sep 17 00:00:00 2001 From: tornadocontrib Date: Mon, 21 Oct 2024 23:11:07 +0000 Subject: [PATCH] Optional tree generation --- dist/cli.js | 201 +++++++++++++++++++++++++--------- dist/services/nodeEvents.d.ts | 4 +- package.json | 2 +- src/program.ts | 2 + src/services/nodeEvents.ts | 9 +- yarn.lock | 4 +- 6 files changed, 162 insertions(+), 60 deletions(-) diff --git a/dist/cli.js b/dist/cli.js index 0d5b4b3..480b4f1 100644 --- a/dist/cli.js +++ b/dist/cli.js @@ -181016,7 +181016,7 @@ const defaultConfig = { multicallContract: "0xcA11bde05977b3631167028862bE2a173976CA11", routerContract: "0xd90e2f925DA726b50C4Ed8D0Fb90Ad053324F31b", echoContract: "0x9B27DD5Bb15d42DC224FCD0B7caEbBe16161Df42", - offchainOracleContract: "0x0AdDd25a91563696D8567Df78D5A01C9a991F9B8", + offchainOracleContract: "0x00000000000D6FFc74A8feb35aF5827bf57f6786", tornContract: "0x77777FeDdddFfC19Ff86DB637967013e6C6A116C", governanceContract: "0x5efda50f22d34F262c29268506C5Fa42cB56A1Ce", stakingRewardsContract: "0x5B3f656C80E8ddb9ec01Dd9018815576E9238c29", @@ -181130,7 +181130,7 @@ const defaultConfig = { multicallContract: "0xcA11bde05977b3631167028862bE2a173976CA11", routerContract: "0x0D5550d52428E7e3175bfc9550207e4ad3859b17", echoContract: "0xa75BF2815618872f155b7C4B0C81bF990f5245E4", - offchainOracleContract: "0x0AdDd25a91563696D8567Df78D5A01C9a991F9B8", + offchainOracleContract: "0x00000000000D6FFc74A8feb35aF5827bf57f6786", tornadoSubgraph: "tornadocash/bsc-tornado-subgraph", subgraphs: {}, rpcUrls: { @@ -181193,7 +181193,7 @@ const defaultConfig = { multicallContract: "0xcA11bde05977b3631167028862bE2a173976CA11", routerContract: "0x0D5550d52428E7e3175bfc9550207e4ad3859b17", echoContract: "0xa75BF2815618872f155b7C4B0C81bF990f5245E4", - offchainOracleContract: "0x0AdDd25a91563696D8567Df78D5A01C9a991F9B8", + offchainOracleContract: "0x00000000000D6FFc74A8feb35aF5827bf57f6786", tornadoSubgraph: "tornadocash/matic-tornado-subgraph", subgraphs: {}, rpcUrls: { @@ -181244,7 +181244,7 @@ const defaultConfig = { multicallContract: "0xcA11bde05977b3631167028862bE2a173976CA11", routerContract: "0x0D5550d52428E7e3175bfc9550207e4ad3859b17", echoContract: "0xa75BF2815618872f155b7C4B0C81bF990f5245E4", - offchainOracleContract: "0x0AdDd25a91563696D8567Df78D5A01C9a991F9B8", + offchainOracleContract: "0x00000000000D6FFc74A8feb35aF5827bf57f6786", ovmGasPriceOracleContract: "0x420000000000000000000000000000000000000F", tornadoSubgraph: "tornadocash/optimism-tornado-subgraph", subgraphs: {}, @@ -181296,7 +181296,7 @@ const defaultConfig = { multicallContract: "0xcA11bde05977b3631167028862bE2a173976CA11", routerContract: "0x0D5550d52428E7e3175bfc9550207e4ad3859b17", echoContract: "0xa75BF2815618872f155b7C4B0C81bF990f5245E4", - offchainOracleContract: "0x0AdDd25a91563696D8567Df78D5A01C9a991F9B8", + offchainOracleContract: "0x00000000000D6FFc74A8feb35aF5827bf57f6786", tornadoSubgraph: "tornadocash/arbitrum-tornado-subgraph", subgraphs: {}, rpcUrls: { @@ -181351,7 +181351,7 @@ const defaultConfig = { multicallContract: "0xcA11bde05977b3631167028862bE2a173976CA11", routerContract: "0x0D5550d52428E7e3175bfc9550207e4ad3859b17", echoContract: "0xa75BF2815618872f155b7C4B0C81bF990f5245E4", - offchainOracleContract: "0x0AdDd25a91563696D8567Df78D5A01C9a991F9B8", + offchainOracleContract: "0x00000000000D6FFc74A8feb35aF5827bf57f6786", tornadoSubgraph: "tornadocash/xdai-tornado-subgraph", subgraphs: {}, rpcUrls: { @@ -181402,7 +181402,7 @@ const defaultConfig = { multicallContract: "0xcA11bde05977b3631167028862bE2a173976CA11", routerContract: "0x0D5550d52428E7e3175bfc9550207e4ad3859b17", echoContract: "0xa75BF2815618872f155b7C4B0C81bF990f5245E4", - offchainOracleContract: "0x0AdDd25a91563696D8567Df78D5A01C9a991F9B8", + offchainOracleContract: "0x00000000000D6FFc74A8feb35aF5827bf57f6786", tornadoSubgraph: "tornadocash/avalanche-tornado-subgraph", subgraphs: {}, rpcUrls: { @@ -181452,6 +181452,7 @@ const defaultConfig = { multicallContract: "0xcA11bde05977b3631167028862bE2a173976CA11", routerContract: "0x1572AFE6949fdF51Cb3E0856216670ae9Ee160Ee", echoContract: "0xa75BF2815618872f155b7C4B0C81bF990f5245E4", + offchainOracleContract: "0x1f89EAF03E5b260Bc6D4Ae3c3334b1B750F3e127", tornContract: "0x3AE6667167C0f44394106E197904519D808323cA", governanceContract: "0xe5324cD7602eeb387418e594B87aCADee08aeCAD", stakingRewardsContract: "0x6d0018890751Efd31feb8166711B16732E2b496b", @@ -182299,10 +182300,15 @@ class BaseEventsService { lastBlock: rpcEvents.lastBlock }; } - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async validateEvents({ events, lastBlock }) { + /* eslint-disable @typescript-eslint/no-unused-vars */ + async validateEvents({ + events, + lastBlock, + hasNewEvents + }) { return void 0; } + /* eslint-enable @typescript-eslint/no-unused-vars */ /** * Handle saving events */ @@ -182332,7 +182338,11 @@ class BaseEventsService { return !hasEvent; }); const lastBlock = newEvents.lastBlock || allEvents[allEvents.length - 1]?.blockNumber; - const validateResult = await this.validateEvents({ events: allEvents, lastBlock }); + const validateResult = await this.validateEvents({ + events: allEvents, + lastBlock, + hasNewEvents: Boolean(newEvents.events.length) + }); if (savedEvents.fromCache || newEvents.events.length) { await this.saveEvents({ events: allEvents, lastBlock }); } @@ -182346,17 +182356,19 @@ class BaseEventsService { class BaseTornadoService extends BaseEventsService { amount; currency; + optionalTree; merkleTreeService; batchTransactionService; batchBlockService; constructor(serviceConstructor) { - const { Tornado: contract, amount, currency, provider, merkleTreeService } = serviceConstructor; + const { Tornado: contract, amount, currency, provider, optionalTree, merkleTreeService } = serviceConstructor; super({ ...serviceConstructor, contract }); this.amount = amount; this.currency = currency; + this.optionalTree = optionalTree; this.merkleTreeService = merkleTreeService; this.batchTransactionService = new BatchTransactionService({ provider, @@ -182431,7 +182443,10 @@ class BaseTornadoService extends BaseEventsService { }); } } - async validateEvents({ events }) { + async validateEvents({ + events, + hasNewEvents + }) { if (events.length && this.getType().toLowerCase() === DEPOSIT) { const depositEvents = events; const lastEvent = depositEvents[depositEvents.length - 1]; @@ -182439,7 +182454,7 @@ class BaseTornadoService extends BaseEventsService { const errMsg = `Deposit events invalid wants ${depositEvents.length - 1} leafIndex have ${lastEvent.leafIndex}`; throw new Error(errMsg); } - if (this.merkleTreeService) { + if (this.merkleTreeService && (!this.optionalTree || hasNewEvents)) { return await this.merkleTreeService.verifyTree(depositEvents); } } @@ -183043,8 +183058,14 @@ async function saveDBEvents({ lastBlock }) { try { + const formattedEvents = events.map((e) => { + return { + eid: `${e.transactionHash}_${e.logIndex}`, + ...e + }; + }); await idb.createMultipleTransactions({ - data: events, + data: formattedEvents, storeName: instanceName }); await idb.putItem({ @@ -183074,8 +183095,12 @@ async function loadDBEvents({ lastBlock: 0 }; } + const events = (await idb.getAll({ storeName: instanceName })).map((e) => { + delete e.eid; + return e; + }); return { - events: await idb.getAll({ storeName: instanceName }), + events, lastBlock: lastBlockStore.blockNumber }; } catch (err) { @@ -189131,7 +189156,7 @@ class IndexedDB { } }; this.dbName = dbName; - this.dbVersion = 34; + this.dbVersion = 35; } async initDB() { try { @@ -189325,24 +189350,42 @@ async function getIndexedDB(netId) { await idb2.initDB(); return idb2; } - const DEPOSIT_INDEXES = [ - { name: "transactionHash", unique: false }, - { name: "commitment", unique: true } + const minimalIndexes = [ + { + name: "blockNumber", + unique: false + }, + { + name: "transactionHash", + unique: false + } ]; - const WITHDRAWAL_INDEXES = [ - { name: "nullifierHash", unique: true } - // keys on which the index is created - ]; - const LAST_EVENT_INDEXES = [{ name: "name", unique: false }]; const defaultState = [ { - name: "encrypted_events", - keyPath: "transactionHash" + name: `echo_${netId}`, + keyPath: "eid", + indexes: [ + ...minimalIndexes, + { + name: "address", + unique: false + } + ] + }, + { + name: `encrypted_notes_${netId}`, + keyPath: "eid", + indexes: minimalIndexes }, { name: "lastEvents", keyPath: "name", - indexes: LAST_EVENT_INDEXES + indexes: [ + { + name: "name", + unique: false + } + ] } ]; const config = getConfig(netId); @@ -189350,33 +189393,68 @@ async function getIndexedDB(netId) { const stores = [...defaultState]; if (netId === NetId.MAINNET) { stores.push({ - name: "register_events", - keyPath: "ensName" + name: `registered_${netId}`, + keyPath: "ensName", + indexes: [ + ...minimalIndexes, + { + name: "relayerAddress", + unique: false + } + ] + }); + stores.push({ + name: `governance_${netId}`, + keyPath: "eid", + indexes: [ + ...minimalIndexes, + { + name: "event", + unique: false + } + ] }); } Object.entries(tokens).forEach(([token, { instanceAddress }]) => { Object.keys(instanceAddress).forEach((amount) => { if (nativeCurrency === token) { - stores.push({ - name: `stringify_bloom_${netId}_${token}_${amount}`, - keyPath: "hashBloom" - }); + stores.push( + { + name: `stringify_bloom_${netId}_${token}_${amount}`, + keyPath: "hashBloom", + indexes: [] + }, + { + name: `stringify_tree_${netId}_${token}_${amount}`, + keyPath: "hashTree", + indexes: [] + } + ); } stores.push( { name: `deposits_${netId}_${token}_${amount}`, keyPath: "leafIndex", // the key by which it refers to the object must be in all instances of the storage - indexes: DEPOSIT_INDEXES + indexes: [ + ...minimalIndexes, + { + name: "commitment", + unique: true + } + ] }, { name: `withdrawals_${netId}_${token}_${amount}`, - keyPath: "blockNumber", - indexes: WITHDRAWAL_INDEXES - }, - { - name: `stringify_tree_${netId}_${token}_${amount}`, - keyPath: "hashTree" + keyPath: "eid", + indexes: [ + ...minimalIndexes, + { + name: "nullifierHash", + unique: true + } + // keys on which the index is created + ] } ); }); @@ -189589,16 +189667,19 @@ class TokenPriceOracle { oracle; multicall; provider; + fallbackPrice; constructor(provider, multicall2, oracle) { this.provider = provider; this.multicall = multicall2; this.oracle = oracle; + this.fallbackPrice = parseEther("0.0001"); } buildCalls(tokens) { return tokens.map(({ tokenAddress }) => ({ contract: this.oracle, name: "getRateToEth", - params: [tokenAddress, true] + params: [tokenAddress, true], + allowFailure: true })); } buildStable(stablecoinAddress) { @@ -189611,32 +189692,42 @@ class TokenPriceOracle { { contract: this.oracle, name: "getRateToEth", - params: [stablecoin.target, true] + params: [stablecoin.target, true], + allowFailure: true } ]; } async fetchPrice(tokenAddress, decimals) { if (!this.oracle) { - return new Promise((resolve) => resolve(parseEther("0.0001"))); + return new Promise((resolve) => resolve(this.fallbackPrice)); + } + try { + const price = await this.oracle.getRateToEth(tokenAddress, true); + return price * BigInt(10 ** decimals) / BigInt(10 ** 18); + } catch (err) { + console.log(`Failed to fetch oracle price for ${tokenAddress}, will use fallback price ${this.fallbackPrice}`); + console.log(err); + return this.fallbackPrice; } - const price = await this.oracle.getRateToEth(tokenAddress, true); - return price * BigInt(10 ** decimals) / BigInt(10 ** 18); } async fetchPrices(tokens) { if (!this.oracle) { - return new Promise((resolve) => resolve(tokens.map(() => parseEther("0.0001")))); + return new Promise((resolve) => resolve(tokens.map(() => this.fallbackPrice))); } const prices = await multicall(this.multicall, this.buildCalls(tokens)); return prices.map((price, index) => { + if (!price) { + price = this.fallbackPrice; + } return price * BigInt(10 ** tokens[index].decimals) / BigInt(10 ** 18); }); } async fetchEthUSD(stablecoinAddress) { if (!this.oracle) { - return new Promise((resolve) => resolve(1e4)); + return new Promise((resolve) => resolve(10 ** 18 / Number(this.fallbackPrice))); } const [decimals, price] = await multicall(this.multicall, this.buildStable(stablecoinAddress)); - const ethPrice = price * BigInt(10n ** decimals) / BigInt(10 ** 18); + const ethPrice = (price || this.fallbackPrice) * BigInt(10n ** decimals) / BigInt(10 ** 18); return 1 / Number(formatEther(ethPrice)); } } @@ -190237,8 +190328,12 @@ class NodeTornadoService extends BaseTornadoService { async getEventsFromCache() { return await getEventsFromCache(this); } - async validateEvents({ events }) { - const tree = await super.validateEvents({ events }); + async validateEvents({ + events, + lastBlock, + hasNewEvents + }) { + const tree = await super.validateEvents({ events, lastBlock, hasNewEvents }); if (tree && this.currency === this.nativeCurrency && this.treeCache) { const merkleTree = tree; await this.treeCache.createTree(events, merkleTree); @@ -191592,7 +191687,8 @@ Connected with Tovarish Relayer ${tovarishClient.selectedRelayer.url} currency, Tornado, merkleWorkerPath - }) + }), + optionalTree: true }); const withdrawalsService = new NodeTornadoService({ ...TornadoServiceConstructor, @@ -191755,7 +191851,8 @@ Connected with Tovarish Relayer ${tovarishClient.selectedRelayer.url} amount, currency, userDirectory: SAVED_TREE_DIR - }) + }), + optionalTree: true }); const withdrawalsService = new NodeTornadoService({ ...TornadoServiceConstructor, diff --git a/dist/services/nodeEvents.d.ts b/dist/services/nodeEvents.d.ts index 48e1f94..2ee8fd1 100644 --- a/dist/services/nodeEvents.d.ts +++ b/dist/services/nodeEvents.d.ts @@ -21,8 +21,8 @@ export declare class NodeTornadoService extends BaseTornadoService { updateGraphProgress({ type, fromBlock, toBlock, count }: Parameters[0]): void; getEventsFromDB(): Promise>; getEventsFromCache(): Promise>; - validateEvents({ events }: { - events: (DepositsEvents | WithdrawalsEvents)[]; + validateEvents({ events, lastBlock, hasNewEvents, }: BaseEvents & { + hasNewEvents?: boolean; }): Promise; saveEvents({ events, lastBlock }: BaseEvents): Promise; } diff --git a/package.json b/package.json index 22bb15a..0beec62 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "optionalDependencies": {}, "devDependencies": { "@colors/colors": "^1.6.0", - "@tornado/core": "git+https://git.tornado.ws/tornadocontrib/tornado-core.git#dfb20aa61753c4f29ab144517ff48959e1b33c40", + "@tornado/core": "git+https://git.tornado.ws/tornadocontrib/tornado-core.git#9f4044d11033e99e0a863989864d1578a93de531", "@typechain/ethers-v6": "^0.5.1", "@types/figlet": "^1.7.0", "@typescript-eslint/eslint-plugin": "^8.9.0", diff --git a/src/program.ts b/src/program.ts index 725bd11..fa909b1 100644 --- a/src/program.ts +++ b/src/program.ts @@ -1305,6 +1305,7 @@ export function tornadoProgram() { Tornado, merkleWorkerPath, }), + optionalTree: true, }); const withdrawalsService = new NodeTornadoService({ @@ -1506,6 +1507,7 @@ export function tornadoProgram() { currency, userDirectory: SAVED_TREE_DIR, }), + optionalTree: true, }); const withdrawalsService = new NodeTornadoService({ diff --git a/src/services/nodeEvents.ts b/src/services/nodeEvents.ts index df62124..339ee18 100644 --- a/src/services/nodeEvents.ts +++ b/src/services/nodeEvents.ts @@ -17,7 +17,6 @@ import { BaseEchoService, CachedRelayers, toFixedHex, - DEPOSIT, } from '@tornado/core'; import type { BaseEvents, @@ -183,8 +182,12 @@ export class NodeTornadoService extends BaseTornadoService { return await getEventsFromCache(this); } - async validateEvents({ events }: { events: (DepositsEvents | WithdrawalsEvents)[] }): Promise { - const tree = await super.validateEvents({ events }); + async validateEvents({ + events, + lastBlock, + hasNewEvents, + }: BaseEvents & { hasNewEvents?: boolean }): Promise { + const tree = await super.validateEvents({ events, lastBlock, hasNewEvents }); if (tree && this.currency === this.nativeCurrency && this.treeCache) { const merkleTree = tree as unknown as MerkleTree; diff --git a/yarn.lock b/yarn.lock index 622b176..2f1d41c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -785,9 +785,9 @@ "@openzeppelin/contracts-v3" "npm:@openzeppelin/contracts@3.2.0-rc.0" ethers "^6.13.4" -"@tornado/core@git+https://git.tornado.ws/tornadocontrib/tornado-core.git#dfb20aa61753c4f29ab144517ff48959e1b33c40": +"@tornado/core@git+https://git.tornado.ws/tornadocontrib/tornado-core.git#9f4044d11033e99e0a863989864d1578a93de531": version "1.0.19" - resolved "git+https://git.tornado.ws/tornadocontrib/tornado-core.git#dfb20aa61753c4f29ab144517ff48959e1b33c40" + resolved "git+https://git.tornado.ws/tornadocontrib/tornado-core.git#9f4044d11033e99e0a863989864d1578a93de531" dependencies: "@metamask/eth-sig-util" "^7.0.3" "@tornado/contracts" "git+https://git.tornado.ws/tornadocontrib/tornado-contracts.git#ece511f424dc811c3aec149a4bf0e3731c0598a4"