Update tree generation

This commit is contained in:
Tornado Contrib 2024-10-21 11:51:15 +00:00
parent 1217d411c2
commit 53e60da285
Signed by: tornadocontrib
GPG Key ID: 60B4DF1A076C64B1
6 changed files with 201 additions and 63 deletions

197
dist/cli.js vendored

@ -182300,7 +182300,8 @@ class BaseEventsService {
}; };
} }
// eslint-disable-next-line @typescript-eslint/no-unused-vars // eslint-disable-next-line @typescript-eslint/no-unused-vars
validateEvents({ events, lastBlock }) { async validateEvents({ events, lastBlock }) {
return void 0;
} }
/** /**
* Handle saving events * Handle saving events
@ -182331,29 +182332,32 @@ class BaseEventsService {
return !hasEvent; return !hasEvent;
}); });
const lastBlock = newEvents.lastBlock || allEvents[allEvents.length - 1]?.blockNumber; const lastBlock = newEvents.lastBlock || allEvents[allEvents.length - 1]?.blockNumber;
this.validateEvents({ events: allEvents, lastBlock }); const validateResult = await this.validateEvents({ events: allEvents, lastBlock });
if (savedEvents.fromCache || newEvents.events.length) { if (savedEvents.fromCache || newEvents.events.length) {
await this.saveEvents({ events: allEvents, lastBlock }); await this.saveEvents({ events: allEvents, lastBlock });
} }
return { return {
events: allEvents, events: allEvents,
lastBlock lastBlock,
validateResult
}; };
} }
} }
class BaseTornadoService extends BaseEventsService { class BaseTornadoService extends BaseEventsService {
amount; amount;
currency; currency;
merkleTreeService;
batchTransactionService; batchTransactionService;
batchBlockService; batchBlockService;
constructor(serviceConstructor) { constructor(serviceConstructor) {
const { Tornado: contract, amount, currency, provider } = serviceConstructor; const { Tornado: contract, amount, currency, provider, merkleTreeService } = serviceConstructor;
super({ super({
...serviceConstructor, ...serviceConstructor,
contract contract
}); });
this.amount = amount; this.amount = amount;
this.currency = currency; this.currency = currency;
this.merkleTreeService = merkleTreeService;
this.batchTransactionService = new BatchTransactionService({ this.batchTransactionService = new BatchTransactionService({
provider, provider,
onProgress: this.updateTransactionProgress onProgress: this.updateTransactionProgress
@ -182427,15 +182431,20 @@ class BaseTornadoService extends BaseEventsService {
}); });
} }
} }
validateEvents({ events }) { async validateEvents({ events }) {
if (events.length && this.getType().toLowerCase() === DEPOSIT) { if (events.length && this.getType().toLowerCase() === DEPOSIT) {
const lastEvent = events[events.length - 1]; const depositEvents = events;
if (lastEvent.leafIndex !== events.length - 1) { const lastEvent = depositEvents[depositEvents.length - 1];
const errMsg = `Deposit events invalid wants ${events.length - 1} leafIndex have ${lastEvent.leafIndex}`; if (lastEvent.leafIndex !== depositEvents.length - 1) {
const errMsg = `Deposit events invalid wants ${depositEvents.length - 1} leafIndex have ${lastEvent.leafIndex}`;
throw new Error(errMsg); throw new Error(errMsg);
} }
if (this.merkleTreeService) {
return await this.merkleTreeService.verifyTree(depositEvents);
} }
} }
return void 0;
}
async getLatestEvents({ fromBlock }) { async getLatestEvents({ fromBlock }) {
if (this.tovarishClient?.selectedRelayer) { if (this.tovarishClient?.selectedRelayer) {
const { events, lastSyncBlock: lastBlock } = await this.tovarishClient.getEvents({ const { events, lastSyncBlock: lastBlock } = await this.tovarishClient.getEvents({
@ -183142,6 +183151,134 @@ class DBTornadoService extends BaseTornadoService {
}); });
} }
} }
class DBEchoService extends BaseEchoService {
staticUrl;
idb;
zipDigest;
constructor(params) {
super(params);
this.staticUrl = params.staticUrl;
this.idb = params.idb;
}
async getEventsFromDB() {
return await loadDBEvents({
idb: this.idb,
instanceName: this.getInstanceName()
});
}
async getEventsFromCache() {
return await loadRemoteEvents({
staticUrl: this.staticUrl,
instanceName: this.getInstanceName(),
deployedBlock: this.deployedBlock,
zipDigest: this.zipDigest
});
}
async saveEvents({ events, lastBlock }) {
await saveDBEvents({
idb: this.idb,
instanceName: this.getInstanceName(),
events,
lastBlock
});
}
}
class DBEncryptedNotesService extends BaseEncryptedNotesService {
staticUrl;
idb;
zipDigest;
constructor(params) {
super(params);
this.staticUrl = params.staticUrl;
this.idb = params.idb;
}
async getEventsFromDB() {
return await loadDBEvents({
idb: this.idb,
instanceName: this.getInstanceName()
});
}
async getEventsFromCache() {
return await loadRemoteEvents({
staticUrl: this.staticUrl,
instanceName: this.getInstanceName(),
deployedBlock: this.deployedBlock,
zipDigest: this.zipDigest
});
}
async saveEvents({ events, lastBlock }) {
await saveDBEvents({
idb: this.idb,
instanceName: this.getInstanceName(),
events,
lastBlock
});
}
}
class DBGovernanceService extends BaseGovernanceService {
staticUrl;
idb;
zipDigest;
constructor(params) {
super(params);
this.staticUrl = params.staticUrl;
this.idb = params.idb;
}
async getEventsFromDB() {
return await loadDBEvents({
idb: this.idb,
instanceName: this.getInstanceName()
});
}
async getEventsFromCache() {
return await loadRemoteEvents({
staticUrl: this.staticUrl,
instanceName: this.getInstanceName(),
deployedBlock: this.deployedBlock,
zipDigest: this.zipDigest
});
}
async saveEvents({ events, lastBlock }) {
await saveDBEvents({
idb: this.idb,
instanceName: this.getInstanceName(),
events,
lastBlock
});
}
}
class DBRegistryService extends BaseRegistryService {
staticUrl;
idb;
zipDigest;
constructor(params) {
super(params);
this.staticUrl = params.staticUrl;
this.idb = params.idb;
}
async getEventsFromDB() {
return await loadDBEvents({
idb: this.idb,
instanceName: this.getInstanceName()
});
}
async getEventsFromCache() {
return await loadRemoteEvents({
staticUrl: this.staticUrl,
instanceName: this.getInstanceName(),
deployedBlock: this.deployedBlock,
zipDigest: this.zipDigest
});
}
async saveEvents({ events, lastBlock }) {
await saveDBEvents({
idb: this.idb,
instanceName: this.getInstanceName(),
events,
lastBlock
});
}
}
const dist_abi$8 = [ const dist_abi$8 = [
{ {
@ -189412,15 +189549,18 @@ class MerkleTreeService {
Creating deposit tree for ${this.netId} ${this.amount} ${this.currency.toUpperCase()} would take a while Creating deposit tree for ${this.netId} ${this.amount} ${this.currency.toUpperCase()} would take a while
` `
); );
console.time("Created tree in"); const timeStart = Date.now();
const tree = await this.createTree(events.map(({ commitment }) => commitment)); const tree = await this.createTree(events.map(({ commitment }) => commitment));
console.timeEnd("Created tree in");
console.log("");
const isKnownRoot = await this.Tornado.isKnownRoot(toFixedHex(BigInt(tree.root))); const isKnownRoot = await this.Tornado.isKnownRoot(toFixedHex(BigInt(tree.root)));
if (!isKnownRoot) { if (!isKnownRoot) {
const errMsg = `Deposit Event ${this.netId} ${this.amount} ${this.currency} is invalid`; const errMsg = `Deposit Event ${this.netId} ${this.amount} ${this.currency} is invalid`;
throw new Error(errMsg); throw new Error(errMsg);
} }
console.log(
`
Created ${this.netId} ${this.amount} ${this.currency.toUpperCase()} tree in ${Date.now() - timeStart}ms
`
);
return tree; return tree;
} }
} }
@ -190048,15 +190188,13 @@ class NodeTornadoService extends BaseTornadoService {
cacheDirectory; cacheDirectory;
userDirectory; userDirectory;
nativeCurrency; nativeCurrency;
merkleTreeService;
treeCache; treeCache;
constructor(serviceConstructor) { constructor(serviceConstructor) {
super(serviceConstructor); super(serviceConstructor);
const { cacheDirectory, userDirectory, nativeCurrency, merkleTreeService, treeCache } = serviceConstructor; const { cacheDirectory, userDirectory, nativeCurrency, treeCache } = serviceConstructor;
this.cacheDirectory = cacheDirectory; this.cacheDirectory = cacheDirectory;
this.userDirectory = userDirectory; this.userDirectory = userDirectory;
this.nativeCurrency = nativeCurrency; this.nativeCurrency = nativeCurrency;
this.merkleTreeService = merkleTreeService;
this.treeCache = treeCache; this.treeCache = treeCache;
} }
updateEventProgress({ type, fromBlock, toBlock, count }) { updateEventProgress({ type, fromBlock, toBlock, count }) {
@ -190099,15 +190237,17 @@ class NodeTornadoService extends BaseTornadoService {
async getEventsFromCache() { async getEventsFromCache() {
return await getEventsFromCache(this); return await getEventsFromCache(this);
} }
async saveEvents({ events, lastBlock }) { async validateEvents({ events }) {
if (this.userDirectory && this.merkleTreeService) { const tree = await super.validateEvents({ events });
const tree = await this.merkleTreeService.verifyTree(events); if (tree && this.currency === this.nativeCurrency && this.treeCache) {
if (this.currency === this.nativeCurrency && this.treeCache) { const merkleTree = tree;
await this.treeCache.createTree(events, tree); await this.treeCache.createTree(events, merkleTree);
console.log(`${this.getInstanceName()}: Updated tree cache with root ${toFixedHex(BigInt(tree.root))} console.log(`${this.getInstanceName()}: Updated tree cache with root ${toFixedHex(BigInt(merkleTree.root))}
`); `);
} }
return tree;
} }
async saveEvents({ events, lastBlock }) {
const eventTable = new (cli_table3_default())(); const eventTable = new (cli_table3_default())();
eventTable.push( eventTable.push(
[{ colSpan: 2, content: `${this.getType()}s`, hAlign: "center" }], [{ colSpan: 2, content: `${this.getType()}s`, hAlign: "center" }],
@ -191207,21 +191347,20 @@ Connected with Tovarish Relayer ${tovarishClient.selectedRelayer.url}
}; };
const depositsService = new NodeTornadoService({ const depositsService = new NodeTornadoService({
...TornadoServiceConstructor, ...TornadoServiceConstructor,
type: "Deposit" type: "Deposit",
}); merkleTreeService: new MerkleTreeService({
const withdrawalsService = new NodeTornadoService({
...TornadoServiceConstructor,
type: "Withdrawal"
});
const merkleTreeService = new MerkleTreeService({
netId, netId,
amount, amount,
currency, currency,
Tornado, Tornado,
merkleWorkerPath merkleWorkerPath
})
}); });
const depositEvents = (await depositsService.updateEvents()).events; const withdrawalsService = new NodeTornadoService({
const tree = await merkleTreeService.verifyTree(depositEvents); ...TornadoServiceConstructor,
type: "Withdrawal"
});
const { events: depositEvents, validateResult: tree } = await depositsService.updateEvents();
const withdrawalEvents = (await withdrawalsService.updateEvents()).events; const withdrawalEvents = (await withdrawalsService.updateEvents()).events;
const depositEvent = depositEvents.find(({ commitment }) => commitment === commitmentHex); const depositEvent = depositEvents.find(({ commitment }) => commitment === commitmentHex);
const withdrawalEvent = withdrawalEvents.find(({ nullifierHash }) => nullifierHash === nullifierHex); const withdrawalEvent = withdrawalEvents.find(({ nullifierHash }) => nullifierHash === nullifierHex);

@ -1,5 +1,5 @@
import { BatchBlockOnProgress, BatchEventOnProgress, BaseTornadoService, BaseEncryptedNotesService, BaseGovernanceService, BaseRegistryService, BaseTornadoServiceConstructor, BaseEncryptedNotesServiceConstructor, BaseGovernanceServiceConstructor, BaseRegistryServiceConstructor, BaseEchoServiceConstructor, BaseEchoService, CachedRelayers } from '@tornado/core'; import { BatchBlockOnProgress, BatchEventOnProgress, BaseTornadoService, BaseEncryptedNotesService, BaseGovernanceService, BaseRegistryService, BaseTornadoServiceConstructor, BaseEncryptedNotesServiceConstructor, BaseGovernanceServiceConstructor, BaseRegistryServiceConstructor, BaseEchoServiceConstructor, BaseEchoService, CachedRelayers } from '@tornado/core';
import type { BaseEvents, DepositsEvents, WithdrawalsEvents, EncryptedNotesEvents, RegistersEvents, AllGovernanceEvents, EchoEvents, MerkleTreeService } from '@tornado/core'; import type { BaseEvents, DepositsEvents, WithdrawalsEvents, EncryptedNotesEvents, RegistersEvents, AllGovernanceEvents, EchoEvents } from '@tornado/core';
import { TreeCache } from './treeCache'; import { TreeCache } from './treeCache';
export type NodeServiceConstructor = { export type NodeServiceConstructor = {
cacheDirectory: string; cacheDirectory: string;
@ -7,14 +7,12 @@ export type NodeServiceConstructor = {
}; };
export type NodeTornadoServiceConstructor = BaseTornadoServiceConstructor & NodeServiceConstructor & { export type NodeTornadoServiceConstructor = BaseTornadoServiceConstructor & NodeServiceConstructor & {
nativeCurrency: string; nativeCurrency: string;
merkleTreeService?: MerkleTreeService;
treeCache?: TreeCache; treeCache?: TreeCache;
}; };
export declare class NodeTornadoService extends BaseTornadoService { export declare class NodeTornadoService extends BaseTornadoService {
cacheDirectory: string; cacheDirectory: string;
userDirectory: string; userDirectory: string;
nativeCurrency: string; nativeCurrency: string;
merkleTreeService?: MerkleTreeService;
treeCache?: TreeCache; treeCache?: TreeCache;
constructor(serviceConstructor: NodeTornadoServiceConstructor); constructor(serviceConstructor: NodeTornadoServiceConstructor);
updateEventProgress({ type, fromBlock, toBlock, count }: Parameters<BatchEventOnProgress>[0]): void; updateEventProgress({ type, fromBlock, toBlock, count }: Parameters<BatchEventOnProgress>[0]): void;
@ -23,6 +21,9 @@ export declare class NodeTornadoService extends BaseTornadoService {
updateGraphProgress({ type, fromBlock, toBlock, count }: Parameters<BatchEventOnProgress>[0]): void; updateGraphProgress({ type, fromBlock, toBlock, count }: Parameters<BatchEventOnProgress>[0]): void;
getEventsFromDB(): Promise<BaseEvents<DepositsEvents | WithdrawalsEvents>>; getEventsFromDB(): Promise<BaseEvents<DepositsEvents | WithdrawalsEvents>>;
getEventsFromCache(): Promise<import("@tornado/core").CachedEvents<DepositsEvents | WithdrawalsEvents>>; getEventsFromCache(): Promise<import("@tornado/core").CachedEvents<DepositsEvents | WithdrawalsEvents>>;
validateEvents<S>({ events }: {
events: (DepositsEvents | WithdrawalsEvents)[];
}): Promise<S>;
saveEvents({ events, lastBlock }: BaseEvents<DepositsEvents | WithdrawalsEvents>): Promise<void>; saveEvents({ events, lastBlock }: BaseEvents<DepositsEvents | WithdrawalsEvents>): Promise<void>;
} }
export type NodeEchoServiceConstructor = BaseEchoServiceConstructor & NodeServiceConstructor; export type NodeEchoServiceConstructor = BaseEchoServiceConstructor & NodeServiceConstructor;

@ -51,7 +51,7 @@
"optionalDependencies": {}, "optionalDependencies": {},
"devDependencies": { "devDependencies": {
"@colors/colors": "^1.6.0", "@colors/colors": "^1.6.0",
"@tornado/core": "git+https://git.tornado.ws/tornadocontrib/tornado-core.git#ddf306a835a0907eed1a2a7aab4ed41e34fd9037", "@tornado/core": "git+https://git.tornado.ws/tornadocontrib/tornado-core.git#dfb20aa61753c4f29ab144517ff48959e1b33c40",
"@typechain/ethers-v6": "^0.5.1", "@typechain/ethers-v6": "^0.5.1",
"@types/figlet": "^1.7.0", "@types/figlet": "^1.7.0",
"@typescript-eslint/eslint-plugin": "^8.9.0", "@typescript-eslint/eslint-plugin": "^8.9.0",

@ -60,7 +60,6 @@ import {
NetIdType, NetIdType,
getInstanceByAddress, getInstanceByAddress,
getConfig, getConfig,
Config,
enabledChains, enabledChains,
substring, substring,
NoteAccount, NoteAccount,
@ -73,6 +72,7 @@ import {
ReverseRecords__factory, ReverseRecords__factory,
numberFormatter, numberFormatter,
} from '@tornado/core'; } from '@tornado/core';
import type { MerkleTree } from '@tornado/fixed-merkle-tree';
import * as packageJson from '../package.json'; import * as packageJson from '../package.json';
import { import {
parseUrl, parseUrl,
@ -983,6 +983,13 @@ export function tornadoProgram() {
const depositsService = new NodeTornadoService({ const depositsService = new NodeTornadoService({
...TornadoServiceConstructor, ...TornadoServiceConstructor,
type: 'Deposit', type: 'Deposit',
merkleTreeService: new MerkleTreeService({
netId,
amount,
currency,
Tornado,
merkleWorkerPath,
}),
}); });
const withdrawalsService = new NodeTornadoService({ const withdrawalsService = new NodeTornadoService({
@ -990,22 +997,11 @@ export function tornadoProgram() {
type: 'Withdrawal', type: 'Withdrawal',
}); });
const merkleTreeService = new MerkleTreeService({ const { events: depositEvents, validateResult: tree } = await depositsService.updateEvents<MerkleTree>();
netId,
amount,
currency,
Tornado,
merkleWorkerPath,
});
const depositEvents = (await depositsService.updateEvents()).events as DepositsEvents[];
// Create tree using node workers which would spawn another dedicated thread to create trees
const tree = await merkleTreeService.verifyTree(depositEvents);
const withdrawalEvents = (await withdrawalsService.updateEvents()).events as WithdrawalsEvents[]; const withdrawalEvents = (await withdrawalsService.updateEvents()).events as WithdrawalsEvents[];
const depositEvent = depositEvents.find(({ commitment }) => commitment === commitmentHex); const depositEvent = (depositEvents as DepositsEvents[]).find(({ commitment }) => commitment === commitmentHex);
const withdrawalEvent = withdrawalEvents.find(({ nullifierHash }) => nullifierHash === nullifierHex); const withdrawalEvent = withdrawalEvents.find(({ nullifierHash }) => nullifierHash === nullifierHex);

@ -17,6 +17,7 @@ import {
BaseEchoService, BaseEchoService,
CachedRelayers, CachedRelayers,
toFixedHex, toFixedHex,
DEPOSIT,
} from '@tornado/core'; } from '@tornado/core';
import type { import type {
BaseEvents, BaseEvents,
@ -26,10 +27,10 @@ import type {
RegistersEvents, RegistersEvents,
AllGovernanceEvents, AllGovernanceEvents,
EchoEvents, EchoEvents,
MerkleTreeService,
BaseEventsService, BaseEventsService,
MinimalEvents, MinimalEvents,
} from '@tornado/core'; } from '@tornado/core';
import type { MerkleTree } from '@tornado/fixed-merkle-tree';
import { TreeCache } from './treeCache'; import { TreeCache } from './treeCache';
import { saveUserFile, loadSavedEvents, loadCachedEvents, existsAsync } from './data'; import { saveUserFile, loadSavedEvents, loadCachedEvents, existsAsync } from './data';
@ -114,7 +115,6 @@ export type NodeServiceConstructor = {
export type NodeTornadoServiceConstructor = BaseTornadoServiceConstructor & export type NodeTornadoServiceConstructor = BaseTornadoServiceConstructor &
NodeServiceConstructor & { NodeServiceConstructor & {
nativeCurrency: string; nativeCurrency: string;
merkleTreeService?: MerkleTreeService;
treeCache?: TreeCache; treeCache?: TreeCache;
}; };
@ -123,19 +123,17 @@ export class NodeTornadoService extends BaseTornadoService {
userDirectory: string; userDirectory: string;
nativeCurrency: string; nativeCurrency: string;
merkleTreeService?: MerkleTreeService;
treeCache?: TreeCache; treeCache?: TreeCache;
constructor(serviceConstructor: NodeTornadoServiceConstructor) { constructor(serviceConstructor: NodeTornadoServiceConstructor) {
super(serviceConstructor); super(serviceConstructor);
const { cacheDirectory, userDirectory, nativeCurrency, merkleTreeService, treeCache } = serviceConstructor; const { cacheDirectory, userDirectory, nativeCurrency, treeCache } = serviceConstructor;
this.cacheDirectory = cacheDirectory; this.cacheDirectory = cacheDirectory;
this.userDirectory = userDirectory; this.userDirectory = userDirectory;
this.nativeCurrency = nativeCurrency; this.nativeCurrency = nativeCurrency;
this.merkleTreeService = merkleTreeService;
this.treeCache = treeCache; this.treeCache = treeCache;
} }
@ -185,17 +183,21 @@ export class NodeTornadoService extends BaseTornadoService {
return await getEventsFromCache<DepositsEvents | WithdrawalsEvents>(this); return await getEventsFromCache<DepositsEvents | WithdrawalsEvents>(this);
} }
async validateEvents<S>({ events }: { events: (DepositsEvents | WithdrawalsEvents)[] }): Promise<S> {
const tree = await super.validateEvents<S>({ events });
if (tree && this.currency === this.nativeCurrency && this.treeCache) {
const merkleTree = tree as unknown as MerkleTree;
await this.treeCache.createTree(events as DepositsEvents[], merkleTree);
console.log(`${this.getInstanceName()}: Updated tree cache with root ${toFixedHex(BigInt(merkleTree.root))}\n`);
}
return tree;
}
async saveEvents({ events, lastBlock }: BaseEvents<DepositsEvents | WithdrawalsEvents>) { async saveEvents({ events, lastBlock }: BaseEvents<DepositsEvents | WithdrawalsEvents>) {
if (this.userDirectory && this.merkleTreeService) {
const tree = await this.merkleTreeService.verifyTree(events as DepositsEvents[]);
if (this.currency === this.nativeCurrency && this.treeCache) {
await this.treeCache.createTree(events as DepositsEvents[], tree);
console.log(`${this.getInstanceName()}: Updated tree cache with root ${toFixedHex(BigInt(tree.root))}\n`);
}
}
const eventTable = new Table(); const eventTable = new Table();
eventTable.push( eventTable.push(

@ -785,9 +785,9 @@
"@openzeppelin/contracts-v3" "npm:@openzeppelin/contracts@3.2.0-rc.0" "@openzeppelin/contracts-v3" "npm:@openzeppelin/contracts@3.2.0-rc.0"
ethers "^6.13.4" ethers "^6.13.4"
"@tornado/core@git+https://git.tornado.ws/tornadocontrib/tornado-core.git#ddf306a835a0907eed1a2a7aab4ed41e34fd9037": "@tornado/core@git+https://git.tornado.ws/tornadocontrib/tornado-core.git#dfb20aa61753c4f29ab144517ff48959e1b33c40":
version "1.0.19" version "1.0.19"
resolved "git+https://git.tornado.ws/tornadocontrib/tornado-core.git#ddf306a835a0907eed1a2a7aab4ed41e34fd9037" resolved "git+https://git.tornado.ws/tornadocontrib/tornado-core.git#dfb20aa61753c4f29ab144517ff48959e1b33c40"
dependencies: dependencies:
"@metamask/eth-sig-util" "^7.0.3" "@metamask/eth-sig-util" "^7.0.3"
"@tornado/contracts" "git+https://git.tornado.ws/tornadocontrib/tornado-contracts.git#ece511f424dc811c3aec149a4bf0e3731c0598a4" "@tornado/contracts" "git+https://git.tornado.ws/tornadocontrib/tornado-contracts.git#ece511f424dc811c3aec149a4bf0e3731c0598a4"