efficient bulk insert

(cherry picked from commit c83a4a4dc0)

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: Node.js CI
+name: build
 
 on:
   pull_request:
@@ -16,6 +16,14 @@ jobs:
           node-version: 14.7.0
       - run: npm ci
       - run: npm run test
+      - name: Telegram Failure Notification
+        uses: appleboy/telegram-action@0.0.7
+        if: failure()
+        with:
+          message: ❗ Build failed for [${{ github.repository }}](https://github.com/${{ github.repository }}/actions) because of ${{ github.actor }}
+          format: markdown
+          to: ${{ secrets.TELEGRAM_CHAT_ID }}
+          token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
 
   lint:
     runs-on: ubuntu-latest
@@ -26,6 +34,14 @@ jobs:
           node-version: 14.7.0
       - run: npm ci
       - run: npm run lint
+      - name: Telegram Failure Notification
+        uses: appleboy/telegram-action@0.0.7
+        if: failure()
+        with:
+          message: ❗ Build failed for [${{ github.repository }}](https://github.com/${{ github.repository }}/actions) because of ${{ github.actor }}
+          format: markdown
+          to: ${{ secrets.TELEGRAM_CHAT_ID }}
+          token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
 
   publish:
     runs-on: ubuntu-latest
@@ -65,3 +81,12 @@ jobs:
           message: 🚀 Published [${{ steps.vars.outputs.repo_name }}](https://github.com/${{ github.repository }}) version [${{ steps.vars.outputs.version }}](https://www.npmjs.com/package/${{ steps.vars.outputs.repo_name }}/v/${{ steps.vars.outputs.version }}) to npm
           debug: true
           format: markdown
+
+      - name: Telegram Failure Notification
+        uses: appleboy/telegram-action@0.0.7
+        if: failure()
+        with:
+          message: ❗ Failed to publish [${{ steps.vars.outputs.repo_name }}](https://github.com/${{ github.repository }}/actions) because of ${{ env.GITHUB_ACTOR }}
+          format: markdown
+          to: ${{ secrets.TELEGRAM_CHAT_ID }}
+          token: ${{ secrets.TELEGRAM_BOT_TOKEN }}

.prettierrc

@@ -0,0 +1,7 @@
+{
+  "semi": false,
+  "arrowParens": "always",
+  "singleQuote": true,
+  "printWidth": 110,
+  "trailingComma": "all"
+}

README.md

@@ -1,4 +1,4 @@
-# Merkle Tree [![GitHub Workflow Status](https://img.shields.io/github/workflow/status/tornadocash/fixed-merkle-tree/Node.js%20CI)](https://github.com/tornadocash/fixed-merkle-tree/actions) [![npm](https://img.shields.io/npm/v/fixed-merkle-tree)](https://www.npmjs.com/package/fixed-merkle-tree)
+# Merkle Tree [![GitHub Workflow Status](https://img.shields.io/github/workflow/status/tornadocash/fixed-merkle-tree/build)](https://github.com/tornadocash/fixed-merkle-tree/actions) [![npm](https://img.shields.io/npm/v/fixed-merkle-tree)](https://www.npmjs.com/package/fixed-merkle-tree)
 
 This is a fixed depth merkle tree implementation with sequential inserts
 
@@ -9,8 +9,8 @@ const MerkleTree = require('MerkleTree')
 const tree = new MerkleTree(10, [1, 2, 3, 4, 5])
 tree.insert(6)
 tree.update(3, 42)
-const proof = tree.proof(tree.indexOf(2))
+const path = tree.path(tree.indexOf(2))
-console.log(proof)
+console.log(path)
 
 // output:
 {

package.json

@@ -1,19 +1,26 @@
 {
   "name": "fixed-merkle-tree",
-  "version": "0.2.1",
+  "version": "0.6.0",
   "description": "Fixed depth merkle tree implementation with sequential inserts",
+  "repository": "https://github.com/tornadocash/fixed-merkle-tree.git",
   "main": "src/merkleTree.js",
   "scripts": {
     "test": "mocha",
     "lint": "eslint ."
   },
-  "keywords": ["merkle", "tree", "merkleTree"],
+  "keywords": [
+    "merkle",
+    "tree",
+    "merkleTree"
+  ],
   "author": "Roman Semenov <semenov.roma@gmail.com>",
   "license": "ISC",
-  "files": ["src/*"],
+  "files": [
+    "src/*"
+  ],
   "dependencies": {
     "snarkjs": "git+https://github.com/tornadocash/snarkjs.git#869181cfaf7526fe8972073d31655493a04326d5",
-    "circomlib": "git+https://github.com/tornadocash/circomlib.git#c372f14d324d57339c88451834bf2824e73bbdbc"
+    "circomlib": "git+https://github.com/tornadocash/circomlib.git#5beb6aee94923052faeecea40135d45b6ce6172c"
   },
   "devDependencies": {
     "babel-eslint": "^10.1.0",

src/merkleTree.js

@@ -24,17 +24,19 @@ class MerkleTree {
    */
   constructor(levels, elements = [], { hashFunction, zeroElement = DEFAULT_ZERO } = {}) {
     this.levels = levels
-    this.capacity = 2 << levels
+    this.capacity = 2 ** levels
-    this.zeroElement = zeroElement
+    if (elements.length > this.capacity) {
+      throw new Error('Tree is full')
+    }
     this._hash = hashFunction || defaultHash
-
+    this.zeroElement = zeroElement
     this._zeros = []
-    this._layers = []
+    this._zeros[0] = zeroElement
-    this._layers[0] = elements
-    this._zeros[0] = this.zeroElement
     for (let i = 1; i <= levels; i++) {
       this._zeros[i] = this._hash(this._zeros[i - 1], this._zeros[i - 1])
     }
+    this._layers = []
+    this._layers[0] = elements.slice()
     this._rebuild()
   }
 
@@ -44,9 +46,9 @@ class MerkleTree {
       for (let i = 0; i < Math.ceil(this._layers[level - 1].length / 2); i++) {
         this._layers[level][i] = this._hash(
           this._layers[level - 1][i * 2],
-          i * 2 + 1 < this._layers[level - 1].length ?
+          i * 2 + 1 < this._layers[level - 1].length
-            this._layers[level - 1][i * 2 + 1] :
+            ? this._layers[level - 1][i * 2 + 1]
-            this._zeros[level - 1],
+            : this._zeros[level - 1],
         )
       }
     }
@@ -72,15 +74,30 @@ class MerkleTree {
   }
 
   /**
-   * Insert multiple elements into the tree. Tree will be fully rebuilt during this operation.
+   * Insert multiple elements into the tree.
    * @param {Array} elements Elements to insert
    */
   bulkInsert(elements) {
     if (this._layers[0].length + elements.length > this.capacity) {
       throw new Error('Tree is full')
     }
-    this._layers[0].push(...elements)
+    // First we insert all elements except the last one
-    this._rebuild()
+    // updating only full subtree hashes (all layers where inserted element has odd index)
+    // the last element will update the full path to the root making the tree consistent again
+    for (let i = 0; i < elements.length - 1; i++) {
+      this._layers[0].push(elements[i])
+      let level = 0
+      let index = this._layers[0].length - 1
+      while (index % 2 === 1) {
+        level++
+        index >>= 1
+        this._layers[level][index] = this._hash(
+          this._layers[level - 1][index * 2],
+          this._layers[level - 1][index * 2 + 1],
+        )
+      }
+    }
+    this.insert(elements[elements.length - 1])
   }
 
   /**
@@ -89,7 +106,7 @@ class MerkleTree {
    * @param element Updated element value
    */
   update(index, element) {
-    if (index < 0 || index > this._layers[0].length || index >= this.capacity) {
+    if (isNaN(Number(index)) || index < 0 || index > this._layers[0].length || index >= this.capacity) {
       throw new Error('Insert index out of bounds: ' + index)
     }
     this._layers[0][index] = element
@@ -97,29 +114,28 @@ class MerkleTree {
       index >>= 1
       this._layers[level][index] = this._hash(
         this._layers[level - 1][index * 2],
-        index * 2 + 1 < this._layers[level - 1].length ?
+        index * 2 + 1 < this._layers[level - 1].length
-          this._layers[level - 1][index * 2 + 1] :
+          ? this._layers[level - 1][index * 2 + 1]
-          this._zeros[level - 1],
+          : this._zeros[level - 1],
       )
     }
   }
 
   /**
-   * Get merkle proof for a leaf
+   * Get merkle path to a leaf
-   * @param index Leaf index to generate proof for
+   * @param {number} index Leaf index to generate path for
    * @returns {{pathElements: Object[], pathIndex: number[]}} An object containing adjacent elements and left-right index
    */
-  proof(index) {
+  path(index) {
-    if (index < 0 || index >= this._layers[0].length) {
+    if (isNaN(Number(index)) || index < 0 || index >= this._layers[0].length) {
       throw new Error('Index out of bounds: ' + index)
     }
     const pathElements = []
     const pathIndices = []
     for (let level = 0; level < this.levels; level++) {
       pathIndices[level] = index % 2
-      pathElements[level] = (index ^ 1) < this._layers[level].length ?
+      pathElements[level] =
-        this._layers[level][index ^ 1] :
+        (index ^ 1) < this._layers[level].length ? this._layers[level][index ^ 1] : this._zeros[level]
-        this._zeros[level]
       index >>= 1
     }
     return {
@@ -131,11 +147,16 @@ class MerkleTree {
   /**
    * Find an element in the tree
    * @param element An element to find
+   * @param comparator A function that checks leaf value equality
    * @returns {number} Index if element is found, otherwise -1
    */
-  indexOf(element) {
+  indexOf(element, comparator) {
+    if (comparator) {
+      return this._layers[0].findIndex((el) => comparator(element, el))
+    } else {
       return this._layers[0].indexOf(element)
     }
+  }
 
   /**
    * Returns a copy of non-zero tree elements
@@ -144,6 +165,44 @@ class MerkleTree {
   elements() {
     return this._layers[0].slice()
   }
+
+  /**
+   * Returns a copy of n-th zero elements array
+   * @returns {Object[]}
+   */
+  zeros() {
+    return this._zeros.slice()
+  }
+
+  /**
+   * Serialize entire tree state including intermediate layers into a plain object
+   * Deserializing it back will not require to recompute any hashes
+   * Elements are not converted to a plain type, this is responsibility of the caller
+   */
+  serialize() {
+    return {
+      levels: this.levels,
+      _zeros: this._zeros,
+      _layers: this._layers,
+    }
+  }
+
+  /**
+   * Deserialize data into a MerkleTree instance
+   * Make sure to provide the same hashFunction as was used in the source tree,
+   * otherwise the tree state will be invalid
+   *
+   * @param data
+   * @param hashFunction
+   * @returns {MerkleTree}
+   */
+  static deserialize(data, hashFunction) {
+    const instance = Object.assign(Object.create(this.prototype), data)
+    instance._hash = hashFunction || defaultHash
+    instance.capacity = 2 ** instance.levels
+    instance.zeroElement = instance._zeros[0]
+    return instance
+  }
 }
 
 module.exports = MerkleTree

test/merkleTree.test.js

@@ -1,9 +1,6 @@
 const MerkleTree = require('../src/merkleTree')
 require('chai').should()
 
-// todo negative test cases for:
-//   - full tree
-//   - invalid indexes
 describe('MerkleTree', () => {
   describe('#constructor', () => {
     it('should have correct zero root', () => {
@@ -13,17 +10,26 @@ describe('MerkleTree', () => {
 
     it('should have correct 1 element root', () => {
       const tree = new MerkleTree(10, [1])
-      return tree.root().should.equal('8423266420989796135179818298985240707844287090553672312129988553683991994663')
+      tree.root().should.equal('8423266420989796135179818298985240707844287090553672312129988553683991994663')
     })
 
     it('should have correct even elements root', () => {
       const tree = new MerkleTree(10, [1, 2])
-      return tree.root().should.equal('6632020347849276860492323008882350357301732786233864934344775324188835172576')
+      tree.root().should.equal('6632020347849276860492323008882350357301732786233864934344775324188835172576')
     })
 
     it('should have correct odd elements root', () => {
       const tree = new MerkleTree(10, [1, 2, 3])
-      return tree.root().should.equal('13605252518346649016266481317890801910232739395710162921320863289825142055129')
+      tree.root().should.equal('13605252518346649016266481317890801910232739395710162921320863289825142055129')
+    })
+
+    it('should be able to create a full tree', () => {
+      new MerkleTree(2, [1, 2, 3, 4])
+    })
+
+    it('should fail to create tree with too many elements', () => {
+      const call = () => new MerkleTree(2, [1, 2, 3, 4, 5])
+      call.should.throw('Tree is full')
     })
   })
 
@@ -31,19 +37,75 @@ describe('MerkleTree', () => {
     it('should insert into empty tree', () => {
       const tree = new MerkleTree(10)
       tree.insert(42)
-      return tree.root().should.equal('5305397050004975530787056746976521882221645950652996479084366175595194436378')
+      tree.root().should.equal('5305397050004975530787056746976521882221645950652996479084366175595194436378')
     })
 
     it('should insert into odd tree', () => {
       const tree = new MerkleTree(10, [1])
       tree.insert(42)
-      return tree.root().should.equal('4732716818150428188641303198013632061441036732749853605989871103991103096471')
+      tree.root().should.equal('4732716818150428188641303198013632061441036732749853605989871103991103096471')
     })
 
     it('should insert into even tree', () => {
       const tree = new MerkleTree(10, [1, 2])
       tree.insert(42)
-      return tree.root().should.equal('6204016789747878948181936326719724987136198810274146408545977300318734508764')
+      tree.root().should.equal('6204016789747878948181936326719724987136198810274146408545977300318734508764')
+    })
+
+    it('should insert last element', () => {
+      const tree = new MerkleTree(2, [1, 2, 3])
+      tree.insert(4)
+    })
+
+    it('should fail to insert when tree is full', () => {
+      const tree = new MerkleTree(2, [1, 2, 3, 4])
+      const call = () => tree.insert(5)
+      call.should.throw('Tree is full')
+    })
+  })
+
+  describe('#bulkInsert', () => {
+    it('should work', () => {
+      const tree = new MerkleTree(10, [1, 2, 3])
+      tree.bulkInsert([4, 5, 6])
+      tree.root().should.equal('10132905325673518287563057607527946096399700874345297651940963130460267058606')
+    })
+
+    it('should give the same result as sequental inserts', () => {
+      const initialArray = [
+        [1],
+        [1, 2],
+        [1, 2, 3],
+        [1, 2, 3, 4],
+      ]
+      const insertedArray = [
+        [11],
+        [11, 12],
+        [11, 12, 13],
+        [11, 12, 13, 14],
+      ]
+      for (const initial of initialArray) {
+        for (const inserted of insertedArray) {
+          const tree1 = new MerkleTree(10, initial)
+          const tree2 = new MerkleTree(10, initial)
+          tree1.bulkInsert(inserted)
+          for (const item of inserted) {
+            tree2.insert(item)
+          }
+          tree1.root().should.equal(tree2.root())
+        }
+      }
+    }).timeout(10000)
+
+    it('should work with max elements', () => {
+      const tree = new MerkleTree(2, [1, 2])
+      tree.bulkInsert([3, 4])
+    })
+
+    it('should fail to insert too many elements', () => {
+      const tree = new MerkleTree(2, [1, 2])
+      const call = () => tree.bulkInsert([3, 4, 5])
+      call.should.throw('Tree is full')
     })
   })
 
@@ -51,25 +113,44 @@ describe('MerkleTree', () => {
     it('should update first element', () => {
       const tree = new MerkleTree(10, [1, 2, 3, 4, 5])
       tree.update(0, 42)
-      return tree.root().should.equal('153077538697962715163231177553585573790587443799974092612333826693999310199')
+      tree.root().should.equal('153077538697962715163231177553585573790587443799974092612333826693999310199')
     })
 
     it('should update last element', () => {
       const tree = new MerkleTree(10, [1, 2, 3, 4, 5])
       tree.update(4, 42)
-      return tree.root().should.equal('1955192134603843666100093417117434845771298375724087600313714421260719033775')
+      tree.root().should.equal('1955192134603843666100093417117434845771298375724087600313714421260719033775')
     })
 
     it('should update odd element', () => {
       const tree = new MerkleTree(10, [1, 2, 3, 4, 5])
       tree.update(1, 42)
-      return tree.root().should.equal('6642888742811380760154112624880866754768235565211186414088321870395007150538')
+      tree.root().should.equal('6642888742811380760154112624880866754768235565211186414088321870395007150538')
     })
 
     it('should update even element', () => {
       const tree = new MerkleTree(10, [1, 2, 3, 4, 5])
       tree.update(2, 42)
-      return tree.root().should.equal('11739358667442647096377238675718917508981868161724701476635082606510350785683')
+      tree.root().should.equal('11739358667442647096377238675718917508981868161724701476635082606510350785683')
+    })
+
+    it('should update extra element', () => {
+      const tree = new MerkleTree(10, [1, 2, 3, 4])
+      tree.update(4, 5)
+      tree.root().should.equal('6341751103515285836339987888606244815365572869367801108789753151704260302930')
+    })
+
+    it('should fail to update incorrect index', () => {
+      const tree = new MerkleTree(10, [1, 2, 3, 4, 5]);
+      (() => tree.update(-1, 42)).should.throw('Insert index out of bounds: -1');
+      (() => tree.update(6, 42)).should.throw('Insert index out of bounds: 6');
+      (() => tree.update('qwe', 42)).should.throw('Insert index out of bounds: qwe')
+    })
+
+    it('should fail to update over capacity', () => {
+      const tree = new MerkleTree(2, [1, 2, 3, 4])
+      const call = () => tree.update(4, 42)
+      call.should.throw('Insert index out of bounds: 4')
     })
   })
 
@@ -85,12 +166,12 @@ describe('MerkleTree', () => {
     })
   })
 
-  describe('#proof', () => {
+  describe('#path', () => {
     it('should work for even index', () => {
       const tree = new MerkleTree(10, [1, 2, 3, 4, 5])
-      const proof = tree.proof(2)
+      const path = tree.path(2)
-      proof.pathIndices.should.be.deep.equal([0, 1, 0, 0, 0, 0, 0, 0, 0, 0])
+      path.pathIndices.should.be.deep.equal([0, 1, 0, 0, 0, 0, 0, 0, 0, 0])
-      proof.pathElements.should.be.deep.equal([
+      path.pathElements.should.be.deep.equal([
         4,
         '19814528709687996974327303300007262407299502847885145507292406548098437687919',
         '21305827034995891902714687670641862055126514524916463201449278400604999416145',
@@ -106,9 +187,9 @@ describe('MerkleTree', () => {
 
     it('should work for odd index', () => {
       const tree = new MerkleTree(10, [1, 2, 3, 4, 5])
-      const proof = tree.proof(3)
+      const path = tree.path(3)
-      proof.pathIndices.should.be.deep.equal([1, 1, 0, 0, 0, 0, 0, 0, 0, 0])
+      path.pathIndices.should.be.deep.equal([1, 1, 0, 0, 0, 0, 0, 0, 0, 0])
-      proof.pathElements.should.be.deep.equal([
+      path.pathElements.should.be.deep.equal([
         3,
         '19814528709687996974327303300007262407299502847885145507292406548098437687919',
         '21305827034995891902714687670641862055126514524916463201449278400604999416145',
@@ -121,5 +202,45 @@ describe('MerkleTree', () => {
         '155962837046691114236524362966874066300454611955781275944230309195800494087',
       ])
     })
+
+    it('should fail on incorrect index', () => {
+      const tree = new MerkleTree(10, [1, 2, 3, 4]);
+      (() => tree.path(-1)).should.throw('Index out of bounds: -1');
+      (() => tree.path(5)).should.throw('Index out of bounds: 5');
+      (() => tree.path('qwe')).should.throw('Index out of bounds: qwe')
+    })
+
+    it('should work for correct string index', () => {
+      const tree = new MerkleTree(10, [1, 2, 3, 4, 5])
+      const path = tree.path('2')
+      path.pathIndices.should.be.deep.equal([0, 1, 0, 0, 0, 0, 0, 0, 0, 0])
+      path.pathElements.should.be.deep.equal([
+        4,
+        '19814528709687996974327303300007262407299502847885145507292406548098437687919',
+        '21305827034995891902714687670641862055126514524916463201449278400604999416145',
+        '14506027710748750947258687001455876266559341618222612722926156490737302846427',
+        '4766583705360062980279572762279781527342845808161105063909171241304075622345',
+        '16640205414190175414380077665118269450294358858897019640557533278896634808665',
+        '13024477302430254842915163302704885770955784224100349847438808884122720088412',
+        '11345696205391376769769683860277269518617256738724086786512014734609753488820',
+        '17235543131546745471991808272245772046758360534180976603221801364506032471936',
+        '155962837046691114236524362966874066300454611955781275944230309195800494087',
+      ])
+    })
+  })
+
+  describe('#serialize', () => {
+    it('should work', () => {
+      const src = new MerkleTree(10, [1, 2, 3])
+      const data = src.serialize()
+      const dst = MerkleTree.deserialize(data)
+
+      src.root().should.equal(dst.root())
+
+      src.insert(10)
+      dst.insert(10)
+
+      src.root().should.equal(dst.root())
+    })
   })
 })