efficient bulk insert

(cherry picked from commit c83a4a4dc0)

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: Node.js CI
+name: build
 
 on:
   pull_request:
@@ -16,6 +16,14 @@ jobs:
           node-version: 14.7.0
       - run: npm ci
       - run: npm run test
+      - name: Telegram Failure Notification
+        uses: appleboy/telegram-action@0.0.7
+        if: failure()
+        with:
+          message: ❗ Build failed for [${{ github.repository }}](https://github.com/${{ github.repository }}/actions) because of ${{ github.actor }}
+          format: markdown
+          to: ${{ secrets.TELEGRAM_CHAT_ID }}
+          token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
 
   lint:
     runs-on: ubuntu-latest
@@ -26,6 +34,14 @@ jobs:
           node-version: 14.7.0
       - run: npm ci
       - run: npm run lint
+      - name: Telegram Failure Notification
+        uses: appleboy/telegram-action@0.0.7
+        if: failure()
+        with:
+          message: ❗ Build failed for [${{ github.repository }}](https://github.com/${{ github.repository }}/actions) because of ${{ github.actor }}
+          format: markdown
+          to: ${{ secrets.TELEGRAM_CHAT_ID }}
+          token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
 
   publish:
     runs-on: ubuntu-latest
@@ -65,3 +81,12 @@ jobs:
           message: 🚀 Published [${{ steps.vars.outputs.repo_name }}](https://github.com/${{ github.repository }}) version [${{ steps.vars.outputs.version }}](https://www.npmjs.com/package/${{ steps.vars.outputs.repo_name }}/v/${{ steps.vars.outputs.version }}) to npm
           debug: true
           format: markdown
+
+      - name: Telegram Failure Notification
+        uses: appleboy/telegram-action@0.0.7
+        if: failure()
+        with:
+          message: ❗ Failed to publish [${{ steps.vars.outputs.repo_name }}](https://github.com/${{ github.repository }}/actions) because of ${{ env.GITHUB_ACTOR }}
+          format: markdown
+          to: ${{ secrets.TELEGRAM_CHAT_ID }}
+          token: ${{ secrets.TELEGRAM_BOT_TOKEN }}

.prettierrc

@@ -0,0 +1,7 @@
+{
+  "semi": false,
+  "arrowParens": "always",
+  "singleQuote": true,
+  "printWidth": 110,
+  "trailingComma": "all"
+}

README.md

@@ -1,4 +1,4 @@
-# Merkle Tree [![GitHub Workflow Status](https://img.shields.io/github/workflow/status/tornadocash/fixed-merkle-tree/Node.js%20CI)](https://github.com/tornadocash/fixed-merkle-tree/actions) [![npm](https://img.shields.io/npm/v/fixed-merkle-tree)](https://www.npmjs.com/package/fixed-merkle-tree)
+# Merkle Tree [![GitHub Workflow Status](https://img.shields.io/github/workflow/status/tornadocash/fixed-merkle-tree/build)](https://github.com/tornadocash/fixed-merkle-tree/actions) [![npm](https://img.shields.io/npm/v/fixed-merkle-tree)](https://www.npmjs.com/package/fixed-merkle-tree)
 
 This is a fixed depth merkle tree implementation with sequential inserts
 

package.json

@@ -1,19 +1,26 @@
 {
   "name": "fixed-merkle-tree",
-  "version": "0.3.2",
+  "version": "0.6.0",
   "description": "Fixed depth merkle tree implementation with sequential inserts",
+  "repository": "https://github.com/tornadocash/fixed-merkle-tree.git",
   "main": "src/merkleTree.js",
   "scripts": {
     "test": "mocha",
     "lint": "eslint ."
   },
-  "keywords": ["merkle", "tree", "merkleTree"],
+  "keywords": [
+    "merkle",
+    "tree",
+    "merkleTree"
+  ],
   "author": "Roman Semenov <semenov.roma@gmail.com>",
   "license": "ISC",
-  "files": ["src/*"],
+  "files": [
+    "src/*"
+  ],
   "dependencies": {
     "snarkjs": "git+https://github.com/tornadocash/snarkjs.git#869181cfaf7526fe8972073d31655493a04326d5",
-    "circomlib": "git+https://github.com/tornadocash/circomlib.git#c372f14d324d57339c88451834bf2824e73bbdbc"
+    "circomlib": "git+https://github.com/tornadocash/circomlib.git#5beb6aee94923052faeecea40135d45b6ce6172c"
   },
   "devDependencies": {
     "babel-eslint": "^10.1.0",

src/merkleTree.js

@@ -24,20 +24,19 @@ class MerkleTree {
    */
   constructor(levels, elements = [], { hashFunction, zeroElement = DEFAULT_ZERO } = {}) {
     this.levels = levels
-    this.capacity = 1 << levels
+    this.capacity = 2 ** levels
-    this.zeroElement = zeroElement
-    this._hash = hashFunction || defaultHash
     if (elements.length > this.capacity) {
       throw new Error('Tree is full')
     }
-
+    this._hash = hashFunction || defaultHash
+    this.zeroElement = zeroElement
     this._zeros = []
-    this._layers = []
+    this._zeros[0] = zeroElement
-    this._layers[0] = elements
-    this._zeros[0] = this.zeroElement
     for (let i = 1; i <= levels; i++) {
       this._zeros[i] = this._hash(this._zeros[i - 1], this._zeros[i - 1])
     }
+    this._layers = []
+    this._layers[0] = elements.slice()
     this._rebuild()
   }
 
@@ -47,9 +46,9 @@ class MerkleTree {
       for (let i = 0; i < Math.ceil(this._layers[level - 1].length / 2); i++) {
         this._layers[level][i] = this._hash(
           this._layers[level - 1][i * 2],
-          i * 2 + 1 < this._layers[level - 1].length ?
+          i * 2 + 1 < this._layers[level - 1].length
-            this._layers[level - 1][i * 2 + 1] :
+            ? this._layers[level - 1][i * 2 + 1]
-            this._zeros[level - 1],
+            : this._zeros[level - 1],
         )
       }
     }
@@ -75,15 +74,30 @@ class MerkleTree {
   }
 
   /**
-   * Insert multiple elements into the tree. Tree will be fully rebuilt during this operation.
+   * Insert multiple elements into the tree.
    * @param {Array} elements Elements to insert
    */
   bulkInsert(elements) {
     if (this._layers[0].length + elements.length > this.capacity) {
       throw new Error('Tree is full')
     }
-    this._layers[0].push(...elements)
+    // First we insert all elements except the last one
-    this._rebuild()
+    // updating only full subtree hashes (all layers where inserted element has odd index)
+    // the last element will update the full path to the root making the tree consistent again
+    for (let i = 0; i < elements.length - 1; i++) {
+      this._layers[0].push(elements[i])
+      let level = 0
+      let index = this._layers[0].length - 1
+      while (index % 2 === 1) {
+        level++
+        index >>= 1
+        this._layers[level][index] = this._hash(
+          this._layers[level - 1][index * 2],
+          this._layers[level - 1][index * 2 + 1],
+        )
+      }
+    }
+    this.insert(elements[elements.length - 1])
   }
 
   /**
@@ -100,9 +114,9 @@ class MerkleTree {
       index >>= 1
       this._layers[level][index] = this._hash(
         this._layers[level - 1][index * 2],
-        index * 2 + 1 < this._layers[level - 1].length ?
+        index * 2 + 1 < this._layers[level - 1].length
-          this._layers[level - 1][index * 2 + 1] :
+          ? this._layers[level - 1][index * 2 + 1]
-          this._zeros[level - 1],
+          : this._zeros[level - 1],
       )
     }
   }
@@ -120,9 +134,8 @@ class MerkleTree {
     const pathIndices = []
     for (let level = 0; level < this.levels; level++) {
       pathIndices[level] = index % 2
-      pathElements[level] = (index ^ 1) < this._layers[level].length ?
+      pathElements[level] =
-        this._layers[level][index ^ 1] :
+        (index ^ 1) < this._layers[level].length ? this._layers[level][index ^ 1] : this._zeros[level]
-        this._zeros[level]
       index >>= 1
     }
     return {
@@ -152,6 +165,44 @@ class MerkleTree {
   elements() {
     return this._layers[0].slice()
   }
+
+  /**
+   * Returns a copy of n-th zero elements array
+   * @returns {Object[]}
+   */
+  zeros() {
+    return this._zeros.slice()
+  }
+
+  /**
+   * Serialize entire tree state including intermediate layers into a plain object
+   * Deserializing it back will not require to recompute any hashes
+   * Elements are not converted to a plain type, this is responsibility of the caller
+   */
+  serialize() {
+    return {
+      levels: this.levels,
+      _zeros: this._zeros,
+      _layers: this._layers,
+    }
+  }
+
+  /**
+   * Deserialize data into a MerkleTree instance
+   * Make sure to provide the same hashFunction as was used in the source tree,
+   * otherwise the tree state will be invalid
+   *
+   * @param data
+   * @param hashFunction
+   * @returns {MerkleTree}
+   */
+  static deserialize(data, hashFunction) {
+    const instance = Object.assign(Object.create(this.prototype), data)
+    instance._hash = hashFunction || defaultHash
+    instance.capacity = 2 ** instance.levels
+    instance.zeroElement = instance._zeros[0]
+    return instance
+  }
 }
 
 module.exports = MerkleTree

test/merkleTree.test.js

@@ -71,6 +71,32 @@ describe('MerkleTree', () => {
       tree.root().should.equal('10132905325673518287563057607527946096399700874345297651940963130460267058606')
     })
 
+    it('should give the same result as sequental inserts', () => {
+      const initialArray = [
+        [1],
+        [1, 2],
+        [1, 2, 3],
+        [1, 2, 3, 4],
+      ]
+      const insertedArray = [
+        [11],
+        [11, 12],
+        [11, 12, 13],
+        [11, 12, 13, 14],
+      ]
+      for (const initial of initialArray) {
+        for (const inserted of insertedArray) {
+          const tree1 = new MerkleTree(10, initial)
+          const tree2 = new MerkleTree(10, initial)
+          tree1.bulkInsert(inserted)
+          for (const item of inserted) {
+            tree2.insert(item)
+          }
+          tree1.root().should.equal(tree2.root())
+        }
+      }
+    }).timeout(10000)
+
     it('should work with max elements', () => {
       const tree = new MerkleTree(2, [1, 2])
       tree.bulkInsert([3, 4])
@@ -202,4 +228,19 @@ describe('MerkleTree', () => {
       ])
     })
   })
+
+  describe('#serialize', () => {
+    it('should work', () => {
+      const src = new MerkleTree(10, [1, 2, 3])
+      const data = src.serialize()
+      const dst = MerkleTree.deserialize(data)
+
+      src.root().should.equal(dst.root())
+
+      src.insert(10)
+      dst.insert(10)
+
+      src.root().should.equal(dst.root())
+    })
+  })
 })