event retry & err conditioning

This commit is contained in:
gozzy 2022-12-02 21:02:43 +00:00
parent 90ef7362bf
commit 064c481c02

@ -292,6 +292,8 @@ class EventService {
i !== 5, i !== 5,
i i
) )
} else {
throw new Error(`Failed to fetch block ${toBlock}`)
} }
} }
@ -310,23 +312,16 @@ class EventService {
} }
} }
createBatchRequest({ batchIndex, batchSize, batchBlocks, blockDenom, type }) { createBatchRequest(batchArray) {
return new Array(batchSize).fill('').map( return batchArray.map(
(_, i) => (e, i) =>
new Promise(async (resolve) => { new Promise(async (resolve) => {
const toBlock = batchBlocks[batchIndex * batchSize + i] try {
const fromBlock = toBlock - blockDenom const { events } = await this.getEventsPartFromRpc({ ...e }, true)
resolve(events)
const batchEvents = await this.getEventsPartFromRpc( } catch (e) {
{ resolve({ isFailedBatch: true, ...e })
fromBlock, }
toBlock,
type
},
true
)
resolve(batchEvents.events)
}) })
) )
} }
@ -335,30 +330,47 @@ class EventService {
try { try {
const batchSize = 10 const batchSize = 10
const blockRange = 10000 const blockRange = 10000
const { blockDifference, currentBlockNumber } = await this.getBlocksDiff({ fromBlock })
let [events, failed] = [[], []]
let lastBlock = fromBlock
const { blockDifference, currentBlockNumber } = await this.getBlocksDiff({ fromBlock })
const batchDigest = blockDifference === 0 ? 1 : Math.ceil(blockDifference / blockRange) const batchDigest = blockDifference === 0 ? 1 : Math.ceil(blockDifference / blockRange)
const blockDenom = Math.ceil(blockDifference / batchDigest) const blockDenom = Math.ceil(blockDifference / batchDigest)
const batchCount = Math.ceil(batchDigest / batchSize) const batchCount = Math.ceil(batchDigest / batchSize)
const blocks = new Array(batchCount * batchSize).fill('')
const batchBlocks = blocks.map((_, i) => (i + 1) * blockDenom + fromBlock)
let events = []
if (fromBlock < currentBlockNumber) { if (fromBlock < currentBlockNumber) {
this.updateEventProgress(0, type) await this.updateEventProgress(0, type)
for (let batchIndex = 0; batchIndex < batchCount; batchIndex++) { for (let batchIndex = 0; batchIndex < batchCount; batchIndex++) {
const batch = await Promise.all( const isLastBatch = batchIndex === batchCount - 1
this.createBatchRequest({ batchIndex, batchBlocks, blockDenom, batchSize, type }) const params = new Array(batchSize).fill('').map((_, i) => {
) const toBlock = (i + 1) * blockDenom + lastBlock
const fromBlock = toBlock - blockDenom
return { fromBlock, toBlock, type }
})
const batch = await Promise.all(this.createBatchRequest(params))
this.updateEventProgress(batchIndex / batchCount, type) lastBlock = params[batchSize - 1].toBlock
events = events.concat(batch) events = events.concat(batch.filter((e) => !e.isFailedBatch))
failed = failed.concat(batch.filter((e) => e.isFailedBatch))
const progressIndex = batchIndex - failed.length / batchSize
if (isLastBatch && failed.length !== 0) {
const fbatch = await Promise.all(this.createBatchRequest(failed))
const isFailedBatch = fbatch.filter((e) => e.isFailedBatch).length !== 0
if (isFailedBatch) {
throw new Error('Failed to batch events')
} else {
events = events.concat(fbatch)
}
}
await this.updateEventProgress(progressIndex / batchCount, type)
await sleep(200) await sleep(200)
} }
events = flattenNArray(events) events = flattenNArray(events)
return { return {