From cda55646d2964291d3f7fe44b44b482dafef35eb Mon Sep 17 00:00:00 2001 From: nikugogoi Date: Fri, 3 Dec 2021 16:24:13 +0530 Subject: [PATCH] Update uni-info-watcher with changes in uniswap subgraph mapping (#298) * Update mapping code with latest subgraph changes * Add mapping code for TickDayData entity --- packages/uni-info-watcher/src/database.ts | 32 +++++++ .../src/entity/TickDayData.ts | 37 ++++++++ .../src/get-prev-entity.test.ts | 14 +-- packages/uni-info-watcher/src/indexer.ts | 92 ++++++++++++++++--- .../src/utils/interval-updates.ts | 28 ++++++ .../uni-info-watcher/src/utils/pricing.ts | 46 +++++++--- packages/uni-info-watcher/src/utils/tick.ts | 17 ++++ 7 files changed, 232 insertions(+), 34 deletions(-) create mode 100644 packages/uni-info-watcher/src/entity/TickDayData.ts diff --git a/packages/uni-info-watcher/src/database.ts b/packages/uni-info-watcher/src/database.ts index a651da7a..1e9a6ee9 100644 --- a/packages/uni-info-watcher/src/database.ts +++ b/packages/uni-info-watcher/src/database.ts @@ -44,6 +44,7 @@ import { PositionSnapshot } from './entity/PositionSnapshot'; import { BlockProgress } from './entity/BlockProgress'; import { Block } from './events'; import { SyncStatus } from './entity/SyncStatus'; +import { TickDayData } from './entity/TickDayData'; export class Database implements DatabaseInterface { _config: ConnectionOptions @@ -396,6 +397,30 @@ export class Database implements DatabaseInterface { return entity; } + async getTickDayData (queryRunner: QueryRunner, { id, blockHash }: DeepPartial): Promise { + const repo = queryRunner.manager.getRepository(TickDayData); + const whereOptions: FindConditions = { id }; + + if (blockHash) { + whereOptions.blockHash = blockHash; + } + + const findOptions = { + where: whereOptions, + order: { + blockNumber: 'DESC' + } + }; + + let entity = await repo.findOne(findOptions as FindOneOptions); + + if (!entity && findOptions.where.blockHash) { + entity = await this._baseDatabase.getPrevEntityVersion(queryRunner, repo, findOptions); + } + + return entity; + } + async getTransaction (queryRunner: QueryRunner, { id, blockHash }: DeepPartial): Promise { const repo = queryRunner.manager.getRepository(Transaction); const whereOptions: FindConditions = { id }; @@ -515,6 +540,13 @@ export class Database implements DatabaseInterface { return repo.save(tick); } + async saveTickDayData (queryRunner: QueryRunner, tickDayData: TickDayData, block: Block): Promise { + const repo = queryRunner.manager.getRepository(TickDayData); + tickDayData.blockNumber = block.number; + tickDayData.blockHash = block.hash; + return repo.save(tickDayData); + } + async savePosition (queryRunner: QueryRunner, position: Position, block: Block): Promise { const repo = queryRunner.manager.getRepository(Position); position.blockNumber = block.number; diff --git a/packages/uni-info-watcher/src/entity/TickDayData.ts b/packages/uni-info-watcher/src/entity/TickDayData.ts new file mode 100644 index 00000000..5e810a41 --- /dev/null +++ b/packages/uni-info-watcher/src/entity/TickDayData.ts @@ -0,0 +1,37 @@ +// +// Copyright 2021 Vulcanize, Inc. +// + +import { Entity, PrimaryColumn, Column, ManyToOne } from 'typeorm'; +import { bigintTransformer } from '@vulcanize/util'; + +import { Pool } from './Pool'; +import { Tick } from './Tick'; + +@Entity() +export class TickDayData { + @PrimaryColumn('varchar') + id!: string; + + // https://typeorm.io/#/entities/primary-columns + @PrimaryColumn('varchar', { length: 66 }) + blockHash!: string + + @Column('integer') + blockNumber!: number; + + @Column('integer') + date!: number + + @ManyToOne(() => Pool, { onDelete: 'CASCADE' }) + pool!: Pool; + + @ManyToOne(() => Tick, { onDelete: 'CASCADE' }) + tick!: Tick + + @Column('numeric', { transformer: bigintTransformer }) + liquidityGross!: bigint; + + @Column('numeric', { transformer: bigintTransformer }) + liquidityNet!: bigint; +} diff --git a/packages/uni-info-watcher/src/get-prev-entity.test.ts b/packages/uni-info-watcher/src/get-prev-entity.test.ts index 1904f5fc..2e061f78 100644 --- a/packages/uni-info-watcher/src/get-prev-entity.test.ts +++ b/packages/uni-info-watcher/src/get-prev-entity.test.ts @@ -113,7 +113,7 @@ describe('getPrevEntityVersion', () => { const searchedToken = await db.getToken(dbTx, { id: token00.id, blockHash: head.hash }); expect(searchedToken).to.not.be.empty; expect(searchedToken?.id).to.be.equal(token00.id); - expect(searchedToken?.txCount).to.be.equal(token00.txCount.toString()); + expect(searchedToken?.txCount).to.be.equal(token00.txCount); expect(searchedToken?.blockNumber).to.be.equal(token00.blockNumber); expect(searchedToken?.blockHash).to.be.equal(token00.blockHash); @@ -189,7 +189,7 @@ describe('getPrevEntityVersion', () => { const searchedToken = await db.getToken(dbTx, { id: token00.id, blockHash: head.hash }); expect(searchedToken).to.not.be.empty; expect(searchedToken?.id).to.be.equal(token02.id); - expect(searchedToken?.txCount).to.be.equal(token02.txCount.toString()); + expect(searchedToken?.txCount).to.be.equal(token02.txCount); expect(searchedToken?.blockNumber).to.be.equal(token02.blockNumber); expect(searchedToken?.blockHash).to.be.equal(token02.blockHash); @@ -255,7 +255,7 @@ describe('getPrevEntityVersion', () => { const searchedToken = await db.getToken(dbTx, { id: token00.id, blockHash: head.hash }); expect(searchedToken).to.not.be.empty; expect(searchedToken?.id).to.be.equal(token30.id); - expect(searchedToken?.txCount).to.be.equal(token30.txCount.toString()); + expect(searchedToken?.txCount).to.be.equal(token30.txCount); expect(searchedToken?.blockNumber).to.be.equal(token30.blockNumber); expect(searchedToken?.blockHash).to.be.equal(token30.blockHash); @@ -325,7 +325,7 @@ describe('getPrevEntityVersion', () => { const searchedToken = await db.getToken(dbTx, { id: token08.id, blockHash: head.hash }); expect(searchedToken).to.not.be.empty; expect(searchedToken?.id).to.be.equal(token30.id); - expect(searchedToken?.txCount).to.be.equal(token30.txCount.toString()); + expect(searchedToken?.txCount).to.be.equal(token30.txCount); expect(searchedToken?.blockNumber).to.be.equal(token30.blockNumber); expect(searchedToken?.blockHash).to.be.equal(token30.blockHash); @@ -448,7 +448,7 @@ describe('getPrevEntityVersion', () => { const searchedToken = await db.getToken(dbTx, { id: tokenA00.id, blockHash: head.hash }); expect(searchedToken).to.not.be.empty; expect(searchedToken?.id).to.be.equal(tokenA00.id); - expect(searchedToken?.txCount).to.be.equal(tokenA00.txCount.toString()); + expect(searchedToken?.txCount).to.be.equal(tokenA00.txCount); expect(searchedToken?.blockNumber).to.be.equal(tokenA00.blockNumber); expect(searchedToken?.blockHash).to.be.equal(tokenA00.blockHash); @@ -525,7 +525,7 @@ describe('getPrevEntityVersion', () => { const searchedToken = await db.getToken(dbTx, { id: tokenA00.id, blockHash: head.hash }); expect(searchedToken).to.not.be.empty; expect(searchedToken?.id).to.be.equal(tokenA30.id); - expect(searchedToken?.txCount).to.be.equal(tokenA30.txCount.toString()); + expect(searchedToken?.txCount).to.be.equal(tokenA30.txCount); expect(searchedToken?.blockNumber).to.be.equal(tokenA30.blockNumber); expect(searchedToken?.blockHash).to.be.equal(tokenA30.blockHash); @@ -607,7 +607,7 @@ describe('getPrevEntityVersion', () => { const searchedToken = await db.getToken(dbTx, { id: tokenA08.id, blockHash: head.hash }); expect(searchedToken).to.not.be.empty; expect(searchedToken?.id).to.be.equal(tokenA31.id); - expect(searchedToken?.txCount).to.be.equal(tokenA31.txCount.toString()); + expect(searchedToken?.txCount).to.be.equal(tokenA31.txCount); expect(searchedToken?.blockNumber).to.be.equal(tokenA31.blockNumber); expect(searchedToken?.blockHash).to.be.equal(tokenA31.blockHash); diff --git a/packages/uni-info-watcher/src/indexer.ts b/packages/uni-info-watcher/src/indexer.ts index ec588d6e..65569008 100644 --- a/packages/uni-info-watcher/src/indexer.ts +++ b/packages/uni-info-watcher/src/indexer.ts @@ -6,7 +6,7 @@ import assert from 'assert'; import debug from 'debug'; import { DeepPartial, QueryRunner } from 'typeorm'; import JSONbig from 'json-bigint'; -import { providers, utils } from 'ethers'; +import { providers, utils, BigNumber } from 'ethers'; import { Client as UniClient } from '@vulcanize/uni-watcher'; import { Client as ERC20Client } from '@vulcanize/erc20-watcher'; @@ -14,10 +14,10 @@ import { EthClient } from '@vulcanize/ipld-eth-client'; import { IndexerInterface, Indexer as BaseIndexer, QueryOptions, OrderDirection, BlockHeight, Relation, GraphDecimal } from '@vulcanize/util'; import { findEthPerToken, getEthPriceInUSD, getTrackedAmountUSD, sqrtPriceX96ToTokenPrices, WHITELIST_TOKENS } from './utils/pricing'; -import { updatePoolDayData, updatePoolHourData, updateTokenDayData, updateTokenHourData, updateUniswapDayData } from './utils/interval-updates'; +import { updatePoolDayData, updatePoolHourData, updateTickDayData, updateTokenDayData, updateTokenHourData, updateUniswapDayData } from './utils/interval-updates'; import { Token } from './entity/Token'; import { convertTokenToDecimal, loadTransaction, safeDiv } from './utils'; -import { createTick } from './utils/tick'; +import { createTick, feeTierToTickSpacing } from './utils/tick'; import { Position } from './entity/Position'; import { Database } from './database'; import { Event } from './entity/Event'; @@ -31,6 +31,7 @@ import { Swap } from './entity/Swap'; import { PositionSnapshot } from './entity/PositionSnapshot'; import { SyncStatus } from './entity/SyncStatus'; import { BlockProgress } from './entity/BlockProgress'; +import { Tick } from './entity/Tick'; const SYNC_DELTA = 5; @@ -411,6 +412,12 @@ export class Indexer implements IndexerInterface { token1 = await this._initToken(block, token1Address); } + // Bail if we couldn't figure out the decimals. + if (token0.decimals === null || token1.decimals === null) { + log('mybug the decimal on token was null'); + return; + } + // Save entities to DB. const dbTx = await this._db.createTransactionRunner(); @@ -690,7 +697,10 @@ export class Indexer implements IndexerInterface { await this._db.saveTick(dbTx, upperTick, block) ]); - // Skipping update inner tick vars and tick day data as they are not queried. + // Update inner tick vars and save the ticks. + await this._updateTickFeeVarsAndSave(dbTx, lowerTick, block); + await this._updateTickFeeVarsAndSave(dbTx, upperTick, block); + await dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -806,6 +816,8 @@ export class Indexer implements IndexerInterface { await updateTokenHourData(this._db, dbTx, token0, { block }); await updatePoolDayData(this._db, dbTx, { block, contractAddress }); await updatePoolHourData(this._db, dbTx, { block, contractAddress }); + await this._updateTickFeeVarsAndSave(dbTx, lowerTick, block); + await this._updateTickFeeVarsAndSave(dbTx, upperTick, block); [token0, token1] = await Promise.all([ this._db.saveToken(dbTx, token0, block), @@ -865,6 +877,9 @@ export class Indexer implements IndexerInterface { assert(token0 && token1, 'Pool tokens not found.'); + const oldTick = pool.tick; + assert(oldTick); + // Amounts - 0/1 are token deltas. Can be positive or negative. const amount0 = convertTokenToDecimal(BigInt(swapEvent.amount0), BigInt(token0.decimals)); const amount1 = convertTokenToDecimal(BigInt(swapEvent.amount1), BigInt(token1.decimals)); @@ -1055,7 +1070,44 @@ export class Indexer implements IndexerInterface { poolDayData.pool = pool; await this._db.savePoolDayData(dbTx, poolDayData, block); - // Skipping update of inner vars of current or crossed ticks as they are not queried. + // Update inner vars of current or crossed ticks. + const newTick = pool.tick; + assert(newTick); + const tickSpacing = feeTierToTickSpacing(pool.feeTier); + const modulo = newTick % tickSpacing; + + if (modulo === BigInt(0)) { + // Current tick is initialized and needs to be updated. + this._loadTickUpdateFeeVarsAndSave(dbTx, Number(newTick), block, contractAddress); + } + + const numIters = BigInt( + BigNumber.from(oldTick - newTick) + .abs() + .div(tickSpacing) + .toString() + ); + + if (numIters > BigInt(100)) { + // In case more than 100 ticks need to be updated ignore the update in + // order to avoid timeouts. From testing this behavior occurs only upon + // pool initialization. This should not be a big issue as the ticks get + // updated later. For early users this error also disappears when calling + // collect. + } else if (newTick > oldTick) { + const firstInitialized = oldTick + tickSpacing - modulo; + + for (let i = firstInitialized; i < newTick; i = i + tickSpacing) { + this._loadTickUpdateFeeVarsAndSave(dbTx, Number(i), block, contractAddress); + } + } else if (newTick < oldTick) { + const firstInitialized = oldTick - modulo; + + for (let i = firstInitialized; i >= newTick; i = i - tickSpacing) { + this._loadTickUpdateFeeVarsAndSave(dbTx, Number(i), block, contractAddress); + } + } + await dbTx.commitTransaction(); } catch (error) { await dbTx.rollbackTransaction(); @@ -1162,7 +1214,6 @@ export class Indexer implements IndexerInterface { return; } - // Temp fix from Subgraph mapping code. if (utils.getAddress(position.pool.id) === utils.getAddress('0x8fe8d9bb8eeba3ed688069c3d6b556c9ca258248')) { return; } @@ -1177,13 +1228,6 @@ export class Indexer implements IndexerInterface { position = await this._db.savePosition(dbTx, position, block); } - const token0 = position.token0; - const token1 = position.token1; - const amount0 = convertTokenToDecimal(BigInt(event.amount0), BigInt(token0.decimals)); - const amount1 = convertTokenToDecimal(BigInt(event.amount1), BigInt(token1.decimals)); - position.collectedFeesToken0 = position.collectedFeesToken0.plus(amount0); - position.collectedFeesToken1 = position.collectedFeesToken1.plus(amount1); - await this._db.savePosition(dbTx, position, block); await this._savePositionSnapshot(dbTx, position, block, tx); @@ -1225,6 +1269,28 @@ export class Indexer implements IndexerInterface { } } + async _updateTickFeeVarsAndSave (dbTx: QueryRunner, tick: Tick, block: Block): Promise { + // Skipping update feeGrowthOutside0X128 and feeGrowthOutside1X128 data as they are not queried. + + await updateTickDayData(this._db, dbTx, tick, { block }); + } + + async _loadTickUpdateFeeVarsAndSave (dbTx:QueryRunner, tickId: number, block: Block, contractAddress: string): Promise { + const poolAddress = contractAddress; + + const tick = await this._db.getTick( + dbTx, + { + id: poolAddress.concat('#').concat(tickId.toString()), + blockHash: block.hash + } + ); + + if (tick) { + await this._updateTickFeeVarsAndSave(dbTx, tick, block); + } + } + async _getPosition (block: Block, contractAddress: string, tx: Transaction, tokenId: bigint): Promise { const { hash: blockHash } = block; let position = await this._db.getPosition({ id: tokenId.toString(), blockHash }); diff --git a/packages/uni-info-watcher/src/utils/interval-updates.ts b/packages/uni-info-watcher/src/utils/interval-updates.ts index 6ea8167d..049758c6 100644 --- a/packages/uni-info-watcher/src/utils/interval-updates.ts +++ b/packages/uni-info-watcher/src/utils/interval-updates.ts @@ -10,6 +10,8 @@ import { Database } from '../database'; import { Factory } from '../entity/Factory'; import { PoolDayData } from '../entity/PoolDayData'; import { PoolHourData } from '../entity/PoolHourData'; +import { Tick } from '../entity/Tick'; +import { TickDayData } from '../entity/TickDayData'; import { Token } from '../entity/Token'; import { TokenDayData } from '../entity/TokenDayData'; import { TokenHourData } from '../entity/TokenHourData'; @@ -229,3 +231,29 @@ export const updateTokenHourData = async (db: Database, dbTx: QueryRunner, token tokenHourData.totalValueLockedUSD = token.totalValueLockedUSD; return db.saveTokenHourData(dbTx, tokenHourData, block); }; + +export const updateTickDayData = async (db: Database, dbTx: QueryRunner, tick: Tick, event: { block: Block }): Promise => { + const { block } = event; + const timestamp = block.timestamp; + const dayID = Math.floor(timestamp / 86400); + const dayStartTimestamp = dayID * 86400; + + const tickDayDataID = tick.id + .concat('-') + .concat(dayID.toString()); + + let tickDayData = await db.getTickDayData(dbTx, { id: tickDayDataID, blockHash: block.hash }); + + if (!tickDayData) { + tickDayData = new TickDayData(); + tickDayData.id = tickDayDataID; + tickDayData.date = dayStartTimestamp; + tickDayData.pool = tick.pool; + tickDayData.tick = tick; + } + + tickDayData.liquidityGross = tick.liquidityGross; + tickDayData.liquidityNet = tick.liquidityNet; + + return db.saveTickDayData(dbTx, tickDayData, block); +}; diff --git a/packages/uni-info-watcher/src/utils/pricing.ts b/packages/uni-info-watcher/src/utils/pricing.ts index 8350a615..36949179 100644 --- a/packages/uni-info-watcher/src/utils/pricing.ts +++ b/packages/uni-info-watcher/src/utils/pricing.ts @@ -44,6 +44,15 @@ export const WHITELIST_TOKENS: string[] = [ '0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9' // AAVE ]; +const STABLE_COINS: string[] = [ + '0x6b175474e89094c44da98b954eedeac495271d0f', + '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', + '0xdac17f958d2ee523a2206206994597c13d831ec7', + '0x0000000000085d4780b73119b644ae5ecd22b376', + '0x956f47f50a910163d8bf957cf5846d573e7f87ca', + '0x4dd28568d05f09b02220b09c2cb307bfd837cb95' +]; + const MINIMUM_ETH_LOCKED = new GraphDecimal(52); const Q192 = 2 ** 192; @@ -95,23 +104,32 @@ export const findEthPerToken = async (db: Database, dbTx: QueryRunner, token: To // Need to update this to actually detect best rate based on liquidity distribution. let largestLiquidityETH = new GraphDecimal(0); let priceSoFar = new GraphDecimal(0); + const bundle = await db.getBundle(dbTx, { id: '1' }); + assert(bundle); - for (let i = 0; i < whiteList.length; ++i) { - const poolAddress = whiteList[i].id; - const pool = await db.getPool(dbTx, { id: poolAddress }); - assert(pool); + // hardcoded fix for incorrect rates + // if whitelist includes token - get the safe price + if (STABLE_COINS.includes(token.id)) { + priceSoFar = safeDiv(new GraphDecimal(1), bundle.ethPriceUSD); + } else { + for (let i = 0; i < whiteList.length; ++i) { + const poolAddress = whiteList[i].id; + const pool = await db.getPool(dbTx, { id: poolAddress }); + assert(pool); - if (BigNumber.from(pool.liquidity).gt(0)) { - if (pool.token0.id === token.id) { - // Whitelist token is token1. - const token1 = pool.token1; - // Get the derived ETH in pool. - const ethLocked = pool.totalValueLockedToken1.times(token1.derivedETH); + if (BigNumber.from(pool.liquidity).gt(0)) { + if (pool.token0.id === token.id) { + // whitelist token is token1 + const token1 = pool.token1; - if (ethLocked.gt(largestLiquidityETH) && ethLocked.gt(MINIMUM_ETH_LOCKED)) { - largestLiquidityETH = ethLocked; - // token1 per our token * Eth per token1 - priceSoFar = pool.token1Price.times(token1.derivedETH); + // get the derived ETH in pool + const ethLocked = pool.totalValueLockedToken1.times(token1.derivedETH); + + if (ethLocked.gt(largestLiquidityETH) && ethLocked.gt(MINIMUM_ETH_LOCKED)) { + largestLiquidityETH = ethLocked; + // token1 per our token * Eth per token1 + priceSoFar = pool.token1Price.times(token1.derivedETH); + } } } if (pool.token1.id === token.id) { diff --git a/packages/uni-info-watcher/src/utils/tick.ts b/packages/uni-info-watcher/src/utils/tick.ts index 5a38b3cc..e191fe40 100644 --- a/packages/uni-info-watcher/src/utils/tick.ts +++ b/packages/uni-info-watcher/src/utils/tick.ts @@ -27,3 +27,20 @@ export const createTick = async (db: Database, dbTx: QueryRunner, tickId: string return db.saveTick(dbTx, tick, block); }; + +export const feeTierToTickSpacing = (feeTier: bigint): bigint => { + if (feeTier === BigInt(10000)) { + return BigInt(200); + } + if (feeTier === BigInt(3000)) { + return BigInt(60); + } + if (feeTier === BigInt(500)) { + return BigInt(10); + } + if (feeTier === BigInt(100)) { + return BigInt(1); + } + + throw Error('Unexpected fee tier'); +};