Update uni-info-watcher with changes in uniswap subgraph mapping (#298)

* Update mapping code with latest subgraph changes

* Add mapping code for TickDayData entity

packages/uni-info-watcher/src/database.ts

@@ -44,6 +44,7 @@ import { PositionSnapshot } from './entity/PositionSnapshot';
 import { BlockProgress } from './entity/BlockProgress';
 import { Block } from './events';
 import { SyncStatus } from './entity/SyncStatus';
+import { TickDayData } from './entity/TickDayData';
 
 export class Database implements DatabaseInterface {
   _config: ConnectionOptions
@@ -396,6 +397,30 @@ export class Database implements DatabaseInterface {
     return entity;
   }
 
+  async getTickDayData (queryRunner: QueryRunner, { id, blockHash }: DeepPartial<TickDayData>): Promise<TickDayData | undefined> {
+    const repo = queryRunner.manager.getRepository(TickDayData);
+    const whereOptions: FindConditions<TickDayData> = { id };
+
+    if (blockHash) {
+      whereOptions.blockHash = blockHash;
+    }
+
+    const findOptions = {
+      where: whereOptions,
+      order: {
+        blockNumber: 'DESC'
+      }
+    };
+
+    let entity = await repo.findOne(findOptions as FindOneOptions<TickDayData>);
+
+    if (!entity && findOptions.where.blockHash) {
+      entity = await this._baseDatabase.getPrevEntityVersion(queryRunner, repo, findOptions);
+    }
+
+    return entity;
+  }
+
   async getTransaction (queryRunner: QueryRunner, { id, blockHash }: DeepPartial<Transaction>): Promise<Transaction | undefined> {
     const repo = queryRunner.manager.getRepository(Transaction);
     const whereOptions: FindConditions<Transaction> = { id };
@@ -515,6 +540,13 @@ export class Database implements DatabaseInterface {
     return repo.save(tick);
   }
 
+  async saveTickDayData (queryRunner: QueryRunner, tickDayData: TickDayData, block: Block): Promise<TickDayData> {
+    const repo = queryRunner.manager.getRepository(TickDayData);
+    tickDayData.blockNumber = block.number;
+    tickDayData.blockHash = block.hash;
+    return repo.save(tickDayData);
+  }
+
   async savePosition (queryRunner: QueryRunner, position: Position, block: Block): Promise<Position> {
     const repo = queryRunner.manager.getRepository(Position);
     position.blockNumber = block.number;

packages/uni-info-watcher/src/entity/TickDayData.ts

@@ -0,0 +1,37 @@
+//
+// Copyright 2021 Vulcanize, Inc.
+//
+
+import { Entity, PrimaryColumn, Column, ManyToOne } from 'typeorm';
+import { bigintTransformer } from '@vulcanize/util';
+
+import { Pool } from './Pool';
+import { Tick } from './Tick';
+
+@Entity()
+export class TickDayData {
+  @PrimaryColumn('varchar')
+  id!: string;
+
+  // https://typeorm.io/#/entities/primary-columns
+  @PrimaryColumn('varchar', { length: 66 })
+  blockHash!: string
+
+  @Column('integer')
+  blockNumber!: number;
+
+  @Column('integer')
+  date!: number
+
+  @ManyToOne(() => Pool, { onDelete: 'CASCADE' })
+  pool!: Pool;
+
+  @ManyToOne(() => Tick, { onDelete: 'CASCADE' })
+  tick!: Tick
+
+  @Column('numeric', { transformer: bigintTransformer })
+  liquidityGross!: bigint;
+
+  @Column('numeric', { transformer: bigintTransformer })
+  liquidityNet!: bigint;
+}

packages/uni-info-watcher/src/get-prev-entity.test.ts

@@ -113,7 +113,7 @@ describe('getPrevEntityVersion', () => {
       const searchedToken = await db.getToken(dbTx, { id: token00.id, blockHash: head.hash });
       expect(searchedToken).to.not.be.empty;
       expect(searchedToken?.id).to.be.equal(token00.id);
-      expect(searchedToken?.txCount).to.be.equal(token00.txCount.toString());
+      expect(searchedToken?.txCount).to.be.equal(token00.txCount);
       expect(searchedToken?.blockNumber).to.be.equal(token00.blockNumber);
       expect(searchedToken?.blockHash).to.be.equal(token00.blockHash);
 
@@ -189,7 +189,7 @@ describe('getPrevEntityVersion', () => {
       const searchedToken = await db.getToken(dbTx, { id: token00.id, blockHash: head.hash });
       expect(searchedToken).to.not.be.empty;
       expect(searchedToken?.id).to.be.equal(token02.id);
-      expect(searchedToken?.txCount).to.be.equal(token02.txCount.toString());
+      expect(searchedToken?.txCount).to.be.equal(token02.txCount);
       expect(searchedToken?.blockNumber).to.be.equal(token02.blockNumber);
       expect(searchedToken?.blockHash).to.be.equal(token02.blockHash);
 
@@ -255,7 +255,7 @@ describe('getPrevEntityVersion', () => {
       const searchedToken = await db.getToken(dbTx, { id: token00.id, blockHash: head.hash });
       expect(searchedToken).to.not.be.empty;
       expect(searchedToken?.id).to.be.equal(token30.id);
-      expect(searchedToken?.txCount).to.be.equal(token30.txCount.toString());
+      expect(searchedToken?.txCount).to.be.equal(token30.txCount);
       expect(searchedToken?.blockNumber).to.be.equal(token30.blockNumber);
       expect(searchedToken?.blockHash).to.be.equal(token30.blockHash);
 
@@ -325,7 +325,7 @@ describe('getPrevEntityVersion', () => {
       const searchedToken = await db.getToken(dbTx, { id: token08.id, blockHash: head.hash });
       expect(searchedToken).to.not.be.empty;
       expect(searchedToken?.id).to.be.equal(token30.id);
-      expect(searchedToken?.txCount).to.be.equal(token30.txCount.toString());
+      expect(searchedToken?.txCount).to.be.equal(token30.txCount);
       expect(searchedToken?.blockNumber).to.be.equal(token30.blockNumber);
       expect(searchedToken?.blockHash).to.be.equal(token30.blockHash);
 
@@ -448,7 +448,7 @@ describe('getPrevEntityVersion', () => {
       const searchedToken = await db.getToken(dbTx, { id: tokenA00.id, blockHash: head.hash });
       expect(searchedToken).to.not.be.empty;
       expect(searchedToken?.id).to.be.equal(tokenA00.id);
-      expect(searchedToken?.txCount).to.be.equal(tokenA00.txCount.toString());
+      expect(searchedToken?.txCount).to.be.equal(tokenA00.txCount);
       expect(searchedToken?.blockNumber).to.be.equal(tokenA00.blockNumber);
       expect(searchedToken?.blockHash).to.be.equal(tokenA00.blockHash);
 
@@ -525,7 +525,7 @@ describe('getPrevEntityVersion', () => {
       const searchedToken = await db.getToken(dbTx, { id: tokenA00.id, blockHash: head.hash });
       expect(searchedToken).to.not.be.empty;
       expect(searchedToken?.id).to.be.equal(tokenA30.id);
-      expect(searchedToken?.txCount).to.be.equal(tokenA30.txCount.toString());
+      expect(searchedToken?.txCount).to.be.equal(tokenA30.txCount);
       expect(searchedToken?.blockNumber).to.be.equal(tokenA30.blockNumber);
       expect(searchedToken?.blockHash).to.be.equal(tokenA30.blockHash);
 
@@ -607,7 +607,7 @@ describe('getPrevEntityVersion', () => {
       const searchedToken = await db.getToken(dbTx, { id: tokenA08.id, blockHash: head.hash });
       expect(searchedToken).to.not.be.empty;
       expect(searchedToken?.id).to.be.equal(tokenA31.id);
-      expect(searchedToken?.txCount).to.be.equal(tokenA31.txCount.toString());
+      expect(searchedToken?.txCount).to.be.equal(tokenA31.txCount);
       expect(searchedToken?.blockNumber).to.be.equal(tokenA31.blockNumber);
       expect(searchedToken?.blockHash).to.be.equal(tokenA31.blockHash);
 

packages/uni-info-watcher/src/indexer.ts

@@ -6,7 +6,7 @@ import assert from 'assert';
 import debug from 'debug';
 import { DeepPartial, QueryRunner } from 'typeorm';
 import JSONbig from 'json-bigint';
-import { providers, utils } from 'ethers';
+import { providers, utils, BigNumber } from 'ethers';
 
 import { Client as UniClient } from '@vulcanize/uni-watcher';
 import { Client as ERC20Client } from '@vulcanize/erc20-watcher';
@@ -14,10 +14,10 @@ import { EthClient } from '@vulcanize/ipld-eth-client';
 import { IndexerInterface, Indexer as BaseIndexer, QueryOptions, OrderDirection, BlockHeight, Relation, GraphDecimal } from '@vulcanize/util';
 
 import { findEthPerToken, getEthPriceInUSD, getTrackedAmountUSD, sqrtPriceX96ToTokenPrices, WHITELIST_TOKENS } from './utils/pricing';
-import { updatePoolDayData, updatePoolHourData, updateTokenDayData, updateTokenHourData, updateUniswapDayData } from './utils/interval-updates';
+import { updatePoolDayData, updatePoolHourData, updateTickDayData, updateTokenDayData, updateTokenHourData, updateUniswapDayData } from './utils/interval-updates';
 import { Token } from './entity/Token';
 import { convertTokenToDecimal, loadTransaction, safeDiv } from './utils';
-import { createTick } from './utils/tick';
+import { createTick, feeTierToTickSpacing } from './utils/tick';
 import { Position } from './entity/Position';
 import { Database } from './database';
 import { Event } from './entity/Event';
@@ -31,6 +31,7 @@ import { Swap } from './entity/Swap';
 import { PositionSnapshot } from './entity/PositionSnapshot';
 import { SyncStatus } from './entity/SyncStatus';
 import { BlockProgress } from './entity/BlockProgress';
+import { Tick } from './entity/Tick';
 
 const SYNC_DELTA = 5;
 
@@ -411,6 +412,12 @@ export class Indexer implements IndexerInterface {
       token1 = await this._initToken(block, token1Address);
     }
 
+    // Bail if we couldn't figure out the decimals.
+    if (token0.decimals === null || token1.decimals === null) {
+      log('mybug the decimal on token was null');
+      return;
+    }
+
     // Save entities to DB.
     const dbTx = await this._db.createTransactionRunner();
 
@@ -690,7 +697,10 @@ export class Indexer implements IndexerInterface {
         await this._db.saveTick(dbTx, upperTick, block)
       ]);
 
-      // Skipping update inner tick vars and tick day data as they are not queried.
+      // Update inner tick vars and save the ticks.
+      await this._updateTickFeeVarsAndSave(dbTx, lowerTick, block);
+      await this._updateTickFeeVarsAndSave(dbTx, upperTick, block);
+
       await dbTx.commitTransaction();
     } catch (error) {
       await dbTx.rollbackTransaction();
@@ -806,6 +816,8 @@ export class Indexer implements IndexerInterface {
       await updateTokenHourData(this._db, dbTx, token0, { block });
       await updatePoolDayData(this._db, dbTx, { block, contractAddress });
       await updatePoolHourData(this._db, dbTx, { block, contractAddress });
+      await this._updateTickFeeVarsAndSave(dbTx, lowerTick, block);
+      await this._updateTickFeeVarsAndSave(dbTx, upperTick, block);
 
       [token0, token1] = await Promise.all([
         this._db.saveToken(dbTx, token0, block),
@@ -865,6 +877,9 @@ export class Indexer implements IndexerInterface {
 
       assert(token0 && token1, 'Pool tokens not found.');
 
+      const oldTick = pool.tick;
+      assert(oldTick);
+
       // Amounts - 0/1 are token deltas. Can be positive or negative.
       const amount0 = convertTokenToDecimal(BigInt(swapEvent.amount0), BigInt(token0.decimals));
       const amount1 = convertTokenToDecimal(BigInt(swapEvent.amount1), BigInt(token1.decimals));
@@ -1055,7 +1070,44 @@ export class Indexer implements IndexerInterface {
       poolDayData.pool = pool;
       await this._db.savePoolDayData(dbTx, poolDayData, block);
 
-      // Skipping update of inner vars of current or crossed ticks as they are not queried.
+      // Update inner vars of current or crossed ticks.
+      const newTick = pool.tick;
+      assert(newTick);
+      const tickSpacing = feeTierToTickSpacing(pool.feeTier);
+      const modulo = newTick % tickSpacing;
+
+      if (modulo === BigInt(0)) {
+        // Current tick is initialized and needs to be updated.
+        this._loadTickUpdateFeeVarsAndSave(dbTx, Number(newTick), block, contractAddress);
+      }
+
+      const numIters = BigInt(
+        BigNumber.from(oldTick - newTick)
+          .abs()
+          .div(tickSpacing)
+          .toString()
+      );
+
+      if (numIters > BigInt(100)) {
+        // In case more than 100 ticks need to be updated ignore the update in
+        // order to avoid timeouts. From testing this behavior occurs only upon
+        // pool initialization. This should not be a big issue as the ticks get
+        // updated later. For early users this error also disappears when calling
+        // collect.
+      } else if (newTick > oldTick) {
+        const firstInitialized = oldTick + tickSpacing - modulo;
+
+        for (let i = firstInitialized; i < newTick; i = i + tickSpacing) {
+          this._loadTickUpdateFeeVarsAndSave(dbTx, Number(i), block, contractAddress);
+        }
+      } else if (newTick < oldTick) {
+        const firstInitialized = oldTick - modulo;
+
+        for (let i = firstInitialized; i >= newTick; i = i - tickSpacing) {
+          this._loadTickUpdateFeeVarsAndSave(dbTx, Number(i), block, contractAddress);
+        }
+      }
+
       await dbTx.commitTransaction();
     } catch (error) {
       await dbTx.rollbackTransaction();
@@ -1162,7 +1214,6 @@ export class Indexer implements IndexerInterface {
       return;
     }
 
-    // Temp fix from Subgraph mapping code.
     if (utils.getAddress(position.pool.id) === utils.getAddress('0x8fe8d9bb8eeba3ed688069c3d6b556c9ca258248')) {
       return;
     }
@@ -1177,13 +1228,6 @@ export class Indexer implements IndexerInterface {
         position = await this._db.savePosition(dbTx, position, block);
       }
 
-      const token0 = position.token0;
-      const token1 = position.token1;
-      const amount0 = convertTokenToDecimal(BigInt(event.amount0), BigInt(token0.decimals));
-      const amount1 = convertTokenToDecimal(BigInt(event.amount1), BigInt(token1.decimals));
-      position.collectedFeesToken0 = position.collectedFeesToken0.plus(amount0);
-      position.collectedFeesToken1 = position.collectedFeesToken1.plus(amount1);
-
       await this._db.savePosition(dbTx, position, block);
 
       await this._savePositionSnapshot(dbTx, position, block, tx);
@@ -1225,6 +1269,28 @@ export class Indexer implements IndexerInterface {
     }
   }
 
+  async _updateTickFeeVarsAndSave (dbTx: QueryRunner, tick: Tick, block: Block): Promise<void> {
+    // Skipping update feeGrowthOutside0X128 and feeGrowthOutside1X128 data as they are not queried.
+
+    await updateTickDayData(this._db, dbTx, tick, { block });
+  }
+
+  async _loadTickUpdateFeeVarsAndSave (dbTx:QueryRunner, tickId: number, block: Block, contractAddress: string): Promise<void> {
+    const poolAddress = contractAddress;
+
+    const tick = await this._db.getTick(
+      dbTx,
+      {
+        id: poolAddress.concat('#').concat(tickId.toString()),
+        blockHash: block.hash
+      }
+    );
+
+    if (tick) {
+      await this._updateTickFeeVarsAndSave(dbTx, tick, block);
+    }
+  }
+
   async _getPosition (block: Block, contractAddress: string, tx: Transaction, tokenId: bigint): Promise<Position | null> {
     const { hash: blockHash } = block;
     let position = await this._db.getPosition({ id: tokenId.toString(), blockHash });

packages/uni-info-watcher/src/utils/interval-updates.ts

@@ -10,6 +10,8 @@ import { Database } from '../database';
 import { Factory } from '../entity/Factory';
 import { PoolDayData } from '../entity/PoolDayData';
 import { PoolHourData } from '../entity/PoolHourData';
+import { Tick } from '../entity/Tick';
+import { TickDayData } from '../entity/TickDayData';
 import { Token } from '../entity/Token';
 import { TokenDayData } from '../entity/TokenDayData';
 import { TokenHourData } from '../entity/TokenHourData';
@@ -229,3 +231,29 @@ export const updateTokenHourData = async (db: Database, dbTx: QueryRunner, token
   tokenHourData.totalValueLockedUSD = token.totalValueLockedUSD;
   return db.saveTokenHourData(dbTx, tokenHourData, block);
 };
+
+export const updateTickDayData = async (db: Database, dbTx: QueryRunner, tick: Tick, event: { block: Block }): Promise<TickDayData> => {
+  const { block } = event;
+  const timestamp = block.timestamp;
+  const dayID = Math.floor(timestamp / 86400);
+  const dayStartTimestamp = dayID * 86400;
+
+  const tickDayDataID = tick.id
+    .concat('-')
+    .concat(dayID.toString());
+
+  let tickDayData = await db.getTickDayData(dbTx, { id: tickDayDataID, blockHash: block.hash });
+
+  if (!tickDayData) {
+    tickDayData = new TickDayData();
+    tickDayData.id = tickDayDataID;
+    tickDayData.date = dayStartTimestamp;
+    tickDayData.pool = tick.pool;
+    tickDayData.tick = tick;
+  }
+
+  tickDayData.liquidityGross = tick.liquidityGross;
+  tickDayData.liquidityNet = tick.liquidityNet;
+
+  return db.saveTickDayData(dbTx, tickDayData, block);
+};

packages/uni-info-watcher/src/utils/pricing.ts

@@ -44,6 +44,15 @@ export const WHITELIST_TOKENS: string[] = [
   '0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9' // AAVE
 ];
 
+const STABLE_COINS: string[] = [
+  '0x6b175474e89094c44da98b954eedeac495271d0f',
+  '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48',
+  '0xdac17f958d2ee523a2206206994597c13d831ec7',
+  '0x0000000000085d4780b73119b644ae5ecd22b376',
+  '0x956f47f50a910163d8bf957cf5846d573e7f87ca',
+  '0x4dd28568d05f09b02220b09c2cb307bfd837cb95'
+];
+
 const MINIMUM_ETH_LOCKED = new GraphDecimal(52);
 const Q192 = 2 ** 192;
 
@@ -95,23 +104,32 @@ export const findEthPerToken = async (db: Database, dbTx: QueryRunner, token: To
   // Need to update this to actually detect best rate based on liquidity distribution.
   let largestLiquidityETH = new GraphDecimal(0);
   let priceSoFar = new GraphDecimal(0);
+  const bundle = await db.getBundle(dbTx, { id: '1' });
+  assert(bundle);
 
-  for (let i = 0; i < whiteList.length; ++i) {
+  // hardcoded fix for incorrect rates
-    const poolAddress = whiteList[i].id;
+  // if whitelist includes token - get the safe price
-    const pool = await db.getPool(dbTx, { id: poolAddress });
+  if (STABLE_COINS.includes(token.id)) {
-    assert(pool);
+    priceSoFar = safeDiv(new GraphDecimal(1), bundle.ethPriceUSD);
+  } else {
+    for (let i = 0; i < whiteList.length; ++i) {
+      const poolAddress = whiteList[i].id;
+      const pool = await db.getPool(dbTx, { id: poolAddress });
+      assert(pool);
 
-    if (BigNumber.from(pool.liquidity).gt(0)) {
+      if (BigNumber.from(pool.liquidity).gt(0)) {
-      if (pool.token0.id === token.id) {
+        if (pool.token0.id === token.id) {
-        // Whitelist token is token1.
+          // whitelist token is token1
-        const token1 = pool.token1;
+          const token1 = pool.token1;
-        // Get the derived ETH in pool.
-        const ethLocked = pool.totalValueLockedToken1.times(token1.derivedETH);
 
-        if (ethLocked.gt(largestLiquidityETH) && ethLocked.gt(MINIMUM_ETH_LOCKED)) {
+          // get the derived ETH in pool
-          largestLiquidityETH = ethLocked;
+          const ethLocked = pool.totalValueLockedToken1.times(token1.derivedETH);
-          // token1 per our token * Eth per token1
+
-          priceSoFar = pool.token1Price.times(token1.derivedETH);
+          if (ethLocked.gt(largestLiquidityETH) && ethLocked.gt(MINIMUM_ETH_LOCKED)) {
+            largestLiquidityETH = ethLocked;
+            // token1 per our token * Eth per token1
+            priceSoFar = pool.token1Price.times(token1.derivedETH);
+          }
         }
       }
       if (pool.token1.id === token.id) {

packages/uni-info-watcher/src/utils/tick.ts

@@ -27,3 +27,20 @@ export const createTick = async (db: Database, dbTx: QueryRunner, tickId: string
 
   return db.saveTick(dbTx, tick, block);
 };
+
+export const feeTierToTickSpacing = (feeTier: bigint): bigint => {
+  if (feeTier === BigInt(10000)) {
+    return BigInt(200);
+  }
+  if (feeTier === BigInt(3000)) {
+    return BigInt(60);
+  }
+  if (feeTier === BigInt(500)) {
+    return BigInt(10);
+  }
+  if (feeTier === BigInt(100)) {
+    return BigInt(1);
+  }
+
+  throw Error('Unexpected fee tier');
+};