Bugfix/orderbook data cummulative vol (#1833)

* chore: move market depth queries to .graphql file

* chore: fix orderbook data updates

* chore: allow to overlap subscription update with stored data in market depth

libs/market-depth/src/lib/market-depth-provider.ts

@@ -1,6 +1,8 @@
 import { makeDataProvider } from '@vegaprotocol/react-helpers';
 import { updateLevels } from './orderbook-data';
 import type { Update } from '@vegaprotocol/react-helpers';
+import { captureException } from '@sentry/react';
+
 import {
   MarketDepthDocument,
   MarketDepthUpdateDocument,
@@ -23,18 +25,24 @@ const update: Update<
     if (delta.marketId !== data.id) {
       continue;
     }
-    /*
+    const currentSequenceNumber = Number(delta.sequenceNumber);
-    const sequenceNumber = Number(delta.sequenceNumber);
+    if (currentSequenceNumber <= sequenceNumbers[delta.marketId]) {
-    if (sequenceNumber <= sequenceNumbers[delta.marketId]) {
       return data;
     }
-    if (sequenceNumber - 1 !== sequenceNumbers[delta.marketId]) {
+    if (
-      sequenceNumbers[delta.marketId] = 0;
+      delta.previousSequenceNumber !==
+      sequenceNumbers[delta.marketId].toString()
+    ) {
+      captureException(
+        new Error(
+          `Sequence number gap in marketsDepthUpdate for {data.id}, {sequenceNumbers[delta.marketId]} - {delta.previousSequenceNumber}`
+        )
+      );
+      delete sequenceNumbers[delta.marketId];
       reload();
       return;
     }
-    sequenceNumbers[delta.marketId] = sequenceNumber;
+    sequenceNumbers[delta.marketId] = Number(currentSequenceNumber);
-    */
     const updatedData = {
       ...data,
       depth: { ...data.depth },

libs/market-depth/src/lib/orderbook-data.ts

@@ -1,5 +1,7 @@
 import groupBy from 'lodash/groupBy';
 import uniqBy from 'lodash/uniqBy';
+import reverse from 'lodash/reverse';
+import cloneDeep from 'lodash/cloneDeep';
 import { VolumeType } from '@vegaprotocol/react-helpers';
 import { MarketTradingMode } from '@vegaprotocol/types';
 import type { MarketData } from '@vegaprotocol/market-list';
@@ -56,7 +58,7 @@ const toPercentValue = (value?: number) => Math.ceil((value ?? 0) * 100);
 const updateRelativeData = (data: OrderbookRowData[]) => {
   const { bid, ask, cumulativeVol } = getMaxVolumes(data);
   const maxBidAsk = Math.max(bid, ask);
-  data.forEach((data) => {
+  data.forEach((data, i) => {
     data.relativeAsk = toPercentValue(data.ask / maxBidAsk);
     data.relativeBid = toPercentValue(data.bid / maxBidAsk);
     data.cumulativeVol.relativeAsk = toPercentValue(
@@ -68,6 +70,20 @@ const updateRelativeData = (data: OrderbookRowData[]) => {
   });
 };
 
+const updateCumulativeVolume = (data: OrderbookRowData[]) => {
+  if (data.length > 1) {
+    const maxIndex = data.length - 1;
+    for (let i = 0; i <= maxIndex; i++) {
+      data[i].cumulativeVol.bid =
+        data[i].bid + (i !== 0 ? data[i - 1].cumulativeVol.bid : 0);
+    }
+    for (let i = maxIndex; i >= 0; i--) {
+      data[i].cumulativeVol.ask =
+        data[i].ask + (i !== maxIndex ? data[i + 1].cumulativeVol.ask : 0);
+    }
+  }
+};
+
 export const createPartialRow = (
   price: string,
   volume = 0,
@@ -157,6 +173,7 @@ export const compactRows = (
         (i !== maxIndex ? orderbookData[i + 1].cumulativeVol.ask : 0);
     }
   }
+  updateCumulativeVolume(orderbookData);
   // count relative volumes
   updateRelativeData(orderbookData);
   return orderbookData;
@@ -175,45 +192,26 @@ const partiallyUpdateCompactedRows = (
   dataType: VolumeType,
   data: OrderbookRowData[],
   delta: PriceLevelFieldsFragment,
-  resolution: number,
+  resolution: number
-  modifiedIndex: number
+) => {
-): [number, OrderbookRowData[]] => {
   const { price } = delta;
   const volume = Number(delta.volume);
   const priceLevel = getPriceLevel(price, resolution);
   const isAskDataType = dataType === VolumeType.ask;
   const volKey = isAskDataType ? 'ask' : 'bid';
-  const oppositeVolKey = isAskDataType ? 'bid' : 'ask';
   const volByLevelKey = isAskDataType ? 'askByLevel' : 'bidByLevel';
-  const resolveModifiedIndex = isAskDataType ? Math.max : Math.min;
   let index = data.findIndex((row) => row.price === priceLevel);
   if (index !== -1) {
-    modifiedIndex = resolveModifiedIndex(modifiedIndex, index);
+    data[index][volKey] =
-    data[index] = {
+      data[index][volKey] - (data[index][volByLevelKey][price] || 0) + volume;
-      ...data[index],
+    data[index][volByLevelKey][price] = volume;
-      [volKey]:
-        data[index][volKey] - (data[index][volByLevelKey][price] || 0) + volume,
-      [volByLevelKey]: {
-        ...data[index][volByLevelKey],
-        [price]: volume,
-      },
-    };
-    return [modifiedIndex, [...data]];
   } else {
     const newData: OrderbookRowData = createRow(priceLevel, volume, dataType);
     index = data.findIndex((row) => BigInt(row.price) < BigInt(priceLevel));
     if (index !== -1) {
-      newData.cumulativeVol[oppositeVolKey] =
+      data.splice(index, 0, newData);
-        data[index + (isAskDataType ? 0 : 1)]?.cumulativeVol[oppositeVolKey] ??
-        0;
-      modifiedIndex = resolveModifiedIndex(modifiedIndex, index);
-      return [
-        modifiedIndex,
-        [...data.slice(0, index), newData, ...data.slice(index)],
-      ];
     } else {
-      modifiedIndex = data.length - 1;
+      data.push(newData);
-      return [modifiedIndex, [...data, newData]];
     }
   }
 };
@@ -228,57 +226,19 @@ const partiallyUpdateCompactedRows = (
  * @returns void
  */
 export const updateCompactedRows = (
-  rows: OrderbookRowData[],
+  rows: Readonly<OrderbookRowData[]>,
-  sell: PriceLevelFieldsFragment[] | null,
+  sell: Readonly<PriceLevelFieldsFragment[]> | null,
-  buy: PriceLevelFieldsFragment[] | null,
+  buy: Readonly<PriceLevelFieldsFragment[]> | null,
   resolution: number
 ) => {
-  let sellModifiedIndex = -1;
+  const data = cloneDeep(rows as OrderbookRowData[]);
-  let data = [...rows];
+  uniqBy(reverse(sell || []), 'price')?.forEach((delta) => {
-  uniqBy(sell?.reverse(), 'price')?.forEach((delta) => {
+    partiallyUpdateCompactedRows(VolumeType.ask, data, delta, resolution);
-    [sellModifiedIndex, data] = partiallyUpdateCompactedRows(
-      VolumeType.ask,
-      data,
-      delta,
-      resolution,
-      sellModifiedIndex
-    );
   });
-  let buyModifiedIndex = data.length;
+  uniqBy(reverse(buy || []), 'price')?.forEach((delta) => {
-  uniqBy(buy?.reverse(), 'price')?.forEach((delta) => {
+    partiallyUpdateCompactedRows(VolumeType.bid, data, delta, resolution);
-    [buyModifiedIndex, data] = partiallyUpdateCompactedRows(
-      VolumeType.bid,
-      data,
-      delta,
-      resolution,
-      buyModifiedIndex
-    );
   });
-
+  updateCumulativeVolume(data);
-  // update cumulative ask only below highest modified price level
-  if (sellModifiedIndex !== -1) {
-    for (let i = Math.min(sellModifiedIndex, data.length - 2); i >= 0; i--) {
-      data[i] = {
-        ...data[i],
-        cumulativeVol: {
-          ...data[i].cumulativeVol,
-          ask: data[i + 1].cumulativeVol.ask + data[i].ask,
-        },
-      };
-    }
-  }
-  // update cumulative bid only above lowest modified price level
-  if (buyModifiedIndex !== data.length) {
-    for (let i = Math.max(buyModifiedIndex, 1), l = data.length; i < l; i++) {
-      data[i] = {
-        ...data[i],
-        cumulativeVol: {
-          ...data[i].cumulativeVol,
-          bid: data[i - 1].cumulativeVol.bid + data[i].bid,
-        },
-      };
-    }
-  }
   let index = 0;
   // remove levels that do not have any volume
   while (index < data.length) {