Add a sushiswap-subgraph stack (#496)

* Add a sushiswap-graph stack

* Fix shared volume name in fixturenet-lotus

* Export Lotus miner node multiaddr after initialization

* Add deployment scripts for sushiswap contracts

* Skip contracts deployment if already exists

* Rename stack to sushiswap-subgraph

* Fix postgres and use Lotus fixturenet in graph-node stack

* Add sushiswap v3 subgraph stack

* Fixes to deploy sushiswap subgraph to graph-node

* Add graph-node container in sushiswap subgraph stack

---------

Co-authored-by: Nabarun <nabarun@deepstacksoft.com>

app/data/compose/docker-compose-contract-sushiswap.yml

@@ -0,0 +1,49 @@
+version: '3.2'
+
+services:
+  # Deploys the core (UniswapV3Factory) contract
+  sushiswap-v3-core:
+    image: cerc/sushiswap-v3-core:local
+    restart: on-failure
+    env_file:
+      # Defaults
+      - ../config/contract-sushiswap/deployment-params.env
+    environment:
+      # Overrides
+      ETH_RPC_ENDPOINT: ${ETH_RPC_ENDPOINT}
+      CHAIN_ID: ${CHAIN_ID}
+      ACCOUNT_PRIVATE_KEY: ${ACCOUNT_PRIVATE_KEY}
+      DEPLOY: ${DEPLOY}
+    volumes:
+      - ../config/network/wait-for-it.sh:/app/wait-for-it.sh
+      - ../config/contract-sushiswap/deploy-core-contracts.sh:/app/deploy-core-contracts.sh
+      - sushiswap_core_deployment:/app/deployments/docker
+    command: ["bash", "-c", "/app/deploy-core-contracts.sh && tail -f"]
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  # Deploys the periphery (NFPM, token, etc.) contracts
+  sushiswap-v3-periphery:
+    image: cerc/sushiswap-v3-periphery:local
+    restart: on-failure
+    env_file:
+      # Defaults
+      - ../config/contract-sushiswap/deployment-params.env
+    environment:
+      # Overrides
+      ETH_RPC_ENDPOINT: ${ETH_RPC_ENDPOINT}
+      CHAIN_ID: ${CHAIN_ID}
+      ACCOUNT_PRIVATE_KEY: ${ACCOUNT_PRIVATE_KEY}
+      DEPLOY: ${DEPLOY}
+    volumes:
+      - ../config/network/wait-for-it.sh:/app/wait-for-it.sh
+      - ../config/contract-sushiswap/deploy-periphery-contracts.sh:/app/deploy-periphery-contracts.sh
+      - sushiswap_core_deployment:/app/core-deployments/docker
+      - sushiswap_periphery_deployment:/app/deployments/docker
+    command: ["bash", "-c", "/app/deploy-periphery-contracts.sh && tail -f"]
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  sushiswap_core_deployment:
+  sushiswap_periphery_deployment:

app/data/compose/docker-compose-fixturenet-graph-node.yml

@@ -1,6 +1,15 @@
 services:
   graph-node:
     image: cerc/graph-node:local
+    depends_on:
+      db:
+        condition: service_healthy
+      ipfs:
+        condition: service_healthy
+      lotus-node-1:
+        condition: service_healthy
+    extra_hosts:
+      - host.docker.internal:host-gateway
     environment:
       ipfs: ipfs:5001
       postgres_host: db
@@ -8,11 +17,20 @@ services:
       postgres_user: graph-node
       postgres_pass: password
       postgres_db: graph-node
+      # TODO: Get endpoint from env
+      ethereum: 'lotus-fixturenet:http://lotus-node-1:1234/rpc/v1'
+      GRAPH_LOG: info
     ports:
       - "8000"
       - "8001"
       - "8020"
       - "8030"
+    healthcheck:
+      test: ["CMD", "nc", "-vz", "localhost", "8020"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 3s
   ipfs:
     image: ipfs/kubo:master-2023-02-20-714a968
     volumes:
@@ -30,6 +48,12 @@ services:
       POSTGRES_USER: "graph-node"
       POSTGRES_DB: "graph-node"
       POSTGRES_PASSWORD: "password"
+      POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C"
+    command:
+      [
+        "postgres",
+        "-cshared_preload_libraries=pg_stat_statements"
+      ]
     healthcheck:
       test: ["CMD", "nc", "-v", "localhost", "5432"]
       interval: 30s

app/data/compose/docker-compose-fixturenet-lotus.yml

@@ -10,7 +10,7 @@ services:
       - ../config/fixturenet-lotus/setup-miner.sh:/docker-entrypoint-scripts.d/setup-miner.sh
       - ../config/fixturenet-lotus/fund-account.sh:/fund-account.sh
       - lotus_miner_params:/var/tmp/filecoin-proof-parameters
-      - lotus-shared:/root/.lotus-shared
+      - lotus_shared:/root/.lotus-shared
       - lotus_miner_data:/root/data
     entrypoint: ["sh", "/docker-entrypoint-scripts.d/setup-miner.sh"]
     ports:
@@ -30,13 +30,13 @@ services:
     volumes:
       - ../config/fixturenet-lotus/setup-node.sh:/docker-entrypoint-scripts.d/setup-node.sh
       - lotus_node_1_params:/var/tmp/filecoin-proof-parameters
-      - lotus-shared:/root/.lotus-shared
+      - lotus_shared:/root/.lotus-shared
       - lotus_node_1_data:/root/data
     healthcheck:
       test: ["CMD", "nc", "-vz", "localhost", "1234"]
       interval: 30s
       timeout: 10s
-      retries: 10
+      retries: 60
       start_period: 3s
     depends_on:
       - lotus-miner
@@ -58,13 +58,13 @@ services:
     volumes:
       - ../config/fixturenet-lotus/setup-node.sh:/docker-entrypoint-scripts.d/setup-node.sh
       - lotus_node_2_params:/var/tmp/filecoin-proof-parameters
-      - lotus-shared:/root/.lotus-shared
+      - lotus_shared:/root/.lotus-shared
       - lotus_node_2_data:/root/data
     healthcheck:
       test: ["CMD", "nc", "-vz", "localhost", "1234"]
       interval: 30s
       timeout: 10s
-      retries: 10
+      retries: 60
       start_period: 3s
     depends_on:
       - lotus-miner
@@ -79,7 +79,7 @@ volumes:
   lotus_miner_params:
   lotus_node_1_params:
   lotus_node_2_params:
-  lotus-shared:
+  lotus_shared:
   lotus_miner_data:
   lotus_node_1_data:
   lotus_node_2_data:

app/data/compose/docker-compose-sushiswap-subgraph-v3.yml

@@ -0,0 +1,26 @@
+version: '3.2'
+
+services:
+  # Deploys the sushiswap v3 subgraph
+  sushiswap-subgraph-v3:
+    image: cerc/sushiswap-subgraphs:local
+    restart: on-failure
+    depends_on:
+      graph-node:
+        condition: service_healthy
+    environment:
+      - APP=v3
+      - NETWORK=lotus-fixturenet
+    command: ["bash", "-c", "./run-v3.sh"]
+    working_dir: /app/subgraphs/v3
+    volumes:
+      - ../config/sushiswap-subgraph-v3/lotus-fixturenet.js.template:/app/config/lotus-fixturenet.js.template
+      - ../config/sushiswap-subgraph-v3/run-v3.sh:/app/subgraphs/v3/run-v3.sh
+      - sushiswap_core_deployment:/app/subgraphs/v3/core-deployments/docker
+      - sushiswap_periphery_deployment:/app/subgraphs/v3/deployments/docker
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  sushiswap_core_deployment:
+  sushiswap_periphery_deployment:

app/data/config/contract-sushiswap/deploy-core-contracts.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e
+
+# Chain config
+ETH_RPC_ENDPOINT="${ETH_RPC_ENDPOINT:-${DEFAULT_ETH_RPC_ENDPOINT}}"
+CHAIN_ID="${CHAIN_ID:-${DEFAULT_CHAIN_ID}}"
+ACCOUNT_PRIVATE_KEY="${ACCOUNT_PRIVATE_KEY:-${DEFAULT_ACCOUNT_PRIVATE_KEY}}"
+
+# Option
+DEPLOY="${DEPLOY:-${DEFAULT_DEPLOY}}"
+
+# Create a .env file
+echo "ETH_RPC_ENDPOINT=$ETH_RPC_ENDPOINT" > .env
+echo "CHAIN_ID=$CHAIN_ID" >> .env
+echo "ACCOUNT_PRIVATE_KEY=$ACCOUNT_PRIVATE_KEY" >> .env
+
+echo "Using RPC endpoint ${ETH_RPC_ENDPOINT}"
+
+# Wait for the RPC endpoint to be up
+endpoint=${ETH_RPC_ENDPOINT#http://}
+endpoint=${endpoint#https://}
+RPC_HOST=$(echo "$endpoint" | awk -F'[:/]' '{print $1}')
+RPC_PORT=$(echo "$endpoint" | awk -F'[:/]' '{print $2}')
+./wait-for-it.sh -h "${RPC_HOST}" -p "${RPC_PORT}" -s -t 0
+
+if [ "$DEPLOY" = true ] && [ ! -e "/app/deployments/docker/UniswapV3Factory.json" ]; then
+  echo "Performing core contract deployments..."
+  pnpm hardhat --network docker deploy --tags UniswapV3Factory
+else
+  echo "Skipping contract deployments"
+fi
+
+echo "Done"

app/data/config/contract-sushiswap/deploy-periphery-contracts.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -e
+
+# Chain config
+ETH_RPC_ENDPOINT="${ETH_RPC_ENDPOINT:-${DEFAULT_ETH_RPC_ENDPOINT}}"
+CHAIN_ID="${CHAIN_ID:-${DEFAULT_CHAIN_ID}}"
+ACCOUNT_PRIVATE_KEY="${ACCOUNT_PRIVATE_KEY:-${DEFAULT_ACCOUNT_PRIVATE_KEY}}"
+
+# Option
+DEPLOY="${DEPLOY:-${DEFAULT_DEPLOY}}"
+
+# Create a .env file
+echo "ETH_RPC_ENDPOINT=$ETH_RPC_ENDPOINT" > .env
+echo "CHAIN_ID=$CHAIN_ID" >> .env
+echo "ACCOUNT_PRIVATE_KEY=$ACCOUNT_PRIVATE_KEY" >> .env
+
+echo "Using RPC endpoint $ETH_RPC_ENDPOINT"
+
+# Wait for the RPC endpoint to be up
+endpoint=${ETH_RPC_ENDPOINT#http://}
+endpoint=${endpoint#https://}
+RPC_HOST=$(echo "$endpoint" | awk -F'[:/]' '{print $1}')
+RPC_PORT=$(echo "$endpoint" | awk -F'[:/]' '{print $2}')
+./wait-for-it.sh -h "${RPC_HOST}" -p "${RPC_PORT}" -s -t 0
+
+if [ "$DEPLOY" = true ] && [ ! -e "/app/deployments/docker/NonfungiblePositionManager.json" ]; then
+  # Loop until the factory deployment is detected
+  echo "Waiting for core deployments to occur"
+  while [ ! -f /app/core-deployments/docker/UniswapV3Factory.json ]; do
+    sleep 5
+  done
+
+  echo "Reading factory address from core deployments"
+  FACTORY_ADDRESS=$(jq -r '.address' /app/core-deployments/docker/UniswapV3Factory.json)
+
+  echo "Using UniswapV3Factory at $FACTORY_ADDRESS"
+  echo "FACTORY_ADDRESS=$FACTORY_ADDRESS" >> .env
+
+  echo "Performing periphery contract deployments..."
+  yarn hardhat --network docker deploy --tags NonfungiblePositionManager
+else
+  echo "Skipping contract deployments"
+fi
+
+echo "Done"

app/data/config/contract-sushiswap/deployment-params.env

@@ -0,0 +1,11 @@
+# Chain config
+
+DEFAULT_ETH_RPC_ENDPOINT="http://lotus-node-1:1234/rpc/v1"
+DEFAULT_CHAIN_ID=31415926
+
+# From app/data/config/fixturenet-lotus/fund-account.sh
+DEFAULT_ACCOUNT_PRIVATE_KEY="0xc05fd3613bcd62a4f25e5eba1f464d0b76d74c3f771a7c2f13e26ad6439444b3"
+
+# Options
+
+DEFAULT_DEPLOY=true

app/data/config/fixturenet-lotus/setup-miner.sh

@@ -33,8 +33,8 @@ while ! grep -q "started ChainNotify channel" /var/log/lotus.log ; do
 done
 echo "Daemon started."
 
-# publish bootnode peer info to shared volume
+# copy genesis file to shared volume
-lotus net listen | grep "$(ip addr | grep inet | grep -v '127.0.0.1' | sort | head -1 | awk '{print $2}' | cut -d '/' -f1)" | head -1 > /root/.lotus-shared/miner.addr
+cp /devgen.car /root/.lotus-shared
 
 # if miner not already initialized
 if [ ! -d $LOTUS_MINER_PATH ]; then
@@ -47,6 +47,9 @@ if [ ! -d $LOTUS_MINER_PATH ]; then
   lotus-miner init --genesis-miner --actor=t01000 --sector-size=2KiB --pre-sealed-sectors=/root/data/.genesis-sectors --pre-sealed-metadata=/root/data/.genesis-sectors/pre-seal-t01000.json --nosync
 fi
 
+# publish bootnode peer info to shared volume
+lotus net listen | grep "$(ip addr | grep inet | grep -v '127.0.0.1' | sort | head -1 | awk '{print $2}' | cut -d '/' -f1)" | head -1 > /root/.lotus-shared/miner.addr
+
 # start miner
 nohup lotus-miner run --nosync &
 

app/data/config/sushiswap-subgraph-v3/lotus-fixturenet.js.template

@@ -0,0 +1,20 @@
+module.exports = {
+  network: 'lotus-fixturenet',
+  v3: {
+    factory: {
+      address: 'FACTORY_ADDRESS',
+      startBlock: FACTORY_BLOCK
+    },
+    positionManager: {
+      address: 'NFPM_ADDRESS',
+      startBlock: NFPM_BLOCK
+    },
+    native: { address: 'NATIVE_ADDRESS' },
+    whitelistedTokenAddresses: [
+      'NATIVE_ADDRESS',
+    ],
+    stableTokenAddresses: [
+    ],
+    minimumEthLocked: 1.5
+  }
+}

app/data/config/sushiswap-subgraph-v3/run-v3.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -e
+
+# Loop until the NFPM deployment is detected
+echo "Waiting for sushiswap-periphery deployments to occur"
+while [ ! -f ./deployments/docker/NonfungiblePositionManager.json ]; do
+  sleep 5
+done
+
+echo "Reading contract addresses and block numbers from deployments"
+FACTORY_ADDRESS=$(jq -r '.address' ./core-deployments/docker/UniswapV3Factory.json)
+FACTORY_BLOCK=$(jq -r '.receipt.blockNumber' ./core-deployments/docker/UniswapV3Factory.json)
+NATIVE_ADDRESS=$(jq -r '.address' ./deployments/docker/WFIL.json)
+NFPM_ADDRESS=$(jq -r '.address' ./deployments/docker/NonfungiblePositionManager.json)
+NFPM_BLOCK=$(jq -r '.receipt.blockNumber' ./deployments/docker/NonfungiblePositionManager.json)
+
+# Read the JavaScript file content
+file_content=$(</app/config/lotus-fixturenet.js.template)
+
+# Replace uppercase words with environment variables
+echo "Reading values in lotus-fixturenet config"
+replaced_content=$(echo "$file_content" | sed -e "s/FACTORY_ADDRESS/$FACTORY_ADDRESS/g" \
+                                              -e "s/FACTORY_BLOCK/$FACTORY_BLOCK/g" \
+                                              -e "s/NFPM_ADDRESS/$NFPM_ADDRESS/g" \
+                                              -e "s/NFPM_BLOCK/$NFPM_BLOCK/g" \
+                                              -e "s/NATIVE_ADDRESS/$NATIVE_ADDRESS/g")
+
+# Write the replaced content back to the JavaScript file
+echo "$replaced_content" > /app/config/lotus-fixturenet.js
+
+
+echo "Building subgraph and deploying to graph-node..."
+pnpm run generate
+pnpm run build
+pnpm exec graph create --node http://graph-node:8020/ sushiswap/v3-lotus
+pnpm exec graph deploy --node http://graph-node:8020/ --ipfs http://ipfs:5001 --version-label 0.1.0 sushiswap/v3-lotus
+
+echo "Done"

app/data/container-build/cerc-sushiswap-subgraphs/Dockerfile

@@ -0,0 +1,11 @@
+FROM node:18.15.0-alpine3.16
+
+RUN apk --update --no-cache add git alpine-sdk bash jq
+RUN curl -L https://unpkg.com/@pnpm/self-installer | node
+
+WORKDIR /app
+
+COPY . .
+
+RUN echo "Installing dependencies..." && \
+  pnpm install

app/data/container-build/cerc-sushiswap-subgraphs/build.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# Build cerc/sushiswap-subgraphs
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/sushiswap-subgraphs:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/subgraphs

app/data/container-build/cerc-sushiswap-v3-core/Dockerfile

@@ -1,6 +1,6 @@
 FROM node:18.15.0-alpine3.16
 
-RUN apk --update --no-cache add git python3 alpine-sdk bash
+RUN apk --update --no-cache add git python3 alpine-sdk bash jq
 RUN curl -L https://unpkg.com/@pnpm/self-installer | node
 
 WORKDIR /app

app/data/container-build/cerc-sushiswap-v3-periphery/Dockerfile

@@ -1,6 +1,6 @@
 FROM node:18.15.0-alpine3.16
 
-RUN apk --update --no-cache add git python3 alpine-sdk bash
+RUN apk --update --no-cache add git python3 alpine-sdk bash jq
 
 WORKDIR /app
 

app/data/container-image-list.txt

@@ -48,3 +48,4 @@ cerc/sushiswap-v3-core
 cerc/sushiswap-v3-periphery
 cerc/watcher-sushiswap
 cerc/graph-node
+cerc/sushiswap-subgraphs

app/data/pod-list.txt

@@ -31,3 +31,4 @@ mainnet-go-opera
 lasso
 reth
 watcher-sushiswap
+contract-sushiswap

app/data/repository-list.txt

@@ -42,3 +42,4 @@ git.vdb.to/cerc-io/plugeth-statediff
 github.com/cerc-io/sushiswap-v3-core
 github.com/cerc-io/sushiswap-v3-periphery
 github.com/graphprotocol/graph-node
+github.com/sushiswap/subgraphs

app/data/stacks/fixturenet-graph-node/stack.yml

@@ -2,9 +2,11 @@ version: "1.0"
 name: fixturenet-graph-node
 description: "A graph-node fixturenet"
 repos:
+  - github.com/filecoin-project/lotus
   - github.com/graphprotocol/graph-node
 containers:
+  - cerc/lotus
   - cerc/graph-node
 pods:
+  - fixturenet-lotus
   - fixturenet-graph-node
-

app/data/stacks/sushiswap-subgraph/README.md

@@ -0,0 +1,63 @@
+# SushiSwap Graph
+
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack sushiswap-subgraph setup-repositories
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack sushiswap-subgraph build-containers
+```
+
+## Deploy
+
+Deploy the stack:
+
+```bash
+laconic-so --stack sushiswap-subgraph deploy --cluster sushigraph up
+```
+
+After all services have started, wait and check that the subgraph has been deployed to graph-node
+
+```bash
+laconic-so --stack sushiswap-subgraph deploy --cluster sushigraph logs -f sushiswap-subgraph-v3
+
+# Expected end output
+# ...
+# sushigraph-sushiswap-subgraph-v3-1  | - Deploying to Graph node http://graph-node:8020/
+# sushigraph-sushiswap-subgraph-v3-1  | Deployed to http://graph-node:8000/subgraphs/name/sushiswap/v3-lotus/graphql
+# sushigraph-sushiswap-subgraph-v3-1  |
+# sushigraph-sushiswap-subgraph-v3-1  | Subgraph endpoints:
+# sushigraph-sushiswap-subgraph-v3-1  | Queries (HTTP):     http://graph-node:8000/subgraphs/name/sushiswap/v3-lotus
+# sushigraph-sushiswap-subgraph-v3-1  |
+# sushigraph-sushiswap-subgraph-v3-1  | Done
+```
+
+## Clean up
+
+Stop all the services running in background run:
+
+```bash
+laconic-so --stack sushiswap-subgraph deploy --cluster sushigraph down
+```
+
+Clear volumes created by this stack:
+
+```bash
+# List all relevant volumes
+docker volume ls -q --filter "name=sushigraph"
+
+# Remove all the listed volumes
+docker volume rm $(docker volume ls -q --filter "name=sushigraph")
+
+# WARNING: After removing volumes with Lotus params
+# They will be downloaded again on restart
+
+# To remove volumes that do not contain Lotus params
+docker volume rm $(docker volume ls -q --filter "name=sushigraph" | grep -v "params$")
+```

app/data/stacks/sushiswap-subgraph/stack.yml

@@ -0,0 +1,28 @@
+version: "1.0"
+name: sushiswap-subgraph
+description: "An end-to-end SushiSwap Subgraph stack"
+repos:
+  ## fixturenet-lotus repo
+  - github.com/filecoin-project/lotus
+  ## graph-node repo
+  - github.com/graphprotocol/graph-node
+  ## sushiswap repos
+  - github.com/cerc-io/sushiswap-v3-core@watcher-ts
+  - github.com/cerc-io/sushiswap-v3-periphery@watcher-ts
+  ## subgraph repo
+  - github.com/sushiswap/subgraphs
+containers:
+  ## fixturenet-lotus image
+  - cerc/lotus
+  ## fixturenet-graph-node image
+  - cerc/graph-node
+  ## sushiswap contract deployment images
+  - cerc/sushiswap-v3-core
+  - cerc/sushiswap-v3-periphery
+  ## sushiswap subgraphs image
+  - cerc/sushiswap-subgraphs
+pods:
+  - fixturenet-lotus
+  - fixturenet-graph-node
+  - contract-sushiswap
+  - sushiswap-subgraph-v3