Update sushiswap-subgraph stack to point to filecoin endpoint (#509)

* Update sushiswap-subgraph stack to point to filecoin endpoint

* Deploy blocks subgraph in sushiswap-subgraph stack

* Update subgraph config

* Remove duplicate nativePricePool

* Enable debug logs in graph-node and update instructions

* Two additional miner nodes in fixturenet-lotus

* Revert experimental change for additional miner nodes

* Set ETHEREUM_REORG_THRESHOLD env for graph-node to catch up to head

* Take ETH RPC endpoint for graph-node from env

* Set default values for RPC endpoint in graph-node

* Rename fixturenet-graph-node pod to graph-node

* Clean up sushiswap-subgraph stack

* Use deployment command in sushiswap-subgraph stack

* Add a separate fixturenet-sushiswap-subgraph stack

* Fix pods in fixturenet-sushiswap-subgraph

* Fix fixturenet subgraph deployment commands and instructions

---------

Co-authored-by: Nabarun <nabarun@deepstacksoft.com>

app/data/compose/docker-compose-fixturenet-sushiswap-subgraph-v3.yml

@@ -0,0 +1,27 @@
+version: '3.2'
+
+services:
+  # Deploys the sushiswap v3 subgraph
+  sushiswap-subgraph-v3:
+    image: cerc/sushiswap-subgraphs:local
+    restart: on-failure
+    depends_on:
+      graph-node:
+        condition: service_healthy
+    environment:
+      - APP=v3
+      - NETWORK=lotus-fixturenet
+    command: ["bash", "-c", "./v3/run-v3.sh && ./blocks/run-blocks.sh"]
+    working_dir: /app/subgraphs
+    volumes:
+      - ../config/fixturenet-sushiswap-subgraph-v3/lotus-fixturenet.js.template:/app/config/lotus-fixturenet.js.template
+      - ../config/fixturenet-sushiswap-subgraph-v3/run-blocks.sh:/app/subgraphs/blocks/run-blocks.sh
+      - ../config/fixturenet-sushiswap-subgraph-v3/run-v3.sh:/app/subgraphs/v3/run-v3.sh
+      - sushiswap_core_deployment:/app/subgraphs/v3/core-deployments/docker
+      - sushiswap_periphery_deployment:/app/subgraphs/v3/periphery-deployments/docker
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+volumes:
+  sushiswap_core_deployment:
+  sushiswap_periphery_deployment:

app/data/compose/docker-compose-graph-node.yml

@@ -6,8 +6,9 @@ services:
         condition: service_healthy
       ipfs:
         condition: service_healthy
-      lotus-node-1:
-        condition: service_healthy
+      # Uncomment when running against fixturenet-lotus to wait for the Lotus node to come up
+      # lotus-node-1:
+      #   condition: service_healthy
     extra_hosts:
       - host.docker.internal:host-gateway
     environment:
@@ -17,9 +18,9 @@ services:
       postgres_user: graph-node
       postgres_pass: password
       postgres_db: graph-node
-      # TODO: Get endpoint from env
-      ethereum: 'lotus-fixturenet:http://lotus-node-1:1234/rpc/v1'
-      GRAPH_LOG: info
+      ethereum: ${NETWORK:-filecoin}:${ETH_RPC_ENDPOINT:-https://archive.lotus.vdb.to/rpc/v1}
+      GRAPH_LOG: debug
+      ETHEREUM_REORG_THRESHOLD: 3
     ports:
       - "8000"
       - "8001"

app/data/compose/docker-compose-mainnet-eth-api-proxy.yml

@@ -0,0 +1,15 @@
+version: '3.8'
+
+services:
+  redis:
+    hostname: mainnet-eth-api-proxy-redis
+    image: redis:7-alpine
+    command: redis-server --save "" --appendonly no --maxmemory-policy allkeys-lru --maxmemory 1G
+
+  eth-api-proxy:
+    hostname: mainnet-eth-api-proxy
+    image: cerc/eth-api-proxy:local
+    env_file:
+      - ../config/mainnet-eth-api-proxy/ethpxy.env
+    ports:
+      - 8547

app/data/compose/docker-compose-mainnet-eth-keycloak.yml

@@ -0,0 +1,51 @@
+version: '3.8'
+
+services:
+  keycloak-db:
+    image: postgres:14-alpine
+    env_file:
+      - ../config/mainnet-eth-keycloak/keycloak.env
+    healthcheck:
+      test: ["CMD", "nc", "-v", "localhost", "5432"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 3s
+    volumes:
+      - mainnet_eth_keycloak_db:/var/lib/postgresql/data
+    ports:
+      - 5432
+
+  keycloak:
+    image: cerc/keycloak:local
+    env_file:
+      - ../config/mainnet-eth-keycloak/keycloak.env
+    environment:
+      JAVA_OPTS_APPEND: "-Dkeycloak.migration.action=import -Dkeycloak.migration.provider=dir -Dkeycloak.migration.dir=/import -Dkeycloak.migration.strategy=IGNORE_EXISTING"
+    volumes:
+      - ../config/mainnet-eth-keycloak/import:/import
+    ports:
+      - 8080
+    command: ["start"]
+    depends_on:
+      keycloak-db:
+        condition: service_healthy
+
+  keycloak-reg-ui:
+    image: cerc/keycloak-reg-ui:local
+    env_file:
+      - ../config/mainnet-eth-keycloak/keycloak.env
+    volumes:
+      - ../config/mainnet-eth-keycloak/ui:/config
+    ports:
+      - 80
+
+  keycloak-reg-api:
+    image: cerc/keycloak-reg-api:local
+    env_file:
+      - ../config/mainnet-eth-keycloak/keycloak.env
+    ports:
+      - 9292
+
+volumes:
+  mainnet_eth_keycloak_db:

app/data/compose/docker-compose-mainnet-eth-metrics.yml

@@ -0,0 +1,25 @@
+version: "3.2"
+services:
+  prometheus:
+    restart: always
+    image: prom/prometheus
+    depends_on:
+      mainnet-eth-geth-1:
+        condition: service_healthy
+    env_file:
+      - ../config/mainnet-eth-metrics/metrics.env
+    volumes:
+      - ../config/mainnet-eth-metrics/prometheus/etc:/etc/prometheus
+    ports:
+      - "9090"
+  grafana:
+    restart: always
+    image: grafana/grafana
+    env_file:
+      - ../config/mainnet-eth-metrics/metrics.env
+    volumes:
+      - ../config/mainnet-eth-metrics/grafana/etc/provisioning/dashboards:/etc/grafana/provisioning/dashboards
+      - ../config/mainnet-eth-metrics/grafana/etc/provisioning/datasources:/etc/grafana/provisioning/datasources
+      - ../config/mainnet-eth-metrics/grafana/etc/dashboards:/etc/grafana/dashboards
+    ports:
+      - "3000"

app/data/compose/docker-compose-mainnet-eth.yml

@@ -6,16 +6,13 @@ services:
     hostname: mainnet-eth-geth-1
     cap_add:
       - SYS_PTRACE
-    environment:
-      CERC_REMOTE_DEBUG: "true"
-      CERC_RUN_STATEDIFF: ${CERC_RUN_STATEDIFF:-detect}
-      CERC_STATEDIFF_DB_NODE_ID: 1
-      CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
     image: cerc/go-ethereum:local
     entrypoint: /bin/sh
     command: -c "/opt/run-geth.sh"
+    env_file:
+      - ../config/mainnet-eth/geth.env
     volumes:
-      - mainnet_eth_geth_1_data:/root/ethdata
+      - mainnet_eth_geth_1_data:/data
       - mainnet_eth_config_data:/etc/mainnet-eth
       - ../config/mainnet-eth/scripts/run-geth.sh:/opt/run-geth.sh
     healthcheck:
@@ -25,30 +22,48 @@ services:
       retries: 10
       start_period: 3s
     ports:
+      # http api
       - "8545"
+      # ws api
+      - "8546"
+      # ws el
+      - "8551"
+      # p2p
+      - "30303"
+      - "30303/udp"
+      # debugging
       - "40000"
+      # metrics
       - "6060"
 
   mainnet-eth-lighthouse-1:
     restart: always
     hostname: mainnet-eth-lighthouse-1
     healthcheck:
-      test: ["CMD", "wget", "--tries=1", "--connect-timeout=1", "--quiet", "-O", "-", "http://localhost:8001/eth/v2/beacon/blocks/head"]
+      test: ["CMD", "wget", "--tries=1", "--connect-timeout=1", "--quiet", "-O", "-", "http://localhost:5052/eth/v2/beacon/blocks/head"]
       interval: 30s
       timeout: 10s
       retries: 10
       start_period: 30s
     environment:
-      EXECUTION_ENDPOINT: "http://mainnet-eth-geth-1:8551"
+      LIGHTHOUSE_EXECUTION_ENDPOINT: "http://mainnet-eth-geth-1:8551"
+    env_file:
+      - ../config/mainnet-eth/lighthouse.env
     image: cerc/lighthouse:local
     entrypoint: /bin/sh
     command: -c "/opt/run-lighthouse.sh"
     volumes:
-      - mainnet_eth_lighthouse_1_data:/var/lighthouse-data-dir
+      - mainnet_eth_lighthouse_1_data:/data
       - mainnet_eth_config_data:/etc/mainnet-eth
       - ../config/mainnet-eth/scripts/run-lighthouse.sh:/opt/run-lighthouse.sh
     ports:
-      - "8001"
+      # api
+      - "5052"
+      # metrics
+      - "5054"
+      # p2p
+      - "9000"
+      - "9000/udp"
 
 volumes:
   mainnet_eth_config_data:

app/data/compose/docker-compose-sushiswap-subgraph-v3.yml

@@ -10,17 +10,12 @@ services:
         condition: service_healthy
     environment:
       - APP=v3
-      - NETWORK=lotus-fixturenet
-    command: ["bash", "-c", "./run-v3.sh"]
-    working_dir: /app/subgraphs/v3
+      - NETWORK=filecoin
+    command: ["bash", "-c", "./blocks/run-blocks.sh && ./v3/run-v3.sh"]
+    working_dir: /app/subgraphs
     volumes:
-      - ../config/sushiswap-subgraph-v3/lotus-fixturenet.js.template:/app/config/lotus-fixturenet.js.template
+      - ../config/sushiswap-subgraph-v3/filecoin.js:/app/config/filecoin.js
+      - ../config/sushiswap-subgraph-v3/run-blocks.sh:/app/subgraphs/blocks/run-blocks.sh
       - ../config/sushiswap-subgraph-v3/run-v3.sh:/app/subgraphs/v3/run-v3.sh
-      - sushiswap_core_deployment:/app/subgraphs/v3/core-deployments/docker
-      - sushiswap_periphery_deployment:/app/subgraphs/v3/deployments/docker
     extra_hosts:
       - "host.docker.internal:host-gateway"
-
-volumes:
-  sushiswap_core_deployment:
-  sushiswap_periphery_deployment:

app/data/config/fixturenet-lotus/setup-miner.sh

@@ -23,7 +23,6 @@ if [ ! -f /root/data/localnet.json ]; then
 fi
 
 # start daemon
-# /root/.lotus-shared/devgen.car path
 nohup lotus daemon --lotus-make-genesis=/root/.lotus-shared/devgen.car --profile=bootstrapper --genesis-template=/root/data/localnet.json --bootstrap=false > /var/log/lotus.log 2>&1 &
 
 # Loop until the daemon is started
@@ -33,9 +32,6 @@ while ! grep -q "started ChainNotify channel" /var/log/lotus.log ; do
 done
 echo "Daemon started."
 
-# copy genesis file to shared volume
-cp /devgen.car /root/.lotus-shared
-
 # if miner not already initialized
 if [ ! -d $LOTUS_MINER_PATH ]; then
   # initialize miner
@@ -44,6 +40,7 @@ if [ ! -d $LOTUS_MINER_PATH ]; then
   # fund a known account for usage
   /fund-account.sh
 
+  echo "Initializing miner..."
   lotus-miner init --genesis-miner --actor=t01000 --sector-size=2KiB --pre-sealed-sectors=/root/data/.genesis-sectors --pre-sealed-metadata=/root/data/.genesis-sectors/pre-seal-t01000.json --nosync
 fi
 

app/data/config/fixturenet-sushiswap-subgraph-v3/lotus-fixturenet.js.template

@@ -1,5 +1,9 @@
 module.exports = {
   network: 'lotus-fixturenet',
+  blocks: {
+    address: '0x0000000000000000000000000000000000000000',
+    startBlock: 0,
+  },
   v3: {
     factory: {
       address: 'FACTORY_ADDRESS',

app/data/config/fixturenet-sushiswap-subgraph-v3/run-blocks.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+echo "Building blocks subgraph and deploying to graph-node..."
+
+cd blocks
+
+pnpm run generate
+pnpm run build
+
+pnpm exec graph create --node http://graph-node:8020/ sushiswap/blocks
+pnpm exec graph deploy --node http://graph-node:8020/ --ipfs http://ipfs:5001 --version-label 0.1.0 sushiswap/blocks
+
+echo "Done"

app/data/config/fixturenet-sushiswap-subgraph-v3/run-v3.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -e
+
+cd v3
+
+# Loop until the NFPM deployment is detected
+echo "Waiting for sushiswap-periphery deployments to occur"
+while [ ! -f ./periphery-deployments/docker/NonfungiblePositionManager.json ]; do
+  sleep 5
+done
+
+echo "Reading contract addresses and block numbers from deployments"
+FACTORY_ADDRESS=$(jq -r '.address' ./core-deployments/docker/UniswapV3Factory.json)
+FACTORY_BLOCK=$(jq -r '.receipt.blockNumber' ./core-deployments/docker/UniswapV3Factory.json)
+NATIVE_ADDRESS=$(jq -r '.address' ./periphery-deployments/docker/WFIL.json)
+NFPM_ADDRESS=$(jq -r '.address' ./periphery-deployments/docker/NonfungiblePositionManager.json)
+NFPM_BLOCK=$(jq -r '.receipt.blockNumber' ./periphery-deployments/docker/NonfungiblePositionManager.json)
+
+# Read the JavaScript file content
+file_content=$(</app/config/lotus-fixturenet.js.template)
+
+# Replace uppercase words with environment variables
+echo "Reading values in lotus-fixturenet config"
+replaced_content=$(echo "$file_content" | sed -e "s/FACTORY_ADDRESS/$FACTORY_ADDRESS/g" \
+                                              -e "s/FACTORY_BLOCK/$FACTORY_BLOCK/g" \
+                                              -e "s/NFPM_ADDRESS/$NFPM_ADDRESS/g" \
+                                              -e "s/NFPM_BLOCK/$NFPM_BLOCK/g" \
+                                              -e "s/NATIVE_ADDRESS/$NATIVE_ADDRESS/g")
+
+# Write the replaced content back to the JavaScript file
+echo "$replaced_content" > /app/config/lotus-fixturenet.js
+
+echo "Building v3 subgraph and deploying to graph-node..."
+
+pnpm run generate
+pnpm run build
+
+pnpm exec graph create --node http://graph-node:8020/ sushiswap/v3-lotus
+pnpm exec graph deploy --node http://graph-node:8020/ --ipfs http://ipfs:5001 --version-label 0.1.0 sushiswap/v3-lotus
+
+echo "Done"

app/data/config/mainnet-eth-api-proxy/ethpxy.env

@@ -0,0 +1,6 @@
+CERC_ETHPXY_LISTEN_PORT=8547
+CERC_ETHPXY_LISTEN_ADDR='0.0.0.0'
+CERC_ETHPXY_REDIS_URL='redis://mainnet-eth-api-proxy-redis:6379'
+CERC_ETHPXY_REDIS_MAX_AGE=120000
+CERC_ETHPXY_GETH_WS_URL='ws://mainnet-eth-geth-1:8546'
+CERC_ETHPXY_GETH_HTTP_URL='http://mainnet-eth-geth-1:8545'

app/data/config/mainnet-eth-keycloak/keycloak.env

@@ -0,0 +1,29 @@
+POSTGRES_DB=keycloak
+POSTGRES_USER=keycloak
+POSTGRES_PASSWORD=keycloak
+KC_DB=postgres
+KC_DB_URL_HOST=keycloak-db
+KC_DB_URL_DATABASE=${POSTGRES_DB}
+KC_DB_USERNAME=${POSTGRES_USER}
+KC_DB_PASSWORD=${POSTGRES_PASSWORD}
+KC_DB_SCHEMA=public
+KC_HOSTNAME=localhost
+KC_HTTP_ENABLED="true"
+KC_HTTP_RELATIVE_PATH="/auth"
+KC_HOSTNAME_STRICT_HTTPS="false"
+KEYCLOAK_ADMIN=admin
+KEYCLOAK_ADMIN_PASSWORD=admin
+X_API_CHECK_REALM=cerc
+X_API_CHECK_CLIENT_ID="%user_id%"
+
+
+# keycloak-reg-api
+CERC_KCUSERREG_LISTEN_PORT=9292
+CERC_KCUSERREG_LISTEN_ADDR='0.0.0.0'
+CERC_KCUSERREG_API_URL='http://keycloak:8080/auth'
+CERC_KCUSERREG_REG_USER="${KEYCLOAK_ADMIN}"
+CERC_KCUSERREG_REG_PW="${KEYCLOAK_ADMIN_PASSWORD}"
+CERC_KCUSERREG_REG_CLIENT_ID='admin-cli'
+CERC_KCUSERREG_TARGET_REALM=cerc
+CERC_KCUSERREG_TARGET_GROUPS=eth
+CERC_KCUSERREG_CREATE_ENABLED=true

app/data/config/mainnet-eth-keycloak/nginx.example

@@ -0,0 +1,107 @@
+server {
+    listen 80;
+    server_name my.example.com;
+
+    # See: https://github.com/acmesh-official/acme.sh/wiki/Stateless-Mode
+    # and  https://datatracker.ietf.org/doc/html/rfc8555
+    location ~ ^/\.well-known/acme-challenge/([-_a-zA-Z0-9]+)$ {
+      default_type text/plain;
+      return 200 "$1.MY_ACCOUNT_THUMBPRINT_GOES_HERE";
+    }
+
+    location / {
+      return 301 https://$host$request_uri;
+    }
+}
+
+upstream geth-pool {
+  keepalive 100;
+  hash $user_id consistent;
+  server server-a:8545;
+  server server-b:8545;
+  server server-c:8545;
+}
+
+# self-reg happens on one server for clarity
+upstream reg-ui-pool {
+  keepalive 100;
+  server server-a:8085;
+}
+
+upstream reg-api-pool {
+  keepalive 100;
+  server server-a:8086;
+}
+
+# auth uses server-a if available
+upstream auth-pool {
+  keepalive 100;
+  server server-a:8080;
+  server server-b:8080 backup;
+  server server-c:8080 backup;
+}
+
+log_format upstreamlog '[$time_local] $remote_addr $user_id - $server_name $host to: $upstream_addr: $request $status upstream_response_time $upstream_response_time msec $msec request_time $request_time';
+proxy_cache_path                /var/cache/nginx/auth_cache levels=1 keys_zone=auth_cache:1m max_size=5m inactive=60m;
+server {
+    listen 443 ssl http2;
+    server_name my.example.com;
+    access_log /var/log/nginx/my.example.com-access.log upstreamlog;
+    error_log /var/log/nginx/my.example.com-error.log;
+
+    ssl_certificate /etc/nginx/ssl/my.example.com/cert.pem;
+    ssl_certificate_key /etc/nginx/ssl/my.example.com/key.pem;
+
+    error_page 500 502 503 504 /50x.html;
+    location = /50x.html {
+        root /usr/share/nginx/html;
+    }
+
+    #rewrite ^/?$ /newuser/;
+    rewrite ^/?$ https://www.example.com/;
+
+
+    # geth-pool ETH API
+    location ~ ^/v1/eth/?([^/]*)$ {
+        set $apiKey $1;
+        if ($apiKey = '') {
+          set $apiKey $http_X_API_KEY;
+        }
+        auth_request /auth;
+        auth_request_set $user_id $sent_http_x_user_id;
+        rewrite /.*$ / break;
+
+        client_max_body_size 3m;
+        client_body_buffer_size 3m;
+        proxy_buffer_size          32k;
+        proxy_buffers              16 32k;
+        proxy_busy_buffers_size    96k;
+
+        proxy_pass  http://geth-pool;
+        proxy_set_header X-Original-Remote-Addr $remote_addr;
+        proxy_set_header X-User-Id $user_id;
+    }
+    
+    # keycloak
+    location = /auth {
+        internal;
+        proxy_cache auth_cache;
+        proxy_cache_key "$apiKey";
+        proxy_cache_valid 200 300s;
+        proxy_cache_valid 401 30s;
+        proxy_pass http://auth-pool/auth/realms/cerc/check?memberOf=eth&apiKey=$apiKey;
+        proxy_pass_request_body off;
+        proxy_set_header Content-Length "";
+        proxy_set_header X-Original-URI $request_uri;
+        proxy_set_header X-Original-Remote-Addr $remote_addr;
+        proxy_set_header X-Original-Host $host;
+    }
+
+    location /newuser/ {
+        proxy_pass  http://reg-ui-pool/;
+    }
+
+    location /user-api/ {
+        proxy_pass  http://reg-api-pool/;
+    }
+}

app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/keycloak-mirror.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import sys
+
+import psycopg
+import random
+
+from subprocess import Popen
+from fabric import Connection
+
+
+def dump_src_db_to_file(db_host, db_port, db_user, db_password, db_name, file_name):
+    command = f"pg_dump -h {db_host} -p {db_port} -U {db_user} -d {db_name} -c --inserts -f {file_name}"
+    my_env = os.environ.copy()
+    my_env["PGPASSWORD"] = db_password
+    print(f"Exporting from {db_host}:{db_port}/{db_name} to {file_name}... ", end="")
+    ret = Popen(command, shell=True, env=my_env).wait()
+    print("DONE")
+    return ret
+
+
+def establish_ssh_tunnel(ssh_host, ssh_port, ssh_user, db_host, db_port):
+    local_port = random.randint(11000, 12000)
+    conn = Connection(host=ssh_host, port=ssh_port, user=ssh_user)
+    fw = conn.forward_local(
+        local_port=local_port, remote_port=db_port, remote_host=db_host
+    )
+    return conn, fw, local_port
+
+
+def load_db_from_file(db_host, db_port, db_user, db_password, db_name, file_name):
+    connstr = "host=%s port=%s user=%s password=%s sslmode=disable dbname=%s" % (
+        db_host,
+        db_port,
+        db_user,
+        db_password,
+        db_name,
+    )
+    with psycopg.connect(connstr) as conn:
+        with conn.cursor() as cur:
+            print(
+                f"Importing from {file_name} to {db_host}:{db_port}/{db_name}... ",
+                end="",
+            )
+            cur.execute(open(file_name, "rt").read())
+            print("DONE")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--src-dbhost", help="DB hostname", default="localhost")
+    parser.add_argument("--src-dbport", help="DB port", default=5432, type=int)
+    parser.add_argument("--src-dbuser", help="DB username", default="keycloak")
+    parser.add_argument("--src-dbpw", help="DB password", required=True)
+    parser.add_argument("--src-dbname", help="dbname", default="keycloak")
+
+    parser.add_argument(
+        "--dst-file", help="Destination filename", default="keycloak-mirror.sql"
+    )
+
+    parser.add_argument("--live-import", help="run the import", action="store_true")
+
+    parser.add_argument("--dst-dbhost", help="DB hostname", default="localhost")
+    parser.add_argument("--dst-dbport", help="DB port", default=5432, type=int)
+    parser.add_argument("--dst-dbuser", help="DB username", default="keycloak")
+    parser.add_argument("--dst-dbpw", help="DB password")
+    parser.add_argument("--dst-dbname", help="dbname", default="keycloak")
+
+    parser.add_argument("--ssh-host", help="SSH hostname")
+    parser.add_argument("--ssh-port", help="SSH port", default=22, type=int)
+    parser.add_argument("--ssh-user", help="SSH user")
+
+    args = parser.parse_args()
+
+    if args.live_import and not args.dst_dbpw:
+        print("--dst-dbpw is required if importing", file=sys.stderr)
+        sys.exit(2)
+
+    remove_sql_file = False
+    if args.dst_dbhost and not args.dst_file:
+        remove_sql_file = True
+
+    dst_file = args.dst_file
+    if not dst_file:
+        dst_file = "keycloak-mirror.sql"
+
+    dump_src_db_to_file(
+        args.src_dbhost,
+        args.src_dbport,
+        args.src_dbuser,
+        args.src_dbpw,
+        args.src_dbname,
+        dst_file,
+    )
+
+    if args.live_import:
+        try:
+            if args.ssh_host:
+                dst_dbport = random.randint(11000, 12000)
+                print(
+                    f"Establishing SSH tunnel from 127.0.0.1:{dst_dbport} to {args.ssh_host}->{args.dst_dbhost}:{args.dst_dbport}... ",
+                    end="",
+                )
+                with Connection(
+                    host=args.ssh_host, port=args.ssh_port, user=args.ssh_user
+                ).forward_local(
+                    local_port=dst_dbport,
+                    remote_port=args.dst_dbport,
+                    remote_host=args.dst_dbhost,
+                ):
+                    print("DONE")
+
+                    load_db_from_file(
+                        args.dst_dbhost,
+                        args.dst_dbport,
+                        args.dst_dbuser,
+                        args.dst_dbpw,
+                        args.dst_dbname,
+                        dst_file,
+                    )
+            else:
+                load_db_from_file(
+                    args.dst_dbhost,
+                    args.dst_dbport,
+                    args.dst_dbuser,
+                    args.dst_dbpw,
+                    args.dst_dbname,
+                    dst_file,
+                )
+
+        finally:
+            if args.live_import:
+                print(f"Removing {dst_file}... ", end="")
+                os.remove(dst_file)
+                print("DONE")

app/data/config/mainnet-eth-keycloak/scripts/keycloak-mirror/requirements.txt

@@ -0,0 +1,3 @@
+fabric
+psycopg~=3.1.8
+psycopg_binary

app/data/config/mainnet-eth-keycloak/ui/config.yml

@@ -0,0 +1,4 @@
+web:
+  path: ''
+api:
+  url: 'http://keycloak-reg-api:9292'

app/data/config/mainnet-eth-metrics/grafana/etc/provisioning/dashboards/dashboards.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+providers:
+  - name: dashboards
+    type: file
+    updateIntervalSeconds: 10
+    options:
+      path: /etc/grafana/dashboards
+      foldersFromFilesStructure: true

app/data/config/mainnet-eth-metrics/grafana/etc/provisioning/datasources/prometheus.yml

@@ -0,0 +1,19 @@
+apiVersion: 1
+
+datasources:
+  - id: 1
+    uid: jZUuGao4k
+    orgId: 1
+    name: Prometheus
+    type: prometheus
+    typeName: Prometheus
+    typeLogoUrl: public/app/plugins/datasource/prometheus/img/prometheus_logo.svg
+    access: proxy
+    url: http://prometheus:9090
+    user: ""
+    database: ""
+    basicAuth: false
+    isDefault: true
+    jsonData:
+      httpMethod: POST
+    readOnly: false

app/data/config/mainnet-eth-metrics/metrics.env

@@ -0,0 +1,2 @@
+# grafana
+GF_SECURITY_ADMIN_PASSWORD=changeme6325

app/data/config/mainnet-eth-metrics/prometheus/etc/prometheus.yml

@@ -0,0 +1,19 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 15s
+
+scrape_configs:
+  # geth
+  - job_name: 'geth'
+    metrics_path: /debug/metrics/prometheus
+    scheme: http
+    static_configs:
+    - targets: ['mainnet-eth-geth-1:6060']
+
+  # keycloak
+  - job_name: 'keycloak'
+    scrape_interval: 5s
+    metrics_path: /auth/realms/cerc/metrics
+    scheme: http
+    static_configs:
+      - targets: ['keycloak:8080']

app/data/config/mainnet-eth/geth.env

@@ -0,0 +1,57 @@
+# Enable remote debugging using dlv
+CERC_REMOTE_DEBUG=false
+
+# Enable startup script debug output.
+CERC_SCRIPT_DEBUG=false
+
+# Simple toggle to choose either a 'full' node or an 'archive' node
+# (controls the values of --syncmode --gcmode --snapshot)
+CERC_GETH_MODE_QUICK_SET=full
+
+# Optional custom node name.
+# GETH_NODE_NAME=""
+
+# Specify any other geth CLI options.
+GETH_OPTS=""
+
+# --cache
+GETH_CACHE=1024
+
+# --cache.database
+GETH_CACHE_DB=50
+
+# --cache.gc
+GETH_CACHE_GC=25
+
+# --cache.trie
+GETH_CACHE_TRIE=15
+j
+# --datadir
+GETH_DATADIR="/data"
+
+# --http.api
+GETH_HTTP_API="eth,web3,net"
+
+# --authrpc.jwtsecret
+GETH_JWTSECRET="/etc/mainnet-eth/jwtsecret"
+
+# --maxpeers
+GETH_MAX_PEERS=100
+
+# --rpc.evmtimeout
+GETH_RPC_EVMTIMEOUT=0
+
+# --rpc.gascap
+GETH_RPC_GASCAP=0
+
+# --txlookuplimit
+GETH_TXLOOKUPLIMIT=0
+
+# --verbosity
+GETH_VERBOSITY=3
+
+# --log.vmodule
+GETH_VMODULE="rpc/*=4"
+
+# --ws.api
+GETH_WS_API="eth,web3,net"

app/data/config/mainnet-eth/lighthouse.env

@@ -0,0 +1,33 @@
+# Enable startup script debug output.
+CERC_SCRIPT_DEBUG=false
+
+# Specify any other lighthouse CLI options.
+LIGHTHOUSE_OPTS=""
+
+# Override the advertised public IP (optional)
+# --enr-address
+#LIGHTHOUSE_ENR_ADDRESS=""
+
+# --checkpoint-sync-url
+LIGHTHOUSE_CHECKPOINT_SYNC_URL="https://beaconstate.ethstaker.cc"
+
+# --checkpoint-sync-url-timeout
+LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT=300
+
+# --datadir
+LIGHTHOUSE_DATADIR=/data
+
+# --debug-level
+LIGHTHOUSE_DEBUG_LEVEL=info
+
+# --http-port
+LIGHTHOUSE_HTTP_PORT=5052
+
+# --execution-jwt
+LIGHTHOUSE_JWTSECRET=/etc/mainnet-eth/jwtsecret
+
+# --metrics-port
+LIGHTHOUSE_METRICS_PORT=5054
+
+# --port --enr-udp-port --enr-tcp-port
+LIGHTHOUSE_NETWORK_PORT=9000

app/data/config/mainnet-eth/scripts/run-geth.sh

@@ -1,12 +1,10 @@
 #!/bin/sh
-if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then
     set -x
 fi
 
-CERC_ETH_DATADIR=/root/ethdata
-
 START_CMD="geth"
-if [ "true" == "$CERC_REMOTE_DEBUG" ] && [ -x "/usr/local/bin/dlv" ]; then
+if [[ "true" == "$CERC_REMOTE_DEBUG" ]] && [[ -x "/usr/local/bin/dlv" ]]; then
     START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --"
 fi
 
@@ -22,29 +20,44 @@ cleanup() {
     wait
     echo "Done"
 }
-
 trap 'cleanup' SIGINT SIGTERM
 
+MODE_FLAGS=""
+if [[ "$CERC_GETH_MODE_QUICK_SET" = "archive" ]]; then
+  MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-full} --gcmode=${GETH_GC_MODE:-archive} --snapshot=${GETH_SNAPSHOT:-false}"
+else
+  MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-snap} --gcmode=${GETH_GC_MODE:-full} --snapshot=${GETH_SNAPSHOT:-true}"
+fi
+
 $START_CMD \
-    --datadir="${CERC_ETH_DATADIR}" \
-    --authrpc.addr="0.0.0.0" \
-    --authrpc.port 8551 \
-    --authrpc.vhosts="*" \
-    --authrpc.jwtsecret="/etc/mainnet-eth/jwtsecret" \
-    --ws \
-    --ws.addr="0.0.0.0" \
-    --ws.origins="*" \
-    --ws.api="${CERC_GETH_WS_APIS:-eth,web3,net,admin,personal,debug,statediff}" \
-    --http.corsdomain="*" \
-    --gcmode full \
-    --txlookuplimit=0 \
-    --cache.preimages \
-    --syncmode=snap \
-    &
+  $MODE_FLAGS \
+  --datadir="${GETH_DATADIR}"\
+  --identity="${GETH_NODE_NAME}" \
+  --maxpeers=${GETH_MAX_PEERS} \
+  --cache=${GETH_CACHE} \
+  --cache.gc=${GETH_CACHE_GC} \
+  --cache.database=${GETH_CACHE_DB} \
+  --cache.trie=${GETH_CACHE_TRIE} \
+  --authrpc.addr='0.0.0.0' \
+  --authrpc.vhosts='*'  \
+  --authrpc.jwtsecret="${GETH_JWTSECRET}" \
+  --http \
+  --http.addr='0.0.0.0' \
+  --http.api="${GETH_HTTP_API}" \
+  --http.vhosts='*' \
+  --metrics \
+  --metrics.addr='0.0.0.0' \
+  --ws \
+  --ws.addr='0.0.0.0' \
+  --ws.api="${GETH_WS_API}" \
+  --rpc.gascap=${GETH_RPC_GASCAP} \
+  --rpc.evmtimeout=${GETH_RPC_EVMTIMEOUT} \
+  --txlookuplimit=${GETH_TXLOOKUPLIMIT} \
+  --verbosity=${GETH_VERBOSITY} \
+  --log.vmodule="${GETH_VMODULE}" \
+  ${GETH_OPTS} &
 
 geth_pid=$!
-
-
 wait $geth_pid
 
 if [ "true" == "$CERC_KEEP_RUNNING_AFTER_GETH_EXIT" ]; then

app/data/config/mainnet-eth/scripts/run-lighthouse.sh

@@ -1,22 +1,30 @@
 #!/bin/bash
-if [[ -n "$CERC_SCRIPT_DEBUG" ]]; then
+if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then
     set -x
 fi
 
-DEBUG_LEVEL=${CERC_LIGHTHOUSE_DEBUG_LEVEL:-info}
+ENR_OPTS=""
+if [[ -n "$LIGHTHOUSE_ENR_ADDRESS" ]]; then
+  ENR_OPTS="--enr-address $LIGHTHOUSE_ENR_ADDRESS"
+fi
 
-data_dir=/var/lighthouse-data-dir
-
-network_port=9001
-http_port=8001
-authrpc_port=8551
-
-exec lighthouse \
-  bn \
-  --debug-level $DEBUG_LEVEL \
-  --datadir $data_dir \
-  --network mainnet \
-  --execution-endpoint $EXECUTION_ENDPOINT \
-  --execution-jwt /etc/mainnet-eth/jwtsecret \
+exec lighthouse bn \
+  --checkpoint-sync-url "$LIGHTHOUSE_CHECKPOINT_SYNC_URL" \
+  --checkpoint-sync-url-timeout ${LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT} \
+  --datadir "$LIGHTHOUSE_DATADIR" \
+  --debug-level $LIGHTHOUSE_DEBUG_LEVEL \
   --disable-deposit-contract-sync \
-  --checkpoint-sync-url https://beaconstate.ethstaker.cc
+  --disable-upnp \
+  --enr-tcp-port $LIGHTHOUSE_NETWORK_PORT \
+  --enr-udp-port $LIGHTHOUSE_NETWORK_PORT \
+  --execution-endpoint "$LIGHTHOUSE_EXECUTION_ENDPOINT" \
+  --execution-jwt /etc/mainnet-eth/jwtsecret \
+  --http \
+  --http-address 0.0.0.0 \
+  --http-port $LIGHTHOUSE_HTTP_PORT \
+  --metrics \
+  --metrics-address=0.0.0.0 \
+  --metrics-port $LIGHTHOUSE_METRICS_PORT \
+  --network mainnet \
+  --port $LIGHTHOUSE_NETWORK_PORT \
+  $ENR_OPTS $LIGHTHOUSE_OPTS

app/data/config/sushiswap-subgraph-v3/filecoin.js

@@ -0,0 +1,30 @@
+const NATIVE_ADDRESS = '0x60e1773636cf5e4a227d9ac24f20feca034ee25a'
+const USDC_ADDRESS = '0xeb466342c4d449bc9f53a865d5cb90586f405215'
+const DAI_ADDRESS = '0x5c7e299cf531eb66f2a1df637d37abb78e6200c7'
+
+module.exports = {
+  network: 'filecoin',
+  blocks: {
+    address: '0x719e14fcb364bb05649bd525eb6c4a2d0d4ea2b7',
+    startBlock: 2867000,
+  },
+  v3: {
+    factory: { // 0xb4fbf271143f4fbf7b91a5ded31805e42b2208d6
+      address: '0xc35dadb65012ec5796536bd9864ed8773abc74c4',
+      startBlock: 2867560,
+    },
+    positionManager: {
+      address: '0xf4d73326c13a4fc5fd7a064217e12780e9bd62c3',
+      startBlock: 2868037,
+    },
+    native: { address: NATIVE_ADDRESS },
+    whitelistedTokenAddresses: [
+      NATIVE_ADDRESS,
+      USDC_ADDRESS,
+      DAI_ADDRESS,
+    ],
+    stableTokenAddresses: [USDC_ADDRESS, DAI_ADDRESS],
+    nativePricePool: '0x1d1375281265e4dd496d90455f7c82f4fbd85cc2',
+    minimumEthLocked: 250
+  },
+}

app/data/config/sushiswap-subgraph-v3/run-blocks.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+echo "Building blocks subgraph and deploying to graph-node..."
+
+cd blocks
+
+pnpm run generate
+pnpm run build
+
+pnpm exec graph create --node http://graph-node:8020/ sushiswap/blocks
+pnpm exec graph deploy --node http://graph-node:8020/ --ipfs http://ipfs:5001 --version-label 0.1.0 sushiswap/blocks
+
+echo "Done"

app/data/config/sushiswap-subgraph-v3/run-v3.sh

@@ -2,38 +2,14 @@
 
 set -e
 
-# Loop until the NFPM deployment is detected
-echo "Waiting for sushiswap-periphery deployments to occur"
-while [ ! -f ./deployments/docker/NonfungiblePositionManager.json ]; do
-  sleep 5
-done
+echo "Building v3 subgraph and deploying to graph-node..."
 
-echo "Reading contract addresses and block numbers from deployments"
-FACTORY_ADDRESS=$(jq -r '.address' ./core-deployments/docker/UniswapV3Factory.json)
-FACTORY_BLOCK=$(jq -r '.receipt.blockNumber' ./core-deployments/docker/UniswapV3Factory.json)
-NATIVE_ADDRESS=$(jq -r '.address' ./deployments/docker/WFIL.json)
-NFPM_ADDRESS=$(jq -r '.address' ./deployments/docker/NonfungiblePositionManager.json)
-NFPM_BLOCK=$(jq -r '.receipt.blockNumber' ./deployments/docker/NonfungiblePositionManager.json)
+cd v3
 
-# Read the JavaScript file content
-file_content=$(</app/config/lotus-fixturenet.js.template)
-
-# Replace uppercase words with environment variables
-echo "Reading values in lotus-fixturenet config"
-replaced_content=$(echo "$file_content" | sed -e "s/FACTORY_ADDRESS/$FACTORY_ADDRESS/g" \
-                                              -e "s/FACTORY_BLOCK/$FACTORY_BLOCK/g" \
-                                              -e "s/NFPM_ADDRESS/$NFPM_ADDRESS/g" \
-                                              -e "s/NFPM_BLOCK/$NFPM_BLOCK/g" \
-                                              -e "s/NATIVE_ADDRESS/$NATIVE_ADDRESS/g")
-
-# Write the replaced content back to the JavaScript file
-echo "$replaced_content" > /app/config/lotus-fixturenet.js
-
-
-echo "Building subgraph and deploying to graph-node..."
 pnpm run generate
 pnpm run build
-pnpm exec graph create --node http://graph-node:8020/ sushiswap/v3-lotus
-pnpm exec graph deploy --node http://graph-node:8020/ --ipfs http://ipfs:5001 --version-label 0.1.0 sushiswap/v3-lotus
+
+pnpm exec graph create --node http://graph-node:8020/ sushiswap/v3-filecoin
+pnpm exec graph deploy --node http://graph-node:8020/ --ipfs http://ipfs:5001 --version-label 0.1.0 sushiswap/v3-filecoin
 
 echo "Done"

app/data/container-build/cerc-eth-api-proxy/build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Build cerc/eth-api-proxy
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/eth-api-proxy:local ${build_command_args} ${CERC_REPO_BASE_DIR}/eth-api-proxy

app/data/container-build/cerc-keycloak-reg-api/build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Build cerc/keycloak-reg-api
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/keycloak-reg-api:local ${build_command_args} ${CERC_REPO_BASE_DIR}/keycloak-reg-api

app/data/container-build/cerc-keycloak-reg-ui/build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Build cerc/keycloak-reg-ui
+
+source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
+
+# See: https://stackoverflow.com/a/246128/1701505
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+docker build -t cerc/keycloak-reg-ui:local ${build_command_args} ${CERC_REPO_BASE_DIR}/keycloak-reg-ui

app/data/pod-list.txt

@@ -32,3 +32,6 @@ lasso
 reth
 watcher-sushiswap
 contract-sushiswap
+graph-node
+sushiswap-subgraph-v3
+fixturenet-sushiswap-subgraph-v3

app/data/stacks/fixturenet-graph-node/stack.yml

@@ -9,4 +9,4 @@ containers:
   - cerc/graph-node
 pods:
   - fixturenet-lotus
-  - fixturenet-graph-node
+  - graph-node

app/data/stacks/fixturenet-sushiswap-subgraph/README.md

@@ -0,0 +1,179 @@
+# Fixturenet SushiSwap Subgraph
+
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack fixturenet-sushiswap-subgraph setup-repositories --pull
+```
+
+Checkout to a non-default branch in the cloned repos if required:
+
+```bash
+# Default repo base dir
+cd ~/cerc
+
+# Example
+cd graph-node
+git checkout <your-branch> && git pull
+
+# Remove the corresponding docker image if it already exists
+docker image rm cerc/graph-node:local
+# Remove any dangling images
+docker image prune
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack fixturenet-sushiswap-subgraph build-containers
+```
+
+## Deploy
+
+
+Create an env file with the following contents to be used in the next step:
+
+```bash
+# Network and ETH RPC endpoint to run graph-node against
+NETWORK=lotus-fixturenet
+ETH_RPC_ENDPOINT=http://lotus-node-1:1234/rpc/v1
+```
+
+Uncomment the dependency on `lotus-node-1` in the [graph-node compose file](../../compose/docker-compose-graph-node.yml)
+
+Deploy the stack:
+
+```bash
+laconic-so --stack fixturenet-sushiswap-subgraph deploy --cluster sushigraph --env-file <PATH_TO_ENV_FILE> up
+
+# Note: Remove any existing volumes for the cluster for a fresh start
+```
+
+After all services have started:
+
+* Follow `graph-node` logs:
+
+  ```bash
+  laconic-so --stack fixturenet-sushiswap-subgraph deploy --cluster sushigraph logs -f graph-node
+  ```
+
+* Check that the subgraphs have been deployed:
+
+  ```bash
+  laconic-so --stack fixturenet-sushiswap-subgraph deploy --cluster sushigraph logs -f sushiswap-subgraph-v3
+
+  # Expected output:
+  # .
+  # .
+  # sushigraph-sushiswap-subgraph-v3-1  | - Deploying to Graph node http://graph-node:8020/
+  # sushigraph-sushiswap-subgraph-v3-1  | Deployed to http://graph-node:8000/subgraphs/name/sushiswap/v3-lotus/graphql
+  # sushigraph-sushiswap-subgraph-v3-1  |
+  # sushigraph-sushiswap-subgraph-v3-1  |
+  # sushigraph-sushiswap-subgraph-v3-1  | Subgraph endpoints:
+  # sushigraph-sushiswap-subgraph-v3-1  | Queries (HTTP):     http://graph-node:8000/subgraphs/name/sushiswap/v3-lotus
+  # .
+  # .
+  # sushigraph-sushiswap-subgraph-v3-1  | - Deploying to Graph node http://graph-node:8020/
+  # sushigraph-sushiswap-subgraph-v3-1  | Deployed to http://graph-node:8000/subgraphs/name/sushiswap/blocks/graphql
+  # sushigraph-sushiswap-subgraph-v3-1  |
+  # sushigraph-sushiswap-subgraph-v3-1  |
+  # sushigraph-sushiswap-subgraph-v3-1  | Subgraph endpoints:
+  # sushigraph-sushiswap-subgraph-v3-1  | Queries (HTTP):     http://graph-node:8000/subgraphs/name/sushiswap/blocks
+  # sushigraph-sushiswap-subgraph-v3-1  |
+  # sushigraph-sushiswap-subgraph-v3-1  |
+  # sushigraph-sushiswap-subgraph-v3-1  | Done
+  ```
+
+After `graph-node` has fetched the latest blocks from upstream, use the subgraph (GQL) endpoints for querying:
+
+```bash
+# Find out the mapped host port for the subgraph endpoint
+laconic-so --stack fixturenet-sushiswap-subgraph deploy --cluster sushigraph port graph-node 8000
+# 0.0.0.0:HOST_PORT
+
+# Blocks subgraph endpoint:
+http://127.0.0.1:<HOST_PORT>/subgraphs/name/sushiswap/blocks/graphql
+
+# v3 subgraph endpoint:
+http://127.0.0.1:<HOST_PORT>/subgraphs/name/sushiswap/v3-lotus/graphql
+```
+
+## Run
+
+* Deploy an ERC20 token:
+
+  ```bash
+  docker exec -it sushigraph-sushiswap-v3-periphery-1 yarn hardhat --network docker deploy --tags TestERC20
+
+  # Deploy two tokens and set the addresses to variables TOKEN1_ADDRESS and TOKEN2_ADDRESS
+  export TOKEN1_ADDRESS=<TOKEN1_ADDRESS>
+  export TOKEN2_ADDRESS=<TOKEN2_ADDRESS>
+  ```
+
+* Get contract address of factory deployed:
+
+  ```bash
+  docker exec -it sushigraph-sushiswap-v3-core-1 jq -r '.address' /app/deployments/docker/UniswapV3Factory.json
+
+  # Set the address to variable FACTORY_ADDRESS
+  export FACTORY_ADDRESS=<FACTORY_ADDRESS>
+  ```
+
+* Create a pool:
+
+  ```bash
+  docker exec -it sushigraph-sushiswap-v3-core-1 pnpm run pool:create:docker --factory $FACTORY_ADDRESS --token0 $TOKEN1_ADDRESS --token1 $TOKEN2_ADDRESS --fee 500
+
+  # Set the created pool address to variable POOL_ADDRESS
+  export POOL_ADDRESS=<POOL_ADDRESS>
+  ```
+
+* Initialize the pool:
+
+  ```bash
+  docker exec -it sushigraph-sushiswap-v3-core-1 pnpm run pool:initialize:docker --sqrt-price 4295128939 --pool $POOL_ADDRESS
+  ```
+
+* Set the recipient address to the contract deployer:
+
+  ```bash
+  export RECIPIENT=0xD375B03bd3A2434A9f675bEC4Ccd68aC5e67C743
+  ```
+
+* Trigger pool `Mint` event:
+
+  ```bash
+  docker exec -it sushigraph-sushiswap-v3-core-1 pnpm run pool:mint:docker --pool $POOL_ADDRESS --recipient $RECIPIENT --amount 10
+  ```
+
+* Trigger pool `Burn` event:
+
+  ```bash
+  docker exec -it sushigraph-sushiswap-v3-core-1 pnpm run pool:burn:docker --pool $POOL_ADDRESS --amount 10
+  ```
+
+## Clean up
+
+Stop all the services running in background run:
+
+```bash
+laconic-so --stack fixturenet-sushiswap-subgraph deploy --cluster sushigraph down
+```
+
+Clear volumes created by this stack:
+
+```bash
+# List all relevant volumes
+docker volume ls -q --filter "name=sushigraph"
+
+# Remove all the listed volumes
+docker volume rm $(docker volume ls -q --filter "name=sushigraph")
+
+# WARNING: To avoid refetching the Lotus proof params on the next run,
+# avoid removing the corresponding volumes
+
+# To remove volumes that do not contain Lotus params
+docker volume rm $(docker volume ls -q --filter "name=sushigraph" | grep -v "params$")
+```

app/data/stacks/fixturenet-sushiswap-subgraph/stack.yml

@@ -0,0 +1,28 @@
+version: "1.0"
+name: fixturenet-sushiswap-subgraph
+description: "An end-to-end SushiSwap Subgraph stack"
+repos:
+  # fixturenet-lotus repo
+  - github.com/filecoin-project/lotus
+  # graph-node repo
+  - github.com/graphprotocol/graph-node
+  # sushiswap repos
+  - github.com/cerc-io/sushiswap-v3-core@watcher-ts
+  - github.com/cerc-io/sushiswap-v3-periphery@watcher-ts
+  # sushiswap subgraph repo
+  - github.com/sushiswap/subgraphs
+containers:
+  # fixturenet-lotus image
+  - cerc/lotus
+  # graph-node image
+  - cerc/graph-node
+  # sushiswap contract deployment images
+  - cerc/sushiswap-v3-core
+  - cerc/sushiswap-v3-periphery
+  # sushiswap subgraphs image
+  - cerc/sushiswap-subgraphs
+pods:
+  - fixturenet-lotus
+  - graph-node
+  - contract-sushiswap
+  - fixturenet-sushiswap-subgraph-v3

app/data/stacks/mainnet-eth/deploy/commands.py

@@ -23,7 +23,7 @@ def setup(ctx):
     return None
 
 
-def create(ctx):
+def create(ctx, extra_args):
     # Generate the JWT secret and save to its config file
     secret = token_hex(32)
     jwt_file_path = ctx.deployment_dir.joinpath("data", "mainnet_eth_config_data", "jwtsecret")

app/data/stacks/mainnet-eth/stack.yml

@@ -5,11 +5,22 @@ repos:
   - github.com/cerc-io/go-ethereum
   - github.com/cerc-io/lighthouse
   - github.com/dboreham/foundry
+  - git.vdb.to/cerc-io/keycloak-reg-api
+  - git.vdb.to/cerc-io/keycloak-reg-ui
+  - github.com/vulcanize/eth-api-proxy
 containers:
   - cerc/go-ethereum
   - cerc/lighthouse
   - cerc/lighthouse-cli
   - cerc/foundry
+  - cerc/keycloak
+  - cerc/webapp-base
+  - cerc/keycloak-reg-api
+  - cerc/keycloak-reg-ui
+  - cerc/eth-api-proxy
 pods:
   - mainnet-eth
+  - mainnet-eth-keycloak
+  - mainnet-eth-metrics
+  - mainnet-eth-api-proxy
   - foundry

app/data/stacks/sushiswap-subgraph/README.md

@@ -1,11 +1,27 @@
-# SushiSwap Graph
+# SushiSwap Subgraph
 
 ## Setup
 
 Clone required repositories:
 
 ```bash
-laconic-so --stack sushiswap-subgraph setup-repositories
+laconic-so --stack sushiswap-subgraph setup-repositories --pull
+```
+
+Checkout to a non-default branch in the cloned repos if required:
+
+```bash
+# Default repo base dir
+cd ~/cerc
+
+# Example
+cd graph-node
+git checkout <your-branch> && git pull
+
+# Remove the corresponding docker image if it already exists
+docker image rm cerc/graph-node:local
+# Remove any dangling images
+docker image prune
 ```
 
 Build the container images:
@@ -14,76 +30,87 @@ Build the container images:
 laconic-so --stack sushiswap-subgraph build-containers
 ```
 
-## Deploy
+## Create a deployment
+
+Initialize deployment and create "spec" file:
+
+```bash
+laconic-so --stack sushiswap-subgraph deploy init --output sushiswap-subgraph-spec.yml
+```
+
+We need to assign a fixed port `8000` for graph-node subgraph GQL endpoint. The values can be
+customized by editing the "spec" file generated by `laconic-so deploy init`.
+```
+$ cat sushiswap-subgraph-spec.yml
+stack: sushiswap-subgraph
+ports:
+  graph-node:
+   - '8000:8000'
+   - '8001'
+   - '8020'
+   - '8030'
+...
+```
+
+Create deployment:
+
+```bash
+laconic-so deploy create --spec-file sushiswap-subgraph-spec.yml --deployment-dir sushiswap-subgraph-deployment
+```
+
+## Start the stack
 
 Deploy the stack:
 
 ```bash
-laconic-so --stack sushiswap-subgraph deploy --cluster sushigraph up
+laconic-so deployment --dir sushiswap-subgraph-deployment start
+
+# Note: Remove any existing volumes for the cluster for a fresh start
 ```
 
-After all services have started, wait and check that the subgraph has been deployed to graph-node
+After all services have started:
+
+* Follow `graph-node` logs:
+
+  ```bash
+  laconic-so deployment --dir sushiswap-subgraph-deployment logs -f graph-node
+  ```
+
+* Check that the subgraphs have been deployed:
+
+  ```bash
+  laconic-so deployment --dir sushiswap-subgraph-deployment logs -f sushiswap-subgraph-v3
+
+  # Expected output:
+  # .
+  # .
+  # sushigraph-sushiswap-subgraph-v3-1  | - Deploying to Graph node http://graph-node:8020/
+  # sushigraph-sushiswap-subgraph-v3-1  | Deployed to http://graph-node:8000/subgraphs/name/sushiswap/blocks/graphql
+  # sushigraph-sushiswap-subgraph-v3-1  |
+  # sushigraph-sushiswap-subgraph-v3-1  |
+  # sushigraph-sushiswap-subgraph-v3-1  | Subgraph endpoints:
+  # sushigraph-sushiswap-subgraph-v3-1  | Queries (HTTP):     http://graph-node:8000/subgraphs/name/sushiswap/blocks
+  # .
+  # .
+  # sushigraph-sushiswap-subgraph-v3-1  | - Deploying to Graph node http://graph-node:8020/
+  # sushigraph-sushiswap-subgraph-v3-1  | Deployed to http://graph-node:8000/subgraphs/name/sushiswap/v3-filecoin/graphql
+  # sushigraph-sushiswap-subgraph-v3-1  |
+  # sushigraph-sushiswap-subgraph-v3-1  |
+  # sushigraph-sushiswap-subgraph-v3-1  | Subgraph endpoints:
+  # sushigraph-sushiswap-subgraph-v3-1  | Queries (HTTP):     http://graph-node:8000/subgraphs/name/sushiswap/v3-filecoin
+  # sushigraph-sushiswap-subgraph-v3-1  |
+  # sushigraph-sushiswap-subgraph-v3-1  |
+  # sushigraph-sushiswap-subgraph-v3-1  | Done
+  ```
+
+After `graph-node` has fetched the latest blocks from upstream, use the subgraph (GQL) endpoints for querying:
 
 ```bash
-laconic-so --stack sushiswap-subgraph deploy --cluster sushigraph logs -f sushiswap-subgraph-v3
+# Blocks subgraph endpoint:
+http://127.0.0.1:8000/subgraphs/name/sushiswap/blocks/graphql
 
-# Expected end output
-# ...
-# sushigraph-sushiswap-subgraph-v3-1  | - Deploying to Graph node http://graph-node:8020/
-# sushigraph-sushiswap-subgraph-v3-1  | Deployed to http://graph-node:8000/subgraphs/name/sushiswap/v3-lotus/graphql
-# sushigraph-sushiswap-subgraph-v3-1  |
-# sushigraph-sushiswap-subgraph-v3-1  | Subgraph endpoints:
-# sushigraph-sushiswap-subgraph-v3-1  | Queries (HTTP):     http://graph-node:8000/subgraphs/name/sushiswap/v3-lotus
-# sushigraph-sushiswap-subgraph-v3-1  |
-# sushigraph-sushiswap-subgraph-v3-1  | Done
-```
-
-## Run
-
-To check graph-node logs:
-```bash
-laconic-so --stack sushiswap-subgraph deploy --cluster sushigraph logs -f graph-node
-```
-
-To deploy tokens run:
-```bash
-docker exec -it sushigraph-sushiswap-v3-periphery-1 yarn hardhat --network docker deploy --tags TestERC20
-```
-This can be run multiple times to deploy ERC20 tokens
-
-Take note of the deployed token addresses to use later
-
-Get contract address of factory deployed:
-```bash
-docker exec -it sushigraph-sushiswap-v3-core-1 jq -r '.address' /app/deployments/docker/UniswapV3Factory.json
-```
-Set it to environment variable `FACTORY_ADDRESS` to use later
-
-To create a pool:
-```bash
-docker exec -it sushigraph-sushiswap-v3-core-1 pnpm run pool:create:docker --factory $FACTORY_ADDRESS --token0 $TOKEN1_ADDRESS --token1 $TOKEN2_ADDRESS --fee 500
-```
-
-Set the created pool address to environment variable `POOL_ADDRESS` to use later
-
-To initialize pool:
-```bash
-docker exec -it sushigraph-sushiswap-v3-core-1 pnpm run pool:initialize:docker --sqrt-price 4295128939 --pool $POOL_ADDRESS
-```
-
-Set the recipient address to the contract deployer:
-```bash
-export RECIPIENT=0xD375B03bd3A2434A9f675bEC4Ccd68aC5e67C743
-```
-
-Trigger pool mint event:
-```bash
-docker exec -it sushigraph-sushiswap-v3-core-1 pnpm run pool:mint:docker --pool $POOL_ADDRESS --recipient $RECIPIENT --amount 10
-```
-
-Trigger pool burn event:
-```bash
-docker exec -it sushigraph-sushiswap-v3-core-1 pnpm run pool:burn:docker --pool $POOL_ADDRESS --amount 10
+# v3 subgraph endpoint:
+http://127.0.0.1:8000/subgraphs/name/sushiswap/v3-filecoin/graphql
 ```
 
 ## Clean up
@@ -91,21 +118,11 @@ docker exec -it sushigraph-sushiswap-v3-core-1 pnpm run pool:burn:docker --pool
 Stop all the services running in background run:
 
 ```bash
-laconic-so --stack sushiswap-subgraph deploy --cluster sushigraph down
+laconic-so deployment --dir sushiswap-subgraph-deployment stop
 ```
 
 Clear volumes created by this stack:
 
 ```bash
-# List all relevant volumes
-docker volume ls -q --filter "name=sushigraph"
-
-# Remove all the listed volumes
-docker volume rm $(docker volume ls -q --filter "name=sushigraph")
-
-# WARNING: After removing volumes with Lotus params
-# They will be downloaded again on restart
-
-# To remove volumes that do not contain Lotus params
-docker volume rm $(docker volume ls -q --filter "name=sushigraph" | grep -v "params$")
+laconic-so deployment --dir sushiswap-subgraph-deployment stop --delete-volumes
 ```

app/data/stacks/sushiswap-subgraph/stack.yml

@@ -1,28 +1,16 @@
 version: "1.0"
 name: sushiswap-subgraph
-description: "An end-to-end SushiSwap Subgraph stack"
+description: "SushiSwap Subgraph stack"
 repos:
-  ## fixturenet-lotus repo
-  - github.com/filecoin-project/lotus
-  ## graph-node repo
+  # graph-node repo
   - github.com/graphprotocol/graph-node
-  ## sushiswap repos
-  - github.com/cerc-io/sushiswap-v3-core@watcher-ts
-  - github.com/cerc-io/sushiswap-v3-periphery@watcher-ts
-  ## subgraph repo
+  # sushiswap subgraph repo
   - github.com/sushiswap/subgraphs
 containers:
-  ## fixturenet-lotus image
-  - cerc/lotus
-  ## fixturenet-graph-node image
+  # graph-node image
   - cerc/graph-node
-  ## sushiswap contract deployment images
-  - cerc/sushiswap-v3-core
-  - cerc/sushiswap-v3-periphery
-  ## sushiswap subgraphs image
+  # sushiswap subgraphs image
   - cerc/sushiswap-subgraphs
 pods:
-  - fixturenet-lotus
-  - fixturenet-graph-node
-  - contract-sushiswap
+  - graph-node
   - sushiswap-subgraph-v3

app/deployment.py

@@ -30,6 +30,8 @@ class DeploymentContext:
 @click.option("--dir", required=True, help="path to deployment directory")
 @click.pass_context
 def command(ctx, dir):
+    '''create a deployment'''
+
     # Check that --stack wasn't supplied
     if ctx.parent.obj.stack:
         print("Error: --stack can't be supplied with the deployment command")