Implement --check-only (#546)

* Simple self-update feature

* Update quick install script for self-update

app/data/compose/docker-compose-graph-node.yml

@@ -6,9 +6,6 @@ services:
         condition: service_healthy
       ipfs:
         condition: service_healthy
-      # Uncomment when running against fixturenet-lotus to wait for the Lotus node to come up
-      # lotus-node-1:
-      #   condition: service_healthy
     extra_hosts:
       - host.docker.internal:host-gateway
     environment:
@@ -18,9 +15,13 @@ services:
       postgres_user: graph-node
       postgres_pass: password
       postgres_db: graph-node
-      ethereum: ${NETWORK:-filecoin}:${ETH_RPC_ENDPOINT:-https://archive.lotus.vdb.to/rpc/v1}
+      ethereum: ${ETH_NETWORKS:-lotus-fixturenet:http://lotus-node-1:1234/rpc/v1}
       GRAPH_LOG: debug
       ETHEREUM_REORG_THRESHOLD: 3
+    entrypoint: ["bash", "-c"]
+    # Wait for ETH RPC endpoint to be up when running with fixturenet-lotus
+    command: |
+      "wait_for ${ETH_RPC_HOST:-lotus-node-1}:${ETH_RPC_PORT:-1234} -t 1800 -- start"
     ports:
       - "8000"
       - "8001"

app/data/compose/docker-compose-mobymask-app-v3.yml

@@ -14,6 +14,7 @@ services:
       CERC_RELAY_NODES: ${CERC_RELAY_NODES}
       CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
       CERC_PUBSUB: ${CERC_PUBSUB}
+      CERC_GOSSIPSUB_DIRECT_PEERS: ${CERC_GOSSIPSUB_DIRECT_PEERS}
       CERC_RELEASE: "laconic-v3"
       CERC_NA_ADDRESS: ${CERC_NA_ADDRESS}
       CERC_VPA_ADDRESS: ${CERC_VPA_ADDRESS}

app/data/config/fixturenet-sushiswap-subgraph-v3/lotus-fixturenet.js.template

@@ -17,8 +17,8 @@ module.exports = {
     whitelistedTokenAddresses: [
       'NATIVE_ADDRESS',
     ],
-    stableTokenAddresses: [
-    ],
+    stableTokenAddresses: [],
+    nativePricePool: '0x0000000000000000000000000000000000000000',
     minimumEthLocked: 1.5
   }
 }

app/data/config/watcher-mobymask-v3/mobymask-app-start.sh

@@ -10,6 +10,7 @@ CERC_DEPLOYED_CONTRACT="${CERC_DEPLOYED_CONTRACT:-${DEFAULT_CERC_DEPLOYED_CONTRA
 CERC_RELAY_NODES="${CERC_RELAY_NODES:-${DEFAULT_CERC_RELAY_NODES}}"
 CERC_DENY_MULTIADDRS="${CERC_DENY_MULTIADDRS:-${DEFAULT_CERC_DENY_MULTIADDRS}}"
 CERC_PUBSUB="${CERC_PUBSUB:-${DEFAULT_CERC_PUBSUB}}"
+CERC_GOSSIPSUB_DIRECT_PEERS="${CERC_GOSSIPSUB_DIRECT_PEERS:-${DEFAULT_CERC_GOSSIPSUB_DIRECT_PEERS}}"
 
 echo "Using CERC_RELAY_NODES $CERC_RELAY_NODES"
 
@@ -49,7 +50,8 @@ jq --arg address "$CERC_DEPLOYED_CONTRACT" \
   --argjson relayNodes "$CERC_RELAY_NODES" \
   --argjson denyMultiaddrs "$CERC_DENY_MULTIADDRS" \
   --arg pubsub "$CERC_PUBSUB" \
-  '.address = $address | .chainId = $chainId | .relayNodes = $relayNodes | .peer.denyMultiaddrs = $denyMultiaddrs | .peer.pubsub = $pubsub' \
+  --argjson directPeers "$CERC_GOSSIPSUB_DIRECT_PEERS" \
+  '.address = $address | .chainId = $chainId | .relayNodes = $relayNodes | .peer.denyMultiaddrs = $denyMultiaddrs | .peer.pubsub = $pubsub | .peer.directPeers = $directPeers' \
   /app/src/mobymask-app-config.json > /app/src/utils/config.json
 
 yarn install

app/data/config/watcher-mobymask-v3/mobymask-params.env

@@ -28,6 +28,9 @@ DEFAULT_CERC_DENY_MULTIADDRS=[]
 # Type of pubsub to be used
 DEFAULT_CERC_PUBSUB=""
 
+# Set of direct peers to be used when pubsub is set to gossipsub
+DEFAULT_CERC_GOSSIPSUB_DIRECT_PEERS=[]
+
 # Set deployed Nitro addresses to avoid deploying them in the stack
 DEFAULT_CERC_NA_ADDRESS=
 DEFAULT_CERC_VPA_ADDRESS=

app/data/container-build/cerc-builder-js/build-npm-package-local-dependencies.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # Usage: build-npm-package-local-dependencies.sh <registry-url> <publish-with-this-version>
 # Runs build-npm-package.sh after first fixing up yarn.lock to use a local
-# npm registry for all packages in a specific scope (currently @cerc-io, @lirewine and @muknsys)
+# npm registry for all packages in a specific scope (currently @cerc-io and @lirewine)
 if [ -n "$CERC_SCRIPT_DEBUG" ]; then
     set -x
 fi
@@ -18,7 +18,7 @@ set -e
 local_npm_registry_url=$1
 package_publish_version=$2
 # If we need to handle an additional scope, add it to the list below:
-npm_scopes_to_handle=("@cerc-io" "@lirewine" "@muknsys")
+npm_scopes_to_handle=("@cerc-io" "@lirewine")
 for npm_scope_for_local in ${npm_scopes_to_handle[@]}
 do
     # We need to configure the local registry

app/data/container-build/cerc-builder-js/build-npm-package.sh

@@ -24,7 +24,6 @@ package_name=$( cat package.json | jq -r .name )
 local_npm_registry_url=$1
 npm config set @cerc-io:registry ${local_npm_registry_url}
 npm config set @lirewine:registry ${local_npm_registry_url}
-npm config set @muknsys:registry ${local_npm_registry_url}
 # Workaround bug in npm unpublish where it needs the url to be of the form //<foo> and not http://<foo>
 local_npm_registry_url_fixed=$( echo ${local_npm_registry_url} | sed -e 's/^http[s]\{0,1\}://')
 npm config set -- ${local_npm_registry_url_fixed}:_authToken ${CERC_NPM_AUTH_TOKEN}

app/data/stacks/fixturenet-graph-node/README.md

@@ -1,3 +0,0 @@
-# Graph-Node Fixturenet
-
-Experimental

app/data/stacks/fixturenet-graph-node/stack.yml

@@ -1,12 +0,0 @@
-version: "1.0"
-name: fixturenet-graph-node
-description: "A graph-node fixturenet"
-repos:
-  - github.com/filecoin-project/lotus
-  - github.com/graphprotocol/graph-node
-containers:
-  - cerc/lotus
-  - cerc/graph-node
-pods:
-  - fixturenet-lotus
-  - graph-node

app/data/stacks/fixturenet-sushiswap-subgraph/README.md

@@ -32,21 +32,10 @@ laconic-so --stack fixturenet-sushiswap-subgraph build-containers
 
 ## Deploy
 
-
-Create an env file with the following contents to be used in the next step:
-
-```bash
-# Network and ETH RPC endpoint to run graph-node against
-NETWORK=lotus-fixturenet
-ETH_RPC_ENDPOINT=http://lotus-node-1:1234/rpc/v1
-```
-
-Uncomment the dependency on `lotus-node-1` in the [graph-node compose file](../../compose/docker-compose-graph-node.yml)
-
 Deploy the stack:
 
 ```bash
-laconic-so --stack fixturenet-sushiswap-subgraph deploy --cluster sushigraph --env-file <PATH_TO_ENV_FILE> up
+laconic-so --stack fixturenet-sushiswap-subgraph deploy --cluster sushigraph up
 
 # Note: Remove any existing volumes for the cluster for a fresh start
 ```
@@ -154,6 +143,42 @@ http://127.0.0.1:<HOST_PORT>/subgraphs/name/sushiswap/v3-lotus/graphql
   docker exec -it sushigraph-sushiswap-v3-core-1 pnpm run pool:burn:docker --pool $POOL_ADDRESS --amount 10
   ```
 
+* Query the sushiswap v3-lotus subgraph GQL after running above commands
+
+  ```graphql
+  {
+    _meta {
+      block {
+        number
+      }
+      deployment
+      hasIndexingErrors
+    }
+    
+    factories {
+      poolCount
+      id
+    }
+    
+    pools {
+      id
+      token0 {
+        id
+        name
+        symbol
+      }
+      mints {
+        id
+        owner
+      }
+      burns {
+        id
+        owner
+      }
+    }
+  }
+  ```
+
 ## Clean up
 
 Stop all the services running in background run:

app/data/stacks/graph-node/README.md

@@ -0,0 +1,157 @@
+# Graph Node
+
+## Setup
+
+Clone required repositories:
+
+```bash
+laconic-so --stack graph-node setup-repositories --pull
+```
+
+Checkout to a non-default branch in the cloned repos if required:
+
+```bash
+# Default repo base dir
+cd ~/cerc
+
+# Example
+cd graph-node
+git checkout <your-branch> && git pull
+
+# Remove the corresponding docker image if it already exists
+docker image rm cerc/graph-node:local
+# Remove any dangling images
+docker image prune
+```
+
+Build the container images:
+
+```bash
+laconic-so --stack graph-node build-containers
+```
+
+## Create a deployment
+
+Initialize deployment and create "spec" file:
+
+```bash
+laconic-so --stack graph-node deploy init --output graph-node-spec.yml
+```
+
+We need to assign fixed ports: `8000` for subgraph GQL endpoint, `8020` for subgraph deployment and `5001` for IPFS. The values can be
+customized by editing the "spec" file generated by `laconic-so deploy init`.
+```
+$ cat graph-node-spec.yml
+stack: graph-node
+ports:
+  graph-node:
+    - '8000:8000'
+    - '8001'
+    - '8020:8020'
+    - '8030'
+  ipfs:
+    - '8080'
+    - '4001'
+    - '5001:5001'
+...
+```
+
+Create deployment:
+
+```bash
+laconic-so deploy create --spec-file graph-node-spec.yml --deployment-dir graph-node-deployment
+```
+
+## Start the stack
+
+Create an env file with the following values to be set before starting the stack:
+
+```bash
+# Set ETH RPC endpoint the graph node will use
+
+# Host and port of the ETH RPC endpoint to check before starting graph-node
+export ETH_RPC_HOST=
+export ETH_RPC_PORT=
+
+# The etherum network(s) graph-node will connect to
+# Set this to a space-separated list of the networks where each entry has the form NAME:URL
+export ETH_NETWORKS=
+```
+
+Example env file:
+
+```bash
+export ETH_RPC_HOST=filecoin.chainup.net
+export ETH_RPC_PORT=443
+
+export ETH_NETWORKS=filecoin:https://filecoin.chainup.net/rpc/v1
+```
+
+Set the environment variables:
+
+```bash
+source <PATH_TO_ENV_FILE>
+```
+
+Deploy the stack:
+
+```bash
+laconic-so deployment --dir graph-node-deployment start
+
+# Note: Remove any existing volumes in the cluster for a fresh start
+```
+
+After all services have started, follow `graph-node` logs:
+
+```bash
+laconic-so deployment --dir graph-node-deployment logs -f graph-node
+```
+
+Subgraphs can now be deployed to the graph-node.
+Follow [Deploy the Subgraph](https://github.com/graphprotocol/graph-node/blob/v0.32.0/docs/getting-started.md#24-deploy-the-subgraph) section in graph-node docs for an existing subgraph.
+
+## Set environment variables
+
+* The graph-node environment variable `ETHEREUM_REORG_THRESHOLD` can be set in the deployment compose file
+  ```bash
+  $ cat graph-node-deployment/compose/docker-compose-graph-node.yml
+  services:
+    graph-node:
+      image: cerc/graph-node:local
+      ...
+      environment:
+        ...
+        GRAPH_LOG: debug
+        ETHEREUM_REORG_THRESHOLD: 3
+  ```
+  Change `ETHEREUM_REORG_THRESHOLD` to desired value
+
+  * To restart graph-node with updated values
+    * Stop the stack first
+      ```bash
+      laconic-so deployment --dir graph-node-deployment stop
+      ```
+    * Start the stack again
+      ```
+      laconic-so deployment --dir graph-node-deployment start
+      ```
+  * To check if environment variable has been updated in graph-node container
+      ```bash
+      $ laconic-so deployment --dir graph-node-deployment exec graph-node bash
+      root@dc4d3abe1615:/# echo $ETHEREUM_REORG_THRESHOLD
+      16
+      ```
+
+## Clean up
+
+Stop all the services running in background run:
+
+```bash
+laconic-so deployment --dir graph-node-deployment stop
+```
+
+Clear volumes created by this stack:
+
+```bash
+laconic-so deployment --dir graph-node-deployment stop --delete-volumes
+```

app/data/stacks/graph-node/deploy-subgraph.md

@@ -0,0 +1,68 @@
+# Deploying Subgraph
+
+## Setup
+
+We will use the [ethereum-gravatar](https://github.com/graphprotocol/graph-tooling/tree/%40graphprotocol/graph-cli%400.58.0/examples/ethereum-gravatar) example subgraph from `graphprotocol/graph-tooling` repo
+
+- Clone the repo
+  ```bash
+  git clone git@github.com:graphprotocol/graph-tooling.git
+  
+  cd graph-tooling
+  ```
+
+- Install dependencies
+  ```bash
+  pnpm install
+  ```
+
+- Change directory to example-subgraph
+  ```bash
+  cd examples/ethereum-gravatar
+  ```
+
+## Deploy
+
+The following steps should be similar for every subgraph
+
+- Change the network and address in `subgraph.yaml`
+  ```yaml
+  ...
+  dataSources:
+  - kind: ethereum/contract
+    name: Gravity
+    network: <NETWORK_NAME>
+    source:
+      address: '<CONTRACT_ADDRESS>'
+      abi: Gravity
+      startBlock: <START_BLOCK>
+  ...
+  ```
+  - `CONTRACT_ADDRESS` is the address of the deployed contract on the desired network
+  - `START_BLOCK` is the block number after which we want to process events
+  - `NETWORK_NAME` is the name of the network specified when deploying graph-node
+    - When deploying graph-node `ETH_NETWORKS` env is set to a space-separated list of the networks where each entry has the form `NAME:URL`
+    - The `NAME` can be used in subgraph to specify which network to use
+    - More details can be seen in [Start the stack](./README.md#start-the-stack) section
+
+- Build the subgraph
+  ```bash
+  pnpm codegen
+  pnpm build
+  ```
+
+- Create and deploy the subgraph
+  ```bash
+  pnpm graph create example --node <GRAPH_NODE_DEPLOY_ENDPOINT>
+  
+  pnpm graph deploy example --ipfs <GRAPH_NODE_IPFS_ENDPOINT> --node <GRAPH_NODE_DEPLOY_ENDPOINT>
+  ```
+  - `GRAPH_NODE_DEPLOY_ENDPOINT` and `GRAPH_NODE_IPFS_ENDPOINT` will be available after graph-node has been deployed
+  - More details can be seen in [Create a deployment](./README.md#create-a-deployment) section 
+
+- The subgraph GQL endpoint will be seen after deploy command runs successfully
+
+- To remove the subgraph
+  ```bash
+  pnpm graph remove --node <GRAPH_NODE_DEPLOY_ENDPOINT> example
+  ```

app/data/stacks/graph-node/stack.yml

@@ -0,0 +1,9 @@
+version: "1.0"
+name: graph-node
+description: "Stack for running graph-node"
+repos:
+  - github.com/graphprotocol/graph-node
+containers:
+  - cerc/graph-node
+pods:
+  - graph-node

app/data/stacks/mobymask-v3/web-app.md

@@ -45,6 +45,9 @@ Create and update an env file to be used in the next step ([defaults](../../conf
   # (Optional) Type of pubsub to be used ("floodsub" | "gossipsub")
   CERC_PUBSUB=""
 
+  # (Optional) Set of direct peers to be used when pubsub is set to gossipsub
+  CERC_GOSSIPSUB_DIRECT_PEERS=[]
+
   # Set Nitro addresses
   CERC_NA_ADDRESS=
   CERC_VPA_ADDRESS=

app/update.py

@@ -0,0 +1,90 @@
+# Copyright © 2023 Vulcanize
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+import click
+import datetime
+import filecmp
+import os
+from pathlib import Path
+import requests
+import sys
+import stat
+import shutil
+import validators
+from app.util import get_yaml
+
+
+def _download_url(url: str, file_path: Path):
+    r = requests.get(url, stream=True)
+    r.raw.decode_content = True
+    with open(file_path, 'wb') as f:
+        shutil.copyfileobj(r.raw, f)
+
+
+def _error_exit(s: str):
+    print(s)
+    sys.exit(1)
+
+
+# Note at present this probably won't work on non-Unix based OSes like Windows
+@click.command()
+@click.option("--check-only", is_flag=True, default=False, help="only check, don't update")
+@click.pass_context
+def command(ctx, check_only):
+    '''update shiv binary from a distribution url'''
+    # Get the distribution URL from config
+    config_key = 'distribution-url'
+    config_file_path = Path(os.path.expanduser("~/.laconic-so/config.yml"))
+    if not config_file_path.exists():
+        _error_exit(f"Error: Config file: {config_file_path} not found")
+    yaml = get_yaml()
+    config = yaml.load(open(config_file_path, "r"))
+    if "distribution-url" not in config:
+        _error_exit(f"Error: {config_key} not defined in {config_file_path}")
+    distribution_url = config[config_key]
+    # Sanity check the URL
+    if not validators.url(distribution_url):
+        _error_exit(f"ERROR: distribution url: {distribution_url} is not valid")
+    # Figure out the filename for ourselves
+    shiv_binary_path = Path(sys.argv[0])
+    timestamp_filename = f"laconic-so-download-{datetime.datetime.now().strftime('%y%m%d-%H%M%S')}"
+    temp_download_path = shiv_binary_path.parent.joinpath(timestamp_filename)
+    # Download the file to a temp filename
+    if ctx.obj.verbose:
+        print(f"Downloading from: {distribution_url} to {temp_download_path}")
+    _download_url(distribution_url, temp_download_path)
+    # Set the executable bit
+    existing_mode = os.stat(temp_download_path)
+    os.chmod(temp_download_path, existing_mode.st_mode | stat.S_IXUSR)
+    # Switch the new file for the path we ran from
+    # Check if the downloaded file is identical to the existing one
+    same = filecmp.cmp(temp_download_path, shiv_binary_path)
+    if same:
+        if not ctx.obj.quiet or check_only:
+            print("No update available, latest version already installed")
+    else:
+        if not ctx.obj.quiet:
+            print("Update available")
+        if check_only:
+            if not ctx.obj.quiet:
+                print("Check-only node, update not installed")
+        else:
+            if not ctx.obj.quiet:
+                print("Installing...")
+            if ctx.obj.verbose:
+                print(f"Replacing: {shiv_binary_path} with {temp_download_path}")
+            os.replace(temp_download_path, shiv_binary_path)
+            if not ctx.obj.quiet:
+                print("Done. Run \"laconic-so version\" to see the newly installed version")

cli.py

@@ -22,6 +22,7 @@ from app import build_npms
 from app import deploy
 from app import version
 from app import deployment
+from app import update
 
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
@@ -48,3 +49,4 @@ cli.add_command(deploy.command, "deploy")  # deploy is an alias for deploy-syste
 cli.add_command(deploy.command, "deploy-system")
 cli.add_command(deployment.command, "deployment")
 cli.add_command(version.command, "version")
+cli.add_command(update.command, "update")

requirements.txt

@@ -7,3 +7,4 @@ PyYAML>=6.0.1
 ruamel.yaml>=0.17.32
 pydantic==1.10.9
 tomli==2.0.1
+validators==0.22.0

scripts/quick-install-linux.sh

@@ -5,6 +5,9 @@ fi
 
 install_dir=~/bin
 
+# Skip the package install stuff if so directed
+if ! [[ -n "$CERC_SO_INSTALL_SKIP_PACKAGES" ]]; then
+
 # First display a reasonable warning to the user unless run with -y
 if ! [[ $# -eq 1 && $1 == "-y" ]]; then
   echo "**************************************************************************************"
@@ -128,13 +131,20 @@ sudo apt -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin d
 # Allow the current user to use Docker
 sudo usermod -aG docker $USER
 
+# End of long if block: Skip the package install stuff if so directed
+fi
+
 echo "**************************************************************************************"
 echo "Installing laconic-so"
 # install latest `laconic-so`
+distribution_url=https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
 install_filename=${install_dir}/laconic-so
 mkdir -p  ${install_dir}
-curl -L -o ${install_filename} https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
+curl -L -o ${install_filename} ${distribution_url}
 chmod +x ${install_filename}
+# Set up config file for self-update feature
+mkdir ~/.laconic-so
+echo "distribution-url: ${distribution_url}" >  ~/.laconic-so/config.yml
 
 echo "**************************************************************************************"
 # Check if our PATH line is already there