Compare commits

..
Author SHA1 Message Date
dboreham 5beaf69a36 Implement --check-only (#546) 2023-09-27 14:51:52 -06:00
dboreham 06c4a77afe Simple self-update feature (#545)
* Simple self-update feature

* Update quick install script for self-update
2023-09-27 14:24:41 -06:00
nabarun aafadbbed6 Add readme to deploy subgraphs (#543) 2023-09-27 12:15:51 +05:30
nabarun be519e9ca0 Split graph-node stack to run independently (#541)
* Split graph-node stack to run independently

* Fix sushiswap subgraph config in fixturenet stack
2023-09-26 17:31:00 +05:30
Zach e56a910260 remove old npm scope (#538) 2023-09-25 09:43:06 -04:00
prathamesh0andShreerang Kale e3dc75118b Add support for direct peers when pubsub is set to gossipsub (#539)
Co-authored-by: Shreerang Kale <shreerangkale@gmail.com>
2023-09-25 14:55:53 +05:30
roysc e0b5318ebe Plugeth stack fixX (#537) 2023-09-20 14:26:58 -05:00
Zach 2425aa8556 fix readme (#519) 2023-09-20 10:31:14 -04:00
20 changed files with 403 additions and 45 deletions
@@ -20,7 +20,7 @@ services:
- SYS_PTRACE
environment:
CERC_REMOTE_DEBUG: ${CERC_REMOTE_DEBUG:-true}
CERC_RUN_STATEDIFF: "detect"
CERC_RUN_STATEDIFF: ${CERC_RUN_STATEDIFF:-detect}
CERC_STATEDIFF_DB_NODE_ID: 1
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
env_file:
@@ -6,9 +6,6 @@ services:
condition: service_healthy
ipfs:
condition: service_healthy
# Uncomment when running against fixturenet-lotus to wait for the Lotus node to come up
# lotus-node-1:
# condition: service_healthy
extra_hosts:
- host.docker.internal:host-gateway
environment:
@@ -18,9 +15,13 @@ services:
postgres_user: graph-node
postgres_pass: password
postgres_db: graph-node
ethereum: ${NETWORK:-filecoin}:${ETH_RPC_ENDPOINT:-https://archive.lotus.vdb.to/rpc/v1}
ethereum: ${ETH_NETWORKS:-lotus-fixturenet:http://lotus-node-1:1234/rpc/v1}
GRAPH_LOG: debug
ETHEREUM_REORG_THRESHOLD: 3
entrypoint: ["bash", "-c"]
# Wait for ETH RPC endpoint to be up when running with fixturenet-lotus
command: |
"wait_for ${ETH_RPC_HOST:-lotus-node-1}:${ETH_RPC_PORT:-1234} -t 1800 -- start"
ports:
- "8000"
- "8001"
@@ -14,6 +14,7 @@ services:
CERC_RELAY_NODES: ${CERC_RELAY_NODES}
CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS}
CERC_PUBSUB: ${CERC_PUBSUB}
CERC_GOSSIPSUB_DIRECT_PEERS: ${CERC_GOSSIPSUB_DIRECT_PEERS}
CERC_RELEASE: "laconic-v3"
CERC_NA_ADDRESS: ${CERC_NA_ADDRESS}
CERC_VPA_ADDRESS: ${CERC_VPA_ADDRESS}
@@ -17,8 +17,8 @@ module.exports = {
whitelistedTokenAddresses: [
'NATIVE_ADDRESS',
],
stableTokenAddresses: [
],
stableTokenAddresses: [],
nativePricePool: '0x0000000000000000000000000000000000000000',
minimumEthLocked: 1.5
}
}
@@ -10,6 +10,7 @@ CERC_DEPLOYED_CONTRACT="${CERC_DEPLOYED_CONTRACT:-${DEFAULT_CERC_DEPLOYED_CONTRA
CERC_RELAY_NODES="${CERC_RELAY_NODES:-${DEFAULT_CERC_RELAY_NODES}}"
CERC_DENY_MULTIADDRS="${CERC_DENY_MULTIADDRS:-${DEFAULT_CERC_DENY_MULTIADDRS}}"
CERC_PUBSUB="${CERC_PUBSUB:-${DEFAULT_CERC_PUBSUB}}"
CERC_GOSSIPSUB_DIRECT_PEERS="${CERC_GOSSIPSUB_DIRECT_PEERS:-${DEFAULT_CERC_GOSSIPSUB_DIRECT_PEERS}}"
echo "Using CERC_RELAY_NODES $CERC_RELAY_NODES"
@@ -49,7 +50,8 @@ jq --arg address "$CERC_DEPLOYED_CONTRACT" \
--argjson relayNodes "$CERC_RELAY_NODES" \
--argjson denyMultiaddrs "$CERC_DENY_MULTIADDRS" \
--arg pubsub "$CERC_PUBSUB" \
'.address = $address | .chainId = $chainId | .relayNodes = $relayNodes | .peer.denyMultiaddrs = $denyMultiaddrs | .peer.pubsub = $pubsub' \
--argjson directPeers "$CERC_GOSSIPSUB_DIRECT_PEERS" \
'.address = $address | .chainId = $chainId | .relayNodes = $relayNodes | .peer.denyMultiaddrs = $denyMultiaddrs | .peer.pubsub = $pubsub | .peer.directPeers = $directPeers' \
/app/src/mobymask-app-config.json > /app/src/utils/config.json
yarn install
@@ -28,6 +28,9 @@ DEFAULT_CERC_DENY_MULTIADDRS=[]
# Type of pubsub to be used
DEFAULT_CERC_PUBSUB=""
# Set of direct peers to be used when pubsub is set to gossipsub
DEFAULT_CERC_GOSSIPSUB_DIRECT_PEERS=[]
# Set deployed Nitro addresses to avoid deploying them in the stack
DEFAULT_CERC_NA_ADDRESS=
DEFAULT_CERC_VPA_ADDRESS=
@@ -1,7 +1,7 @@
#!/bin/bash
# Usage: build-npm-package-local-dependencies.sh <registry-url> <publish-with-this-version>
# Runs build-npm-package.sh after first fixing up yarn.lock to use a local
# npm registry for all packages in a specific scope (currently @cerc-io, @lirewine and @muknsys)
# npm registry for all packages in a specific scope (currently @cerc-io and @lirewine)
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
@@ -18,7 +18,7 @@ set -e
local_npm_registry_url=$1
package_publish_version=$2
# If we need to handle an additional scope, add it to the list below:
npm_scopes_to_handle=("@cerc-io" "@lirewine" "@muknsys")
npm_scopes_to_handle=("@cerc-io" "@lirewine")
for npm_scope_for_local in ${npm_scopes_to_handle[@]}
do
# We need to configure the local registry
@@ -24,7 +24,6 @@ package_name=$( cat package.json | jq -r .name )
local_npm_registry_url=$1
npm config set @cerc-io:registry ${local_npm_registry_url}
npm config set @lirewine:registry ${local_npm_registry_url}
npm config set @muknsys:registry ${local_npm_registry_url}
# Workaround bug in npm unpublish where it needs the url to be of the form //<foo> and not http://<foo>
local_npm_registry_url_fixed=$( echo ${local_npm_registry_url} | sed -e 's/^http[s]\{0,1\}://')
npm config set -- ${local_npm_registry_url_fixed}:_authToken ${CERC_NPM_AUTH_TOKEN}
@@ -1,3 +0,0 @@
# Graph-Node Fixturenet
Experimental
@@ -1,12 +0,0 @@
version: "1.0"
name: fixturenet-graph-node
description: "A graph-node fixturenet"
repos:
- github.com/filecoin-project/lotus
- github.com/graphprotocol/graph-node
containers:
- cerc/lotus
- cerc/graph-node
pods:
- fixturenet-lotus
- graph-node
@@ -4,6 +4,8 @@ Testing a "Loaded" fixturenet with console.
Instructions for deploying a local Laconic blockchain "fixturenet" for development and testing purposes using laconic-stack-orchestrator.
**Note:** For building some NPMs, access to the @lirewine repositories is required. If you don't have access, see [this tutorial](/docs/laconicd-fixturenet.md) to run this stack
## 1. Install Laconic Stack Orchestrator
Installation is covered in detail [here](https://github.com/cerc-io/stack-orchestrator#user-mode) but if you're on Linux and already have docker installed it should be as simple as:
```
@@ -30,24 +32,24 @@ $ export CERC_NPM_AUTH_TOKEN=<my-token>
## 3. Clone required repositories
```
$ laconic-so --stack fixturenet-laconicd setup-repositories
$ laconic-so --stack fixturenet-laconic-loaded setup-repositories
```
## 4. Build the stack's packages and containers
```
$ laconic-so --stack fixturenet-laconicd build-npms
$ laconic-so --stack fixturenet-laconicd build-containers
$ laconic-so --stack fixturenet-laconic-loaded build-npms
$ laconic-so --stack fixturenet-laconic-loaded build-containers
```
## 5. Deploy the stack
```
$ laconic-so --stack fixturenet-laconicd deploy up
$ laconic-so --stack fixturenet-laconic-loaded deploy up
```
Correct operation should be verified by checking the laconicd container's logs with:
```
$ laconic-so --stack fixturenet-laconicd deploy logs
$ laconic-so --stack fixturenet-laconic-loaded deploy logs
```
## 6. Test with the Registry CLI
```
$ laconic-so --stack fixturenet-laconicd deploy exec cli "laconic cns status"
$ laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns status"
```
## 7. View the laconic console
Get the URL for the console web app with this command (the port number will be different for each deployment):
@@ -32,21 +32,10 @@ laconic-so --stack fixturenet-sushiswap-subgraph build-containers
## Deploy
Create an env file with the following contents to be used in the next step:
```bash
# Network and ETH RPC endpoint to run graph-node against
NETWORK=lotus-fixturenet
ETH_RPC_ENDPOINT=http://lotus-node-1:1234/rpc/v1
```
Uncomment the dependency on `lotus-node-1` in the [graph-node compose file](../../compose/docker-compose-graph-node.yml)
Deploy the stack:
```bash
laconic-so --stack fixturenet-sushiswap-subgraph deploy --cluster sushigraph --env-file <PATH_TO_ENV_FILE> up
laconic-so --stack fixturenet-sushiswap-subgraph deploy --cluster sushigraph up
# Note: Remove any existing volumes for the cluster for a fresh start
```
@@ -154,6 +143,42 @@ http://127.0.0.1:<HOST_PORT>/subgraphs/name/sushiswap/v3-lotus/graphql
docker exec -it sushigraph-sushiswap-v3-core-1 pnpm run pool:burn:docker --pool $POOL_ADDRESS --amount 10
```
* Query the sushiswap v3-lotus subgraph GQL after running above commands
```graphql
{
_meta {
block {
number
}
deployment
hasIndexingErrors
}
factories {
poolCount
id
}
pools {
id
token0 {
id
name
symbol
}
mints {
id
owner
}
burns {
id
owner
}
}
}
```
## Clean up
Stop all the services running in background run:
+157
View File
@@ -0,0 +1,157 @@
# Graph Node
## Setup
Clone required repositories:
```bash
laconic-so --stack graph-node setup-repositories --pull
```
Checkout to a non-default branch in the cloned repos if required:
```bash
# Default repo base dir
cd ~/cerc
# Example
cd graph-node
git checkout <your-branch> && git pull
# Remove the corresponding docker image if it already exists
docker image rm cerc/graph-node:local
# Remove any dangling images
docker image prune
```
Build the container images:
```bash
laconic-so --stack graph-node build-containers
```
## Create a deployment
Initialize deployment and create "spec" file:
```bash
laconic-so --stack graph-node deploy init --output graph-node-spec.yml
```
We need to assign fixed ports: `8000` for subgraph GQL endpoint, `8020` for subgraph deployment and `5001` for IPFS. The values can be
customized by editing the "spec" file generated by `laconic-so deploy init`.
```
$ cat graph-node-spec.yml
stack: graph-node
ports:
graph-node:
- '8000:8000'
- '8001'
- '8020:8020'
- '8030'
ipfs:
- '8080'
- '4001'
- '5001:5001'
...
```
Create deployment:
```bash
laconic-so deploy create --spec-file graph-node-spec.yml --deployment-dir graph-node-deployment
```
## Start the stack
Create an env file with the following values to be set before starting the stack:
```bash
# Set ETH RPC endpoint the graph node will use
# Host and port of the ETH RPC endpoint to check before starting graph-node
export ETH_RPC_HOST=
export ETH_RPC_PORT=
# The etherum network(s) graph-node will connect to
# Set this to a space-separated list of the networks where each entry has the form NAME:URL
export ETH_NETWORKS=
```
Example env file:
```bash
export ETH_RPC_HOST=filecoin.chainup.net
export ETH_RPC_PORT=443
export ETH_NETWORKS=filecoin:https://filecoin.chainup.net/rpc/v1
```
Set the environment variables:
```bash
source <PATH_TO_ENV_FILE>
```
Deploy the stack:
```bash
laconic-so deployment --dir graph-node-deployment start
# Note: Remove any existing volumes in the cluster for a fresh start
```
After all services have started, follow `graph-node` logs:
```bash
laconic-so deployment --dir graph-node-deployment logs -f graph-node
```
Subgraphs can now be deployed to the graph-node.
Follow [Deploy the Subgraph](https://github.com/graphprotocol/graph-node/blob/v0.32.0/docs/getting-started.md#24-deploy-the-subgraph) section in graph-node docs for an existing subgraph.
## Set environment variables
* The graph-node environment variable `ETHEREUM_REORG_THRESHOLD` can be set in the deployment compose file
```bash
$ cat graph-node-deployment/compose/docker-compose-graph-node.yml
services:
graph-node:
image: cerc/graph-node:local
...
environment:
...
GRAPH_LOG: debug
ETHEREUM_REORG_THRESHOLD: 3
```
Change `ETHEREUM_REORG_THRESHOLD` to desired value
* To restart graph-node with updated values
* Stop the stack first
```bash
laconic-so deployment --dir graph-node-deployment stop
```
* Start the stack again
```
laconic-so deployment --dir graph-node-deployment start
```
* To check if environment variable has been updated in graph-node container
```bash
$ laconic-so deployment --dir graph-node-deployment exec graph-node bash
root@dc4d3abe1615:/# echo $ETHEREUM_REORG_THRESHOLD
16
```
## Clean up
Stop all the services running in background run:
```bash
laconic-so deployment --dir graph-node-deployment stop
```
Clear volumes created by this stack:
```bash
laconic-so deployment --dir graph-node-deployment stop --delete-volumes
```
@@ -0,0 +1,68 @@
# Deploying Subgraph
## Setup
We will use the [ethereum-gravatar](https://github.com/graphprotocol/graph-tooling/tree/%40graphprotocol/graph-cli%400.58.0/examples/ethereum-gravatar) example subgraph from `graphprotocol/graph-tooling` repo
- Clone the repo
```bash
git clone git@github.com:graphprotocol/graph-tooling.git
cd graph-tooling
```
- Install dependencies
```bash
pnpm install
```
- Change directory to example-subgraph
```bash
cd examples/ethereum-gravatar
```
## Deploy
The following steps should be similar for every subgraph
- Change the network and address in `subgraph.yaml`
```yaml
...
dataSources:
- kind: ethereum/contract
name: Gravity
network: <NETWORK_NAME>
source:
address: '<CONTRACT_ADDRESS>'
abi: Gravity
startBlock: <START_BLOCK>
...
```
- `CONTRACT_ADDRESS` is the address of the deployed contract on the desired network
- `START_BLOCK` is the block number after which we want to process events
- `NETWORK_NAME` is the name of the network specified when deploying graph-node
- When deploying graph-node `ETH_NETWORKS` env is set to a space-separated list of the networks where each entry has the form `NAME:URL`
- The `NAME` can be used in subgraph to specify which network to use
- More details can be seen in [Start the stack](./README.md#start-the-stack) section
- Build the subgraph
```bash
pnpm codegen
pnpm build
```
- Create and deploy the subgraph
```bash
pnpm graph create example --node <GRAPH_NODE_DEPLOY_ENDPOINT>
pnpm graph deploy example --ipfs <GRAPH_NODE_IPFS_ENDPOINT> --node <GRAPH_NODE_DEPLOY_ENDPOINT>
```
- `GRAPH_NODE_DEPLOY_ENDPOINT` and `GRAPH_NODE_IPFS_ENDPOINT` will be available after graph-node has been deployed
- More details can be seen in [Create a deployment](./README.md#create-a-deployment) section
- The subgraph GQL endpoint will be seen after deploy command runs successfully
- To remove the subgraph
```bash
pnpm graph remove --node <GRAPH_NODE_DEPLOY_ENDPOINT> example
```
+9
View File
@@ -0,0 +1,9 @@
version: "1.0"
name: graph-node
description: "Stack for running graph-node"
repos:
- github.com/graphprotocol/graph-node
containers:
- cerc/graph-node
pods:
- graph-node
+3
View File
@@ -45,6 +45,9 @@ Create and update an env file to be used in the next step ([defaults](../../conf
# (Optional) Type of pubsub to be used ("floodsub" | "gossipsub")
CERC_PUBSUB=""
# (Optional) Set of direct peers to be used when pubsub is set to gossipsub
CERC_GOSSIPSUB_DIRECT_PEERS=[]
# Set Nitro addresses
CERC_NA_ADDRESS=
CERC_VPA_ADDRESS=
+90
View File
@@ -0,0 +1,90 @@
# Copyright © 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import click
import datetime
import filecmp
import os
from pathlib import Path
import requests
import sys
import stat
import shutil
import validators
from app.util import get_yaml
def _download_url(url: str, file_path: Path):
r = requests.get(url, stream=True)
r.raw.decode_content = True
with open(file_path, 'wb') as f:
shutil.copyfileobj(r.raw, f)
def _error_exit(s: str):
print(s)
sys.exit(1)
# Note at present this probably won't work on non-Unix based OSes like Windows
@click.command()
@click.option("--check-only", is_flag=True, default=False, help="only check, don't update")
@click.pass_context
def command(ctx, check_only):
'''update shiv binary from a distribution url'''
# Get the distribution URL from config
config_key = 'distribution-url'
config_file_path = Path(os.path.expanduser("~/.laconic-so/config.yml"))
if not config_file_path.exists():
_error_exit(f"Error: Config file: {config_file_path} not found")
yaml = get_yaml()
config = yaml.load(open(config_file_path, "r"))
if "distribution-url" not in config:
_error_exit(f"Error: {config_key} not defined in {config_file_path}")
distribution_url = config[config_key]
# Sanity check the URL
if not validators.url(distribution_url):
_error_exit(f"ERROR: distribution url: {distribution_url} is not valid")
# Figure out the filename for ourselves
shiv_binary_path = Path(sys.argv[0])
timestamp_filename = f"laconic-so-download-{datetime.datetime.now().strftime('%y%m%d-%H%M%S')}"
temp_download_path = shiv_binary_path.parent.joinpath(timestamp_filename)
# Download the file to a temp filename
if ctx.obj.verbose:
print(f"Downloading from: {distribution_url} to {temp_download_path}")
_download_url(distribution_url, temp_download_path)
# Set the executable bit
existing_mode = os.stat(temp_download_path)
os.chmod(temp_download_path, existing_mode.st_mode | stat.S_IXUSR)
# Switch the new file for the path we ran from
# Check if the downloaded file is identical to the existing one
same = filecmp.cmp(temp_download_path, shiv_binary_path)
if same:
if not ctx.obj.quiet or check_only:
print("No update available, latest version already installed")
else:
if not ctx.obj.quiet:
print("Update available")
if check_only:
if not ctx.obj.quiet:
print("Check-only node, update not installed")
else:
if not ctx.obj.quiet:
print("Installing...")
if ctx.obj.verbose:
print(f"Replacing: {shiv_binary_path} with {temp_download_path}")
os.replace(temp_download_path, shiv_binary_path)
if not ctx.obj.quiet:
print("Done. Run \"laconic-so version\" to see the newly installed version")
+2
View File
@@ -22,6 +22,7 @@ from app import build_npms
from app import deploy
from app import version
from app import deployment
from app import update
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
@@ -48,3 +49,4 @@ cli.add_command(deploy.command, "deploy") # deploy is an alias for deploy-syste
cli.add_command(deploy.command, "deploy-system")
cli.add_command(deployment.command, "deployment")
cli.add_command(version.command, "version")
cli.add_command(update.command, "update")
+1
View File
@@ -7,3 +7,4 @@ PyYAML>=6.0.1
ruamel.yaml>=0.17.32
pydantic==1.10.9
tomli==2.0.1
validators==0.22.0
+11 -1
View File
@@ -5,6 +5,9 @@ fi
install_dir=~/bin
# Skip the package install stuff if so directed
if ! [[ -n "$CERC_SO_INSTALL_SKIP_PACKAGES" ]]; then
# First display a reasonable warning to the user unless run with -y
if ! [[ $# -eq 1 && $1 == "-y" ]]; then
echo "**************************************************************************************"
@@ -128,13 +131,20 @@ sudo apt -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin d
# Allow the current user to use Docker
sudo usermod -aG docker $USER
# End of long if block: Skip the package install stuff if so directed
fi
echo "**************************************************************************************"
echo "Installing laconic-so"
# install latest `laconic-so`
distribution_url=https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
install_filename=${install_dir}/laconic-so
mkdir -p ${install_dir}
curl -L -o ${install_filename} https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so
curl -L -o ${install_filename} ${distribution_url}
chmod +x ${install_filename}
# Set up config file for self-update feature
mkdir ~/.laconic-so
echo "distribution-url: ${distribution_url}" > ~/.laconic-so/config.yml
echo "**************************************************************************************"
# Check if our PATH line is already there