Compare commits

..
Author SHA1 Message Date
telackey 82785b6823 unused 2024-03-04 23:43:21 -06:00
telackey 583774e32a ; 2024-03-04 23:42:45 -06:00
telackey f91329eef0 lint 2024-03-04 23:41:31 -06:00
telackey 7f0b82b9a9 Update status cmd 2024-03-05 05:40:15 +00:00
telackey 64b395f15a more oops 2024-03-04 23:05:35 -06:00
telackey 240db3ca4c oops 2024-03-04 22:52:57 -06:00
telackey d09fbc3ac4 TLS options 2024-03-04 22:01:09 -06:00
telackey 3c7ff76a83 fmt 2024-03-04 17:09:11 -06:00
telackey 53b29ff69b fmt 2024-03-04 17:08:08 -06:00
telackey 4ec0e38b77 Fix configmaps constant. 2024-03-04 22:01:01 +00:00
telackey 14ee4da1fc comment 2024-03-04 15:27:36 -06:00
telackey e81c95a920 Support wildcard certs. 2024-03-04 15:24:14 -06:00
32 changed files with 317 additions and 722 deletions
@@ -1,30 +0,0 @@
services:
laconic2d:
restart: unless-stopped
image: cerc/laconic2d:local
command: ["sh", "/docker-entrypoint-scripts.d/create-fixturenet.sh"]
volumes:
# The cosmos-sdk node's database directory:
- laconic2d-data:/root/.laconic2d
# TODO: look at folding these scripts into the container
- ../config/fixturenet-laconic2d/create-fixturenet.sh:/docker-entrypoint-scripts.d/create-fixturenet.sh
- ../config/fixturenet-laconic2d/export-mykey.sh:/docker-entrypoint-scripts.d/export-mykey.sh
- ../config/fixturenet-laconic2d/export-myaddress.sh:/docker-entrypoint-scripts.d/export-myaddress.sh
# TODO: determine which of the ports below is really needed
ports:
- "6060"
- "26657"
- "26656"
- "9473"
- "8545"
- "8546"
- "9090"
- "9091"
- "1317"
cli:
image: cerc/laconic-registry-cli:local
volumes:
- ../config/fixturenet-laconic2d/registry-cli-config-template.yml:/registry-cli-config-template.yml
volumes:
laconic2d-data:
@@ -1,76 +0,0 @@
version: '3.2'
services:
ajna-watcher-db:
restart: unless-stopped
image: postgres:14-alpine
environment:
- POSTGRES_USER=vdbm
- POSTGRES_MULTIPLE_DATABASES=ajna-watcher,ajna-watcher-job-queue
- POSTGRES_EXTENSION=ajna-watcher-job-queue:pgcrypto
- POSTGRES_PASSWORD=password
volumes:
- ../config/postgresql/multiple-postgressql-databases.sh:/docker-entrypoint-initdb.d/multiple-postgressql-databases.sh
- ajna_watcher_db_data:/var/lib/postgresql/data
ports:
- "5432"
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "5432"]
interval: 20s
timeout: 5s
retries: 15
start_period: 10s
ajna-watcher-job-runner:
restart: unless-stopped
depends_on:
ajna-watcher-db:
condition: service_healthy
image: cerc/watcher-ajna:local
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
command: ["bash", "./start-job-runner.sh"]
volumes:
- ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
- ../config/watcher-ajna/start-job-runner.sh:/app/start-job-runner.sh
ports:
- "9000"
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "9000"]
interval: 20s
timeout: 5s
retries: 15
start_period: 5s
extra_hosts:
- "host.docker.internal:host-gateway"
ajna-watcher-server:
restart: unless-stopped
depends_on:
ajna-watcher-db:
condition: service_healthy
ajna-watcher-job-runner:
condition: service_healthy
image: cerc/watcher-ajna:local
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
command: ["bash", "./start-server.sh"]
volumes:
- ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
- ../config/watcher-ajna/start-server.sh:/app/start-server.sh
ports:
- "3008"
- "9001"
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "3008"]
interval: 20s
timeout: 5s
retries: 15
start_period: 5s
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
ajna_watcher_db_data:
@@ -1,91 +0,0 @@
#!/bin/bash
KEY="mykey"
CHAINID="laconic_9000-1"
MONIKER="localtestnet"
KEYRING="test"
LOGLEVEL="${LOGLEVEL:-info}"
if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconic2d/data/blockstore.db" ]; then
# validate dependencies are installed
command -v jq > /dev/null 2>&1 || {
echo >&2 "jq not installed. More info: https://stedolan.github.io/jq/download/"
exit 1
}
# remove existing daemon and client
rm -rf $HOME/.laconic2d/*
rm -rf $HOME/.laconic/*
if [ -n "`which make`" ]; then
make install
fi
laconic2d config set client chain-id $CHAINID
laconic2d config set client keyring-backend $KEYRING
# if $KEY exists it should be deleted
laconic2d keys add $KEY --keyring-backend $KEYRING
# Set moniker and chain-id for Ethermint (Moniker can be anything, chain-id must be an integer)
laconic2d init $MONIKER --chain-id $CHAINID --default-denom photon
update_genesis() {
jq "$1" $HOME/.laconic2d/config/genesis.json > $HOME/.laconic2d/config/tmp_genesis.json &&
mv $HOME/.laconic2d/config/tmp_genesis.json $HOME/.laconic2d/config/genesis.json
}
if [[ "$TEST_REGISTRY_EXPIRY" == "true" ]]; then
echo "Setting timers for expiry tests."
update_genesis '.app_state["registry"]["params"]["record_rent_duration"]="60s"'
update_genesis '.app_state["registry"]["params"]["authority_grace_period"]="60s"'
update_genesis '.app_state["registry"]["params"]["authority_rent_duration"]="60s"'
fi
if [[ "$TEST_AUCTION_ENABLED" == "true" ]]; then
echo "Enabling auction and setting timers."
update_genesis '.app_state["registry"]["params"]["authority_auction_enabled"]=true'
update_genesis '.app_state["registry"]["params"]["authority_rent_duration"]="60s"'
update_genesis '.app_state["registry"]["params"]["authority_grace_period"]="300s"'
update_genesis '.app_state["registry"]["params"]["authority_auction_commits_duration"]="60s"'
update_genesis '.app_state["registry"]["params"]["authority_auction_reveals_duration"]="60s"'
fi
# increase block time (?)
update_genesis '.consensus["params"]["block"]["time_iota_ms"]="1000"'
# Set gas limit in genesis
update_genesis '.consensus["params"]["block"]["max_gas"]="10000000"'
# disable produce empty block
if [[ "$OSTYPE" == "darwin"* ]]; then
sed -i '' 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconic2d/config/config.toml
else
sed -i 's/create_empty_blocks = true/create_empty_blocks = false/g' $HOME/.laconic2d/config/config.toml
fi
# Allocate genesis accounts (cosmos formatted addresses)
laconic2d genesis add-genesis-account $KEY 100000000000000000000000000photon --keyring-backend $KEYRING
# Sign genesis transaction
laconic2d genesis gentx $KEY 1000000000000000000000photon --keyring-backend $KEYRING --chain-id $CHAINID
# Collect genesis tx
laconic2d genesis collect-gentxs
# Run this to ensure everything worked and that the genesis file is setup correctly
laconic2d genesis validate
else
echo "Using existing database at $HOME/.laconic2d. To replace, run '`basename $0` clean'"
fi
# Start the node (remove the --pruning=nothing flag if historical queries are not needed)
laconic2d start \
--pruning=nothing \
--log_level $LOGLEVEL \
--minimum-gas-prices=0.0001photon \
--api.enable \
--rpc.laddr="tcp://0.0.0.0:26657" \
--gql-server --gql-playground
@@ -1,2 +0,0 @@
#!/bin/sh
laconic2d keys show mykey | grep address | cut -d ' ' -f 3
@@ -1,2 +0,0 @@
#!/bin/sh
echo y | laconic2d keys export mykey --unarmored-hex --unsafe
@@ -1,9 +0,0 @@
services:
registry:
restEndpoint: 'http://laconic2d:1317'
gqlEndpoint: 'http://laconic2d:9473/api'
userKey: REPLACE_WITH_MYKEY
bondId:
chainId: laconic_9000-1
gas: 350000
fees: 200000aphoton
@@ -50,6 +50,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -126,6 +142,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -202,6 +234,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -278,6 +326,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -354,6 +418,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -430,6 +510,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -506,6 +602,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -582,6 +694,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -660,6 +788,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -736,6 +880,22 @@ groups:
legendFormat: __auto
range: false
refId: latest_external
- refId: latest_indexed
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_indexed
- refId: condition
relativeTimeRange:
from: 600
@@ -1,20 +0,0 @@
#!/bin/sh
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
set -u
echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
# Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
# Write the modified content to a new file
echo "$WATCHER_CONFIG" > environments/local.toml
echo "Running job-runner..."
DEBUG=vulcanize:* exec node --enable-source-maps dist/job-runner.js
@@ -1,20 +0,0 @@
#!/bin/sh
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
set -u
echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
# Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
# Write the modified content to a new file
echo "$WATCHER_CONFIG" > environments/local.toml
echo "Running server..."
DEBUG=vulcanize:* exec node --enable-source-maps dist/server.js
@@ -1,98 +0,0 @@
[server]
host = "0.0.0.0"
port = 3008
kind = "active"
gqlPath = "/"
# Checkpointing state.
checkpointing = true
# Checkpoint interval in number of blocks.
checkpointInterval = 2000
# Enable state creation
# CAUTION: Disable only if state creation is not desired or can be filled subsequently
enableState = false
subgraphPath = "./subgraph-build"
# Interval to restart wasm instance periodically
wasmRestartBlocksInterval = 20
# Interval in number of blocks at which to clear entities cache.
clearEntitiesCacheInterval = 1000
# Max block range for which to return events in eventsInRange GQL query.
# Use -1 for skipping check on block range.
maxEventsBlockRange = 1000
# Flag to specify whether RPC endpoint supports block hash as block tag parameter
rpcSupportsBlockHashParam = false
# GQL cache settings
[server.gqlCache]
enabled = true
# Max in-memory cache size (in bytes) (default 8 MB)
# maxCacheSize
# GQL cache-control max-age settings (in seconds)
maxAge = 15
timeTravelMaxAge = 86400 # 1 day
[metrics]
host = "0.0.0.0"
port = 9000
[metrics.gql]
port = 9001
[database]
type = "postgres"
host = "ajna-watcher-db"
port = 5432
database = "ajna-watcher"
username = "vdbm"
password = "password"
synchronize = true
logging = false
[upstream]
[upstream.ethServer]
rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
# Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
rpcClient = true
# Boolean flag to specify if rpcProviderEndpoint is an FEVM RPC endpoint
isFEVM = true
# Boolean flag to filter event logs by contracts
filterLogsByAddresses = true
# Boolean flag to filter event logs by topics
filterLogsByTopics = true
[upstream.cache]
name = "requests"
enabled = false
deleteOnStart = false
[jobQueue]
dbConnectionString = "postgres://vdbm:password@ajna-watcher-db/ajna-watcher-job-queue"
maxCompletionLagInSecs = 300
jobDelayInMilliSecs = 100
eventsInBatch = 50
subgraphEventsOrder = true
# Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
blockDelayInMilliSecs = 30000
# Boolean to switch between modes of processing events when starting the server.
# Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
# Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
useBlockRanges = true
# Block range in which logs are fetched during historical blocks processing
historicalLogsBlockRange = 2000
# Max block range of historical processing after which it waits for completion of events processing
# If set to -1 historical processing does not wait for events processing and completes till latest canonical block
historicalMaxFetchAhead = 10000
@@ -5,7 +5,7 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
registry_command="laconic registry"
registry_command="laconic cns"
demo_records_dir="scripts/demo-records"
# Check we have funds
@@ -1,4 +0,0 @@
#!/usr/bin/env bash
# Build cerc/laconicd
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
docker build -t cerc/laconic2d:local ${build_command_args} ${CERC_REPO_BASE_DIR}/laconic2d
@@ -11,14 +11,8 @@ CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/D
CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/nextjs-base:local}
docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
rc=$?
if [ $rc -ne 0 ]; then
echo "BUILD FAILED" 1>&2
exit $rc
fi
if [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/nextjs-base:local" ]; then
if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/nextjs-base:local" ]; then
cat <<EOF
#################################################################
@@ -1,10 +0,0 @@
FROM node:18.17.1-alpine3.18
RUN apk --update --no-cache add git python3 alpine-sdk bash curl jq
WORKDIR /app
COPY . .
RUN echo "Installing dependencies and building ajna-watcher-ts" && \
yarn && yarn build
@@ -1,9 +0,0 @@
#!/usr/bin/env bash
# Build cerc/watcher-ajna
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
# See: https://stackoverflow.com/a/246128/1701505
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
docker build -t cerc/watcher-ajna:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/ajna-watcher-ts
@@ -11,14 +11,8 @@ CERC_CONTAINER_BUILD_DOCKERFILE=${CERC_CONTAINER_BUILD_DOCKERFILE:-$SCRIPT_DIR/D
CERC_CONTAINER_BUILD_TAG=${CERC_CONTAINER_BUILD_TAG:-cerc/webapp-base:local}
docker build -t $CERC_CONTAINER_BUILD_TAG ${build_command_args} -f $CERC_CONTAINER_BUILD_DOCKERFILE $CERC_CONTAINER_BUILD_WORK_DIR
rc=$?
if [ $rc -ne 0 ]; then
echo "BUILD FAILED" 1>&2
exit $rc
fi
if [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/webapp-base:local" ]; then
if [ $? -eq 0 ] && [ "$CERC_CONTAINER_BUILD_TAG" != "cerc/webapp-base:local" ]; then
cat <<EOF
#################################################################
@@ -1,118 +0,0 @@
# Ajna Watcher
## Setup
Clone required repositories:
```bash
laconic-so --stack ajna setup-repositories --git-ssh --pull
```
Build the container images:
```bash
laconic-so --stack ajna build-containers
```
## Deploy
Create a spec file for the deployment:
```bash
laconic-so --stack ajna deploy init --output ajna-spec.yml
```
### Ports
Edit `network` in the spec file to map container ports to host ports as required:
```yml
...
network:
ports:
ajna-watcher-db:
- 15432:5432
ajna-watcher-job-runner:
- 9000:9000
ajna-watcher-server:
- 3008:3008
- 9001:9001
```
### Create a deployment
Create a deployment from the spec file:
```bash
laconic-so --stack ajna deploy create --spec-file ajna-spec.yml --deployment-dir ajna-deployment
```
### Configuration
Inside deployment directory, open the `config.env` file and set following env variables:
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher to
CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
```
### Start the deployment
```bash
laconic-so deployment --dir ajna-deployment start
```
* To list down and monitor the running containers:
```bash
# With status
docker ps -a
# Check logs for a container
docker logs -f <CONTAINER_ID>
```
* Open the GQL playground at <http://localhost:3008/graphql>
```graphql
# Example query
query {
_meta {
block {
hash
number
timestamp
}
deployment
hasIndexingErrors
}
accounts {
id
txCount
tokensDelegated
rewardsClaimed
}
}
```
## Clean up
Stop all the ajna services running in background:
```bash
# Only stop the docker containers
laconic-so deployment --dir ajna-deployment stop
# Run 'start' to restart the deployment
```
To stop all the ajna services and also delete data:
```bash
# Stop the docker containers
laconic-so deployment --dir ajna-deployment stop --delete-volumes
# Remove deployment directory (deployment will have to be recreated for a re-run)
rm -r ajna-deployment
```
@@ -1,9 +0,0 @@
version: "1.0"
name: ajna
description: "Ajna watcher stack"
repos:
- git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.1
containers:
- cerc/watcher-ajna
pods:
- watcher-ajna
@@ -49,7 +49,7 @@ $ laconic-so --stack fixturenet-laconic-loaded deploy logs
```
## 6. Test with the Registry CLI
```
$ laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic registry status"
$ laconic-so --stack fixturenet-laconic-loaded deploy exec cli "laconic cns status"
```
## 7. View the laconic console
Get the URL for the console web app with this command (the port number will be different for each deployment):
@@ -2,16 +2,16 @@ version: "1.1"
name: fixturenet-laconic-loaded
description: "A full featured laconic fixturenet"
repos:
- git.vdb.to/cerc-io/laconic2d
- git.vdb.to/cerc-io/laconicd
- github.com/lirewine/debug
- github.com/lirewine/crypto
- github.com/lirewine/gem
- github.com/lirewine/sdk
- git.vdb.to/cerc-io/registry-sdk
- git.vdb.to/cerc-io/laconic-registry-cli@laconic2
- git.vdb.to/cerc-io/laconic-console@laconic2
- git.vdb.to/cerc-io/laconic-sdk
- git.vdb.to/cerc-io/laconic-registry-cli
- git.vdb.to/cerc-io/laconic-console
npms:
- registry-sdk
- laconic-sdk
- laconic-registry-cli
- debug
- crypto
@@ -19,12 +19,12 @@ npms:
- gem
- laconic-console
containers:
- cerc/laconic2d
- cerc/laconicd
- cerc/laconic-registry-cli
- cerc/webapp-base
- cerc/laconic-console-host
pods:
- fixturenet-laconic2d
- fixturenet-laconicd
- fixturenet-laconic-console
config:
cli:
@@ -44,5 +44,5 @@ $ laconic-so --stack fixturenet-laconicd deploy logs
```
## 6. Test with the Registry CLI
```
$ laconic-so --stack fixturenet-laconicd deploy exec cli "laconic registry status"
$ laconic-so --stack fixturenet-laconicd deploy exec cli "laconic cns status"
```
@@ -16,55 +16,26 @@ laconic-so --stack merkl-sushiswap-v3 build-containers
## Deploy
Create a spec file for the deployment:
```bash
laconic-so --stack merkl-sushiswap-v3 deploy init --output merkl-sushiswap-v3-spec.yml
```
### Ports
Edit `network` in the spec file to map container ports to host ports as required:
```
...
network:
ports:
merkl-sushiswap-v3-watcher-db:
- '5432'
merkl-sushiswap-v3-watcher-job-runner:
- 9002:9000
merkl-sushiswap-v3-watcher-server:
- 127.0.0.1:3007:3008
- 9003:9001
```
### Create a deployment
Create a deployment from the spec file:
```bash
laconic-so --stack merkl-sushiswap-v3 deploy create --spec-file merkl-sushiswap-v3-spec.yml --deployment-dir merkl-sushiswap-v3-deployment
```
### Configuration
Inside deployment directory, open the `config.env` file and set following env variables:
Create and update an env file to be used in the next step:
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher
CERC_ETH_RPC_ENDPOINT=
```
### Deploy the stack
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher to
CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
```
### Start the deployment
```bash
laconic-so deployment --dir merkl-sushiswap-v3-deployment start
laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 --env-file <PATH_TO_ENV_FILE> up
```
* To list down and monitor the running containers:
```bash
laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 ps
# With status
docker ps -a
@@ -75,7 +46,6 @@ laconic-so deployment --dir merkl-sushiswap-v3-deployment start
* Open the GQL playground at http://localhost:3007/graphql
```graphql
# Example query
{
_meta {
block {
@@ -84,7 +54,7 @@ laconic-so deployment --dir merkl-sushiswap-v3-deployment start
}
hasIndexingErrors
}
factories {
id
poolCount
@@ -94,21 +64,18 @@ laconic-so deployment --dir merkl-sushiswap-v3-deployment start
## Clean up
Stop all the merkl-sushiswap-v3 services running in background:
Stop all the services running in background:
```bash
# Only stop the docker containers
laconic-so deployment --dir merkl-sushiswap-v3-deployment stop
# Run 'start' to restart the deployment
laconic-so --stack merkl-sushiswap-v3 deploy --cluster merkl_sushiswap_v3 down
```
To stop all the merkl-sushiswap-v3 services and also delete data:
Clear volumes created by this stack:
```bash
# Stop the docker containers
laconic-so deployment --dir merkl-sushiswap-v3-deployment stop --delete-volumes
# List all relevant volumes
docker volume ls -q --filter "name=merkl_sushiswap_v3"
# Remove deployment directory (deployment will have to be recreated for a re-run)
rm -r merkl-sushiswap-v3-deployment
# Remove all the listed volumes
docker volume rm $(docker volume ls -q --filter "name=merkl_sushiswap_v3")
```
@@ -2,7 +2,7 @@ version: "1.0"
name: merkl-sushiswap-v3
description: "SushiSwap v3 watcher stack"
repos:
- github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.7
- github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.6
containers:
- cerc/watcher-merkl-sushiswap-v3
pods:
@@ -1,7 +1,7 @@
version: "0.1"
name: monitoring
repos:
- github.com/cerc-io/watcher-ts@v0.2.81
- github.com/cerc-io/watcher-ts@v0.2.79
containers:
- cerc/watcher-ts
pods:
@@ -55,7 +55,7 @@ ports:
Create deployment:
```bash
laconic-so --stack sushiswap-subgraph deploy create --spec-file sushiswap-subgraph-spec.yml --deployment-dir sushiswap-subgraph-deployment
laconic-so deploy create --spec-file sushiswap-subgraph-spec.yml --deployment-dir sushiswap-subgraph-deployment
```
## Start the stack
@@ -16,55 +16,26 @@ laconic-so --stack sushiswap-v3 build-containers
## Deploy
Create a spec file for the deployment:
```bash
laconic-so --stack sushiswap-v3 deploy init --output sushiswap-v3-spec.yml
```
### Ports
Edit `network` in the spec file to map container ports to host ports as required:
```
...
network:
ports:
sushiswap-v3-watcher-db:
- '5432'
sushiswap-v3-watcher-job-runner:
- 9000:9000
sushiswap-v3-watcher-server:
- 127.0.0.1:3008:3008
- 9001:9001
```
### Create a deployment
Create a deployment from the spec file:
```bash
laconic-so --stack sushiswap-v3 deploy create --spec-file sushiswap-v3-spec.yml --deployment-dir sushiswap-v3-deployment
```
### Configuration
Inside deployment directory, open the `config.env` file and set following env variables:
Create and update an env file to be used in the next step:
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher
CERC_ETH_RPC_ENDPOINT=
```
### Deploy the stack
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher to
CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
```
### Start the deployment
```bash
laconic-so deployment --dir sushiswap-v3-deployment start
laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 --env-file <PATH_TO_ENV_FILE> up
```
* To list down and monitor the running containers:
```bash
laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 ps
# With status
docker ps -a
@@ -72,43 +43,20 @@ laconic-so deployment --dir sushiswap-v3-deployment start
docker logs -f <CONTAINER_ID>
```
* Open the GQL playground at http://localhost:3008/graphql
```graphql
# Example query
{
_meta {
block {
number
timestamp
}
hasIndexingErrors
}
factories {
id
poolCount
}
}
```
## Clean up
Stop all the sushiswap-v3 services running in background:
Stop all the services running in background:
```bash
# Only stop the docker containers
laconic-so deployment --dir sushiswap-v3-deployment stop
# Run 'start' to restart the deployment
laconic-so --stack sushiswap-v3 deploy --cluster sushiswap_v3 down
```
To stop all the sushiswap-v3 services and also delete data:
Clear volumes created by this stack:
```bash
# Stop the docker containers
laconic-so deployment --dir sushiswap-v3-deployment stop --delete-volumes
# List all relevant volumes
docker volume ls -q --filter "name=sushiswap_v3"
# Remove deployment directory (deployment will have to be recreated for a re-run)
rm -r sushiswap-v3-deployment
# Remove all the listed volumes
docker volume rm $(docker volume ls -q --filter "name=sushiswap_v3")
```
@@ -2,7 +2,7 @@ version: "1.0"
name: sushiswap-v3
description: "SushiSwap v3 watcher stack"
repos:
- github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.7
- github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.6
containers:
- cerc/watcher-sushiswap-v3
pods:
+29 -12
View File
@@ -101,7 +101,7 @@ class ClusterInfo:
)
return service
def get_ingress(self, use_tls=False, certificate=None, cluster_issuer="letsencrypt-prod"):
def get_ingress(self, use_tls=False):
# No ingress for a deployment that has no http-proxy defined, for now
http_proxy_info_list = self.spec.get_http_proxy()
ingress = None
@@ -110,13 +110,32 @@ class ClusterInfo:
http_proxy_info = http_proxy_info_list[0]
if opts.o.debug:
print(f"http-proxy: {http_proxy_info}")
# TODO: good enough parsing for webapp deployment for now
host_name = http_proxy_info["host-name"]
tls = None
tls_issuer = None
if use_tls:
tls_info = http_proxy_info.get("tls", {})
tls_hosts = tls_info.get("hosts", [host_name])
tls_issuer = tls_info.get("issuer", "letsencrypt-prod")
tls_secret_name = f"{self.app_name}-tls"
if "secret" in tls_info:
# If an existing secret is specified, unset the issuer so that we don't try to re-request it.
tls_secret_name = tls_info["secret"]
tls_issuer = None
if opts.o.debug:
print(f"TLS hosts/secret: {tls_hosts}/{tls_secret_name}")
tls = [client.V1IngressTLS(
hosts=tls_hosts,
secret_name=tls_secret_name
)]
# TODO: good enough parsing for webapp deployment for now
rules = []
tls = [client.V1IngressTLS(
hosts=certificate["spec"]["dnsNames"] if certificate else [host_name],
secret_name=certificate["spec"]["secretName"] if certificate else f"{self.app_name}-tls"
)] if use_tls else None
paths = []
for route in http_proxy_info["routes"]:
path = route["path"]
@@ -147,17 +166,15 @@ class ClusterInfo:
tls=tls,
rules=rules
)
ingress_annotations = {
annotations = {
"kubernetes.io/ingress.class": "nginx",
}
if not certificate:
ingress_annotations["cert-manager.io/cluster-issuer"] = cluster_issuer
if tls_issuer:
annotations["cert-manager.io/cluster-issuer"] = tls_issuer
ingress = client.V1Ingress(
metadata=client.V1ObjectMeta(
name=f"{self.app_name}-ingress",
annotations=ingress_annotations
annotations=annotations
),
spec=spec
)
+2 -43
View File
@@ -169,39 +169,6 @@ class K8sDeployer(Deployer):
print("Service created:")
print(f"{service_resp}")
def _find_certificate_for_host_name(self, host_name):
all_certificates = self.custom_obj_api.list_namespaced_custom_object(
group="cert-manager.io",
version="v1",
namespace=self.k8s_namespace,
plural="certificates"
)
host_parts = host_name.split(".", 1)
host_as_wild = None
if len(host_parts) == 2:
host_as_wild = f"*.{host_parts[1]}"
now = datetime.utcnow().replace(tzinfo=timezone.utc)
fmt = "%Y-%m-%dT%H:%M:%S%z"
# Walk over all the configured certificates.
for cert in all_certificates["items"]:
dns = cert["spec"]["dnsNames"]
# Check for an exact hostname match or a wildcard match.
if host_name in dns or host_as_wild in dns:
status = cert.get("status", {})
# Check the certificate date.
if "notAfter" in status and "notBefore" in status:
before = datetime.strptime(status["notBefore"], fmt)
after = datetime.strptime(status["notAfter"], fmt)
if before < now < after:
# Check the status is Ready
for condition in status.get("conditions", []):
if "True" == condition.get("status") and "Ready" == condition.get("type"):
return cert
return None
def up(self, detach, services):
if not opts.o.dry_run:
if self.is_kind():
@@ -222,15 +189,8 @@ class K8sDeployer(Deployer):
self._create_volume_data()
self._create_deployment()
http_proxy_info = self.cluster_info.spec.get_http_proxy()
# Note: at present we don't support tls for kind (and enabling tls causes errors)
use_tls = http_proxy_info and not self.is_kind()
certificate = self._find_certificate_for_host_name(http_proxy_info[0]["host-name"]) if use_tls else None
if opts.o.debug:
if certificate:
print(f"Using existing certificate: {certificate}")
ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=use_tls, certificate=certificate)
ingress: client.V1Ingress = self.cluster_info.get_ingress(use_tls=not self.is_kind())
if ingress:
if opts.o.debug:
print(f"Sending this ingress: {ingress}")
@@ -393,8 +353,7 @@ class K8sDeployer(Deployer):
hostname = ingress.spec.rules[0].host
ip = ingress.status.load_balancer.ingress[0].ip
tls = "notBefore: %s; notAfter: %s; names: %s" % (
cert["status"]["notBefore"], cert["status"]["notAfter"], ingress.spec.tls[0].hosts
)
cert["status"]["notBefore"], cert["status"]["notAfter"], ", ".join(ingress.spec.tls[0].hosts))
except: # noqa: E722
pass
@@ -15,6 +15,7 @@
import click
import os
import sys
from pathlib import Path
from urllib.parse import urlparse
from tempfile import NamedTemporaryFile
@@ -23,6 +24,7 @@ from stack_orchestrator.util import error_exit, global_options2
from stack_orchestrator.deploy.deployment_create import init_operation, create_operation
from stack_orchestrator.deploy.deploy import create_deploy_context
from stack_orchestrator.deploy.deploy_types import DeployCommandContext
from stack_orchestrator.deploy.webapp.util import TlsDetails
def _fixup_container_tag(deployment_dir: str, image: str):
@@ -36,16 +38,16 @@ def _fixup_container_tag(deployment_dir: str, image: str):
wfile.write(contents)
def _fixup_url_spec(spec_file_name: str, url: str):
def _fixup_url_spec(spec_file_name: str, url: str, tls_details: TlsDetails = TlsDetails()):
# url is like: https://example.com/path
parsed_url = urlparse(url)
http_proxy_spec = f'''
http-proxy:
http_proxy_spec = f''' http-proxy:
- host-name: {parsed_url.hostname}
routes:
- path: '{parsed_url.path if parsed_url.path else "/"}'
proxy-to: webapp:80
'''
{tls_details.to_yaml(indent=6)}
'''
spec_file_path = Path(spec_file_name)
with open(spec_file_path) as rfile:
contents = rfile.read()
@@ -54,7 +56,8 @@ def _fixup_url_spec(spec_file_name: str, url: str):
wfile.write(contents)
def create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
def create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file,
tls_details: TlsDetails = None):
# Do the equivalent of:
# 1. laconic-so --stack webapp-template deploy --deploy-to k8s init --output webapp-spec.yml
# --config (eqivalent of the contents of my-config.env)
@@ -86,7 +89,7 @@ def create_deployment(ctx, deployment_dir, image, url, kube_config, image_regist
None
)
# Add the TLS and DNS spec
_fixup_url_spec(spec_file_name, url)
_fixup_url_spec(spec_file_name, url, tls_details)
create_operation(
deploy_command_context,
spec_file_name,
@@ -116,8 +119,16 @@ def command(ctx):
@click.option("--image", help="image to deploy", required=True)
@click.option("--url", help="url to serve", required=True)
@click.option("--env-file", help="environment file for webapp")
@click.option("--tls-host", help="Override TLS hostname (eg, '*.mydomain.com')")
@click.option("--tls-secret", help="Override TLS secret name")
@click.option("--tls-issuer", help="TLS issuer to use (default: letsencrypt-prod)")
@click.pass_context
def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file):
def create(ctx, deployment_dir, image, url, kube_config, image_registry, env_file, tls_host, tls_secret, tls_issuer):
'''create a deployment for the specified webapp container'''
return create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file)
if (tls_secret and not tls_host) or (tls_host and not tls_secret):
print("Cannot specify --tls-host without --tls-secret", file=sys.stderr)
sys.exit(2)
return create_deployment(ctx, deployment_dir, image, url, kube_config, image_registry, env_file,
TlsDetails(tls_host, tls_secret, tls_issuer))
@@ -26,7 +26,7 @@ import click
from stack_orchestrator.deploy.images import remote_image_exists, add_tags_to_image
from stack_orchestrator.deploy.webapp import deploy_webapp
from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, TimedLogger,
from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, TimedLogger, TlsDetails,
build_container_image, push_container_image,
file_hash, deploy_to_k8s, publish_deployment,
hostname_for_deployment_request, generate_hostname_for_app,
@@ -44,7 +44,8 @@ def process_app_deployment_request(
kube_config,
image_registry,
force_rebuild,
logger
tls_details,
logger,
):
logger.log("BEGIN - process_app_deployment_request")
@@ -106,7 +107,7 @@ def process_app_deployment_request(
(app_deployment_crn, deployment_dir))
print("deploy_webapp", deployment_dir)
deploy_webapp.create_deployment(ctx, deployment_dir, deployment_container_tag,
f"https://{fqdn}", kube_config, image_registry, env_filename)
f"https://{fqdn}", kube_config, image_registry, env_filename, tls_details)
elif env_filename:
shutil.copyfile(env_filename, deployment_config_file)
@@ -198,11 +199,14 @@ def dump_known_requests(filename, requests, status="SEEN"):
@click.option("--exclude-tags", help="Exclude requests with matching tags (comma-separated).", default="")
@click.option("--force-rebuild", help="Rebuild even if the image already exists.", is_flag=True)
@click.option("--log-dir", help="Output build/deployment logs to directory.", default=None)
@click.option("--tls-host", help="Override TLS hostname (eg, '*.mydomain.com')")
@click.option("--tls-secret", help="Override TLS secret name")
@click.option("--tls-issuer", help="TLS issuer to use (default: letsencrypt-prod)")
@click.pass_context
def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_dir, # noqa: C901
request_id, discover, state_file, only_update_state,
dns_suffix, record_namespace_dns, record_namespace_deployments, dry_run,
include_tags, exclude_tags, force_rebuild, log_dir):
include_tags, exclude_tags, force_rebuild, log_dir, tls_host, tls_secret, tls_issuer):
if request_id and discover:
print("Cannot specify both --request-id and --discover", file=sys.stderr)
sys.exit(2)
@@ -220,6 +224,10 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
print("--dns-suffix, --record-namespace-dns, and --record-namespace-deployments are all required", file=sys.stderr)
sys.exit(2)
if (tls_secret and not tls_host) or (tls_host and not tls_secret):
print("Cannot specify --tls-host without --tls-secret", file=sys.stderr)
sys.exit(2)
# Split CSV and clean up values.
include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
@@ -305,6 +313,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
print("Found %d unsatisfied request(s) to process." % len(requests_to_execute))
if not dry_run:
tls_details = TlsDetails(tls_host, tls_secret, tls_issuer)
for r in requests_to_execute:
dump_known_requests(state_file, [r], "DEPLOYING")
status = "ERROR"
@@ -334,6 +343,7 @@ def command(ctx, kube_config, laconic_config, image_registry, deployment_parent_
kube_config,
image_registry,
force_rebuild,
tls_details,
logger
)
status = "DEPLOYED"
+40 -7
View File
@@ -26,6 +26,40 @@ import uuid
import yaml
class TlsDetails:
def __init__(self, host_or_hosts=None, secret_name: str = None, issuer_name: str = None):
if host_or_hosts:
if isinstance(host_or_hosts, list):
self.hosts = host_or_hosts
else:
self.hosts = [host_or_hosts]
else:
self.hosts = None
self.secret_name = secret_name
self.issuer_name = issuer_name
def to_yaml(self, indent=6):
if not self.hosts and not self.secret_name and not self.issuer_name:
return ""
ret = " " * indent + "tls:\n"
indent += 2
if self.issuer_name:
ret += " " * indent + "issuer: '%s'\n" % self.issuer_name
if self.secret_name:
ret += " " * indent + "secret: '%s'\n" % self.secret_name
if self.hosts:
ret += " " * indent + "hosts:"
indent += 2
for h in self.hosts:
ret += "\n" + " " * indent + "- '%s'" % h
return ret
class AttrDict(dict):
def __init__(self, *args, **kwargs):
super(AttrDict, self).__init__(*args, **kwargs)
@@ -94,7 +128,7 @@ class LaconicRegistryClient:
)
def list_records(self, criteria={}, all=False):
args = ["laconic", "-c", self.config_file, "registry", "record", "list"]
args = ["laconic", "-c", self.config_file, "cns", "record", "list"]
if all:
args.append("--all")
@@ -140,7 +174,7 @@ class LaconicRegistryClient:
if name in self.cache.name_or_id:
return self.cache.name_or_id[name]
args = ["laconic", "-c", self.config_file, "registry", "name", "resolve", name]
args = ["laconic", "-c", self.config_file, "cns", "name", "resolve", name]
parsed = [AttrDict(r) for r in json.loads(logged_cmd(self.log_file, *args))]
if parsed:
@@ -165,7 +199,7 @@ class LaconicRegistryClient:
"laconic",
"-c",
self.config_file,
"registry",
"cns",
"record",
"get",
"--id",
@@ -203,8 +237,7 @@ class LaconicRegistryClient:
print(open(record_fname, 'r').read(), file=self.log_file)
new_record_id = json.loads(
logged_cmd(self.log_file, "laconic", "-c", self.config_file, "registry",
"record", "publish", "--filename", record_fname)
logged_cmd(self.log_file, "laconic", "-c", self.config_file, "cns", "record", "publish", "--filename", record_fname)
)["id"]
for name in names:
self.set_name(name, new_record_id)
@@ -213,10 +246,10 @@ class LaconicRegistryClient:
logged_cmd(self.log_file, "rm", "-rf", tmpdir)
def set_name(self, name, record_id):
logged_cmd(self.log_file, "laconic", "-c", self.config_file, "registry", "name", "set", name, record_id)
logged_cmd(self.log_file, "laconic", "-c", self.config_file, "cns", "name", "set", name, record_id)
def delete_name(self, name):
logged_cmd(self.log_file, "laconic", "-c", self.config_file, "registry", "name", "delete", name)
logged_cmd(self.log_file, "laconic", "-c", self.config_file, "cns", "name", "delete", name)
def file_hash(filename):