Compare commits

..
93 changed files with 909 additions and 4388 deletions
@@ -9,6 +9,10 @@ on:
schedule: # Note: coordinate with other tests to not overload runners at the same time of day
- cron: '2 14 * * *'
# Needed until we can incorporate docker startup into the executor container
env:
DOCKER_HOST: unix:///var/run/dind.sock
jobs:
test:
@@ -9,6 +9,10 @@ on:
schedule: # Note: coordinate with other tests to not overload runners at the same time of day
- cron: '2 14 * * *'
# Needed until we can incorporate docker startup into the executor container
env:
DOCKER_HOST: unix:///var/run/dind.sock
jobs:
test:
@@ -37,6 +41,10 @@ jobs:
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
run: ./scripts/build_shiv_package.sh
- name: Start dockerd # Also needed until we can incorporate into the executor
run: |
dockerd -H $DOCKER_HOST --userland-proxy=false &
sleep 5
- name: "Run fixturenet-eth tests"
run: ./tests/fixturenet-eth-plugeth/run-test.sh
- name: Notify Vulcanize Slack on CI failure
+8
View File
@@ -7,6 +7,10 @@ on:
- '!**'
- '.gitea/workflows/triggers/fixturenet-eth-test'
# Needed until we can incorporate docker startup into the executor container
env:
DOCKER_HOST: unix:///var/run/dind.sock
jobs:
test:
@@ -35,6 +39,10 @@ jobs:
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
run: ./scripts/build_shiv_package.sh
- name: Start dockerd # Also needed until we can incorporate into the executor
run: |
dockerd -H $DOCKER_HOST --userland-proxy=false &
sleep 5
- name: "Run fixturenet-eth tests"
run: ./tests/fixturenet-eth/run-test.sh
- name: Notify Vulcanize Slack on CI failure
+7
View File
@@ -10,6 +10,9 @@ on:
paths-ignore:
- '.gitea/workflows/triggers/*'
# Needed until we can incorporate docker startup into the executor container
env:
DOCKER_HOST: unix:///var/run/dind.sock
jobs:
test:
@@ -38,6 +41,10 @@ jobs:
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
run: ./scripts/build_shiv_package.sh
- name: Start dockerd # Also needed until we can incorporate into the executor
run: |
dockerd -H $DOCKER_HOST --userland-proxy=false &
sleep 5
- name: "Run deploy tests"
run: ./tests/deploy/run-deploy-test.sh
- name: Notify Vulcanize Slack on CI failure
+8
View File
@@ -10,6 +10,10 @@ on:
paths-ignore:
- '.gitea/workflows/triggers/*'
# Needed until we can incorporate docker startup into the executor container
env:
DOCKER_HOST: unix:///var/run/dind.sock
jobs:
test:
name: "Run webapp test suite"
@@ -39,6 +43,10 @@ jobs:
run: ./scripts/build_shiv_package.sh
- name: "Install wget" # 20240109 - Only needed until the executors are updated.
run: apt update && apt install -y wget
- name: Start dockerd # Also needed until we can incorporate into the executor
run: |
dockerd -H $DOCKER_HOST --userland-proxy=false &
sleep 5
- name: "Run webapp tests"
run: ./tests/webapp-test/run-webapp-test.sh
- name: Notify Vulcanize Slack on CI failure
+7
View File
@@ -10,6 +10,9 @@ on:
paths-ignore:
- '.gitea/workflows/triggers/*'
# Needed until we can incorporate docker startup into the executor container
env:
DOCKER_HOST: unix:///var/run/dind.sock
jobs:
test:
@@ -38,6 +41,10 @@ jobs:
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
run: ./scripts/build_shiv_package.sh
- name: Start dockerd # Also needed until we can incorporate into the executor
run: |
dockerd -H $DOCKER_HOST --userland-proxy=false &
sleep 5
- name: "Run smoke tests"
run: ./tests/smoke-test/run-smoke-test.sh
- name: Notify Vulcanize Slack on CI failure
@@ -4,4 +4,3 @@ Trigger
Trigger
Trigger
Trigger
Trigger
+1 -3
View File
@@ -4,11 +4,9 @@ with open("README.md", "r", encoding="utf-8") as fh:
long_description = fh.read()
with open("requirements.txt", "r", encoding="utf-8") as fh:
requirements = fh.read()
with open("stack_orchestrator/data/version.txt", "r", encoding="utf-8") as fh:
version = fh.readlines()[-1].strip(" \n")
setup(
name='laconic-stack-orchestrator',
version=version,
version='1.0.12',
author='Cerc',
author_email='info@cerc.io',
license='GNU Affero General Public License',
+7 -5
View File
@@ -21,6 +21,11 @@ from stack_orchestrator.util import get_parsed_stack_config, warn_exit
def get_containers_in_scope(stack: str):
# See: https://stackoverflow.com/a/20885799/1701505
from stack_orchestrator import data
with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
all_containers = container_list_file.read().splitlines()
containers_in_scope = []
if stack:
stack_config = get_parsed_stack_config(stack)
@@ -28,14 +33,11 @@ def get_containers_in_scope(stack: str):
warn_exit(f"stack {stack} does not define any containers")
containers_in_scope = stack_config['containers']
else:
# See: https://stackoverflow.com/a/20885799/1701505
from stack_orchestrator import data
with importlib.resources.open_text(data, "container-image-list.txt") as container_list_file:
containers_in_scope = container_list_file.read().splitlines()
containers_in_scope = all_containers
if opts.o.verbose:
print(f'Containers: {containers_in_scope}')
if stack:
print(f"Stack: {stack}")
return containers_in_scope
return containers_in_scope
@@ -3,7 +3,7 @@ services:
restart: always
hostname: fixturenet-eth-bootnode-geth
env_file:
- ../config/fixturenet-eth/fixturenet-eth.env
- ../config/fixturenet-plugeth/fixturenet-eth.env
environment:
RUN_BOOTNODE: "true"
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
@@ -24,8 +24,9 @@ services:
CERC_RUN_STATEDIFF: ${CERC_RUN_STATEDIFF:-detect}
CERC_STATEDIFF_DB_NODE_ID: 1
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ALLOW_UNPROTECTED_TXS: ${CERC_ALLOW_UNPROTECTED_TXS:-false}
env_file:
- ../config/fixturenet-eth/fixturenet-eth.env
- ../config/fixturenet-plugeth/fixturenet-eth.env
image: cerc/fixturenet-plugeth-plugeth:local
volumes:
- fixturenet_plugeth_geth_1_data:/root/ethdata
@@ -55,7 +56,7 @@ services:
environment:
CERC_KEEP_RUNNING_AFTER_GETH_EXIT: "true"
env_file:
- ../config/fixturenet-eth/fixturenet-eth.env
- ../config/fixturenet-plugeth/fixturenet-eth.env
image: cerc/fixturenet-plugeth-plugeth:local
depends_on:
- fixturenet-eth-bootnode-geth
@@ -82,7 +83,7 @@ services:
retries: 10
start_period: 30s
env_file:
- ../config/fixturenet-eth/fixturenet-eth.env
- ../config/fixturenet-plugeth/fixturenet-eth.env
environment:
NODE_NUMBER: "1"
ETH1_ENDPOINT: "http://fixturenet-eth-geth-1:8545"
@@ -108,7 +109,7 @@ services:
retries: 10
start_period: 30s
env_file:
- ../config/fixturenet-eth/fixturenet-eth.env
- ../config/fixturenet-plugeth/fixturenet-eth.env
environment:
NODE_NUMBER: "2"
ETH1_ENDPOINT: "http://fixturenet-eth-geth-2:8545"
@@ -6,20 +6,12 @@ services:
restart: always
environment:
GF_SERVER_ROOT_URL: ${GF_SERVER_ROOT_URL}
CERC_GRAFANA_ALERTS_SUBGRAPH_IDS: ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
volumes:
- ../config/monitoring/grafana/provisioning:/etc/grafana/provisioning
- ../config/monitoring/grafana/dashboards:/etc/grafana/dashboards
- ../config/monitoring/update-grafana-alerts-config.sh:/update-grafana-alerts-config.sh
- grafana_storage:/var/lib/grafana
user: root
entrypoint: ["bash", "-c"]
command: |
"/update-grafana-alerts-config.sh && /run.sh"
ports:
- "3000"
extra_hosts:
- "host.docker.internal:host-gateway"
healthcheck:
test: ["CMD", "nc", "-vz", "localhost", "3000"]
interval: 30s
@@ -22,7 +22,6 @@ services:
GRAPH_ETHEREUM_JSON_RPC_TIMEOUT: ${GRAPH_ETHEREUM_JSON_RPC_TIMEOUT:-180}
GRAPH_ETHEREUM_REQUEST_RETRIES: ${GRAPH_ETHEREUM_REQUEST_RETRIES:-10}
GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE: ${GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE:-2000}
GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS: ${GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS:-1000}
entrypoint: ["bash", "-c"]
# Wait for ETH RPC endpoint to be up when running with fixturenet-lotus
command: |
@@ -32,7 +31,6 @@ services:
- "8001"
- "8020"
- "8030"
- "8040"
healthcheck:
test: ["CMD", "nc", "-vz", "localhost", "8020"]
interval: 30s
@@ -1,10 +0,0 @@
services:
laconic-explorer:
restart: unless-stopped
image: cerc/ping-pub:local
environment:
- LACONIC_LACONICD_API_URL=${LACONIC_LACONICD_API_URL:-http://localhost:1317}
- LACONIC_LACONICD_RPC_URL=${LACONIC_LACONICD_RPC_URL:-http://localhost:26657}
- LACONIC_LACONICD_CHAIN_ID=${LACONIC_LACONICD_CHAIN_ID:-chain-id-not-set}
ports:
- "5173"
@@ -28,37 +28,15 @@ services:
extra_hosts:
- "host.docker.internal:host-gateway"
ethereum-chain-head-exporter:
chain-head-exporter:
image: cerc/watcher-ts:local
restart: always
working_dir: /app/packages/cli
environment:
ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT:-https://mainnet.infura.io/v3}
ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
FIL_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT}
ETH_RPC_API_KEY: ${CERC_INFURA_KEY}
command: ["sh", "-c", "yarn export-metrics:chain-heads"]
ports:
- '5000'
extra_hosts:
- "host.docker.internal:host-gateway"
filecoin-chain-head-exporter:
image: cerc/watcher-ts:local
restart: always
working_dir: /app/packages/cli
environment:
ETH_RPC_ENDPOINT: ${CERC_FIL_RPC_ENDPOINT:-https://api.node.glif.io/rpc/v1}
command: ["sh", "-c", "yarn export-metrics:chain-heads"]
ports:
- '5000'
extra_hosts:
- "host.docker.internal:host-gateway"
graph-node-upstream-head-exporter:
image: cerc/watcher-ts:local
restart: always
working_dir: /app/packages/cli
environment:
ETH_RPC_ENDPOINT: ${GRAPH_NODE_RPC_ENDPOINT}
PORT: ${CERC_METRICS_PORT}
command: ["sh", "-c", "yarn export-metrics:chain-heads"]
ports:
- '5000'
@@ -0,0 +1,13 @@
services:
snowballtools-base-backend:
image: cerc/snowballtools-base-backend:local
restart: always
volumes:
- data:/data
- config:/config:ro
ports:
- 8000
volumes:
data:
config:
@@ -29,7 +29,7 @@ services:
image: cerc/watcher-ajna:local
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
command: ["bash", "./start-job-runner.sh"]
volumes:
- ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
ports:
- "9000"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
test: ["CMD", "nc", "-v", "localhost", "9000"]
interval: 20s
timeout: 5s
retries: 15
@@ -55,17 +55,16 @@ services:
image: cerc/watcher-ajna:local
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
command: ["bash", "./start-server.sh"]
volumes:
- ../config/watcher-ajna/watcher-config-template.toml:/app/environments/watcher-config-template.toml
- ../config/watcher-ajna/start-server.sh:/app/start-server.sh
- ajna_watcher_gql_logs_data:/app/gql-logs
ports:
- "3008"
- "9001"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
test: ["CMD", "nc", "-v", "localhost", "3008"]
interval: 20s
timeout: 5s
retries: 15
@@ -75,4 +74,3 @@ services:
volumes:
ajna_watcher_db_data:
ajna_watcher_gql_logs_data:
@@ -32,8 +32,8 @@ services:
condition: service_healthy
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
CERC_HISTORICAL_BLOCK_RANGE: 500
CONTRACT_ADDRESS: 0x223c067F8CF28ae173EE5CafEa60cA44C335fecB
CONTRACT_NAME: Azimuth
@@ -47,7 +47,7 @@ services:
ports:
- "9000"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
test: ["CMD", "nc", "-vz", "localhost", "9000"]
interval: 20s
timeout: 5s
retries: 15
@@ -66,20 +66,18 @@ services:
condition: service_healthy
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
working_dir: /app/packages/azimuth-watcher
command: "./start-server.sh"
volumes:
- ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/azimuth-watcher/environments/watcher-config-template.toml
- ../config/watcher-azimuth/merge-toml.js:/app/packages/azimuth-watcher/merge-toml.js
- ../config/watcher-azimuth/start-server.sh:/app/packages/azimuth-watcher/start-server.sh
- azimuth_watcher_gql_logs_data:/app/packages/azimuth-watcher/gql-logs
ports:
- "3001"
- "9001"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "3001"]
test: ["CMD", "nc", "-vz", "localhost", "3001"]
interval: 20s
timeout: 5s
retries: 15
@@ -96,8 +94,8 @@ services:
condition: service_healthy
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
CONTRACT_ADDRESS: 0x325f68d32BdEe6Ed86E7235ff2480e2A433D6189
CONTRACT_NAME: Censures
STARTING_BLOCK: 6784954
@@ -110,7 +108,7 @@ services:
ports:
- "9002"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "9002"]
test: ["CMD", "nc", "-vz", "localhost", "9002"]
interval: 20s
timeout: 5s
retries: 15
@@ -129,20 +127,18 @@ services:
condition: service_healthy
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
working_dir: /app/packages/censures-watcher
command: "./start-server.sh"
volumes:
- ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/censures-watcher/environments/watcher-config-template.toml
- ../config/watcher-azimuth/merge-toml.js:/app/packages/censures-watcher/merge-toml.js
- ../config/watcher-azimuth/start-server.sh:/app/packages/censures-watcher/start-server.sh
- censures_watcher_gql_logs_data:/app/packages/censures-watcher/gql-logs
ports:
- "3002"
- "9003"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "3002"]
test: ["CMD", "nc", "-vz", "localhost", "3002"]
interval: 20s
timeout: 5s
retries: 15
@@ -159,8 +155,8 @@ services:
condition: service_healthy
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
CONTRACT_ADDRESS: 0xe7e7f69b34D7d9Bd8d61Fb22C33b22708947971A
CONTRACT_NAME: Claims
STARTING_BLOCK: 6784941
@@ -173,7 +169,7 @@ services:
ports:
- "9004"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "9004"]
test: ["CMD", "nc", "-vz", "localhost", "9004"]
interval: 20s
timeout: 5s
retries: 15
@@ -192,20 +188,18 @@ services:
condition: service_healthy
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
working_dir: /app/packages/claims-watcher
command: "./start-server.sh"
volumes:
- ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/claims-watcher/environments/watcher-config-template.toml
- ../config/watcher-azimuth/merge-toml.js:/app/packages/claims-watcher/merge-toml.js
- ../config/watcher-azimuth/start-server.sh:/app/packages/claims-watcher/start-server.sh
- claims_watcher_gql_logs_data:/app/packages/claims-watcher/gql-logs
ports:
- "3003"
- "9005"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "3003"]
test: ["CMD", "nc", "-vz", "localhost", "3003"]
interval: 20s
timeout: 5s
retries: 15
@@ -222,8 +216,8 @@ services:
condition: service_healthy
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
CONTRACT_ADDRESS: 0x8C241098C3D3498Fe1261421633FD57986D74AeA
CONTRACT_NAME: ConditionalStarRelease
STARTING_BLOCK: 6828004
@@ -236,7 +230,7 @@ services:
ports:
- "9006"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "9006"]
test: ["CMD", "nc", "-vz", "localhost", "9006"]
interval: 20s
timeout: 5s
retries: 15
@@ -255,20 +249,18 @@ services:
condition: service_healthy
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
working_dir: /app/packages/conditional-star-release-watcher
command: "./start-server.sh"
volumes:
- ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/conditional-star-release-watcher/environments/watcher-config-template.toml
- ../config/watcher-azimuth/merge-toml.js:/app/packages/conditional-star-release-watcher/merge-toml.js
- ../config/watcher-azimuth/start-server.sh:/app/packages/conditional-star-release-watcher/start-server.sh
- conditional_star_release_watcher_gql_logs_data:/app/packages/conditional-star-release-watcher/gql-logs
ports:
- "3004"
- "9007"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "3004"]
test: ["CMD", "nc", "-vz", "localhost", "3004"]
interval: 20s
timeout: 5s
retries: 15
@@ -285,8 +277,8 @@ services:
condition: service_healthy
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
CONTRACT_ADDRESS: 0xf6b461fE1aD4bd2ce25B23Fe0aff2ac19B3dFA76
CONTRACT_NAME: DelegatedSending
STARTING_BLOCK: 6784956
@@ -299,7 +291,7 @@ services:
ports:
- "9008"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "9008"]
test: ["CMD", "nc", "-vz", "localhost", "9008"]
interval: 20s
timeout: 5s
retries: 15
@@ -318,20 +310,18 @@ services:
condition: service_healthy
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
working_dir: /app/packages/delegated-sending-watcher
command: "./start-server.sh"
volumes:
- ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/delegated-sending-watcher/environments/watcher-config-template.toml
- ../config/watcher-azimuth/merge-toml.js:/app/packages/delegated-sending-watcher/merge-toml.js
- ../config/watcher-azimuth/start-server.sh:/app/packages/delegated-sending-watcher/start-server.sh
- delegated_sending_watcher_gql_logs_data:/app/packages/delegated-sending-watcher/gql-logs
ports:
- "3005"
- "9009"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "3005"]
test: ["CMD", "nc", "-vz", "localhost", "3005"]
interval: 20s
timeout: 5s
retries: 15
@@ -348,8 +338,8 @@ services:
condition: service_healthy
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
CONTRACT_ADDRESS: 0x33EeCbf908478C10614626A9D304bfe18B78DD73
CONTRACT_NAME: Ecliptic
STARTING_BLOCK: 13692129
@@ -362,7 +352,7 @@ services:
ports:
- "9010"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "9010"]
test: ["CMD", "nc", "-vz", "localhost", "9010"]
interval: 20s
timeout: 5s
retries: 15
@@ -381,20 +371,18 @@ services:
condition: service_healthy
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
working_dir: /app/packages/ecliptic-watcher
command: "./start-server.sh"
volumes:
- ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/ecliptic-watcher/environments/watcher-config-template.toml
- ../config/watcher-azimuth/merge-toml.js:/app/packages/ecliptic-watcher/merge-toml.js
- ../config/watcher-azimuth/start-server.sh:/app/packages/ecliptic-watcher/start-server.sh
- ecliptic_watcher_gql_logs_data:/app/packages/ecliptic-watcher/gql-logs
ports:
- "3006"
- "9011"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "3006"]
test: ["CMD", "nc", "-vz", "localhost", "3006"]
interval: 20s
timeout: 5s
retries: 15
@@ -411,8 +399,8 @@ services:
condition: service_healthy
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
CONTRACT_ADDRESS: 0x86cd9cd0992F04231751E3761De45cEceA5d1801
CONTRACT_NAME: LinearStarRelease
STARTING_BLOCK: 6784943
@@ -425,7 +413,7 @@ services:
ports:
- "9012"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "9012"]
test: ["CMD", "nc", "-vz", "localhost", "9012"]
interval: 20s
timeout: 5s
retries: 15
@@ -444,20 +432,18 @@ services:
condition: service_healthy
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
working_dir: /app/packages/linear-star-release-watcher
command: "./start-server.sh"
volumes:
- ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/linear-star-release-watcher/environments/watcher-config-template.toml
- ../config/watcher-azimuth/merge-toml.js:/app/packages/linear-star-release-watcher/merge-toml.js
- ../config/watcher-azimuth/start-server.sh:/app/packages/linear-star-release-watcher/start-server.sh
- linear_star_release_watcher_gql_logs_data:/app/packages/linear-star-release-watcher/gql-logs
ports:
- "3007"
- "9013"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "3007"]
test: ["CMD", "nc", "-vz", "localhost", "3007"]
interval: 20s
timeout: 5s
retries: 15
@@ -474,8 +460,8 @@ services:
condition: service_healthy
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
CONTRACT_ADDRESS: 0x7fEcaB617c868Bb5996d99D95200D2Fa708218e4
CONTRACT_NAME: Polls
STARTING_BLOCK: 6784912
@@ -488,7 +474,7 @@ services:
ports:
- "9014"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "9014"]
test: ["CMD", "nc", "-vz", "localhost", "9014"]
interval: 20s
timeout: 5s
retries: 15
@@ -507,20 +493,18 @@ services:
condition: service_healthy
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_IPLD_ETH_GQL_ENDPOINT: ${CERC_IPLD_ETH_GQL_ENDPOINT}
CERC_IPLD_ETH_RPC: ${CERC_IPLD_ETH_RPC}
CERC_IPLD_ETH_GQL: ${CERC_IPLD_ETH_GQL}
working_dir: /app/packages/polls-watcher
command: "./start-server.sh"
volumes:
- ../config/watcher-azimuth/watcher-config-template.toml:/app/packages/polls-watcher/environments/watcher-config-template.toml
- ../config/watcher-azimuth/merge-toml.js:/app/packages/polls-watcher/merge-toml.js
- ../config/watcher-azimuth/start-server.sh:/app/packages/polls-watcher/start-server.sh
- polls_watcher_gql_logs_data:/app/packages/polls-watcher/gql-logs
ports:
- "3008"
- "9015"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
test: ["CMD", "nc", "-vz", "localhost", "3008"]
interval: 20s
timeout: 5s
retries: 15
@@ -558,7 +542,7 @@ services:
ports:
- "0.0.0.0:4000:4000"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "4000"]
test: ["CMD", "nc", "-vz", "localhost", "4000"]
interval: 20s
timeout: 5s
retries: 15
@@ -568,11 +552,3 @@ services:
volumes:
watcher_db_data:
azimuth_watcher_gql_logs_data:
censures_watcher_gql_logs_data:
claims_watcher_gql_logs_data:
conditional_star_release_watcher_gql_logs_data:
delegated_sending_watcher_gql_logs_data:
ecliptic_watcher_gql_logs_data:
linear_star_release_watcher_gql_logs_data:
polls_watcher_gql_logs_data:
@@ -29,7 +29,7 @@ services:
image: cerc/watcher-merkl-sushiswap-v3:local
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
command: ["bash", "./start-job-runner.sh"]
volumes:
- ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
ports:
- "9002:9000"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
test: ["CMD", "nc", "-v", "localhost", "9000"]
interval: 20s
timeout: 5s
retries: 15
@@ -55,17 +55,16 @@ services:
image: cerc/watcher-merkl-sushiswap-v3:local
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
command: ["bash", "./start-server.sh"]
volumes:
- ../config/watcher-merkl-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
- ../config/watcher-merkl-sushiswap-v3/start-server.sh:/app/start-server.sh
- merkl_sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
ports:
- "127.0.0.1:3007:3008"
- "9003:9001"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
test: ["CMD", "nc", "-v", "localhost", "3008"]
interval: 20s
timeout: 5s
retries: 15
@@ -75,4 +74,3 @@ services:
volumes:
merkl_sushiswap_v3_watcher_db_data:
merkl_sushiswap_v3_watcher_gql_logs_data:
@@ -29,7 +29,7 @@ services:
image: cerc/watcher-sushiswap-v3:local
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
command: ["bash", "./start-job-runner.sh"]
volumes:
- ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
@@ -37,7 +37,7 @@ services:
ports:
- "9000:9000"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "9000"]
test: ["CMD", "nc", "-v", "localhost", "9000"]
interval: 20s
timeout: 5s
retries: 15
@@ -55,17 +55,16 @@ services:
image: cerc/watcher-sushiswap-v3:local
environment:
CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG}
CERC_ETH_RPC_ENDPOINTS: ${CERC_ETH_RPC_ENDPOINTS}
CERC_ETH_RPC_ENDPOINT: ${CERC_ETH_RPC_ENDPOINT}
command: ["bash", "./start-server.sh"]
volumes:
- ../config/watcher-sushiswap-v3/watcher-config-template.toml:/app/environments/watcher-config-template.toml
- ../config/watcher-sushiswap-v3/start-server.sh:/app/start-server.sh
- sushiswap_v3_watcher_gql_logs_data:/app/gql-logs
ports:
- "127.0.0.1:3008:3008"
- "9001:9001"
healthcheck:
test: ["CMD", "nc", "-vz", "127.0.0.1", "3008"]
test: ["CMD", "nc", "-v", "localhost", "3008"]
interval: 20s
timeout: 5s
retries: 15
@@ -75,4 +74,3 @@ services:
volumes:
sushiswap_v3_watcher_db_data:
sushiswap_v3_watcher_gql_logs_data:
@@ -0,0 +1,28 @@
# The password used to access test accounts (eg, via personal_unlockAccount). The password is the same for all accounts.
ACCOUNT_PASSWORD=secret1212
# ENODE of the geth bootnode.
BOOTNODE_KEY="b0ac22adcad37213c7c565810a50f1772291e7b0ce53fb73e7ec2a3c75bc13b5"
ENODE="enode://af22c29c316ad069cf48a09a4ad5cf04a251b411e45098888d114c6dd7f489a13786620d5953738762afa13711d4ffb3b19aa5de772d8af72f851f7e9c5b164a@fixturenet-eth-bootnode-geth:30303"
# JWT shared by geth and lighthouse for authentication.
JWT="0x6cdcac3501046a08e186730dd8bd136cfaf0fdc1fc955f6e15ad3068c0ff2af0"
# URL to download the ENR of the lighthouse bootnode (generated at first start).
ENR_URL="http://fixturenet-eth-bootnode-lighthouse:3000/bootnode/enr.dat"
# DB connection settings for statediffing (see docker-compose-db.yml)
CERC_STATEDIFF_DB_HOST="ipld-eth-db"
CERC_STATEDIFF_DB_PORT=5432
CERC_STATEDIFF_DB_NAME="cerc_testing"
CERC_STATEDIFF_DB_USER="vdbm"
CERC_STATEDIFF_DB_PASSWORD="password"
CERC_STATEDIFF_DB_GOOSE_MIN_VER=${CERC_STATEDIFF_DB_GOOSE_MIN_VER:-18}
CERC_STATEDIFF_DB_LOG_STATEMENTS="${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false}"
CERC_STATEDIFF_WORKERS=2
CERC_GETH_VMODULE="statediff/*=5,rpc/*=5"
CERC_GETH_VERBOSITY=${CERC_GETH_VERBOSITY:-3}
# Used by Lighthouse
SECONDS_PER_ETH1_BLOCK=${SECONDS_PER_ETH1_BLOCK:-3}
File diff suppressed because it is too large Load Diff
@@ -1,20 +0,0 @@
apiVersion: 1
datasources:
- name: Graph Node Postgres
type: postgres
jsonData:
database: graph-node
sslmode: 'disable'
maxOpenConns: 100
maxIdleConns: 100
maxIdleConnsAuto: true
connMaxLifetime: 14400
postgresVersion: 1411 # 903=9.3, 1000=10, 1411=14.11
timescaledb: false
user: graph-node
# # Add URL for graph-node database
# url: graph-node-db:5432
# # Set password for graph-node database
# secureJsonData:
# password: 'password'
@@ -45,18 +45,7 @@ scrape_configs:
metrics_path: /metrics
scheme: http
static_configs:
- targets: ['ethereum-chain-head-exporter:5000']
labels:
instance: 'external'
chain: 'ethereum'
- targets: ['filecoin-chain-head-exporter:5000']
labels:
instance: 'external'
chain: 'filecoin'
- targets: ['graph-node-upstream-head-exporter:5000']
labels:
instance: 'graph-node'
chain: 'filecoin'
- targets: ['chain-head-exporter:5000']
- job_name: 'postgres'
scrape_interval: 30s
@@ -85,11 +74,3 @@ scrape_configs:
# - targets: ['example-host:1317']
params:
format: ['prometheus']
- job_name: graph-node
metrics_path: /metrics
scrape_interval: 30s
scheme: http
static_configs:
# Add graph-node targets to be monitored below
# - targets: ['graph-node:8040']
@@ -1,64 +0,0 @@
apiVersion: 1
groups:
- orgId: 1
name: subgraph
folder: SubgraphAlerts
interval: 30s
rules:
- uid: b2a9144b-6104-46fc-92b5-352f4e643c4c
title: subgraph_head_tracking
condition: condition
data:
- refId: diff
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: ethereum_chain_head_number - on(network) group_right deployment_head{deployment=~"REPLACE_WITH_SUBGRAPH_IDS"}
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: diff
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 15
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: diff
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: threshold
noDataState: OK
execErrState: Alerting
for: 5m
annotations:
summary: Subgraph deployment {{ index $labels "deployment" }} is falling behind head by {{ index $values "diff" }}
labels: {}
isPaused: false
@@ -1,9 +0,0 @@
#!/bin/bash
echo Using CERC_GRAFANA_ALERTS_SUBGRAPH_IDS ${CERC_GRAFANA_ALERTS_SUBGRAPH_IDS}
# Replace subgraph ids in subgraph alerting config
# Note: Requires the grafana container to be run with user root
if [ -n "$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS" ]; then
sed -i "s/REPLACE_WITH_SUBGRAPH_IDS/$CERC_GRAFANA_ALERTS_SUBGRAPH_IDS/g" /etc/grafana/provisioning/alerting/subgraph-alert-rules.yml
fi
@@ -24,7 +24,7 @@ groups:
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="azimuth", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
@@ -100,7 +100,7 @@ groups:
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="censures", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
@@ -176,7 +176,7 @@ groups:
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="claims", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
@@ -252,7 +252,7 @@ groups:
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="conditional_star_release", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
@@ -328,7 +328,7 @@ groups:
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="delegated_sending", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
@@ -404,7 +404,7 @@ groups:
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="ecliptic", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
@@ -480,7 +480,7 @@ groups:
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="linear_star_release", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
@@ -556,7 +556,7 @@ groups:
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="azimuth", instance="polls", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
@@ -634,7 +634,83 @@ groups:
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="sushiswap", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: diff
useBackend: false
- refId: latest_external
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: latest_block_number{chain="filecoin"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_external
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
- uid: merkl_sushiswap_diff_external
title: merkl_sushiswap_watcher_head_tracking
condition: condition
data:
- refId: diff
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="sushi", instance="merkl_sushiswap", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
@@ -712,85 +788,7 @@ groups:
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: diff
useBackend: false
- refId: latest_external
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
editorMode: code
expr: latest_block_number{chain="filecoin"}
hide: false
instant: true
legendFormat: __auto
range: false
refId: latest_external
- refId: condition
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${diff} >= 16
intervalMs: 1000
maxDataPoints: 43200
refId: condition
type: math
noDataState: Alerting
execErrState: Alerting
for: 15m
annotations:
summary: Watcher {{ index $labels "instance" }} of group {{ index $labels "job" }} is falling behind external head by {{ index $values "diff" }}
isPaused: false
# Secured Finance
- uid: secured_finance_diff_external
title: secured_finance_watcher_head_tracking
condition: condition
data:
- refId: diff
relativeTimeRange:
from: 600
to: 0
datasourceUid: PBFA97CFB590B2093
model:
datasource:
type: prometheus
uid: PBFA97CFB590B2093
disableTextWrap: false
editorMode: code
expr: latest_block_number{instance="external"} - on(chain) group_right sync_status_block_number{job="secured-finance", instance="secured-finance", kind="latest_indexed"}
expr: latest_block_number - on(chain) group_right sync_status_block_number{job="ajna", instance="ajna", kind="latest_indexed"}
fullMetaSearch: false
includeNullMetadata: true
instant: true
@@ -6,16 +6,12 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
fi
set -u
echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
# Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
# Write the modified content to a new file
echo "$WATCHER_CONFIG" > environments/local.toml
@@ -6,16 +6,12 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
fi
set -u
echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
# Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
# Write the modified content to a new file
echo "$WATCHER_CONFIG" > environments/local.toml
@@ -2,6 +2,7 @@
host = "0.0.0.0"
port = 3008
kind = "active"
gqlPath = "/"
# Checkpointing state.
checkpointing = true
@@ -10,6 +11,7 @@
checkpointInterval = 2000
# Enable state creation
# CAUTION: Disable only if state creation is not desired or can be filled subsequently
enableState = false
subgraphPath = "./subgraph-build"
@@ -20,30 +22,23 @@
# Interval in number of blocks at which to clear entities cache.
clearEntitiesCacheInterval = 1000
# Max block range for which to return events in eventsInRange GQL query.
# Use -1 for skipping check on block range.
maxEventsBlockRange = 1000
# Flag to specify whether RPC endpoint supports block hash as block tag parameter
rpcSupportsBlockHashParam = false
# Server GQL config
[server.gql]
path = "/"
# GQL cache settings
[server.gqlCache]
enabled = true
# Max block range for which to return events in eventsInRange GQL query.
# Use -1 for skipping check on block range.
maxEventsBlockRange = 1000
# Max in-memory cache size (in bytes) (default 8 MB)
# maxCacheSize
# Log directory for GQL requests
logDir = "./gql-logs"
# GQL cache settings
[server.gql.cache]
enabled = true
# Max in-memory cache size (in bytes) (default 8 MB)
# maxCacheSize
# GQL cache-control max-age settings (in seconds)
maxAge = 15
timeTravelMaxAge = 86400 # 1 day
# GQL cache-control max-age settings (in seconds)
maxAge = 15
timeTravelMaxAge = 86400 # 1 day
[metrics]
host = "0.0.0.0"
@@ -63,7 +58,7 @@
[upstream]
[upstream.ethServer]
rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
# Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
rpcClient = true
@@ -76,10 +71,6 @@
# Boolean flag to filter event logs by topics
filterLogsByTopics = true
# Switch clients if eth_getLogs call takes more than threshold (in secs)
# Set to 0 for disabling switching
getLogsClientSwitchThresholdInSecs = 30
[upstream.cache]
name = "requests"
enabled = false
@@ -94,9 +85,6 @@
# Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
blockDelayInMilliSecs = 30000
# Number of blocks by which block processing lags behind head
blockProcessingOffset = 3
# Boolean to switch between modes of processing events when starting the server.
# Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
# Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -108,6 +96,3 @@
# Max block range of historical processing after which it waits for completion of events processing
# If set to -1 historical processing does not wait for events processing and completes till latest canonical block
historicalMaxFetchAhead = 10000
# Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
maxNewBlockRetries = 3
@@ -4,19 +4,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"
# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
# Replace env variables in template TOML file
# Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")
# Write the modified content to a new file
@@ -4,19 +4,16 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL_ENDPOINT}"
echo "Using IPLD ETH RPC endpoint ${CERC_IPLD_ETH_RPC}"
echo "Using IPLD GQL endpoint ${CERC_IPLD_ETH_GQL}"
echo "Using historicalLogsBlockRange ${CERC_HISTORICAL_BLOCK_RANGE:-2000}"
# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
# Replace env variables in template TOML file
# Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}|g; \
s|REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT|${CERC_IPLD_ETH_GQL_ENDPOINT}|g; \
sed -E "s|REPLACE_WITH_CERC_IPLD_ETH_RPC|${CERC_IPLD_ETH_RPC}|g; \
s|REPLACE_WITH_CERC_IPLD_ETH_GQL|${CERC_IPLD_ETH_GQL}|g; \
s|REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE|${CERC_HISTORICAL_BLOCK_RANGE:-2000}| ")
# Write the modified content to a new file
@@ -1,7 +1,6 @@
[server]
host = "0.0.0.0"
[server.gql]
maxSimultaneousRequests = -1
maxSimultaneousRequests = -1
[metrics]
host = "0.0.0.0"
@@ -14,8 +13,8 @@
[upstream]
[upstream.ethServer]
gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL_ENDPOINT"
rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
gqlApiEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_GQL"
rpcProviderEndpoint = "REPLACE_WITH_CERC_IPLD_ETH_RPC"
[jobQueue]
historicalLogsBlockRange = REPLACE_WITH_CERC_HISTORICAL_BLOCK_RANGE
@@ -6,16 +6,12 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
fi
set -u
echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
# Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
# Write the modified content to a new file
echo "$WATCHER_CONFIG" > environments/local.toml
@@ -6,16 +6,12 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
fi
set -u
echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
# Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
# Write the modified content to a new file
echo "$WATCHER_CONFIG" > environments/local.toml
@@ -2,6 +2,7 @@
host = "0.0.0.0"
port = 3008
kind = "active"
gqlPath = '/'
# Checkpointing state.
checkpointing = true
@@ -21,30 +22,23 @@
# Interval in number of blocks at which to clear entities cache.
clearEntitiesCacheInterval = 1000
# Max block range for which to return events in eventsInRange GQL query.
# Use -1 for skipping check on block range.
maxEventsBlockRange = 1000
# Flag to specify whether RPC endpoint supports block hash as block tag parameter
rpcSupportsBlockHashParam = false
# Server GQL config
[server.gql]
path = "/"
# GQL cache settings
[server.gqlCache]
enabled = true
# Max block range for which to return events in eventsInRange GQL query.
# Use -1 for skipping check on block range.
maxEventsBlockRange = 1000
# Max in-memory cache size (in bytes) (default 8 MB)
# maxCacheSize
# Log directory for GQL requests
logDir = "./gql-logs"
# GQL cache settings
[server.gql.cache]
enabled = true
# Max in-memory cache size (in bytes) (default 8 MB)
# maxCacheSize
# GQL cache-control max-age settings (in seconds)
maxAge = 15
timeTravelMaxAge = 86400 # 1 day
# GQL cache-control max-age settings (in seconds)
maxAge = 15
timeTravelMaxAge = 86400 # 1 day
[metrics]
host = "0.0.0.0"
@@ -64,7 +58,7 @@
[upstream]
[upstream.ethServer]
rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
# Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
rpcClient = true
@@ -75,7 +69,7 @@
# Boolean flag to filter event logs by contracts
filterLogsByAddresses = true
# Boolean flag to filter event logs by topics
filterLogsByTopics = true
filterLogsByTopics = false
[upstream.cache]
name = "requests"
@@ -91,9 +85,6 @@
# Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
blockDelayInMilliSecs = 30000
# Number of blocks by which block processing lags behind head
blockProcessingOffset = 0
# Boolean to switch between modes of processing events when starting the server.
# Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
# Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -105,6 +96,3 @@
# Max block range of historical processing after which it waits for completion of events processing
# If set to -1 historical processing does not wait for events processing and completes till latest canonical block
historicalMaxFetchAhead = 10000
# Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
maxNewBlockRetries = 3
@@ -6,16 +6,12 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
fi
set -u
echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
# Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
# Write the modified content to a new file
echo "$WATCHER_CONFIG" > environments/local.toml
@@ -6,16 +6,12 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
fi
set -u
echo "Using ETH RPC endpoints ${CERC_ETH_RPC_ENDPOINTS}"
echo "Using ETH RPC endpoint ${CERC_ETH_RPC_ENDPOINT}"
# Read in the config template TOML file and modify it
WATCHER_CONFIG_TEMPLATE=$(cat environments/watcher-config-template.toml)
# Convert the comma-separated list in CERC_ETH_RPC_ENDPOINTS to a JSON array
RPC_ENDPOINTS_ARRAY=$(echo "$CERC_ETH_RPC_ENDPOINTS" | tr ',' '\n' | awk '{print "\"" $0 "\""}' | paste -sd, - | sed 's/^/[/; s/$/]/')
WATCHER_CONFIG=$(echo "$WATCHER_CONFIG_TEMPLATE" | \
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS|${RPC_ENDPOINTS_ARRAY}| ")
sed -E "s|REPLACE_WITH_CERC_ETH_RPC_ENDPOINT|${CERC_ETH_RPC_ENDPOINT}| ")
# Write the modified content to a new file
echo "$WATCHER_CONFIG" > environments/local.toml
@@ -2,6 +2,7 @@
host = "0.0.0.0"
port = 3008
kind = "active"
gqlPath = "/"
# Checkpointing state.
checkpointing = true
@@ -10,6 +11,7 @@
checkpointInterval = 2000
# Enable state creation
# CAUTION: Disable only if state creation is not desired or can be filled subsequently
enableState = false
subgraphPath = "./subgraph-build"
@@ -20,30 +22,23 @@
# Interval in number of blocks at which to clear entities cache.
clearEntitiesCacheInterval = 1000
# Max block range for which to return events in eventsInRange GQL query.
# Use -1 for skipping check on block range.
maxEventsBlockRange = 1000
# Flag to specify whether RPC endpoint supports block hash as block tag parameter
rpcSupportsBlockHashParam = false
# Server GQL config
[server.gql]
path = "/"
# GQL cache settings
[server.gqlCache]
enabled = true
# Max block range for which to return events in eventsInRange GQL query.
# Use -1 for skipping check on block range.
maxEventsBlockRange = 1000
# Max in-memory cache size (in bytes) (default 8 MB)
# maxCacheSize
# Log directory for GQL requests
logDir = "./gql-logs"
# GQL cache settings
[server.gql.cache]
enabled = true
# Max in-memory cache size (in bytes) (default 8 MB)
# maxCacheSize
# GQL cache-control max-age settings (in seconds)
maxAge = 15
timeTravelMaxAge = 86400 # 1 day
# GQL cache-control max-age settings (in seconds)
maxAge = 15
timeTravelMaxAge = 86400 # 1 day
[metrics]
host = "0.0.0.0"
@@ -63,7 +58,7 @@
[upstream]
[upstream.ethServer]
rpcProviderEndpoints = REPLACE_WITH_CERC_ETH_RPC_ENDPOINTS
rpcProviderEndpoint = "REPLACE_WITH_CERC_ETH_RPC_ENDPOINT"
# Boolean flag to specify if rpc-eth-client should be used for RPC endpoint instead of ipld-eth-client (ipld-eth-server GQL client)
rpcClient = true
@@ -76,10 +71,6 @@
# Boolean flag to filter event logs by topics
filterLogsByTopics = true
# Switch clients if eth_getLogs call takes more than threshold (in secs)
# Set to 0 for disabling switching
getLogsClientSwitchThresholdInSecs = 30
[upstream.cache]
name = "requests"
enabled = false
@@ -94,9 +85,6 @@
# Filecoin block time: https://docs.filecoin.io/basics/the-blockchain/blocks-and-tipsets#blocktime
blockDelayInMilliSecs = 30000
# Number of blocks by which block processing lags behind head
blockProcessingOffset = 3
# Boolean to switch between modes of processing events when starting the server.
# Setting to true will fetch filtered events and required blocks in a range of blocks and then process them.
# Setting to false will fetch blocks consecutively with its events and then process them (Behaviour is followed in realtime processing near head).
@@ -108,6 +96,3 @@
# Max block range of historical processing after which it waits for completion of events processing
# If set to -1 historical processing does not wait for events processing and completes till latest canonical block
historicalMaxFetchAhead = 10000
# Max number of retries to fetch new block after which watcher will failover to other RPC endpoints
maxNewBlockRetries = 3
@@ -109,6 +109,10 @@ else
OTHER_OPTS="--miner.threads=1"
fi
if [ "$CERC_ALLOW_UNPROTECTED_TXS" == "true" ]; then
OTHER_OPTS+=" --rpc.allow-unprotected-txs"
fi
$START_CMD \
--datadir="${CERC_ETH_DATADIR}" \
--bootnodes="${ENODE}" \
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
# Build cerc/laconic-console-host
# Build cerc/laconic-registry-cli
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
@@ -28,8 +28,6 @@ RUN \
&& su ${USERNAME} -c "umask 0002 && npm install -g semver" \
# Install pnpm
&& su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
# Install bun
&& su ${USERNAME} -c "umask 0002 && npm install -g bun@1.1.x" \
&& npm cache clean --force > /dev/null 2>&1
# [Optional] Uncomment this section to install additional OS packages.
@@ -14,8 +14,6 @@ if [ -z "$CERC_BUILD_TOOL" ]; then
CERC_BUILD_TOOL=pnpm
elif [ -f "yarn.lock" ]; then
CERC_BUILD_TOOL=yarn
elif [ -f "bun.lockb" ]; then
CERC_BUILD_TOOL=bun
else
CERC_BUILD_TOOL=npm
fi
@@ -13,8 +13,6 @@ if [ -z "$CERC_BUILD_TOOL" ]; then
CERC_BUILD_TOOL=pnpm
elif [ -f "yarn.lock" ]; then
CERC_BUILD_TOOL=yarn
elif [ -f "bun.lockb" ]; then
CERC_BUILD_TOOL=bun
else
CERC_BUILD_TOOL=npm
fi
@@ -5,7 +5,7 @@ fi
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
CERC_MAX_GENERATE_TIME=${CERC_MAX_GENERATE_TIME:-120}
CERC_MAX_GENERATE_TIME=${CERC_MAX_GENERATE_TIME:-60}
tpid=""
ctrl_c() {
@@ -20,8 +20,6 @@ if [ -z "$CERC_BUILD_TOOL" ]; then
CERC_BUILD_TOOL=pnpm
elif [ -f "yarn.lock" ]; then
CERC_BUILD_TOOL=yarn
elif [ -f "bun.lockb" ]; then
CERC_BUILD_TOOL=bun
else
CERC_BUILD_TOOL=npm
fi
@@ -1,10 +0,0 @@
FROM cerc/ping-pub-base:local
COPY ./scripts/update-explorer-config.sh /scripts
COPY ./scripts/start-serving-explorer.sh /scripts
COPY ./config/laconic-chaindata-template.json /config/chains/laconic-chaindata-template.json
EXPOSE 5173
WORKDIR /app
CMD ["/scripts/start-serving-explorer.sh"]
@@ -1,8 +0,0 @@
FROM cerc/webapp-base:local
WORKDIR /app
COPY . .
RUN yarn
@@ -1,8 +1,5 @@
#!/usr/bin/env bash
# Build the ping pub image
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
# Two-stage build is to allow us to pick up both the upstream repo's files, and local files here for config
docker build -t cerc/ping-pub-base:local ${build_command_args} -f $SCRIPT_DIR/Dockerfile.base $CERC_REPO_BASE_DIR/explorer
docker build -t cerc/ping-pub:local ${build_command_args} -f $SCRIPT_DIR/Dockerfile $SCRIPT_DIR
docker build -t cerc/ping-pub:local ${build_command_args} -f $CERC_REPO_BASE_DIR/explorer/Dockerfile $CERC_REPO_BASE_DIR/explorer
@@ -1,22 +0,0 @@
{
"chain_name": "LACONIC_LACONICD_CHAIN_ID",
"registry_name": "LACONIC_LACONICD_CHAIN_ID",
"api": [
{"provider": "LX-one-tree-one-seven", "address": "LACONIC_LACONICD_API_URL"}
],
"rpc": [
{"provider": "LX-tendermint-rpc", "address": "LACONIC_LACONICD_RPC_URL"}
],
"sdk_version": "0.45.1",
"coin_type": "118",
"min_tx_fee": "800",
"addr_prefix": "ethm",
"logo": "/logos/cosmos.svg",
"assets": [{
"base": "aphoton",
"symbol": "LNT",
"exponent": "6",
"coingecko_id": "cosmos",
"logo": "/logos/cosmos.svg"
}]
}
@@ -1,13 +0,0 @@
#!/usr/bin/env bash
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
${SCRIPT_DIR}/update-explorer-config.sh
echo "Starting serving explorer"
# Force cache re-build because vite is dumb and can't be restarted otherwise
yarn serve --host --force
@@ -1,52 +0,0 @@
#!/usr/bin/env bash
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
# Verify that we have the config variables we need
if [[ -z ${LACONIC_LACONICD_API_URL} ]]; then
echo "Error: LACONIC_LACONICD_API_URL not defined"
exit 1
fi
if [[ -z ${LACONIC_LACONICD_RPC_URL} ]]; then
echo "Error: LACONIC_LACONICD_RPC_URL not defined"
exit 1
fi
if [[ -z ${LACONIC_LACONICD_CHAIN_ID} ]]; then
echo "Error: LACONIC_LACONICD_CHAIN_ID not defined"
exit 1
fi
# Ping-pub explorer has endlessly confusing behavior where it
# infers the directory from which to load chain configuration files
# by the presence or absense of the substring "testnet" in the host name
# (browser side -- the host name of the host in the address bar of the browser)
# Accordingly we configure our network in both directories in order to
# subvert this lunacy.
explorer_mainnet_config_dir=/app/chains/mainnet
explorer_testnet_config_dir=/app/chains/testnet
config_template_file=/config/chains/laconic-chaindata-template.json
chain_config_name=laconic.json
mainnet_config_file=${explorer_mainnet_config_dir}/${chain_config_name}
testnet_config_file=${explorer_testnet_config_dir}/${chain_config_name}
# Delete the stock config files
rm -f ${explorer_testnet_config_dir}/*
rm -f ${explorer_mainnet_config_dir}/*
# Copy in our template file
cp ${config_template_file} ${mainnet_config_file}
# Update the file with the config variables
sed -i "s#LACONIC_LACONICD_API_URL#${LACONIC_LACONICD_API_URL}#g" ${mainnet_config_file}
sed -i "s#LACONIC_LACONICD_RPC_URL#${LACONIC_LACONICD_RPC_URL}#g" ${mainnet_config_file}
sed -i "s#LACONIC_LACONICD_CHAIN_ID#${LACONIC_LACONICD_CHAIN_ID}#g" ${mainnet_config_file}
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
echo "Updated chaindata file:"
cat ${mainnet_config_file}
fi
# Copy over to the testnet directory
cp ${mainnet_config_file} ${testnet_config_file}
@@ -0,0 +1,6 @@
FROM cerc/snowballtools-base-backend-base:local
WORKDIR /app/packages/backend
COPY run.sh .
ENTRYPOINT ["./run.sh"]
@@ -0,0 +1,26 @@
FROM ubuntu:22.04 as builder
RUN apt update && \
apt install -y --no-install-recommends --no-install-suggests \
ca-certificates curl gnupg
# Node
ARG NODE_MAJOR=20
RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
apt update && apt install -y nodejs
# npm setup
RUN npm config set @cerc-io:registry https://git.vdb.to/api/packages/cerc-io/npm/ && npm install -g yarn
COPY . /app/
WORKDIR /app/
RUN find . -name 'node_modules' | xargs -n1 rm -rf
RUN yarn && yarn build --ignore frontend
FROM cerc/webapp-base:local
COPY --from=builder /app /app
WORKDIR /app/packages/backend
@@ -0,0 +1,10 @@
#!/usr/bin/env bash
# Build cerc/webapp-deployer-backend
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
# See: https://stackoverflow.com/a/246128/1701505
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
docker build -t cerc/snowballtools-base-backend-base:local ${build_command_args} -f ${SCRIPT_DIR}/Dockerfile-base ${CERC_REPO_BASE_DIR}/snowballtools-base
docker build -t cerc/snowballtools-base-backend:local ${build_command_args} ${SCRIPT_DIR}
@@ -0,0 +1,19 @@
#!/bin/bash
LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
if [ -z "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
if [ -f "/config/laconic-hosted-config.yml" ]; then
LACONIC_HOSTED_CONFIG_FILE="/config/laconic-hosted-config.yml"
elif [ -f "/config/config.yml" ]; then
LACONIC_HOSTED_CONFIG_FILE="/config/config.yml"
fi
fi
if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
/scripts/apply-webapp-config.sh $LACONIC_HOSTED_CONFIG_FILE "`pwd`/dist"
fi
/scripts/apply-runtime-env.sh "`pwd`/dist"
yarn start
@@ -6,10 +6,5 @@ WORKDIR /app
COPY . .
# Get the latest Git commit hash and set in package.json
RUN COMMIT_HASH=$(git rev-parse HEAD) && \
jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
mv tmp.json package.json
RUN echo "Installing dependencies and building ajna-watcher-ts" && \
yarn && yarn build
@@ -1,20 +1,11 @@
FROM node:18.16.0-alpine3.16
RUN apk --update --no-cache add git python3 alpine-sdk jq
RUN apk --update --no-cache add git python3 alpine-sdk
WORKDIR /app
COPY . .
# Get the latest Git commit hash and set it in package.json of all watchers
RUN COMMIT_HASH=$(git rev-parse HEAD) && \
find . -name 'package.json' -exec sh -c ' \
for packageFile; do \
jq --arg commitHash "$0" ".commitHash = \$commitHash" "$packageFile" > "$packageFile.tmp" && \
mv "$packageFile.tmp" "$packageFile"; \
done \
' "$COMMIT_HASH" {} \;
RUN echo "Building azimuth-watcher-ts" && \
yarn && yarn build
@@ -6,10 +6,5 @@ WORKDIR /app
COPY . .
# Get the latest Git commit hash and set in package.json
RUN COMMIT_HASH=$(git rev-parse HEAD) && \
jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
mv tmp.json package.json
RUN echo "Installing dependencies and building merkl-sushiswap-v3-watcher-ts" && \
yarn && yarn build
@@ -6,10 +6,5 @@ WORKDIR /app
COPY . .
# Get the latest Git commit hash and set in package.json
RUN COMMIT_HASH=$(git rev-parse HEAD) && \
jq --arg commitHash "$COMMIT_HASH" '.commitHash = $commitHash' package.json > tmp.json && \
mv tmp.json package.json
RUN echo "Installing dependencies and building sushiswap-v3-watcher-ts" && \
yarn && yarn build
@@ -28,13 +28,11 @@ RUN \
&& su ${USERNAME} -c "umask 0002 && npm install -g semver" \
# Install pnpm
&& su ${USERNAME} -c "umask 0002 && npm install -g pnpm" \
# Install bun
&& su ${USERNAME} -c "umask 0002 && npm install -g bun@1.1.x" \
&& npm cache clean --force > /dev/null 2>&1
# [Optional] Uncomment this section to install additional OS packages.
RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
&& apt-get -y install --no-install-recommends jq gettext-base git
&& apt-get -y install --no-install-recommends jq gettext-base
# [Optional] Uncomment if you want to install an additional version of node using nvm
# ARG EXTRA_NODE_VERSION=10
@@ -4,8 +4,6 @@ if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
# TODO: document what this script does
WORK_DIR="${1:-./}"
cd "${WORK_DIR}" || exit 1
@@ -27,8 +27,6 @@ elif [ -f "${WORK_DIR}/package.json" ]; then
CERC_BUILD_TOOL=pnpm
elif [ -f "yarn.lock" ]; then
CERC_BUILD_TOOL=yarn
elif [ -f "bun.lockb" ]; then
CERC_BUILD_TOOL=bun
else
CERC_BUILD_TOOL=npm
fi
@@ -8,27 +8,21 @@ CERC_WEBAPP_FILES_DIR="${CERC_WEBAPP_FILES_DIR:-/data}"
CERC_ENABLE_CORS="${CERC_ENABLE_CORS:-false}"
CERC_SINGLE_PAGE_APP="${CERC_SINGLE_PAGE_APP}"
if [ -z "${CERC_SINGLE_PAGE_APP}" ]; then
# If there is only one HTML file, assume an SPA.
if [ 1 -eq $(find "${CERC_WEBAPP_FILES_DIR}" -name '*.html' | wc -l) ]; then
if [ -z "${CERC_SINGLE_PAGE_APP}" ]; then
if [ 1 -eq $(find "${CERC_WEBAPP_FILES_DIR}" -name '*.html' | wc -l) ] && [ -d "${CERC_WEBAPP_FILES_DIR}/static" ]; then
CERC_SINGLE_PAGE_APP=true
else
CERC_SINGLE_PAGE_APP=false
fi
fi
# ${var,,} is a lower-case comparison
if [ "true" == "${CERC_ENABLE_CORS,,}" ]; then
if [ "true" == "$CERC_ENABLE_CORS" ]; then
CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --cors"
fi
# ${var,,} is a lower-case comparison
if [ "true" == "${CERC_SINGLE_PAGE_APP,,}" ]; then
echo "Serving content as single-page app. If this is wrong, set 'CERC_SINGLE_PAGE_APP=false'"
if [ "true" == "$CERC_SINGLE_PAGE_APP" ]; then
# Create a catchall redirect back to /
CERC_HTTP_EXTRA_ARGS="$CERC_HTTP_EXTRA_ARGS --proxy http://localhost:${CERC_LISTEN_PORT}?"
else
echo "Serving content normally. If this is a single-page app, set 'CERC_SINGLE_PAGE_APP=true'"
fi
LACONIC_HOSTED_CONFIG_FILE=${LACONIC_HOSTED_CONFIG_FILE}
@@ -45,4 +39,4 @@ if [ -f "${LACONIC_HOSTED_CONFIG_FILE}" ]; then
fi
/scripts/apply-runtime-env.sh ${CERC_WEBAPP_FILES_DIR}
http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT} "${CERC_WEBAPP_FILES_DIR}"
http-server $CERC_HTTP_EXTRA_ARGS -p ${CERC_LISTEN_PORT} "${CERC_WEBAPP_FILES_DIR}"
@@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file and set following env v
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher to
CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
```
### Start the deployment
@@ -2,7 +2,7 @@ version: "1.0"
name: ajna
description: "Ajna watcher stack"
repos:
- git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.16
- git.vdb.to/cerc-io/ajna-watcher-ts@v0.1.1
containers:
- cerc/watcher-ajna
pods:
@@ -4,7 +4,7 @@ Instructions to setup and deploy Azimuth Watcher stack
## Setup
Prerequisite: External RPC endpoints
Prerequisite: `ipld-eth-server` RPC and GQL endpoints
Clone required repositories:
@@ -44,42 +44,34 @@ network:
- 0.0.0.0:9000:9000
azimuth-watcher-server:
- 0.0.0.0:3001:3001
- 0.0.0.0:9001:9001
censures-watcher-job-runner:
- 0.0.0.0:9002:9002
censures-watcher-server:
- 0.0.0.0:3002:3002
- 0.0.0.0:9003:9003
claims-watcher-job-runner:
- 0.0.0.0:9004:9004
claims-watcher-server:
- 0.0.0.0:3003:3003
- 0.0.0.0:9005:9005
conditional-star-release-watcher-job-runner:
- 0.0.0.0:9006:9006
conditional-star-release-watcher-server:
- 0.0.0.0:3004:3004
- 0.0.0.0:9007:9007
delegated-sending-watcher-job-runner:
- 0.0.0.0:9008:9008
delegated-sending-watcher-server:
- 0.0.0.0:3005:3005
- 0.0.0.0:9009:9009
ecliptic-watcher-job-runner:
- 0.0.0.0:9010:9010
ecliptic-watcher-server:
- 0.0.0.0:3006:3006
- 0.0.0.0:9011:9011
linear-star-release-watcher-job-runner:
- 0.0.0.0:9012:9012
linear-star-release-watcher-server:
- 0.0.0.0:3007:3007
- 0.0.0.0:9013:9013
polls-watcher-job-runner:
- 0.0.0.0:9014:9014
polls-watcher-server:
- 0.0.0.0:3008:3008
- 0.0.0.0:9015:9015
gateway-server:
- 0.0.0.0:4000:4000
...
@@ -102,7 +94,7 @@ Inside the deployment directory, open the file `config.env` and add variable to
```bash
# External RPC endpoints
CERC_ETH_RPC_ENDPOINTS=https://example-rpc-endpoint-1,https://example-rpc-endpoint-2
CERC_IPLD_ETH_RPC=
```
* NOTE: If RPC endpoint is on the host machine, use `host.docker.internal` as the hostname to access the host port, or use the `ip a` command to find the IP address of the `docker0` interface (this will usually be something like `172.17.0.1` or `172.18.0.1`)
@@ -128,7 +120,4 @@ To stop all azimuth services and also delete data:
```bash
laconic-so deployment --dir azimuth-deployment stop --delete-volumes
# Remove deployment directory (deployment will have to be recreated for a re-run)
rm -r azimuth-deployment
```
@@ -1,7 +1,7 @@
version: "1.0"
name: azimuth
repos:
- github.com/cerc-io/azimuth-watcher-ts@0.1.6
- github.com/cerc-io/azimuth-watcher-ts@v0.1.3
containers:
- cerc/watcher-azimuth
pods:
@@ -0,0 +1,19 @@
# fixturenet-plugeth
A variation of `fixturenet-eth` that uses `plugeth` instead of `go-ethereum`.
See `stacks/fixturenet-eth/README.md` for more information.
## Containers
* cerc/lighthouse
* cerc/fixturenet-eth-plugeth
* cerc/fixturenet-eth-lighthouse
## Deploy the stack
```bash
laconic-so --stack fixturenet-plugeth setup-repositories
laconic-so --stack fixturenet-plugeth build-containers
laconic-so --stack fixturenet-plugeth deploy up
```
@@ -0,0 +1,17 @@
version: "1.0"
name: fixturenet-plugeth
description: "Plugeth Ethereum Fixturenet"
repos:
- git.vdb.to/cerc-io/plugeth@statediff
- git.vdb.to/cerc-io/plugeth-statediff
- git.vdb.to/cerc-io/lighthouse
containers:
- cerc/plugeth-statediff
- cerc/plugeth
- cerc/fixturenet-eth-genesis
- cerc/fixturenet-plugeth-plugeth
- cerc/lighthouse
- cerc/lighthouse-cli
- cerc/fixturenet-eth-lighthouse
pods:
- fixturenet-plugeth
@@ -43,18 +43,16 @@ customized by editing the "spec" file generated by `laconic-so deploy init`.
```
$ cat graph-node-spec.yml
stack: graph-node
network:
ports:
graph-node:
- '8000:8000'
- '8001'
- '8020:8020'
- '8030'
- '8040'
ipfs:
- '8080'
- '4001'
- '5001:5001'
ports:
graph-node:
- '8000:8000'
- '8001'
- '8020:8020'
- '8030'
ipfs:
- '8080'
- '4001'
- '5001:5001'
...
```
@@ -66,7 +64,7 @@ laconic-so --stack graph-node deploy create --spec-file graph-node-spec.yml --de
## Start the stack
Update `config.env` file inside deployment directory with the following values before starting the stack:
Create an env file with the following values to be set before starting the stack:
```bash
# Set ETH RPC endpoint the graph node will use
@@ -90,13 +88,10 @@ export GRAPH_ETHEREUM_REQUEST_RETRIES=
# Maximum number of blocks to scan for triggers in each request (default: 2000)
export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=
# Maximum number of concurrent requests made against Ethereum for requesting transaction receipts during block ingestion (default: 1000)
export GRAPH_ETHEREUM_BLOCK_INGESTOR_MAX_CONCURRENT_JSON_RPC_CALLS_FOR_TXN_RECEIPTS=
# Ref: https://git.vdb.to/cerc-io/graph-node/src/branch/master/docs/environment-variables.md
```
Example `config.env` file:
Example env file:
```bash
export ETH_RPC_HOST=filecoin.chainup.net
@@ -109,6 +104,12 @@ export GRAPH_ETHEREUM_REQUEST_RETRIES=5
export GRAPH_ETHEREUM_MAX_BLOCK_RANGE_SIZE=50
```
Set the environment variables:
```bash
source <PATH_TO_ENV_FILE>
```
Deploy the stack:
```bash
@@ -3,9 +3,7 @@ name: graph-node
description: "Stack for running graph-node"
repos:
- github.com/graphprotocol/graph-node
- github.com/cerc-io/watcher-ts@v0.2.92
containers:
- cerc/graph-node
- cerc/watcher-ts
pods:
- graph-node
@@ -14,11 +14,10 @@
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from stack_orchestrator.util import get_yaml
from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand
from stack_orchestrator.deploy.deployment_context import DeploymentContext
from stack_orchestrator.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
from stack_orchestrator.deploy.stack_state import State
from stack_orchestrator.deploy.deploy_util import VolumeMapping, run_container_command
from stack_orchestrator.opts import opts
from stack_orchestrator.command_types import CommandOptions
from enum import Enum
from pathlib import Path
from shutil import copyfile, copytree
@@ -62,7 +61,7 @@ def _get_node_moniker_from_config(network_dir: Path):
return moniker
def _get_node_key_from_gentx(gentx_file_name: str):
def _get_node_key_from_gentx(options: CommandOptions, gentx_file_name: str):
gentx_file_path = Path(gentx_file_name)
if gentx_file_path.exists():
with open(Path(gentx_file_name), "rb") as f:
@@ -77,24 +76,24 @@ def _comma_delimited_to_list(list_str: str):
return list_str.split(",") if list_str else []
def _get_node_keys_from_gentx_files(gentx_file_list: str):
def _get_node_keys_from_gentx_files(options: CommandOptions, gentx_file_list: str):
node_keys = []
gentx_files = _comma_delimited_to_list(gentx_file_list)
for gentx_file in gentx_files:
node_key = _get_node_key_from_gentx(gentx_file)
node_key = _get_node_key_from_gentx(options, gentx_file)
if node_key:
node_keys.append(node_key)
return node_keys
def _copy_gentx_files(network_dir: Path, gentx_file_list: str):
def _copy_gentx_files(options: CommandOptions, network_dir: Path, gentx_file_list: str):
gentx_files = _comma_delimited_to_list(gentx_file_list)
for gentx_file in gentx_files:
gentx_file_path = Path(gentx_file)
copyfile(gentx_file_path, os.path.join(network_dir, "config", "gentx", os.path.basename(gentx_file_path)))
def _remove_persistent_peers(network_dir: Path):
def _remove_persistent_peers(options: CommandOptions, network_dir: Path):
config_file_path = _config_toml_path(network_dir)
if not config_file_path.exists():
print("Error: config.toml not found")
@@ -108,45 +107,20 @@ def _remove_persistent_peers(network_dir: Path):
output_file.write(config_file_content)
def _insert_persistent_peers(config_dir: Path, new_persistent_peers: str):
def _insert_persistent_peers(options: CommandOptions, config_dir: Path, new_persistent_peers: str):
config_file_path = config_dir.joinpath("config.toml")
if not config_file_path.exists():
print("Error: config.toml not found")
sys.exit(1)
with open(config_file_path, "r") as input_file:
config_file_content = input_file.read()
persistent_peers_pattern = r'^persistent_peers = ""'
persistent_peers_pattern = '^persistent_peers = ""'
replace_with = f"persistent_peers = \"{new_persistent_peers}\""
config_file_content = re.sub(persistent_peers_pattern, replace_with, config_file_content, flags=re.MULTILINE)
with open(config_file_path, "w") as output_file:
output_file.write(config_file_content)
def _enable_cors(config_dir: Path):
config_file_path = config_dir.joinpath("config.toml")
if not config_file_path.exists():
print("Error: config.toml not found")
sys.exit(1)
with open(config_file_path, "r") as input_file:
config_file_content = input_file.read()
cors_pattern = r'^cors_allowed_origins = \[]'
replace_with = 'cors_allowed_origins = ["*"]'
config_file_content = re.sub(cors_pattern, replace_with, config_file_content, flags=re.MULTILINE)
with open(config_file_path, "w") as output_file:
output_file.write(config_file_content)
app_file_path = config_dir.joinpath("app.toml")
if not app_file_path.exists():
print("Error: app.toml not found")
sys.exit(1)
with open(app_file_path, "r") as input_file:
app_file_content = input_file.read()
cors_pattern = r'^enabled-unsafe-cors = false'
replace_with = "enabled-unsafe-cors = true"
app_file_content = re.sub(cors_pattern, replace_with, app_file_content, flags=re.MULTILINE)
with open(app_file_path, "w") as output_file:
output_file.write(app_file_content)
def _phase_from_params(parameters):
phase = SetupPhase.ILLEGAL
if parameters.initialize_network:
@@ -176,7 +150,7 @@ def _phase_from_params(parameters):
def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCommand, extra_args):
options = opts.o
options = command_context.cluster_context.options
currency = "stake" # Does this need to be a parameter?
@@ -263,7 +237,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
print("Error: --gentx-files must be supplied")
sys.exit(1)
# First look in the supplied gentx files for the other nodes' keys
other_node_keys = _get_node_keys_from_gentx_files(parameters.gentx_file_list)
other_node_keys = _get_node_keys_from_gentx_files(options, parameters.gentx_file_list)
# Add those keys to our genesis, with balances we determine here (why?)
for other_node_key in other_node_keys:
outputk, statusk = run_container_command(
@@ -272,7 +246,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
if options.debug:
print(f"Command output: {outputk}")
# Copy the gentx json files into our network dir
_copy_gentx_files(network_dir, parameters.gentx_file_list)
_copy_gentx_files(options, network_dir, parameters.gentx_file_list)
# Now we can run collect-gentxs
output1, status1 = run_container_command(
command_context, "laconicd", f"laconicd collect-gentxs --home {laconicd_home_path_in_container}", mounts)
@@ -281,7 +255,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
print(f"Generated genesis file, please copy to other nodes as required: \
{os.path.join(network_dir, 'config', 'genesis.json')}")
# Last thing, collect-gentxs puts a likely bogus set of persistent_peers in config.toml so we remove that now
_remove_persistent_peers(network_dir)
_remove_persistent_peers(options, network_dir)
# In both cases we validate the genesis file now
output2, status1 = run_container_command(
command_context, "laconicd", f"laconicd validate-genesis --home {laconicd_home_path_in_container}", mounts)
@@ -292,7 +266,7 @@ def setup(command_context: DeployCommandContext, parameters: LaconicStackSetupCo
sys.exit(1)
def create(deployment_context: DeploymentContext, extra_args):
def create(context: DeploymentContext, extra_args):
network_dir = extra_args[0]
if network_dir is None:
print("Error: --network-dir must be supplied")
@@ -311,17 +285,15 @@ def create(deployment_context: DeploymentContext, extra_args):
sys.exit(1)
# Copy the network directory contents into our deployment
# TODO: change this to work with non local paths
deployment_config_dir = deployment_context.deployment_dir.joinpath("data", "laconicd-config")
deployment_config_dir = context.deployment_dir.joinpath("data", "laconicd-config")
copytree(config_dir_path, deployment_config_dir, dirs_exist_ok=True)
# If supplied, add the initial persistent peers to the config file
if extra_args[1]:
initial_persistent_peers = extra_args[1]
_insert_persistent_peers(deployment_config_dir, initial_persistent_peers)
# Enable CORS headers so explorers and so on can talk to the node
_enable_cors(deployment_config_dir)
_insert_persistent_peers(context.command_context.cluster_context.options, deployment_config_dir, initial_persistent_peers)
# Copy the data directory contents into our deployment
# TODO: change this to work with non local paths
deployment_data_dir = deployment_context.deployment_dir.joinpath("data", "laconicd-data")
deployment_data_dir = context.deployment_dir.joinpath("data", "laconicd-data")
copytree(data_dir_path, deployment_data_dir, dirs_exist_ok=True)
@@ -2,15 +2,14 @@ version: "1.0"
name: mainnet-laconic
description: "Mainnet laconic node"
repos:
- git.vdb.to/cerc-io/laconicd
- github.com/lirewine/debug
- github.com/lirewine/crypto
- github.com/lirewine/gem
- github.com/lirewine/sdk
- git.vdb.to/cerc-io/laconic-sdk
- git.vdb.to/cerc-io/laconic-registry-cli
- git.vdb.to/cerc-io/laconic-console
- github.com/ping-pub/explorer
- cerc-io/laconicd
- lirewine/debug
- lirewine/crypto
- lirewine/gem
- lirewine/sdk
- cerc-io/laconic-sdk
- cerc-io/laconic-registry-cli
- cerc-io/laconic-console
npms:
- laconic-sdk
- laconic-registry-cli
@@ -24,8 +23,7 @@ containers:
- cerc/laconic-registry-cli
- cerc/webapp-base
- cerc/laconic-console-host
- cerc/ping-pub
pods:
- mainnet-laconicd
- fixturenet-laconic-console
- laconic-explorer
@@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file and set following env v
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher to
CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
```
### Start the deployment
@@ -2,7 +2,7 @@ version: "1.0"
name: merkl-sushiswap-v3
description: "SushiSwap v3 watcher stack"
repos:
- github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.14
- github.com/cerc-io/merkl-sushiswap-v3-watcher-ts@v0.1.7
containers:
- cerc/watcher-merkl-sushiswap-v3
pods:
@@ -134,29 +134,6 @@ Note: Use `host.docker.internal` as host to access ports on the host machine
Place the dashboard json files in grafana dashboards config directory (`monitoring-deployment/config/monitoring/grafana/dashboards`) in the deployment folder
#### Graph Node Config
For graph-node dashboard postgres datasource needs to be setup in `monitoring-deployment/config/monitoring/grafana/provisioning/datasources/graph-node-postgres.yml` (in deployment folder)
```yml
# graph-node-postgres.yml
...
datasources:
- name: Graph Node Postgres
type: postgres
jsonData:
# Set name to remote graph-node database name
database: graph-node
...
# Set user to remote graph-node database username
user: graph-node
# Add URL for remote graph-node database
url: graph-node-db:5432
# Set password for graph-node database
secureJsonData:
password: 'password'
```
### Env
Set the following env variables in the deployment env config file (`monitoring-deployment/config.env`):
@@ -179,11 +156,6 @@ Set the following env variables in the deployment env config file (`monitoring-d
# Grafana server host URL (used in various links in alerts, etc.)
# (Optional, default: http://localhost:3000)
GF_SERVER_ROOT_URL=
# RPC endpoint used by graph-node for upstream head metric
# (Optional, default: https://mainnet.infura.io/v3)
GRAPH_NODE_RPC_ENDPOINT=
```
## Start the stack
@@ -57,35 +57,35 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
metrics_path: /metrics
scheme: http
static_configs:
- targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_METRICS_PORT', 'AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_GQL_METRICS_PORT']
- targets: ['AZIMUTH_WATCHER_HOST:AZIMUTH_WATCHER_PORT']
labels:
instance: 'azimuth'
chain: 'ethereum'
- targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_METRICS_PORT', 'CENSURES_WATCHER_HOST:CENSURES_WATCHER_GQL_METRICS_PORT']
- targets: ['CENSURES_WATCHER_HOST:CENSURES_WATCHER_PORT']
labels:
instance: 'censures'
chain: 'ethereum'
- targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_METRICS_PORT', 'CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_GQL_METRICS_PORT']
- targets: ['CLAIMS_WATCHER_HOST:CLAIMS_WATCHER_PORT']
labels:
instance: 'claims'
chain: 'ethereum'
- targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_METRICS_PORT', 'CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_GQL_METRICS_PORT']
- targets: ['CONDITIONAL_STAR_RELEASE_WATCHER_HOST:CONDITIONAL_STAR_RELEASE_WATCHER_PORT']
labels:
instance: 'conditional_star_release'
chain: 'ethereum'
- targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_METRICS_PORT', 'DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_GQL_METRICS_PORT']
- targets: ['DELEGATED_SENDING_WATCHER_HOST:DELEGATED_SENDING_WATCHER_PORT']
labels:
instance: 'delegated_sending'
chain: 'ethereum'
- targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_METRICS_PORT', 'ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_GQL_METRICS_PORT']
- targets: ['ECLIPTIC_WATCHER_HOST:ECLIPTIC_WATCHER_PORT']
labels:
instance: 'ecliptic'
chain: 'ethereum'
- targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_METRICS_PORT', 'LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_GQL_METRICS_PORT']
- targets: ['LINEAR_STAR_WATCHER_HOST:LINEAR_STAR_WATCHER_PORT']
labels:
instance: 'linear_star_release'
chain: 'ethereum'
- targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_METRICS_PORT', 'POLLS_WATCHER_HOST:POLLS_WATCHER_GQL_METRICS_PORT']
- targets: ['POLLS_WATCHER_HOST:POLLS_WATCHER_PORT']
labels:
instance: 'polls'
chain: 'ethereum'
@@ -95,45 +95,39 @@ Add the following scrape configs to prometheus config file (`monitoring-watchers
metrics_path: /metrics
scheme: http
static_configs:
- targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_METRICS_PORT', 'SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_GQL_METRICS_PORT']
- targets: ['SUSHISWAP_WATCHER_HOST:SUSHISWAP_WATCHER_PORT']
labels:
instance: 'sushiswap'
chain: 'filecoin'
- targets: ['MERKLE_SUSHISWAP_WATCHER_HOST:MERKLE_SUSHISWAP_WATCHER_PORT']
labels:
instance: 'merkl_sushiswap'
chain: 'filecoin'
- job_name: ajna
scrape_interval: 20s
metrics_path: /metrics
scheme: http
static_configs:
- targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_METRICS_PORT', 'AJNA_WATCHER_HOST:AJNA_WATCHER_GQL_METRICS_PORT']
- targets: ['AJNA_WATCHER_HOST:AJNA_WATCHER_PORT']
labels:
instance: 'ajna'
chain: 'filecoin'
- job_name: graph-node
metrics_path: /metrics
scrape_interval: 30s
static_configs:
- targets: ['GRAPH_NODE_HOST:GRAPH_NODE_HOST_METRICS_PORT']
```
Add scrape config as done above for any additional watcher to add it to the Watchers dashboard.
### Grafana alerts config
Place the pre-configured alerts rules in Grafana provisioning directory:
Place the pre-configured watcher alerts rules in Grafana provisioning directory:
```bash
# watcher alert rules
cp monitoring-watchers-deployment/config/monitoring/watcher-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
# subgraph alert rules
cp monitoring-watchers-deployment/config/monitoring/subgraph-alert-rules.yml monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/
```
Update the alerting contact points config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/contactpoints.yml`) with desired contact points
Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/config/monitoring/grafana/provisioning/alerting/policies.yml`) with appropriate object-matchers:
Add corresponding routes to the notification policies config (`monitoring-watchers-deployment/monitoring/grafana/provisioning/alerting/policies.yaml`) with appropriate object-matchers:
```yml
...
@@ -141,7 +135,7 @@ Add corresponding routes to the notification policies config (`monitoring-watche
- receiver: SlackNotifier
object_matchers:
# Add matchers below
- ['grafana_folder', '=~', 'WatcherAlerts|SubgraphAlerts']
- ['grafana_folder', '=', 'WatcherAlerts']
```
### Env
@@ -155,9 +149,6 @@ Set the following env variables in the deployment env config file (`monitoring-w
# Grafana server host URL to be used
# (Optional, default: http://localhost:3000)
GF_SERVER_ROOT_URL=
# List of subgraph ids to configure alerts for (separated by |)
CERC_GRAFANA_ALERTS_SUBGRAPH_IDS=
```
## Start the stack
@@ -1,7 +1,7 @@
version: "0.1"
name: monitoring
repos:
- github.com/cerc-io/watcher-ts@v0.2.92
- github.com/cerc-io/watcher-ts@v0.2.81
containers:
- cerc/watcher-ts
pods:
@@ -0,0 +1,10 @@
version: "1.0"
name: snowballtools-base-backend
description: "snowballtools-base-backend"
repos:
- github.com/snowball-tools/snowballtools-base
containers:
- cerc/webapp-base
- cerc/snowballtools-base-backend
pods:
- snowballtools-base-backend
@@ -53,7 +53,7 @@ Inside deployment directory, open the `config.env` file and set following env v
```bash
# External Filecoin (ETH RPC) endpoint to point the watcher to
CERC_ETH_RPC_ENDPOINTS=https://example-lotus-endpoint-1/rpc/v1,https://example-lotus-endpoint-2/rpc/v1
CERC_ETH_RPC_ENDPOINT=https://example-lotus-endpoint/rpc/v1
```
### Start the deployment
@@ -2,7 +2,7 @@ version: "1.0"
name: sushiswap-v3
description: "SushiSwap v3 watcher stack"
repos:
- github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.16
- github.com/cerc-io/sushiswap-v3-watcher-ts@v0.1.7
containers:
- cerc/watcher-sushiswap-v3
pods:
+10 -14
View File
@@ -26,15 +26,8 @@ import click
from pathlib import Path
from stack_orchestrator import constants
from stack_orchestrator.opts import opts
from stack_orchestrator.util import (
get_stack_path,
include_exclude_check,
get_parsed_stack_config,
global_options2,
get_dev_root_path,
stack_is_in_deployment,
resolve_compose_file,
)
from stack_orchestrator.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path
from stack_orchestrator.util import resolve_compose_file
from stack_orchestrator.deploy.deployer import Deployer, DeployerException
from stack_orchestrator.deploy.deployer_factory import getDeployer
from stack_orchestrator.deploy.deploy_types import ClusterContext, DeployCommandContext
@@ -67,7 +60,6 @@ def command(ctx, include, exclude, env_file, cluster, deploy_to):
if deploy_to is None:
deploy_to = "compose"
stack = get_stack_path(stack)
ctx.obj = create_deploy_context(global_options2(ctx), None, stack, include, exclude, cluster, env_file, deploy_to)
# Subcommand is executed now, by the magic of click
@@ -282,12 +274,16 @@ def _make_default_cluster_name(deployment, compose_dir, stack, include, exclude)
# stack has to be either PathLike pointing to a stack yml file, or a string with the name of a known stack
def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
dev_root_path = get_dev_root_path(ctx)
# TODO: hack, this should be encapsulated by the deployment context.
deployment = stack_is_in_deployment(stack)
if deployment:
compose_dir = stack.joinpath("compose")
# TODO: huge hack, fix this
# If the caller passed a path for the stack file, then we know that we can get the compose files
# from the same directory
deployment = False
if isinstance(stack, os.PathLike):
compose_dir = stack.parent.joinpath("compose")
deployment = True
else:
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
compose_dir = Path(__file__).absolute().parent.parent.joinpath("data", "compose")
+3 -4
View File
@@ -50,15 +50,15 @@ def command(ctx, dir):
def make_deploy_context(ctx) -> DeployCommandContext:
context: DeploymentContext = ctx.obj
stack_file_path = context.get_stack_file()
env_file = context.get_env_file()
cluster_name = context.get_cluster_id()
if constants.deploy_to_key in context.spec.obj:
deployment_type = context.spec.obj[constants.deploy_to_key]
else:
deployment_type = constants.compose_deploy_type
stack = context.deployment_dir
return create_deploy_context(ctx.parent.parent.obj, context, stack, None, None,
cluster_name, env_file, deployment_type)
return create_deploy_context(ctx.parent.parent.obj, context, stack_file_path, None, None, cluster_name, env_file,
deployment_type)
@command.command()
@@ -123,7 +123,6 @@ def push_images(ctx):
@click.argument('extra_args', nargs=-1) # help: command: port <service1> <service2>
@click.pass_context
def port(ctx, extra_args):
ctx.obj = make_deploy_context(ctx)
port_operation(ctx, extra_args)
@@ -24,7 +24,7 @@ from secrets import token_hex
import sys
from stack_orchestrator import constants
from stack_orchestrator.opts import opts
from stack_orchestrator.util import (get_stack_path, get_parsed_deployment_spec, get_parsed_stack_config,
from stack_orchestrator.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config,
global_options, get_yaml, get_pod_list, get_pod_file_path, pod_has_scripts,
get_pod_script_paths, get_plugin_code_paths, error_exit, env_var_map_from_file,
resolve_config_dir)
@@ -238,11 +238,6 @@ def _find_extra_config_dirs(parsed_pod_file, pod):
config_dir = host_path.split("/")[2]
if config_dir != pod:
config_dirs.add(config_dir)
for env_file in service_info.get("env_file", []):
if env_file.startswith("../config"):
config_dir = env_file.split("/")[2]
if config_dir != pod:
config_dirs.add(config_dir)
return config_dirs
@@ -459,7 +454,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
_check_volume_definitions(parsed_spec)
stack_name = parsed_spec["stack"]
deployment_type = parsed_spec[constants.deploy_to_key]
stack_file = get_stack_path(stack_name).joinpath(constants.stack_file_name)
stack_file = get_stack_file_path(stack_name)
parsed_stack = get_parsed_stack_config(stack_name)
if opts.o.debug:
print(f"parsed spec: {parsed_spec}")
@@ -472,7 +467,7 @@ def create_operation(deployment_command_context, spec_file, deployment_dir, netw
os.mkdir(deployment_dir_path)
# Copy spec file and the stack file into the deployment dir
copyfile(spec_file, deployment_dir_path.joinpath(constants.spec_file_name))
copyfile(stack_file, deployment_dir_path.joinpath(constants.stack_file_name))
copyfile(stack_file, deployment_dir_path.joinpath(os.path.basename(stack_file)))
_create_deployment_file(deployment_dir_path)
# Copy any config varibles from the spec file into an env file suitable for compose
_write_config_file(spec_file, deployment_dir_path.joinpath(constants.config_file_name))
+7 -22
View File
@@ -29,29 +29,16 @@ def _image_needs_pushed(image: str):
return image.endswith(":local")
def _remote_tag_for_image(image: str, remote_repo_url: str):
# Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
major_parts = image.split("/", 2)
image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
(image_name, image_version) = image_name_with_version.split(":")
if image_version == "local":
return f"{remote_repo_url}/{image_name}:deploy"
else:
return image
# Note: do not add any calls this function
def remote_image_exists(remote_repo_url: str, local_tag: str):
docker = DockerClient()
try:
remote_tag = _remote_tag_for_image(local_tag, remote_repo_url)
remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
result = docker.manifest.inspect(remote_tag)
return True if result else False
except Exception: # noqa: E722
return False
# Note: do not add any calls this function
def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
if not additional_tags:
return
@@ -60,20 +47,18 @@ def add_tags_to_image(remote_repo_url: str, local_tag: str, *additional_tags):
raise Exception(f"{local_tag} does not exist in {remote_repo_url}")
docker = DockerClient()
remote_tag = _remote_tag_for_image(local_tag, remote_repo_url)
new_remote_tags = [_remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
remote_tag = remote_tag_for_image(local_tag, remote_repo_url)
new_remote_tags = [remote_tag_for_image(tag, remote_repo_url) for tag in additional_tags]
docker.buildx.imagetools.create(sources=[remote_tag], tags=new_remote_tags)
def remote_tag_for_image_unique(image: str, remote_repo_url: str, deployment_id: str):
def remote_tag_for_image(image: str, remote_repo_url: str):
# Turns image tags of the form: foo/bar:local into remote.repo/org/bar:deploy
major_parts = image.split("/", 2)
image_name_with_version = major_parts[1] if 2 == len(major_parts) else major_parts[0]
(image_name, image_version) = image_name_with_version.split(":")
if image_version == "local":
# Salt the tag with part of the deployment id to make it unique to this deployment
deployment_tag = deployment_id[-8:]
return f"{remote_repo_url}/{image_name}:deploy-{deployment_tag}"
return f"{remote_repo_url}/{image_name}:deploy"
else:
return image
@@ -88,14 +73,14 @@ def push_images_operation(command_context: DeployCommandContext, deployment_cont
docker = DockerClient()
for image in images:
if _image_needs_pushed(image):
remote_tag = remote_tag_for_image_unique(image, remote_repo_url, deployment_context.id)
remote_tag = remote_tag_for_image(image, remote_repo_url)
if opts.o.verbose:
print(f"Tagging {image} to {remote_tag}")
docker.image.tag(image, remote_tag)
# Run docker push commands to upload
for image in images:
if _image_needs_pushed(image):
remote_tag = remote_tag_for_image_unique(image, remote_repo_url, deployment_context.id)
remote_tag = remote_tag_for_image(image, remote_repo_url)
if opts.o.verbose:
print(f"Pushing image {remote_tag}")
docker.image.push(remote_tag)
@@ -26,7 +26,7 @@ from stack_orchestrator.deploy.k8s.helpers import envs_from_environment_variable
from stack_orchestrator.deploy.deploy_util import parsed_pod_files_map_from_file_names, images_for_deployment
from stack_orchestrator.deploy.deploy_types import DeployEnvVars
from stack_orchestrator.deploy.spec import Spec, Resources, ResourceLimits
from stack_orchestrator.deploy.images import remote_tag_for_image_unique
from stack_orchestrator.deploy.images import remote_tag_for_image
DEFAULT_VOLUME_RESOURCES = Resources({
"reservations": {"storage": "2Gi"}
@@ -326,11 +326,8 @@ class ClusterInfo:
if opts.o.debug:
print(f"Merged envs: {envs}")
# Re-write the image tag for remote deployment
# Note self.app_name has the same value as deployment_id
image_to_use = remote_tag_for_image_unique(
image,
self.spec.get_image_registry(),
self.app_name) if self.spec.get_image_registry() is not None else image
image_to_use = remote_tag_for_image(
image, self.spec.get_image_registry()) if self.spec.get_image_registry() is not None else image
volume_mounts = volume_mounts_for_service(self.parsed_pod_yaml_map, service_name)
container = client.V1Container(
name=container_name,
@@ -24,7 +24,7 @@ import uuid
import click
from stack_orchestrator.deploy.images import remote_image_exists
from stack_orchestrator.deploy.images import remote_image_exists, add_tags_to_image
from stack_orchestrator.deploy.webapp import deploy_webapp
from stack_orchestrator.deploy.webapp.util import (LaconicRegistryClient, TimedLogger,
build_container_image, push_container_image,
@@ -99,62 +99,44 @@ def process_app_deployment_request(
deployment_record = laconic.get_record(app_deployment_crn)
deployment_dir = os.path.join(deployment_parent_dir, fqdn)
# At present we use this to generate a unique but stable ID for the app's host container
# TODO: implement support to derive this transparently from the already-unique deployment id
unique_deployment_id = hashlib.md5(fqdn.encode()).hexdigest()[:16]
deployment_config_file = os.path.join(deployment_dir, "config.env")
deployment_container_tag = "laconic-webapp/%s:local" % unique_deployment_id
# TODO: Is there any reason not to simplify the hash input to the app_deployment_crn?
deployment_container_tag = "laconic-webapp/%s:local" % hashlib.md5(deployment_dir.encode()).hexdigest()
app_image_shared_tag = f"laconic-webapp/{app.id}:local"
# b. check for deployment directory (create if necessary)
if not os.path.exists(deployment_dir):
if deployment_record:
raise Exception("Deployment record %s exists, but not deployment dir %s. Please remove name." %
(app_deployment_crn, deployment_dir))
logger.log(f"Creating webapp deployment in: {deployment_dir} with container id: {deployment_container_tag}")
print("deploy_webapp", deployment_dir)
deploy_webapp.create_deployment(ctx, deployment_dir, deployment_container_tag,
f"https://{fqdn}", kube_config, image_registry, env_filename)
elif env_filename:
shutil.copyfile(env_filename, deployment_config_file)
needs_k8s_deploy = False
if force_rebuild:
logger.log("--force-rebuild is enabled so the container will always be built now, even if nothing has changed in the app")
# 6. build container (if needed)
# TODO: add a comment that explains what this code is doing (not clear to me)
if not deployment_record or deployment_record.attributes.application != app.id or force_rebuild:
if not deployment_record or deployment_record.attributes.application != app.id:
needs_k8s_deploy = True
# check if the image already exists
shared_tag_exists = remote_image_exists(image_registry, app_image_shared_tag)
# Note: in the code below, calls to add_tags_to_image() won't work at present.
# This is because SO deployment code in general re-names the container image
# to be unique to the deployment. This is done transparently
# and so when we call add_tags_to_image() here and try to add tags to the remote image,
# we get the image name wrong. Accordingly I've disabled the relevant code for now.
# This is safe because we are running with --force-rebuild at present
if shared_tag_exists and not force_rebuild:
# simply add our unique tag to the existing image and we are done
logger.log(
f"(SKIPPED) Existing image found for this app: {app_image_shared_tag} "
"tagging it with: {deployment_container_tag} to use in this deployment"
)
# add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
logger.log(f"Using existing app image {app_image_shared_tag} for {deployment_container_tag}")
add_tags_to_image(image_registry, app_image_shared_tag, deployment_container_tag)
logger.log("Tag complete")
else:
extra_build_args = [] # TODO: pull from request
logger.log(f"Building container image: {deployment_container_tag}")
logger.log(f"Building container image {deployment_container_tag}")
build_container_image(app, deployment_container_tag, extra_build_args, logger)
logger.log("Build complete")
logger.log(f"Pushing container image: {deployment_container_tag}")
logger.log(f"Pushing container image {deployment_container_tag}")
push_container_image(deployment_dir, logger)
logger.log("Push complete")
# The build/push commands above will use the unique deployment tag, so now we need to add the shared tag.
logger.log(
f"(SKIPPED) Adding global app image tag: {app_image_shared_tag} to newly built image: {deployment_container_tag}"
)
# add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
logger.log(f"Updating app image tag {app_image_shared_tag} from build of {deployment_container_tag}")
add_tags_to_image(image_registry, deployment_container_tag, app_image_shared_tag)
logger.log("Tag complete")
else:
logger.log("Requested app is already deployed, skipping build and image push")
# 7. update config (if needed)
if not deployment_record or file_hash(deployment_config_file) != deployment_record.attributes.meta.config:
+1 -14
View File
@@ -242,7 +242,6 @@ def determine_base_container(clone_dir, app_type="webapp"):
def build_container_image(app_record, tag, extra_build_args=[], logger=None):
tmpdir = tempfile.mkdtemp()
# TODO: determine if this code could be calling into the Python git library like setup-repositories
try:
record_id = app_record["id"]
ref = app_record.attributes.repository_ref
@@ -250,16 +249,6 @@ def build_container_image(app_record, tag, extra_build_args=[], logger=None):
clone_dir = os.path.join(tmpdir, record_id)
logger.log(f"Cloning repository {repo} to {clone_dir} ...")
# Set github credentials if present running a command like:
# git config --global url."https://${TOKEN}:@github.com/".insteadOf "https://github.com/"
github_token = os.environ.get("DEPLOYER_GITHUB_TOKEN")
if github_token:
logger.log("Github token detected, setting it in the git environment")
git_config_args = [
"git", "config", "--global", f"url.https://{github_token}:@github.com/.insteadOf", "https://github.com/"
]
result = subprocess.run(git_config_args, stdout=logger.file, stderr=logger.file)
result.check_returncode()
if ref:
# TODO: Determing branch or hash, and use depth 1 if we can.
git_env = dict(os.environ.copy())
@@ -276,7 +265,6 @@ def build_container_image(app_record, tag, extra_build_args=[], logger=None):
logger.log(f"git checkout failed. Does ref {ref} exist?")
raise e
else:
# TODO: why is this code different vs the branch above (run vs check_call, and no prompt disable)?
result = subprocess.run(["git", "clone", "--depth", "1", repo, clone_dir], stdout=logger.file, stderr=logger.file)
result.check_returncode()
@@ -311,12 +299,11 @@ def push_container_image(deployment_dir, logger):
def deploy_to_k8s(deploy_record, deployment_dir, logger):
if not deploy_record:
command = "start"
command = "up"
else:
command = "update"
logger.log("Deploying to k8s ...")
logger.log(f"Running {command} command on deployment dir: {deployment_dir}")
result = subprocess.run([sys.argv[0], "deployment", "--dir", deployment_dir, command],
stdout=logger.file, stderr=logger.file)
result.check_returncode()
+23 -15
View File
@@ -20,12 +20,14 @@ import os
import sys
from decouple import config
import git
from git.exc import GitCommandError
from tqdm import tqdm
import click
import importlib.resources
from pathlib import Path
import yaml
from stack_orchestrator.constants import stack_file_name
from stack_orchestrator.opts import opts
from stack_orchestrator.util import get_parsed_stack_config, include_exclude_check, error_exit, warn_exit
from stack_orchestrator.util import include_exclude_check, stack_is_external, error_exit, warn_exit
class GitProgress(git.RemoteProgress):
@@ -79,13 +81,9 @@ def _get_repo_current_branch_or_tag(full_filesystem_repo_path):
except TypeError:
# This means that the current ref is not a branch, so possibly a tag
# Let's try to get the tag
try:
current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).git.describe("--tags", "--exact-match")
# Note that git is asymmetric -- the tag you told it to check out may not be the one
# you get back here (if there are multiple tags associated with the same commit)
except GitCommandError:
# If there is no matching branch or tag checked out, just use the current SHA
current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).commit("HEAD").hexsha
current_repo_branch_or_tag = git.Repo(full_filesystem_repo_path).git.describe("--tags", "--exact-match")
# Note that git is assymetric -- the tag you told it to check out may not be the one
# you get back here (if there are multiple tags associated with the same commit)
return current_repo_branch_or_tag, is_branch
@@ -104,7 +102,7 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully
full_filesystem_repo_path
) if is_present else (None, None)
if not opts.o.quiet:
present_text = f"already exists active {'branch' if is_branch else 'ref'}: {current_repo_branch_or_tag}" if is_present \
present_text = f"already exists active {'branch' if is_branch else 'tag'}: {current_repo_branch_or_tag}" if is_present \
else 'Needs to be fetched'
print(f"Checking: {full_filesystem_repo_path}: {present_text}")
# Quick check that it's actually a repo
@@ -122,7 +120,7 @@ def process_repo(pull, check_only, git_ssh, dev_root_path, branches_array, fully
origin = git_repo.remotes.origin
origin.pull(progress=None if opts.o.quiet else GitProgress())
else:
print("skipping pull because this repo is not on a branch")
print("skipping pull because this repo checked out a tag")
else:
print("(git pull skipped)")
if not is_present:
@@ -224,10 +222,20 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches):
repos_in_scope = []
if stack:
stack_config = get_parsed_stack_config(stack)
if "repos" not in stack_config or stack_config["repos"] is None:
warn_exit(f"stack {stack} does not define any repositories")
repos_in_scope = stack_config["repos"]
if stack_is_external(stack):
stack_file_path = Path(stack).joinpath(stack_file_name)
else:
# In order to be compatible with Python 3.8 we need to use this hack to get the path:
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, stack_file_name)
if not stack_file_path.exists():
error_exit(f"stack {stack} does not exist")
with stack_file_path:
stack_config = yaml.safe_load(open(stack_file_path, "r"))
if "repos" not in stack_config or stack_config["repos"] is None:
warn_exit(f"stack {stack} does not define any repositories")
else:
repos_in_scope = stack_config["repos"]
else:
repos_in_scope = all_repos
+21 -22
View File
@@ -20,7 +20,6 @@ import ruamel.yaml
from pathlib import Path
from dotenv import dotenv_values
from typing import Mapping, Set, List
from stack_orchestrator.constants import stack_file_name, deployment_file_name
def include_exclude_check(s, include, exclude):
@@ -34,14 +33,11 @@ def include_exclude_check(s, include, exclude):
return s not in exclude_list
def get_stack_path(stack):
if stack_is_external(stack):
stack_path = Path(stack)
else:
# In order to be compatible with Python 3.8 we need to use this hack to get the path:
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
stack_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack)
return stack_path
def get_stack_file_path(stack):
# In order to be compatible with Python 3.8 we need to use this hack to get the path:
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
stack_file_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack, "stack.yml")
return stack_file_path
def get_dev_root_path(ctx):
@@ -56,14 +52,21 @@ def get_dev_root_path(ctx):
# Caller can pass either the name of a stack, or a path to a stack file
def get_parsed_stack_config(stack):
stack_file_path = get_stack_path(stack).joinpath(stack_file_name)
if stack_file_path.exists():
return get_yaml().load(open(stack_file_path, "r"))
# We try here to generate a useful diagnostic error
# First check if the stack directory is present
if stack_file_path.parent.exists():
error_exit(f"stack.yml file is missing from stack: {stack}")
error_exit(f"stack {stack} does not exist")
stack_file_path = stack if isinstance(stack, os.PathLike) else get_stack_file_path(stack)
try:
with stack_file_path:
stack_config = get_yaml().load(open(stack_file_path, "r"))
return stack_config
except FileNotFoundError as error:
# We try here to generate a useful diagnostic error
# First check if the stack directory is present
stack_directory = stack_file_path.parent
if os.path.exists(stack_directory):
print(f"Error: stack.yml file is missing from stack: {stack}")
else:
print(f"Error: stack: {stack} does not exist")
print(f"Exiting, error: {error}")
sys.exit(1)
def get_pod_list(parsed_stack):
@@ -84,7 +87,7 @@ def get_plugin_code_paths(stack) -> List[Path]:
result: Set[Path] = set()
for pod in pods:
if type(pod) is str:
result.add(get_stack_path(stack))
result.add(get_stack_file_path(stack).parent)
else:
pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"])
result.add(Path(os.path.join(pod_root_dir, "stack")))
@@ -196,10 +199,6 @@ def stack_is_external(stack: str):
return Path(stack).exists() if stack is not None else False
def stack_is_in_deployment(stack: Path):
return stack.joinpath(deployment_file_name).exists()
def get_yaml():
# See: https://stackoverflow.com/a/45701840/1701505
yaml = ruamel.yaml.YAML()
+5 -8
View File
@@ -14,7 +14,7 @@
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import click
from importlib import resources, metadata
import importlib.resources
@click.command()
@@ -24,11 +24,8 @@ def command(ctx):
# See: https://stackoverflow.com/a/20885799/1701505
from stack_orchestrator import data
if resources.is_resource(data, "build_tag.txt"):
with resources.open_text(data, "build_tag.txt") as version_file:
# TODO: code better version that skips comment lines
version_string = version_file.read().splitlines()[1]
else:
version_string = metadata.version("laconic-stack-orchestrator") + "-unknown"
with importlib.resources.open_text(data, "build_tag.txt") as version_file:
# TODO: code better version that skips comment lines
version_string = version_file.read().splitlines()[1]
print(version_string)
print(f"Version: {version_string}")
+4 -4
View File
@@ -22,16 +22,16 @@ echo "$(date +"%Y-%m-%d %T"): Stack started"
# Verify that the fixturenet is up and running
$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd ps
# Wait for the laconid endpoint to come up
echo "Waiting for the RPC endpoint to come up"
docker exec laconicd-laconicd-1 sh -c "curl --retry 20 --retry-delay 3 --retry-connrefused http://127.0.0.1:9473/api"
# Get the fixturenet account address
laconicd_account_address=$(docker exec laconicd-laconicd-1 laconicd keys list | awk '/- address:/ {print $3}')
# Copy over config
docker exec laconicd-cli-1 cp config.yml laconic-registry-cli/
# Wait for the laconid endpoint to come up
echo "Waiting for the RPC endpoint to come up"
docker exec laconicd-laconicd-1 sh -c "curl --retry 20 --retry-delay 3 --retry-connrefused http://127.0.0.1:9473/api"
# Run the tests
echo "Running the tests"
docker exec -e TEST_ACCOUNT=$laconicd_account_address laconicd-cli-1 sh -c 'cd laconic-registry-cli && yarn && yarn test'
-23
View File
@@ -18,20 +18,6 @@ do
rm -rf ${node_network_dir}
fi
done
echo "Deleting any existing deployments..."
for (( i=1 ; i<=$node_count ; i++ ));
do
node_deployment_dir=${node_dir_prefix}${i}-deployment
node_spec_file=${node_dir_prefix}${i}-spec.yml
if [[ -d $node_deployment_dir ]]; then
echo "Deleting ${node_deployment_dir}"
rm -rf ${node_deployment_dir}
fi
if [[ -f $node_spec_file ]]; then
echo "Deleting ${node_spec_file}"
rm ${node_spec_file}
fi
done
echo "Initalizing ${node_count} nodes networks..."
for (( i=1 ; i<=$node_count ; i++ ));
@@ -70,12 +56,3 @@ do
node_network_dir=${node_dir_prefix}${i}
laconic-so --stack mainnet-laconic deploy setup --network-dir ${node_network_dir} --create-network --genesis-file ${genesis_file}
done
# Create deployments
echo "Creating ${node_count} deployments..."
for (( i=1 ; i<=$node_count ; i++ ));
do
node_network_dir=${node_dir_prefix}${i}
laconic-so --stack mainnet-laconic deploy init --output ${node_network_dir}-spec.yml
laconic-so --stack mainnet-laconic deploy create --deployment-dir ${node_network_dir}-deployment --spec-file ${node_dir_prefix}${i}-spec.yml --network-dir ${node_network_dir}
done