Compare commits

..
Author SHA1 Message Date
iskay bd8c78e1a3 update fixturenet-plugeth test 2023-11-02 09:26:47 -04:00
telackey fd5779f967 Fix KeyError accessing config. (#620) 2023-10-31 12:29:19 -05:00
Ian d854dd5c81 Update fixturenet-laconicd.yml 2023-10-30 15:44:46 -04:00
Ian 948f9f4287 Merge pull request #611 from cerc-io/iskay/fixturenet-laconicd-test
add fixturenet-laconicd test
2023-10-30 11:50:33 -04:00
dboreham b92d9cd7dd Update stack README.md to use config directive 2023-10-29 23:16:39 -06:00
dboreham 86076c7ed8 Fix deployer.exec() (#619) 2023-10-29 22:26:15 -06:00
telackey 8cac598679 Split act-runner into its own pod and offer as a distinct stack. (#612)
* Split act-runner into its own pod and offer as a distinct stack.
2023-10-27 13:57:13 -05:00
dboreham 6130eab5cb k8s deploy (#614) 2023-10-27 10:19:44 -06:00
Ian f198f43b3a add newline 2023-10-27 08:48:44 -04:00
iskay 36d1e0eedd add fixturenet-laconicd test 2023-10-26 17:26:42 -04:00
telackey 0f5b1a097b Add plugeth to chain-chunker stack (needed for new verify option). (#610) 2023-10-25 14:47:53 -05:00
telackeyanddboreham 20d633f81c Plugeth-based full mainnet stack. (#592)
* Plugeth-based full mainnet stack.

---------

Co-authored-by: David Boreham <david@bozemanpass.com>
2023-10-25 14:42:52 -05:00
dboreham 5e36e3e2ae Turn off long run time test on push/pr unless explicitly triggered (#606) 2023-10-24 22:28:58 -06:00
dboreham f7eb8b9a38 Rearrange files (#605) 2023-10-24 22:13:09 -06:00
dboreham 5b9b12a223 Rename functions to remove compose prefix (#604) 2023-10-24 16:23:28 -06:00
Ian 567dadef7d update fixturenet-eth test (#600) 2023-10-24 16:21:30 -06:00
dboreham 052f0df4b0 Fix execute call parameter (#603) 2023-10-24 16:16:57 -06:00
dboreham c51671f786 Trigger CI Job after k8s refactor updates 2023-10-24 14:51:04 -06:00
dboreham 573a19a3b7 k8s refactor (#595) 2023-10-24 14:44:48 -06:00
Zach fc051265d8 more demo records for the console (#597)
* increase gas

* create six demo records

* print record

* use moon

* typos

* use git.vdb.t

* use the right moon

* ok
2023-10-24 14:25:25 -04:00
Zach ddebb9c690 bind only to 127.0.0.1 (#598) 2023-10-23 10:51:44 -04:00
Zach fe6c3f92ed typo (#596) 2023-10-21 13:37:21 -04:00
Zach c06be6da81 stack for the laconic.com website (#590)
* website stack

* use a release, update ports

* use deployments feature for website
2023-10-21 13:31:47 -04:00
telackey 3291c16466 Update stacks for migrated laconic repos. (#594) 2023-10-20 11:59:06 -05:00
nabarun 69b071ee7a Add example GQL queries made in Ponder indexers (#593)
* Add example GQL queries made in Ponder indexers

* Use fixturenet chainId in GQL query
2023-10-19 14:55:07 +05:30
nabarunandShreerang Kale 4a90cedeb2 Run multiple Ponder indexers in payment stack (#588)
* Separate ponder indexer and ponder watcher and add second ponder indexer

* Handle review changes

* Update config to point ponder watcher to indexer 2 to indexer 1

* Update Ponder demo

* Use deployed ERC20 contract in second Ponder indexer

* Add order by timestamp in Ponder watcher app entities query

* Upgrade go-nitro version to v0.1.2-ts-port-0.1.9

* Decrease Ponder start block to process contract transfer event at deployment

---------

Co-authored-by: Shreerang Kale <shreerangkale@gmail.com>
2023-10-19 12:26:10 +05:30
95 changed files with 2210 additions and 314 deletions
@@ -0,0 +1,55 @@
name: Fixturenet-Laconicd-Test
on:
push:
branches: '*'
paths:
- '!**'
- '.gitea/workflows/triggers/fixturenet-laconicd-test'
# Needed until we can incorporate docker startup into the executor container
env:
DOCKER_HOST: unix:///var/run/dind.sock
jobs:
test:
name: "Run an Laconicd fixturenet test"
runs-on: ubuntu-latest
steps:
- name: 'Update'
run: apt-get update
- name: 'Setup jq'
run: apt-get install jq -y
- name: 'Check jq'
run: |
which jq
jq --version
- name: "Clone project repository"
uses: actions/checkout@v3
# At present the stock setup-python action fails on Linux/aarch64
# Conditional steps below workaroud this by using deadsnakes for that case only
- name: "Install Python for ARM on Linux"
if: ${{ runner.arch == 'arm64' && runner.os == 'Linux' }}
uses: deadsnakes/action@v3.0.1
with:
python-version: '3.8'
- name: "Install Python cases other than ARM on Linux"
if: ${{ ! (runner.arch == 'arm64' && runner.os == 'Linux') }}
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
run: ./scripts/build_shiv_package.sh
- name: Start dockerd # Also needed until we can incorporate into the executor
run: |
dockerd -H $DOCKER_HOST --userland-proxy=false &
sleep 5
- name: "Run fixturenet-laconicd tests"
run: ./tests/fixturenet-laconicd/run-test.sh
@@ -1,2 +1,2 @@
Change this file to trigger running the fixturenet-eth-test CI job
trigger
@@ -0,0 +1,2 @@
Change this file to trigger running the fixturenet-laconicd-test CI job
+30
View File
@@ -0,0 +1,30 @@
name: Fixturenet-Eth Test
on:
push:
branches: '*'
paths:
- '!**'
- '.github/workflows/triggers/fixturenet-eth-test'
jobs:
test:
name: "Run fixturenet-eth test suite"
runs-on: ubuntu-latest
steps:
- name: "Clone project repository"
uses: actions/checkout@v3
- name: "Install Python"
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
run: ./scripts/build_shiv_package.sh
- name: "Run fixturenet-eth tests"
run: ./tests/fixturenet-eth/run-test.sh
+30
View File
@@ -0,0 +1,30 @@
name: Fixturenet-Laconicd Test
on:
push:
branches: '*'
paths:
- '!**'
- '.github/workflows/triggers/fixturenet-laconicd-test'
jobs:
test:
name: "Run fixturenet-laconicd test suite"
runs-on: ubuntu-latest
steps:
- name: "Clone project repository"
uses: actions/checkout@v3
- name: "Install Python"
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
run: ./scripts/build_shiv_package.sh
- name: "Run fixturenet-laconicd tests"
run: ./tests/fixturenet-laconicd/run-test.sh
+30
View File
@@ -0,0 +1,30 @@
name: Fixturenet-Plugeth Test
on:
push:
branches: '*'
paths:
- '!**'
- '.github/workflows/triggers/fixturenet-plugeth-test'
jobs:
test:
name: "Run fixturenet-plugeth test suite"
runs-on: ubuntu-latest
steps:
- name: "Clone project repository"
uses: actions/checkout@v3
- name: "Install Python"
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: "Print Python version"
run: python3 --version
- name: "Install shiv"
run: pip install shiv
- name: "Generate build version file"
run: ./scripts/create_build_tag_file.sh
- name: "Build local shiv package"
run: ./scripts/build_shiv_package.sh
- name: "Run fixturenet-plugeth tests"
run: ./tests/fixturenet-plugeth/run-test.sh
@@ -0,0 +1,2 @@
Change this file to trigger running the fixturenet-eth-test CI job
@@ -0,0 +1,3 @@
Change this file to trigger running the fixturenet-laconicd-test CI job
trigger
@@ -0,0 +1,3 @@
Change this file to trigger running the fixturenet-plugeth-test CI job
trigger
+1 -1
View File
@@ -7,4 +7,4 @@ __pycache__
*~
package
app/data/build_tag.txt
build
/build
+1 -1
View File
@@ -52,7 +52,7 @@ Version: 1.1.0-7a607c2-202304260513
Save the distribution url to `~/.laconic-so/config.yml`:
```bash
mkdir ~/.laconic-so
echo "distribution-url: https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so" > ~/.laconic-so/config.yml"
echo "distribution-url: https://github.com/cerc-io/stack-orchestrator/releases/latest/download/laconic-so" > ~/.laconic-so/config.yml
```
### Update
+1 -1
View File
@@ -15,7 +15,7 @@
import os
from abc import ABC, abstractmethod
from app.deploy import get_stack_status
from app.deploy.deploy import get_stack_status
from decouple import config
View File
@@ -52,7 +52,7 @@ def command(ctx, include, exclude, force_rebuild, extra_build_args):
continue_on_error = ctx.obj.continue_on_error
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
container_build_dir = Path(__file__).absolute().parent.joinpath("data", "container-build")
container_build_dir = Path(__file__).absolute().parent.parent.joinpath("data", "container-build")
if local_stack:
dev_root_path = os.getcwd()[0:os.getcwd().rindex("stack-orchestrator")]
@@ -0,0 +1,8 @@
version: "3.2"
services:
laconic-dot-com:
image: cerc/laconic-dot-com:local
restart: always
ports:
- "3000"
@@ -0,0 +1,29 @@
version: "3.2"
services:
migrations:
restart: on-failure
depends_on:
ipld-eth-db:
condition: service_healthy
image: cerc/ipld-eth-db:local
env_file:
- ../config/mainnet-eth-ipld-eth-db/db.env
ipld-eth-db:
image: timescale/timescaledb:2.8.1-pg14
restart: always
env_file:
- ../config/mainnet-eth-ipld-eth-db/db.env
volumes:
- mainnet_eth_ipld_eth_db:/var/lib/postgresql/data
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "5432"]
interval: 30s
timeout: 10s
retries: 10
start_period: 3s
ports:
- "5432"
volumes:
mainnet_eth_ipld_eth_db:
@@ -0,0 +1,24 @@
version: "3.7"
services:
ipld-eth-server:
restart: always
depends_on:
ipld-eth-db:
condition: service_healthy
image: cerc/ipld-eth-server:local
env_file:
- ../config/mainnet-eth-ipld-eth-db/db.env
- ../config/mainnet-eth-ipld-eth-server/srv.env
volumes:
- ../config/mainnet-eth-ipld-eth-server/config.toml:/app/config.toml:ro
ports:
- "8081"
- "8082"
- "8090"
- "40001"
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "8081"]
interval: 20s
timeout: 5s
retries: 15
start_period: 5s
@@ -6,7 +6,7 @@ services:
env_file:
- ../config/mainnet-eth-keycloak/keycloak.env
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "5432"]
test: ["CMD", "nc", "-v", "localhost", "35432"]
interval: 30s
timeout: 10s
retries: 10
@@ -14,7 +14,7 @@ services:
volumes:
- mainnet_eth_keycloak_db:/var/lib/postgresql/data
ports:
- 5432
- 35432
keycloak:
image: cerc/keycloak:local
@@ -0,0 +1,72 @@
services:
mainnet-eth-geth-1:
restart: always
hostname: mainnet-eth-geth-1
cap_add:
- SYS_PTRACE
image: cerc/plugeth-with-plugins:local
entrypoint: /bin/sh
command: -c "/opt/run-geth.sh"
env_file:
- ../config/mainnet-eth-ipld-eth-db/db.env
- ../config/mainnet-eth-plugeth/geth.env
volumes:
- mainnet_eth_plugeth_geth_1_data:/data
- mainnet_eth_plugeth_config_data:/etc/mainnet-eth
- ../config/mainnet-eth-plugeth/scripts/run-geth.sh:/opt/run-geth.sh
healthcheck:
test: ["CMD", "nc", "-v", "localhost", "8545"]
interval: 30s
timeout: 10s
retries: 10
start_period: 3s
ports:
# http api
- "8545"
# ws api
- "8546"
# ws el
- "8551"
# p2p
- "30303"
- "30303/udp"
# debugging
- "40000"
# metrics
- "6060"
mainnet-eth-lighthouse-1:
restart: always
hostname: mainnet-eth-lighthouse-1
healthcheck:
test: ["CMD", "wget", "--tries=1", "--connect-timeout=1", "--quiet", "-O", "-", "http://localhost:5052/eth/v2/beacon/blocks/head"]
interval: 30s
timeout: 10s
retries: 10
start_period: 30s
environment:
LIGHTHOUSE_EXECUTION_ENDPOINT: "http://mainnet-eth-geth-1:8551"
env_file:
- ../config/mainnet-eth-plugeth/lighthouse.env
image: cerc/lighthouse:local
entrypoint: /bin/sh
command: -c "/opt/run-lighthouse.sh"
volumes:
- mainnet_eth_plugeth_lighthouse_1_data:/data
- mainnet_eth_plugeth_config_data:/etc/mainnet-eth
- ../config/mainnet-eth-plugeth/scripts/run-lighthouse.sh:/opt/run-lighthouse.sh
ports:
# api
- "5052"
# metrics
- "5054"
# p2p
- "9000"
- "9000/udp"
volumes:
mainnet_eth_plugeth_config_data:
mainnet_eth_plugeth_geth_1_data:
mainnet_eth_plugeth_lighthouse_1_data:
@@ -15,16 +15,16 @@ services:
extra_hosts:
- "host.docker.internal:host-gateway"
ponder-app-indexer:
hostname: ponder-app-indexer
ponder-app-indexer-1:
hostname: ponder-app-indexer-1
restart: unless-stopped
image: cerc/ponder:local
working_dir: /app/examples/token-erc20
environment:
CERC_PONDER_CHAIN_ID: ${PONDER_CHAIN_ID:-99}
CERC_PONDER_RPC_URL_1: ${PONDER_RPC_URL_1:-http://ipld-eth-server-2:8081}
CERC_PONDER_NITRO_PK: ${CERC_PONDER_INDEXER_NITRO_PK:-58368d20ff12f17669c06158c21d885897aa56f9be430edc789614bf9851d53f}
CERC_PONDER_NITRO_CHAIN_PK: ${CERC_PONDER_INDEXER_NITRO_CHAIN_PK:-fb1e9af328c283ca3e2486e7c24d13582b7912057d8b9542ff41503c85bc05c0}
CERC_PONDER_NITRO_PK: ${CERC_PONDER_INDEXER_NITRO_PK_1:-58368d20ff12f17669c06158c21d885897aa56f9be430edc789614bf9851d53f}
CERC_PONDER_NITRO_CHAIN_PK: ${CERC_PONDER_INDEXER_NITRO_CHAIN_PK_1:-fb1e9af328c283ca3e2486e7c24d13582b7912057d8b9542ff41503c85bc05c0}
CERC_PONDER_NITRO_CHAIN_URL: ${CERC_PONDER_NITRO_CHAIN_URL:-http://fixturenet-eth-geth-1:8546}
CERC_RELAY_MULTIADDR: ${CERC_RELAY_MULTIADDR}
CERC_UPSTREAM_NITRO_ADDRESS: ${CERC_UPSTREAM_NITRO_ADDRESS:-0x660a4bEF3fbC863Fcd8D3CDB39242aE513d7D92e}
@@ -33,44 +33,42 @@ services:
command: ["bash", "./ponder-start.sh"]
volumes:
- ../config/ponder/ponder-start.sh:/app/examples/token-erc20/ponder-start.sh
- ../config/ponder/ponder.indexer.config.ts:/app/examples/token-erc20/ponder.config.ts
- ../config/ponder/ponder.indexer-1.config.ts:/app/examples/token-erc20/ponder.config.ts
- ../config/ponder/base-rates-config.json:/app/examples/token-erc20/base-rates-config.json
- peers_ids:/peers
- nitro_deployment:/nitro
- erc20_deployment:/erc20
- ponder_indexer_nitro_data:/app/examples/token-erc20/.ponder/nitro-db
- ponder_indexer_1_nitro_data:/app/examples/token-erc20/.ponder/nitro-db
ports:
- "42070"
- "127.0.0.1:42070:42070"
extra_hosts:
- "host.docker.internal:host-gateway"
ponder-app-watcher:
hostname: ponder-app-watcher
depends_on:
- ponder-app-indexer
ponder-app-indexer-2:
hostname: ponder-app-indexer-2
restart: unless-stopped
image: cerc/ponder:local
working_dir: /app/examples/token-erc20
environment:
CERC_PONDER_CHAIN_ID: ${PONDER_CHAIN_ID:-99}
CERC_PONDER_NITRO_PK: ${CERC_PONDER_WATCHER_NITRO_PK:-febb3b74b0b52d0976f6571d555f4ac8b91c308dfa25c7b58d1e6a7c3f50c781}
CERC_PONDER_NITRO_CHAIN_PK: ${CERC_PONDER_WATCHER_NITRO_CHAIN_PK:-be4aa664815ea3bc3d63118649a733f6c96b243744310806ecb6d96359ab62cf}
CERC_INDEXER_GQL_ENDPOINT: ${CERC_INDEXER_GQL_ENDPOINT:-http://ponder-app-indexer-1:42070/graphql}
CERC_PONDER_NITRO_PK: ${CERC_PONDER_INDEXER_NITRO_PK_2:-0aca28ba64679f63d71e671ab4dbb32aaa212d4789988e6ca47da47601c18fe2}
CERC_PONDER_NITRO_CHAIN_PK: ${CERC_PONDER_INDEXER_NITRO_CHAIN_PK_2:-6177345b77c4069ac4d553f8b43cf68a799ca4bb63eac93d6cf796d63694ebf0}
CERC_PONDER_NITRO_CHAIN_URL: ${CERC_PONDER_NITRO_CHAIN_URL:-http://fixturenet-eth-geth-1:8546}
CERC_RELAY_MULTIADDR: ${CERC_RELAY_MULTIADDR}
CERC_INDEXER_GQL_ENDPOINT: ${CERC_INDEXER_GQL_ENDPOINT:-http://ponder-app-indexer:42070/graphql}
CERC_INDEXER_NITRO_ADDRESS: ${CERC_INDEXER_NITRO_ADDRESS:-0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d}
CERC_INDEXER_NITRO_PAY_AMOUNT: ${CERC_INDEXER_NITRO_PAY_AMOUNT:-50}
CERC_UPSTREAM_NITRO_PAY_AMOUNT: ${CERC_UPSTREAM_NITRO_PAY_AMOUNT:-100}
command: ["bash", "./ponder-start.sh"]
volumes:
- ../config/ponder/ponder-start.sh:/app/examples/token-erc20/ponder-start.sh
- ../config/ponder/ponder.watcher.config.ts:/app/examples/token-erc20/ponder.config.ts
- ../config/ponder/ponder.indexer-2.config.ts:/app/examples/token-erc20/ponder.config.ts
- ../config/ponder/base-rates-config.json:/app/examples/token-erc20/base-rates-config.json
- peers_ids:/peers
- nitro_deployment:/nitro
- erc20_deployment:/erc20
- ponder_watcher_nitro_data:/app/examples/token-erc20/.ponder/nitro-db
- ponder_indexer_2_nitro_data:/app/examples/token-erc20/.ponder/nitro-db
ports:
- "42069"
- "127.0.0.1:42071:42070"
extra_hosts:
- "host.docker.internal:host-gateway"
@@ -78,5 +76,5 @@ volumes:
peers_ids:
nitro_deployment:
erc20_deployment:
ponder_indexer_nitro_data:
ponder_watcher_nitro_data:
ponder_indexer_1_nitro_data:
ponder_indexer_2_nitro_data:
@@ -0,0 +1,39 @@
version: '3.7'
services:
ponder-app-watcher:
hostname: ponder-app-watcher
depends_on:
- ponder-app-indexer-1
restart: unless-stopped
image: cerc/ponder:local
working_dir: /app/examples/token-erc20
environment:
CERC_PONDER_CHAIN_ID: ${PONDER_CHAIN_ID:-99}
CERC_PONDER_NITRO_PK: ${CERC_PONDER_WATCHER_NITRO_PK:-febb3b74b0b52d0976f6571d555f4ac8b91c308dfa25c7b58d1e6a7c3f50c781}
CERC_PONDER_NITRO_CHAIN_PK: ${CERC_PONDER_WATCHER_NITRO_CHAIN_PK:-be4aa664815ea3bc3d63118649a733f6c96b243744310806ecb6d96359ab62cf}
CERC_PONDER_NITRO_CHAIN_URL: ${CERC_PONDER_NITRO_CHAIN_URL:-http://fixturenet-eth-geth-1:8546}
CERC_RELAY_MULTIADDR: ${CERC_RELAY_MULTIADDR}
CERC_INDEXER_GQL_ENDPOINT: ${CERC_INDEXER_GQL_ENDPOINT:-http://ponder-app-indexer-2:42070/graphql}
CERC_INDEXER_NITRO_ADDRESS: ${CERC_INDEXER_NITRO_ADDRESS:-0xB2B22ec3889d11f2ddb1A1Db11e80D20EF367c01}
CERC_INDEXER_NITRO_PAY_AMOUNT: ${CERC_INDEXER_NITRO_PAY_AMOUNT:-50}
command: ["bash", "./ponder-start.sh"]
volumes:
- ../config/ponder/ponder-start.sh:/app/examples/token-erc20/ponder-start.sh
- ../config/ponder/ponder.watcher.config.ts:/app/examples/token-erc20/ponder.config.ts
- ../config/ponder/base-rates-config.json:/app/examples/token-erc20/base-rates-config.json
- peers_ids:/peers
- nitro_deployment:/nitro
- erc20_deployment:/erc20
- ponder_watcher_nitro_data:/app/examples/token-erc20/.ponder/nitro-db
ports:
- "127.0.0.1:42069:42069"
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
peers_ids:
nitro_deployment:
erc20_deployment:
ponder_watcher_nitro_data:
@@ -5,5 +5,5 @@ services:
userKey: REPLACE_WITH_MYKEY
bondId:
chainId: laconic_9000-1
gas: 250000
gas: 350000
fees: 200000aphoton
@@ -0,0 +1,15 @@
DATABASE_HOSTNAME="ipld-eth-db"
DATABASE_NAME="cerc"
DATABASE_PASSWORD="CHANGEME"
DATABASE_PORT=5432
DATABASE_USER="vdbm"
POSTGRES_DB="${DATABASE_NAME}"
POSTGRES_PASSWORD="${DATABASE_PASSWORD}"
POSTGRES_USER="${DATABASE_USER}"
CERC_STATEDIFF_DB_HOST="${DATABASE_HOSTNAME}"
CERC_STATEDIFF_DB_NAME="${DATABASE_NAME}"
CERC_STATEDIFF_DB_PASSWORD="${DATABASE_PASSWORD}"
CERC_STATEDIFF_DB_PORT=${DATABASE_PORT}
CERC_STATEDIFF_DB_USER="${DATABASE_USER}"
@@ -0,0 +1,33 @@
[database]
name = "" # $DATABASE_NAME
hostname = "" # $DATABASE_HOSTNAME
port = 5432 # $DATABASE_PORT
user = "" # $DATABASE_USER
password = "" # $DATABASE_PASSWORD
[log]
level = "info" # $LOG_LEVEL
[server]
ipc = false
ipcPath = "" # $SERVER_IPC_PATH
ws = false
wsPath = "0.0.0.0:8080" # $SERVER_WS_PATH
http = true
httpPath = "0.0.0.0:8081" # $SERVER_HTTP_PATH
graphql = false # $SERVER_GRAPHQL
graphqlPath = "0.0.0.0:8082" # $SERVER_GRAPHQL_PATH
[ethereum]
chainConfig = "" # ETH_CHAIN_CONFIG
chainID = "1" # $ETH_CHAIN_ID
rpcGasCap = "1000000000000" # $ETH_RPC_GAS_CAP
httpPath = "mainnet-eth-geth-1:8545" # $ETH_HTTP_PATH
supportsStateDiff = true # $ETH_SUPPORTS_STATEDIFF
stateDiffTimeout = "4m" # $ETH_STATEDIFF_TIMEOUT
forwardEthCalls = false # $ETH_FORWARD_ETH_CALLS
proxyOnError = true # $ETH_PROXY_ON_ERROR
nodeID = "" # $ETH_NODE_ID
clientName = "" # $ETH_CLIENT_NAME
genesisBlock = "" # $ETH_GENESIS_BLOCK
networkID = "1" # $ETH_NETWORK_ID
@@ -0,0 +1,27 @@
CERC_REMOTE_DEBUG="false"
LOG_LEVEL="debug"
ETH_CHAIN_ID=1
ETH_CLIENT_NAME="Geth"
ETH_FORWARD_ETH_CALLS="false"
ETH_FORWARD_GET_STORAGE_AT="false"
ETH_GENESIS_BLOCK="0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3"
ETH_HTTP_PATH="mainnet-eth-geth-1:8545"
ETH_NETWORK_ID=1
ETH_NODE_ID=1112
ETH_PROXY_ON_ERROR="true"
ETH_RPC_GAS_CAP=1000000000000
ETH_SUPPORTS_STATEDIFF="true"
ETH_STATEDIFF_TIMEOUT=4m
SERVER_HTTP_PATH=0.0.0.0:8081
SERVER_GRAPHQL="false"
SERVER_GRAPHQLPATH=0.0.0.0:8082
METRICS="true"
PROM_HTTP="true"
PROM_HTTP_ADDR="0.0.0.0"
PROM_HTTP_PORT="8090"
VDB_COMMAND="serve"
@@ -1,8 +1,11 @@
POSTGRES_DB=keycloak
POSTGRES_USER=keycloak
POSTGRES_PASSWORD=keycloak
# Don't change this unless you also change the healthcheck in docker-compose-mainnet-eth-keycloak.yml
PGPORT=35432
KC_DB=postgres
KC_DB_URL_HOST=keycloak-db
KC_DB_URL_PORT=${PGPORT}
KC_DB_URL_DATABASE=${POSTGRES_DB}
KC_DB_USERNAME=${POSTGRES_USER}
KC_DB_PASSWORD=${POSTGRES_PASSWORD}
@@ -15,42 +15,49 @@ server {
}
upstream geth-pool {
keepalive 100;
hash $user_id consistent;
server server-a:8545;
server server-b:8545;
server server-c:8545;
server server-a:8545 max_fails=10 fail_timeout=2s;
server server-c:8545 max_fails=10 fail_timeout=2s backup;
server server-b:8545 max_fails=10 fail_timeout=2s backup;
keepalive 200;
}
# self-reg happens on one server for clarity
upstream reg-ui-pool {
keepalive 100;
keepalive 2;
server server-a:8085;
}
upstream reg-api-pool {
keepalive 100;
keepalive 2;
server server-a:8086;
}
# auth uses server-a if available
# auth uses the reg server when available
upstream auth-pool {
keepalive 100;
keepalive 10;
server server-a:8080;
server server-b:8080 backup;
server server-c:8080 backup;
}
log_format upstreamlog '[$time_local] $remote_addr $user_id - $server_name $host to: $upstream_addr: $request $status upstream_response_time $upstream_response_time msec $msec request_time $request_time';
proxy_cache_path /var/cache/nginx/auth_cache levels=1 keys_zone=auth_cache:1m max_size=5m inactive=60m;
log_format upstreamlog '[$time_local] $msec $remote_addr $user_id - $server_name($host) to $upstream_addr: $request $status upstream_response_time $upstream_response_time request_time $request_time';
proxy_cache_path /var/cache/nginx/auth_cache levels=1 keys_zone=auth_cache:1m max_size=5m inactive=60m;
server {
listen 443 ssl http2;
server_name my.example.com;
keepalive_requests 500000;
keepalive_timeout 90s;
http2_max_requests 5000000;
http2_max_concurrent_streams 1024;
http2_idle_timeout 3m;
http2_recv_timeout 30s;
access_log /var/log/nginx/my.example.com-access.log upstreamlog;
error_log /var/log/nginx/my.example.com-error.log;
ssl_certificate /etc/nginx/ssl/my.example.com/cert.pem;
ssl_certificate_key /etc/nginx/ssl/my.example.com/key.pem;
ssl_session_cache shared:SSL:10m;
error_page 500 502 503 504 /50x.html;
location = /50x.html {
@@ -60,7 +67,6 @@ server {
#rewrite ^/?$ /newuser/;
rewrite ^/?$ https://www.example.com/;
# geth-pool ETH API
location ~ ^/v1/eth/?([^/]*)$ {
set $apiKey $1;
@@ -71,8 +77,8 @@ server {
auth_request_set $user_id $sent_http_x_user_id;
rewrite /.*$ / break;
client_max_body_size 3m;
client_body_buffer_size 3m;
client_max_body_size 3m;
client_body_buffer_size 3m;
proxy_buffer_size 32k;
proxy_buffers 16 32k;
proxy_busy_buffers_size 96k;
@@ -80,8 +86,10 @@ server {
proxy_pass http://geth-pool;
proxy_set_header X-Original-Remote-Addr $remote_addr;
proxy_set_header X-User-Id $user_id;
proxy_http_version 1.1;
proxy_set_header Connection "";
}
# keycloak
location = /auth {
internal;
@@ -95,6 +103,8 @@ server {
proxy_set_header X-Original-URI $request_uri;
proxy_set_header X-Original-Remote-Addr $remote_addr;
proxy_set_header X-Original-Host $host;
proxy_http_version 1.1;
proxy_set_header Connection "";
}
location /newuser/ {
@@ -0,0 +1,75 @@
# Enable remote debugging using dlv
CERC_REMOTE_DEBUG=false
# Enable startup script debug output.
CERC_SCRIPT_DEBUG=false
# Simple toggle to choose either a 'full' node or an 'archive' node
# (controls the values of --syncmode --gcmode --snapshot)
CERC_GETH_MODE_QUICK_SET=archive
# Path to plugeth plugins.
CERC_PLUGINS_DIR="/usr/local/lib/plugeth"
# Will turn on statediffing automatically if CERC_STATEDIFF_DB_HOST exists (see ../mainnet-eth-ipld-eth-db/db.env).
CERC_RUN_STATEDIFF="detect"
# The minimum necessary verion of the DB to enable statediffing.
CERC_STATEDIFF_DB_GOOSE_MIN_VER=18
# Whether all statediff-related DB statements should be logged (useful for debugging).
CERC_STATEDIFF_DB_LOG_STATEMENTS=false
# The number of concurrent workers to process state diff objects
CERC_STATEDIFF_WORKERS=16
# Each statediffing node should have a unique node ID.
CERC_STATEDIFF_DB_NODE_ID=1111
# Optional custom node name.
# GETH_NODE_NAME=""
# Specify any other geth CLI options.
GETH_OPTS=""
# --cache
GETH_CACHE=1024
# --cache.database
GETH_CACHE_DB=50
# --cache.gc
GETH_CACHE_GC=25
# --cache.trie
GETH_CACHE_TRIE=15
# --datadir
GETH_DATADIR="/data"
# --http.api
GETH_HTTP_API="eth,web3,net"
# --authrpc.jwtsecret
GETH_JWTSECRET="/etc/mainnet-eth/jwtsecret"
# --maxpeers
GETH_MAX_PEERS=100
# --rpc.evmtimeout
GETH_RPC_EVMTIMEOUT=0
# --rpc.gascap
GETH_RPC_GASCAP=0
# --txlookuplimit
GETH_TXLOOKUPLIMIT=0
# --verbosity
GETH_VERBOSITY=3
# --log.vmodule
GETH_VMODULE="rpc/*=4"
# --ws.api
GETH_WS_API="eth,web3,net"
@@ -0,0 +1,33 @@
# Enable startup script debug output.
CERC_SCRIPT_DEBUG=false
# Specify any other lighthouse CLI options.
LIGHTHOUSE_OPTS=""
# Override the advertised public IP (optional)
# --enr-address
#LIGHTHOUSE_ENR_ADDRESS=""
# --checkpoint-sync-url
LIGHTHOUSE_CHECKPOINT_SYNC_URL="https://beaconstate.ethstaker.cc"
# --checkpoint-sync-url-timeout
LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT=300
# --datadir
LIGHTHOUSE_DATADIR=/data
# --debug-level
LIGHTHOUSE_DEBUG_LEVEL=info
# --http-port
LIGHTHOUSE_HTTP_PORT=5052
# --execution-jwt
LIGHTHOUSE_JWTSECRET=/etc/mainnet-eth/jwtsecret
# --metrics-port
LIGHTHOUSE_METRICS_PORT=5054
# --port --enr-udp-port --enr-tcp-port
LIGHTHOUSE_NETWORK_PORT=9000
+121
View File
@@ -0,0 +1,121 @@
#!/bin/sh
if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then
set -x
fi
START_CMD="geth"
if [[ "true" == "$CERC_REMOTE_DEBUG" ]] && [[ -x "/usr/local/bin/dlv" ]]; then
START_CMD="/usr/local/bin/dlv --listen=:40000 --headless=true --api-version=2 --accept-multiclient exec /usr/local/bin/geth --continue --"
fi
# See https://linuxconfig.org/how-to-propagate-a-signal-to-child-processes-from-a-bash-script
cleanup() {
echo "Signal received, cleaning up..."
# Kill the child process first (CERC_REMOTE_DEBUG=true uses dlv which starts geth as a child process)
pkill -P ${geth_pid}
sleep 2
kill $(jobs -p)
wait
echo "Done"
}
trap 'cleanup' SIGINT SIGTERM
MODE_FLAGS=""
if [[ "$CERC_GETH_MODE_QUICK_SET" = "archive" ]]; then
MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-full} --gcmode=${GETH_GC_MODE:-archive} --snapshot=${GETH_SNAPSHOT:-false}"
else
MODE_FLAGS="--syncmode=${GETH_SYNC_MODE:-snap} --gcmode=${GETH_GC_MODE:-full} --snapshot=${GETH_SNAPSHOT:-true}"
fi
if [[ "${CERC_RUN_STATEDIFF}" == "detect" ]] && [[ -n "$CERC_STATEDIFF_DB_HOST" ]]; then
dig_result=$(dig $CERC_STATEDIFF_DB_HOST +short)
dig_status_code=$?
if [[ $dig_status_code = 0 && -n $dig_result ]]; then
echo "Statediff DB at $CERC_STATEDIFF_DB_HOST"
CERC_RUN_STATEDIFF="true"
else
echo "No statediff DB available."
CERC_RUN_STATEDIFF="false"
fi
fi
STATEDIFF_OPTS=""
if [[ "${CERC_RUN_STATEDIFF}" == "true" ]]; then
ready=0
echo "Waiting for statediff DB..."
while [ $ready -eq 0 ]; do
sleep 1
export PGPASSWORD="$CERC_STATEDIFF_DB_PASSWORD"
result=$(psql -h "$CERC_STATEDIFF_DB_HOST" \
-p "$CERC_STATEDIFF_DB_PORT" \
-U "$CERC_STATEDIFF_DB_USER" \
-d "$CERC_STATEDIFF_DB_NAME" \
-t -c 'select max(version_id) from goose_db_version;' 2>/dev/null | awk '{ print $1 }')
if [ -n "$result" ]; then
echo "DB ready..."
if [[ $result -ge $CERC_STATEDIFF_DB_GOOSE_MIN_VER ]]; then
ready=1
else
echo "DB not at required version (want $CERC_STATEDIFF_DB_GOOSE_MIN_VER, have $result)"
fi
fi
done
STATEDIFF_OPTS="--statediff \
--statediff.db.host=$CERC_STATEDIFF_DB_HOST \
--statediff.db.name=$CERC_STATEDIFF_DB_NAME \
--statediff.db.nodeid=$CERC_STATEDIFF_DB_NODE_ID \
--statediff.db.password=$CERC_STATEDIFF_DB_PASSWORD \
--statediff.db.port=$CERC_STATEDIFF_DB_PORT \
--statediff.db.user=$CERC_STATEDIFF_DB_USER \
--statediff.db.logstatements=${CERC_STATEDIFF_DB_LOG_STATEMENTS:-false} \
--statediff.db.copyfrom=${CERC_STATEDIFF_DB_COPY_FROM:-true} \
--statediff.waitforsync=${CERC_STATEDIFF_WAIT_FO_SYNC:-true} \
--statediff.workers=${CERC_STATEDIFF_WORKERS:-1} \
--statediff.writing=${CERC_STATEDIFF_WRITING:-true}"
if [[ -d "${CERC_PLUGINS_DIR}" ]]; then
# With plugeth, we separate the statediff options by prefixing with ' -- '
STATEDIFF_OPTS="--pluginsdir "${CERC_PLUGINS_DIR}" -- ${STATEDIFF_OPTS}"
fi
fi
$START_CMD \
$MODE_FLAGS \
--datadir="${GETH_DATADIR}"\
--identity="${GETH_NODE_NAME}" \
--maxpeers=${GETH_MAX_PEERS} \
--cache=${GETH_CACHE} \
--cache.gc=${GETH_CACHE_GC} \
--cache.database=${GETH_CACHE_DB} \
--cache.trie=${GETH_CACHE_TRIE} \
--authrpc.addr='0.0.0.0' \
--authrpc.vhosts='*' \
--authrpc.jwtsecret="${GETH_JWTSECRET}" \
--http \
--http.addr='0.0.0.0' \
--http.api="${GETH_HTTP_API}" \
--http.vhosts='*' \
--metrics \
--metrics.addr='0.0.0.0' \
--ws \
--ws.addr='0.0.0.0' \
--ws.api="${GETH_WS_API}" \
--rpc.gascap=${GETH_RPC_GASCAP} \
--rpc.evmtimeout=${GETH_RPC_EVMTIMEOUT} \
--txlookuplimit=${GETH_TXLOOKUPLIMIT} \
--verbosity=${GETH_VERBOSITY} \
--log.vmodule="${GETH_VMODULE}" \
${STATEDIFF_OPTS} \
${GETH_OPTS} &
geth_pid=$!
wait $geth_pid
if [[ "true" == "$CERC_KEEP_RUNNING_AFTER_GETH_EXIT" ]]; then
while [[ 1 -eq 1 ]]; do
sleep 60
done
fi
@@ -0,0 +1,30 @@
#!/bin/bash
if [[ "true" == "$CERC_SCRIPT_DEBUG" ]]; then
set -x
fi
ENR_OPTS=""
if [[ -n "$LIGHTHOUSE_ENR_ADDRESS" ]]; then
ENR_OPTS="--enr-address $LIGHTHOUSE_ENR_ADDRESS"
fi
exec lighthouse bn \
--checkpoint-sync-url "$LIGHTHOUSE_CHECKPOINT_SYNC_URL" \
--checkpoint-sync-url-timeout ${LIGHTHOUSE_CHECKPOINT_SYNC_URL_TIMEOUT} \
--datadir "$LIGHTHOUSE_DATADIR" \
--debug-level $LIGHTHOUSE_DEBUG_LEVEL \
--disable-deposit-contract-sync \
--disable-upnp \
--enr-tcp-port $LIGHTHOUSE_NETWORK_PORT \
--enr-udp-port $LIGHTHOUSE_NETWORK_PORT \
--execution-endpoint "$LIGHTHOUSE_EXECUTION_ENDPOINT" \
--execution-jwt /etc/mainnet-eth/jwtsecret \
--http \
--http-address 0.0.0.0 \
--http-port $LIGHTHOUSE_HTTP_PORT \
--metrics \
--metrics-address=0.0.0.0 \
--metrics-port $LIGHTHOUSE_METRICS_PORT \
--network mainnet \
--port $LIGHTHOUSE_NETWORK_PORT \
$ENR_OPTS $LIGHTHOUSE_OPTS
+1 -1
View File
@@ -25,7 +25,7 @@ GETH_CACHE_GC=25
# --cache.trie
GETH_CACHE_TRIE=15
j
# --datadir
GETH_DATADIR="/data"
@@ -2,7 +2,9 @@
"freeQueriesLimit": 10,
"freeQueriesList": [],
"queries": {
"getLogEvents": "50"
"getLogEvents": "50",
"getEthLogs": "50",
"getEthBlock": "50"
},
"mutations": {}
}
@@ -7,11 +7,16 @@ fi
erc20_address_file="/app/deployment/erc20-address.json"
# Check and exit if a deployment already exists (on restarts)
echo ETH_RPC_URL=${CERC_ETH_RPC_ENDPOINT} > .env
echo ACCOUNT_PRIVATE_KEY=${CERC_PRIVATE_KEY_DEPLOYER} >> .env
# Check and keep container running if a deployment already exists (on restarts)
if [ -f ${erc20_address_file} ]; then
echo "${erc20_address_file} already exists, skipping ERC20 contract deployment"
cat ${erc20_address_file}
exit
# Keep the container running
tail -f
fi
wait_for_chain_endpoint() {
@@ -46,8 +51,6 @@ wait_for_chain_endpoint
echo "Using CERC_PRIVATE_KEY_DEPLOYER from env"
echo ETH_RPC_URL=${CERC_ETH_RPC_ENDPOINT} > .env
echo ACCOUNT_PRIVATE_KEY=${CERC_PRIVATE_KEY_DEPLOYER} >> .env
yarn token:deploy:docker --file ${erc20_address_file}
# Keep the container running
@@ -35,7 +35,7 @@ export const config: Config = {
network: "fixturenet",
abi: "./abis/AdventureGold.json",
address: process.env.ERC20_CONTRACT,
startBlock: 5,
startBlock: 1,
maxBlockRange: 100,
},
],
@@ -0,0 +1,64 @@
import { type Config, AppMode } from "@ponder/core";
import contractAddresses from "./nitro-addresses.json" assert { type: "json" };
export const config: Config = {
networks: [
{
name: "fixturenet",
chainId: Number(process.env.PONDER_CHAIN_ID),
indexerUrl: process.env.INDEXER_GQL_ENDPOINT,
maxRpcRequestConcurrency: 1,
pollingInterval: 5000,
payments: {
nitro: {
address: process.env.INDEXER_NITRO_ADDRESS!,
fundingAmounts: {
// TODO: Pass amounts from env
directFund: "1000000000000",
virtualFund: "1000000000",
},
},
paidRPCMethods: [
"eth_getLogs",
"eth_getBlockByNumber",
"eth_getBlockByHash",
],
amount: process.env.UPSTREAM_NITRO_PAY_AMOUNT!,
},
},
],
contracts: [
{
name: "AdventureGold",
network: "fixturenet",
abi: "./abis/AdventureGold.json",
address: process.env.ERC20_CONTRACT,
startBlock: 1,
maxBlockRange: 100,
},
],
options: {
mode: AppMode.Indexer,
},
nitro: {
privateKey: process.env.PONDER_NITRO_PK!,
chainPrivateKey: process.env.PONDER_NITRO_CHAIN_PK!,
chainUrl: process.env.PONDER_NITRO_CHAIN_URL!,
contractAddresses,
relayMultiAddr: process.env.RELAY_MULTIADDR!,
store: "./.ponder/nitro-db",
payments: {
cache: {
maxAccounts: 1000,
accountTTLInSecs: 1800,
maxVouchersPerAccount: 1000,
voucherTTLInSecs: 300,
maxPaymentChannels: 10000,
paymentChannelTTLInSecs: 1800,
},
ratesFile: "./base-rates-config.json",
requestTimeoutInSecs: 10,
},
},
};
@@ -15,7 +15,7 @@ export const config: Config = {
network: "fixturenet",
abi: "./abis/AdventureGold.json",
address: process.env.ERC20_CONTRACT,
startBlock: 5,
startBlock: 1,
maxBlockRange: 100,
},
],
@@ -2,4 +2,4 @@
# Build a local version of the task executor for act-runner
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/gitea/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR}
docker build -t cerc/act-runner-task-executor:local -f ${CERC_REPO_BASE_DIR}/hosting/act-runner/Dockerfile.task-executor ${build_command_args} ${SCRIPT_DIR}
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
# Build the laconic.com image
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
docker build -t cerc/laconic-dot-com:local -f ${CERC_REPO_BASE_DIR}/laconic.com/Dockerfile ${build_command_args} ${CERC_REPO_BASE_DIR}/laconic.com
@@ -54,8 +54,10 @@ RUN yarn global add @cerc-io/laconic-registry-cli
# Add scripts
RUN mkdir /scripts
RUN mkdir /scripts/demo-records
ENV PATH="${PATH}:/scripts"
COPY ./create-demo-records.sh /scripts
COPY ./demo-records /scripts/demo-records
COPY ./import-key.sh /scripts
COPY ./import-address.sh /scripts
@@ -4,26 +4,26 @@ set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
registry_command="laconic cns"
record_1_filename=demo-record-1.yml
cat <<EOF > ${record_1_filename}
record:
type: WebsiteRegistrationRecord
url: 'https://cerc.io'
repo_registration_record_cid: QmSnuWmxptJZdLJpKRarxBMS2Ju2oANVrgbr2xWbie9b2D
build_artifact_cid: QmP8jTG1m9GSDJLCbeWhVSVgEzCPPwXRdCRuJtQ5Tz9Kc9
tls_cert_cid: QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR
version: 1.0.23
EOF
demo_records_dir="scripts/demo-records"
# Check we have funds
funds_response=$(${registry_command} account get --address $(cat my-address.txt))
funds_balance=$(echo ${funds_response} | jq -r .[0].balance[0].quantity)
echo "Balance is: ${funds_balance}"
# Create a bond
bond_create_result=$(${registry_command} bond create --type aphoton --quantity 1000000000)
bond_id=$(echo ${bond_create_result} | jq -r .bondId)
echo "Created bond with id: ${bond_id}"
# Publish a demo record
publish_response=$(${registry_command} record publish --filename ${record_1_filename} --bond-id ${bond_id})
published_record_id=$(echo ${publish_response} | jq -r .id)
echo "Published ${record_1_filename} with id: ${published_record_id}"
## Publish the demo records
if [ -d $demo_records_dir ]; then
for demo_record in "${demo_records_dir}"/*; do
publish_response=$(${registry_command} record publish --filename ${demo_record} --bond-id ${bond_id})
published_record_id=$(echo ${publish_response} | jq -r .id)
echo "Published ${demo_record} with id: ${published_record_id}"
cat ${demo_record}
done
fi
@@ -0,0 +1,7 @@
record:
type: WebsiteRegistrationRecord
url: 'https://hello-urbit.laconic.com'
repo_registration_record_cid: QmTZQ8ZJS6mALjEM2wY71msFno6zzxFftVCiZELj9xREPx
build_artifact_cid: '~lostex-rabdur-labtul-moltev/hello-urbit'
tls_cert_cid: QmR1acEmQt7Tjmhp9cFtymie2eFcrHURQKt9kGto1TQTW1
version: 0.2.4
@@ -0,0 +1,7 @@
record:
type: WebsiteRegistrationRecord
url: 'https://dev-hello-urbit.laconic.com'
repo_registration_record_cid: QmPbKkK7gKojnGAgKAE6bK5qBY8BcYp2YDB1e4b28NxvVv
build_artifact_cid: '~lostex-rabdur-labtul-moltev/hello-urbit-dev'
tls_cert_cid: QmWbCv4k9Q9P6XptHNJN6EupDyKVLSLRV6W6T2sAYkfBxr
version: 0.2.5
@@ -0,0 +1,7 @@
record:
type: WebsiteRegistrationRecord
url: 'https://app.osmosis.zone'
repo_registration_record_cid: QmSnuWmxptJZdLJpKRarxBMS2Ju2oANVrgbr2xWbie9b2D
build_artifact_cid: '~larryx-woldyr/osmosis:latest'
tls_cert_cid: QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR
version: 1.3.5
@@ -0,0 +1,7 @@
record:
type: WebsiteRegistrationRecord
url: 'https://dev.osmosis.zone'
repo_registration_record_cid: QmPbKkK7gKojnGAgKAE6bK5qBY8BcYp2YDB1e4b28NxvVv
build_artifact_cid: '~larryx-woldyr/osmosis:dev'
tls_cert_cid: QmWbCv4k9Q9P6XptHNJN6EupDyKVLSLRV6W6T2sAYkfBxr
version: 1.3.6
@@ -0,0 +1,7 @@
record:
type: WebsiteRegistrationRecord
url: 'https://pwa-1-1-3.iglootools.xyz'
repo_registration_record_cid: QmSnuWmxptJZdLJpKRarxBMS2Ju2oANVrgbr2xWbie9b2D
build_artifact_cid: https://ipfs.io/ipfs/QmP8jTG1m9GSDJLCbeWhVSVgEzCPPwXRdCRuJtQ5Tz9Kc9
tls_cert_cid: QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR
version: 1.1.3
@@ -0,0 +1,7 @@
record:
type: WebsiteRegistrationRecord
url: 'https://pwa-1-2-3.iglootools.xyz'
repo_registration_record_cid: QmVjYQ59z3fuKfYUV1aXhDR82JAKxdqB5JMPfUsHkJUkJq
build_artifact_cid: https://ipfs.io/ipfs/QmN2yevgjC92gzGi2BY6Q2yBWmR6JNWQ9yscoPPyy1V2Hv
tls_cert_cid: QmTAnmDMSnzqAN3yyXtFD4KpyATkeze62ZkbmXHjJ8xQW8
version: 1.2.3
@@ -0,0 +1,22 @@
# Using the same golang image as used to build plugeth: https://git.vdb.to/cerc-io/plugeth/src/branch/statediff/Dockerfile
FROM golang:1.20-alpine3.18 as delve
# Add delve so that we can do remote debugging.
RUN go install github.com/go-delve/delve/cmd/dlv@latest
FROM cerc/plugeth-statediff:local as statediff
FROM cerc/plugeth:local as plugeth
FROM alpine:3.18
# Install tools often used in scripting, like bash, wget, and jq.
RUN apk add --no-cache ca-certificates bash wget curl python3 bind-tools postgresql-client jq
COPY --from=delve /go/bin/dlv /usr/local/bin/
COPY --from=plugeth /usr/local/bin/geth /usr/local/bin/
# Place all plugeth plugins in /usr/local/lib/plugeth
COPY --from=statediff /usr/local/lib/statediff.so /usr/local/lib/plugeth/
EXPOSE 8545 8546 8551 6060 30303 30303/udp 40000
ENTRYPOINT ["geth"]
@@ -0,0 +1,9 @@
#!/usr/bin/env bash
# Build cerc/cerc-plugeth-with-plugins
set -x
source ${CERC_CONTAINER_BASE_DIR}/build-base.sh
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
docker build -t cerc/plugeth-with-plugins:local -f ${SCRIPT_DIR}/Dockerfile ${build_command_args} $SCRIPT_DIR
+13
View File
@@ -0,0 +1,13 @@
# act-runner stack
## Example
```
$ laconic-so --stack act-runner deploy init --output act-runner-1.yml --config CERC_GITEA_RUNNER_REGISTRATION_TOKEN=FOO
$ laconic-so --stack act-runner deploy create --spec-file act-runner-1.yml --deployment-dir ~/opt/deployments/act-runner-1
$ laconic-so deployment --dir ~/opt/deployments/act-runner-1 up
$ laconic-so --stack act-runner deploy init --output act-runner-2.yml --config CERC_GITEA_RUNNER_REGISTRATION_TOKEN=BAR
$ laconic-so --stack act-runner deploy create --spec-file act-runner-2.yml --deployment-dir ~/opt/deployments/act-runner-2
$ laconic-so deployment --dir ~/opt/deployments/act-runner-2 up
```
+15
View File
@@ -0,0 +1,15 @@
version: "1.1"
name: act-runner
description: "Local act-runner"
repos:
- git.vdb.to/cerc-io/hosting
- gitea.com/gitea/act_runner
containers:
- cerc/act-runner
- cerc/act-runner-task-executor
pods:
- name: act-runner
repository: cerc-io/hosting
path: act-runner
pre_start_command: "pre_start.sh"
post_start_command: "post_start.sh"
+2
View File
@@ -6,8 +6,10 @@ repos:
- git.vdb.to/cerc-io/eth-statediff-service@v5
- git.vdb.to/cerc-io/ipld-eth-db@v5
- git.vdb.to/cerc-io/ipld-eth-server@v5
- git.vdb.to/cerc-io/plugeth@statediff
containers:
- cerc/ipld-eth-state-snapshot
- cerc/eth-statediff-service
- cerc/ipld-eth-db
- cerc/ipld-eth-server
- cerc/plugeth
@@ -2,14 +2,14 @@ version: "1.1"
name: fixturenet-laconic-loaded
description: "A full featured laconic fixturenet"
repos:
- github.com/cerc-io/laconicd
- git.vdb.to/cerc-io/laconicd
- github.com/lirewine/debug
- github.com/lirewine/crypto
- github.com/lirewine/gem
- github.com/lirewine/sdk
- github.com/cerc-io/laconic-sdk
- github.com/cerc-io/laconic-registry-cli
- github.com/cerc-io/laconic-console
- git.vdb.to/cerc-io/laconic-sdk
- git.vdb.to/cerc-io/laconic-registry-cli
- git.vdb.to/cerc-io/laconic-console
npms:
- laconic-sdk
- laconic-registry-cli
@@ -2,9 +2,9 @@ version: "1.0"
name: fixturenet-laconicd
description: "A laconicd fixturenet"
repos:
- github.com/cerc-io/laconicd
- github.com/cerc-io/laconic-sdk
- github.com/cerc-io/laconic-registry-cli
- git.vdb.to/cerc-io/laconicd
- git.vdb.to/cerc-io/laconic-sdk
- git.vdb.to/cerc-io/laconic-registry-cli
npms:
- laconic-sdk
- laconic-registry-cli
@@ -29,7 +29,7 @@ laconic-so --stack fixturenet-payments deploy --cluster payments up
# 32***: geth in statediffing mode and ipld-eth-server(s)
# 4005: in-process go-nitro node's RPC endpoint
# 3005: in-process go-nitro node's p2p TCP endpoint
# 5005: in-process go-nitro node's p2p WS endpoin
# 5005: in-process go-nitro node's p2p WS endpoint
# 4006: out-of-process go-nitro node's RPC endpoint
# 3006: out-of-process go-nitro node's p2p TCP endpoint
# 5006: out-of-process go-nitro node's p2p WS endpoint
@@ -38,7 +38,9 @@ laconic-so --stack fixturenet-payments deploy --cluster payments up
# 9090: MobyMask v3 watcher relay node endpoint
# 8080: MobyMask snap
# 3004: MobyMask v3 app
# 32***: geth with statediffing
# 42070: Ponder indexer-1
# 42071: Ponder indexer-2
# 42069: Ponder watcher
```
If running in the cloud, ensure all the of the above ports are open. The geth port can be retrieved with:
@@ -60,7 +62,8 @@ This will allow you to access the entirety of the app as if it were running loca
## Demo
Follow the [demo](./demo.md) to try out end-to-end payments.
- Follow the [mobymask-demo](./mobymask-demo.md) to try out MobyMask end-to-end payments.
- Follow the [ponder-demo](./ponder-demo.md) to try out Ponder end-to-end payments.
## Clean up
@@ -3,13 +3,10 @@
Stack components:
* `ipld-eth-db` database for statediffed data
* Local geth + lighthouse blockchain "fixturenet" running in statediffing mode
* `ipld-eth-server-1` and `ipld-eth-server-2` both of which run an ETH RPC API and a GQL server; they both serve data from `ipld-eth-db`
* `ipld-eth-server-1` runs an in-process go-nitro node for payments required for configured RPC requests
* A go-nitro deployment acting as the remote Nitro node for `ipld-eth-server-2`
* `ipld-eth-server-1` which runs an ETH RPC API and a GQL server; they both serve data from `ipld-eth-db`
* It runs an in-process go-nitro node for payments required for configured RPC requests
* A MobyMask v3 watcher that pays the `ipld-eth-server-1` for ETH RPC requests
* A MobyMask v3 app that pays the watcher for reads (GQL queries) and writes
* An example ERC20 Ponder indexer app that pays the `ipld-eth-server-2` for ETH RPC requests
* An example ERC20 Ponder watcher app that pays the Ponder indexer app for GQL queries
## Setup
@@ -61,12 +58,6 @@ Stack components:
docker logs -f $(docker ps -aq --filter name="ipld-eth-server-1")
```
* In another terminal, check `ipld-eth-server-2`'s logs to keep track of incoming RPC requests from the Ponder app in indexer mode:
```bash
docker logs -f $(docker ps -aq --filter name="ipld-eth-server-2")
```
* MetaMask flask wallet setup for running the MobyMask app:
* Get the geth nodes port mapped to host:
@@ -213,96 +204,6 @@ Stack components:
# }
```
### ERC20 Ponder App
* Run the ponder app in indexer mode:
```bash
docker exec -it payments-ponder-app-indexer-1 bash -c "DEBUG=laconic:payments pnpm start"
# Expected output:
# 08:00:28.701 INFO payment Nitro node setup with address 0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d
# laconic:payments Starting voucher subscription... +0ms
# ...
# 09:58:54.288 INFO payment Creating ledger channel with nitro node 0x660a4bEF3fbC863Fcd8D3CDB39242aE513d7D92e
# ...
# 09:59:14.230 INFO payment Creating payment channel with nitro node 0x660a4bEF3fbC863Fcd8D3CDB39242aE513d7D92e
# ...
# 09:59:14.329 INFO payment Using payment channel 0x10f049519bc3f862e2b26e974be8666886228f30ea54aab06e2f23718afffab0
```
* Export the payment channel id to a variable:
```bash
export PONDER_UPSTREAM_PAYMENT_CHANNEL=<PAYMENT_CHANNEL_ID>
```
* On starting the Ponder app in indexer mode, it creates a payment channel with the `ipld-eth-server-2`'s (external) Nitro node and then starts the historical sync service
* The sync service makes several ETH RPC requests to the `ipld-eth-server-2` to fetch required data; check the `ipld-eth-server-2` logs for charged RPC requests (`eth_getBlockByNumber`, `eth_getLogs`):
```bash
# Expected output:
# ...
# 2023/10/18 05:29:38 INFO Serving a paid RPC request method=eth_getBlockByNumber cost=50 sender=0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d
# time="2023-10-18T05:29:38Z" level=debug msg=START api_method=eth_getBlockByNumber api_params="[latest true]" api_reqid=0 conn="172.26.0.19:42306" user_id= uuid=54992dc3-6d77-11ee-9ede-0242ac1a000f
# WARN [10-18|05:29:38.292] Attempting GerRPCCalls, but default PluginLoader has not been initialized
# time="2023-10-18T05:29:38Z" level=debug msg=END api_method=eth_getBlockByNumber api_params="[latest true]" api_reqid=0 conn="172.26.0.19:42306" duration=22 user_id= uuid=54992dc3-6d77-11ee-9ede-0242ac1a000f
# time="2023-10-18T05:29:40Z" level=debug msg=START api_method=eth_getLogs api_params="[map[address:0x32353a6c91143bfd6c7d363b546e62a9a2489a20 fromBlock:0x5 toBlock:0x68]]" api_reqid=1 conn="172.26.0.19:42306" user_id= uuid=55d19200-6d77-11ee-9ede-0242ac1a000f
# 2023/10/18 05:29:40 INFO Serving a paid RPC request method=eth_getLogs cost=50 sender=0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d
# WARN [10-18|05:29:40.340] Attempting GerRPCCalls, but default PluginLoader has not been initialized
# time="2023-10-18T05:29:40Z" level=debug msg="retrieving log cids for receipt ids"
# time="2023-10-18T05:29:40Z" level=debug msg=END api_method=eth_getLogs api_params="[map[address:0x32353a6c91143bfd6c7d363b546e62a9a2489a20 fromBlock:0x5 toBlock:0x68]]" api_reqid=1 conn="172.26.0.19:42306" duration=20 user_id= uuid=55d19200-6d77-11ee-9ede-0242ac1a000f
# ...
```
* Check the ponder - ipld-eth-server-2 payment channel status:
```bash
docker exec payments-nitro-rpc-client-1 npm exec -c "nitro-rpc-client get-payment-channel $PONDER_UPSTREAM_PAYMENT_CHANNEL -s false -h go-nitro -p 4006"
# Expected output ('PaidSoFar' is non zero):
# {
# ID: '0x1178ac0f2a43e54a122216fa6afdd30333b590e49e50317a1f9274a591da0f96',
# Status: 'Open',
# Balance: {
# AssetAddress: '0x0000000000000000000000000000000000000000',
# Payee: '0xaaa6628ec44a8a742987ef3a114ddfe2d4f7adce',
# Payer: '0x67d5b55604d1af90074fcb69b8c51838fff84f8d',
# PaidSoFar: 215000n,
# RemainingFunds: 999785000n
# }
# }
```
* In another terminal run the ponder app in watcher mode:
```bash
docker exec -it payments-ponder-app-watcher-1 bash -c "DEBUG=laconic:payments pnpm start"
# Expected output:
# 11:23:22.057 DEBUG app Started using config file: ponder.config.ts
# 08:02:12.548 INFO payment Nitro node setup with address 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db
# laconic:payments Starting voucher subscription... +0ms
# 08:02:17.417 INFO payment Creating ledger channel with nitro node 0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d ...
# 08:02:37.135 INFO payment Creating payment channel with nitro node 0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d ...
# 08:02:37.313 INFO payment Using payment channel 0x4b8e67f6a6fcfe114fdd60b85f963344ece4c77d4eea3825688c74b45ff5509b
# ...
# 11:23:22.436 INFO server Started responding as healthy
```
* Check the terminal in which ponder is running in indexer mode. Logs of payment for `eth_getLogs` queries can be seen:
```bash
# ...
# 08:02:37.763 DEBUG realtime Finished processing new head block 89 (network=fixturenet)
# laconic:payments Received a payment voucher of 50 from 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +444ms
# laconic:payments Serving a paid query for 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +1ms
# 08:02:37.804 DEBUG payment Verified payment for GQL queries getLogEvents
# laconic:payments Received a payment voucher of 50 from 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +45ms
# laconic:payments Serving a paid query for 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +0ms
# 08:02:37.849 DEBUG payment Verified payment for GQL queries getLogEvents
```
## Clean Up
@@ -0,0 +1,267 @@
# Demo
Stack components:
* `ipld-eth-db` database for statediffed data
* Local geth + lighthouse blockchain "fixturenet" running in statediffing mode
* `ipld-eth-server-2` which runs an ETH RPC API and a GQL server; they both serve data from `ipld-eth-db`
* A go-nitro deployment acting as the remote Nitro node for `ipld-eth-server-2`
* Example ERC20 Ponder apps
* `ponder-app-indexer-1` that pays the `ipld-eth-server-2` for ETH RPC requests
* `ponder-app-indexer-2` that pays `ponder-app-indexer-1` for GQL queries
* `ponder-app-watcher` that pays `ponder-app-indexer-2` for GQL queries
## Setup
* In a terminal, check `ipld-eth-server-2`'s logs to keep track of incoming RPC requests from the `ponder-app-indexer-1`:
```bash
docker logs -f $(docker ps -aq --filter name="ipld-eth-server-2")
```
## Run
### ERC20 Ponder App
* Run the first indexer Ponder app:
```bash
docker exec -it payments-ponder-app-indexer-1-1 bash -c "DEBUG=laconic:payments pnpm start"
# Expected output:
# 12:57:03.751 INFO payment Nitro node setup with address 0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d
# laconic:payments Starting voucher subscription... +0ms
# ...
# 09:58:54.288 INFO payment Creating ledger channel with nitro node 0x660a4bEF3fbC863Fcd8D3CDB39242aE513d7D92e ...
# 09:59:14.230 INFO payment Creating payment channel with nitro node 0x660a4bEF3fbC863Fcd8D3CDB39242aE513d7D92e ...
# 09:59:14.329 INFO payment Using payment channel 0x1ff59db391b7a55bed723b930ab53c80e7ce857487c1e58771aa5a0737d71625
```
* Export the payment channel id to a variable:
```bash
export PONDER_UPSTREAM_PAYMENT_CHANNEL=<PAYMENT_CHANNEL_ID>
```
* On starting the Ponder app in indexer mode, it creates a payment channel with the `ipld-eth-server-2`'s (external) Nitro node and then starts the historical sync service
* The sync service makes several ETH RPC requests to the `ipld-eth-server-2` to fetch required data; check the `ipld-eth-server-2` logs for charged RPC requests (`eth_getBlockByNumber`, `eth_getLogs`):
```bash
# Expected output:
# ...
# 2023/10/18 05:29:38 INFO Serving a paid RPC request method=eth_getBlockByNumber cost=50 sender=0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d
# time="2023-10-18T05:29:38Z" level=debug msg=START api_method=eth_getBlockByNumber api_params="[latest true]" api_reqid=0 conn="172.26.0.19:42306" user_id= uuid=54992dc3-6d77-11ee-9ede-0242ac1a000f
# WARN [10-18|05:29:38.292] Attempting GerRPCCalls, but default PluginLoader has not been initialized
# time="2023-10-18T05:29:38Z" level=debug msg=END api_method=eth_getBlockByNumber api_params="[latest true]" api_reqid=0 conn="172.26.0.19:42306" duration=22 user_id= uuid=54992dc3-6d77-11ee-9ede-0242ac1a000f
# ...
# 2023/10/18 05:29:40 INFO Serving a paid RPC request method=eth_getLogs cost=50 sender=0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d
# WARN [10-18|05:29:40.340] Attempting GerRPCCalls, but default PluginLoader has not been initialized
# time="2023-10-18T05:29:40Z" level=debug msg="retrieving log cids for receipt ids"
# ...
```
* Check the Ponder - ipld-eth-server-2 payment channel status:
```bash
docker exec payments-nitro-rpc-client-1 npm exec -c "nitro-rpc-client get-payment-channel $PONDER_UPSTREAM_PAYMENT_CHANNEL -s false -h go-nitro -p 4006"
# Expected output ('PaidSoFar' is non zero):
# {
# ID: '0x1ff59db391b7a55bed723b930ab53c80e7ce857487c1e58771aa5a0737d71625',
# Status: 'Open',
# Balance: {
# AssetAddress: '0x0000000000000000000000000000000000000000',
# Payee: '0x660a4bef3fbc863fcd8d3cdb39242ae513d7d92e',
# Payer: '0x67d5b55604d1af90074fcb69b8c51838fff84f8d',
# PaidSoFar: 7200n,
# RemainingFunds: 999992800n
# }
# }
```
* In another terminal run the second indexer Ponder app:
```bash
docker exec -it payments-ponder-app-indexer-2-1 bash -c "DEBUG=laconic:payments pnpm start"
# Expected output:
# 08:00:28.701 INFO payment Nitro node setup with address 0xB2B22ec3889d11f2ddb1A1Db11e80D20EF367c01
# laconic:payments Starting voucher subscription... +0ms
# ...
# 09:58:54.288 INFO payment Creating ledger channel with nitro node 0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d ...
# 09:59:14.230 INFO payment Creating payment channel with nitro node 0x67D5b55604d1aF90074FcB69b8C51838FFF84f8d ...
# 09:59:14.329 INFO payment Using payment channel 0xfbf9d7eb7c18446883c7f57f4c94db5607f414a224b3e921c787db07371d2a70
```
* On starting indexer Ponder app, it creates a payment channel with first indexer and then starts the sync services
* Check logs in `ponder-indexer-1` to see payments made from `ponder-indexer-2` from GQL queries to fetch network data
```bash
# ...
# laconic:payments Received a payment voucher of 100 from 0xB2B22ec3889d11f2ddb1A1Db11e80D20EF367c01 +23ms
# laconic:payments Serving a paid query for 0xB2B22ec3889d11f2ddb1A1Db11e80D20EF367c01 +0ms
# 13:01:05.671 DEBUG payment Verified payment for GQL queries getEthLogs
# laconic:payments Received a payment voucher of 100 from 0xB2B22ec3889d11f2ddb1A1Db11e80D20EF367c01 +20ms
# laconic:payments Serving a paid query for 0xB2B22ec3889d11f2ddb1A1Db11e80D20EF367c01 +0ms
# 13:01:05.691 DEBUG payment Verified payment for GQL queries getEthBlock
# 13:01:07.598 INFO realtime Fetched missing blocks [686, 687] (network=fixturenet)
# 13:01:07.598 DEBUG realtime Started processing new head block 686 (network=fixturenet)
# ...
```
* Following are the example GQL queries (with payment headers) `ponder-indexer-2` makes to `ponder-indexer-1` for fetching data
```graphql
{
getEthLogs(
chainId: 1212,
address: "0x6325439389e0797ab35752b4f43a14c004f22a9c",
fromBlock: 100,
toBlock: 200,
) {
address
blockHash
blockNumber
data
logIndex
removed
topics
transactionHash
transactionIndex
}
getEthBlock(
chainId: 1212,
blockNumber: 150,
fullTransactions: false
) {
baseFeePerGas
difficulty
extraData
gasLimit
gasUsed
hash
logsBloom
miner
mixHash
nonce
number
parentHash
receiptsRoot
sha3Uncles
size
stateRoot
timestamp
totalDifficulty
transactionsRoot
txHashes
}
}
```
* In another terminal run the Ponder app in watcher mode:
```bash
docker exec -it payments-ponder-app-watcher-1 bash -c "DEBUG=laconic:payments pnpm start"
# Expected output:
# 11:23:22.057 DEBUG app Started using config file: ponder.config.ts
# 08:02:12.548 INFO payment Nitro node setup with address 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db
# laconic:payments Starting voucher subscription... +0ms
# 08:02:17.417 INFO payment Creating ledger channel with nitro node 0xB2B22ec3889d11f2ddb1A1Db11e80D20EF367c01 ...
# 08:02:37.135 INFO payment Creating payment channel with nitro node 0xB2B22ec3889d11f2ddb1A1Db11e80D20EF367c01 ...
# 08:02:37.313 INFO payment Using payment channel 0xc48622577dfa389283beb19ed946274eb034587d72e61445dc997304be671f1a
# ...
# 11:23:22.436 INFO server Started responding as healthy
```
* Check the terminal of the second indexer Ponder app. Logs of payment for `getLogEvents` queries can be seen:
```bash
# ...
# 08:02:37.763 DEBUG realtime Finished processing new head block 89 (network=fixturenet)
# laconic:payments Received a payment voucher of 50 from 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +444ms
# laconic:payments Serving a paid query for 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +1ms
# 08:02:37.804 DEBUG payment Verified payment for GQL queries getLogEvents
# laconic:payments Received a payment voucher of 50 from 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +45ms
# laconic:payments Serving a paid query for 0x111A00868581f73AB42FEEF67D235Ca09ca1E8db +0ms
# 08:02:37.849 DEBUG payment Verified payment for GQL queries getLogEvents
```
* Following are the example GQL queries (with payment headers) `ponder-watcher` makes to `ponder-indexer-2` for fetching data
```graphql
getLogEvents(
fromTimestamp: 1695303079,
toTimestamp: 1695304156,
filters: [
{
name: "AdventureGold"
chainId: 1212,
address: ["0x6325439389e0797ab35752b4f43a14c004f22a9c"],
fromBlock: 100,
toBlock: 200,
includeEventSelectors: ["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"]
}
]
) {
events {
transaction {
hash
}
log {
id
}
block {
number
}
}
metadata {
pageEndsAtTimestamp
isLastPage
}
}
```
* Open watcher Ponder app endpoint http://localhost:42069
* Try GQL query to see transfer events
```graphql
{
transferEvents (orderBy: "timestamp", orderDirection: "desc") {
id
amount
from {
id
}
to {
id
}
timestamp
}
}
```
* Transfer an ERC20 token on chain
* Get the deployed ERC20 token address
```bash
export TOKEN_ADDRESS=$(docker exec payments-ponder-er20-contracts-1 jq -r '.address' ./deployment/erc20-address.json)
```
* Transfer token
```bash
docker exec -it payments-ponder-er20-contracts-1 bash -c "yarn token:transfer:docker --token ${TOKEN_ADDRESS} --to 0xe22AD83A0dE117bA0d03d5E94Eb4E0d80a69C62a --amount 5000"
```
* Wait for a log in watcher Ponder app
```bash
06:40:47.567 INFO handlers Processed 1 event (up to Oct 19, 2023)
```
* Check the GQL query again in http://localhost:42069 to see a new `TransferEvent` entity
@@ -10,7 +10,7 @@ repos:
- git.vdb.to/cerc-io/ipld-eth-server@payments
# nitro repo
- github.com/cerc-io/ts-nitro@v0.1.15
- github.com/cerc-io/go-nitro@ts-interop # TODO: Use release
- github.com/cerc-io/go-nitro@v0.1.2-ts-port-0.1.9
# mobymask watcher repos
- github.com/cerc-io/watcher-ts@v0.2.66
# this is mobymask-v3
@@ -55,4 +55,5 @@ pods:
- watcher-mobymask-v3
- mobymask-snap
- mobymask-app-v3
- ponder
- ponder-indexer
- ponder-watcher
@@ -0,0 +1,9 @@
# laconic-dot-com
```
laconic-so --stack laconic-dot-com setup-repositories
laconic-so --stack laconic-dot-com build-containers
laconic-so --stack laconic-dot-com deploy init --output laconic-website-spec.yml --map-ports-to-host localhost-same
laconic-so --stack laconic-dot-com deploy create --spec-file laconic-website-spec.yml --deployment-dir lx-website
laconic-so deployment --dir lx-website start
```
@@ -0,0 +1,8 @@
version: "0.1"
name: laconic-dot-com
repos:
- github.com/LaconicNetwork/laconic.com@v0.0.1
containers:
- cerc/laconic-dot-com
pods:
- laconic-dot-com
@@ -0,0 +1,141 @@
# mainnet-eth-plugeth
Deploys a "head-tracking" mainnet Ethereum stack comprising a [plugeth](https://git.vdb.to/cerc-io/plugeth) execution layer node and a [lighthouse](https://github.com/sigp/lighthouse) consensus layer node, with [plugeth-statediff](https://git.vdb.to/cerc-io/plugeth-statediff) for statediffing, [ipld-eth-db](https://git.vdb.to/cerc-io/ipld-eth-db) for storage, and [ipld-eth-server](https://git.vdb.to/cerc-io/ipld-eth-server) for indexed ETH IPLD objects.
## Clone required repositories
```
$ laconic-so --stack mainnet-eth-plugeth setup-repositories
```
## Build containers
```
$ laconic-so --stack mainnet-eth-plugeth build-containers
```
## Create a deployment
```
$ laconic-so --stack mainnet-eth-plugeth deploy init --map-ports-to-host any-same --output mainnet-eth-plugeth-spec.yml
$ laconic-so --stack mainnet-eth-plugeth deploy create --spec-file mainnet-eth-plugeth-spec.yml --deployment-dir mainnet-eth-plugeth-deployment
```
## Start the stack
```
$ laconic-so deployment --dir mainnet-eth-plugeth-deployment start
```
Display stack status:
```
$ laconic-so deployment --dir mainnet-eth-plugeth-deployment ps
Running containers:
id: f39608eca04d72d6b0f1f3acefc5ebb52908da06e221d20c7138f7e3dff5e423, name: laconic-ef641b4d13eb61ed561b19be67063241-foundry-1, ports:
id: 4052b1eddd886ae0d6b41f9ff22e68a70f267b2bfde10f4b7b79b5bd1eeddcac, name: laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-geth-1-1, ports: 30303/tcp, 30303/udp, 0.0.0.0:49184->40000/tcp, 0.0.0.0:49185->6060/tcp, 0.0.0.0:49186->8545/tcp, 8546/tcp
id: ac331232e597944b621b3b8942ace5dafb14524302cab338ff946c7f6e5a1d52, name: laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1, ports: 0.0.0.0:49187->8001/tcp
```
See stack logs:
```
$ laconic-so deployment --dir mainnet-eth-plugeth-deployment logs
time="2023-07-25T09:46:29-06:00" level=warning msg="The \"CERC_SCRIPT_DEBUG\" variable is not set. Defaulting to a blank string."
laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.362 INFO Logging to file path: "/var/lighthouse-data-dir/beacon/logs/beacon.log"
laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.365 INFO Lighthouse started version: Lighthouse/v4.1.0-693886b
laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.365 INFO Configured for network name: mainnet
laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.366 INFO Data directory initialised datadir: /var/lighthouse-data-dir
laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.366 INFO Deposit contract address: 0x00000000219ab540356cbb839cbe05303d7705fa, deploy_block: 11184524
laconic-ef641b4d13eb61ed561b19be67063241-mainnet-eth-plugeth-lighthouse-1-1 | Jul 25 15:45:13.424 INFO Starting checkpoint sync remote_url: https://beaconstate.ethstaker.cc/, service: beacon
```
## Monitoring stack sync progress
Both go-ethereum and lighthouse will engage in an initial chain sync phase that will last up to several hours depending on hardware performance and network capacity.
Syncing can be monitored by looking for these log messages:
```
Jul 24 12:34:17.001 INFO Downloading historical blocks est_time: 5 days 11 hrs, speed: 14.67 slots/sec, distance: 6932481 slots (137 weeks 3 days), service: slot_notifier
INFO [07-24|12:14:52.493] Syncing beacon headers downloaded=145,920 left=17,617,968 eta=1h23m32.815s
INFO [07-24|12:33:15.238] Syncing: chain download in progress synced=1.86% chain=148.94MiB headers=368,640@95.03MiB bodies=330,081@40.56MiB receipts=330,081@13.35MiB eta=37m54.505s
INFO [07-24|12:35:13.028] Syncing: state download in progress synced=1.32% state=4.64GiB accounts=2,850,314@677.57MiB slots=18,663,070@3.87GiB codes=26662@111.14MiB eta=3h18m0.699s
```
Once synced up these log messages will be observed:
```
INFO Synced slot: 6952515, block: 0x5bcb…f6d9, epoch: 217266, finalized_epoch: 217264, finalized_root: 0x6342…2c5c, exec_hash: 0x8d8c…2443 (verified), peers: 31, service: slot_notifier
INFO [07-25|03:04:48.941] Imported new potential chain segment number=17,767,316 hash=84f6e7..bc2cb0 blocks=1 txs=137 mgas=16.123 elapsed=57.087ms mgasps=282.434 dirty=461.46MiB
INFO [07-25|03:04:49.042] Chain head was updated number=17,767,316 hash=84f6e7..bc2cb0 root=ca58b2..8258c1 elapsed=2.480111ms
```
## Clean up
Stop the stack:
```
$ laconic-so deployment --dir mainnet-eth-plugeth-deployment stop
```
This leaves data volumes in place, allowing the stack to be subsequently re-started.
To permanently *delete* the stack's data volumes run:
```
$ laconic-so deployment --dir mainnet-eth-plugeth-deployment stop --delete-data-volumes
```
After deleting the volumes, any subsequent re-start will begin chain sync from cold.
## Ports
It is usually necessary to expose certain container ports on one or more the host's addresses to allow incoming connections.
Any ports defined in the Docker compose file are exposed by default with random port assignments, bound to "any" interface (IP address 0.0.0.0), but the port mappings can be
customized by editing the "spec" file generated by `laconic-so deploy init`.
In this example, ports `8545` and `5052` have been assigned to a specific addresses/port combination on the host, while
port `40000` has been left with random assignment:
```
$ cat mainnet-eth-plugeth-spec.yml
stack: mainnet-eth-plugeth
ports:
mainnet-eth-plugeth-geth-1:
- '10.10.10.10:8545:8545'
- '40000'
mainnet-eth-plugeth-lighthouse-1:
- '10.10.10.10:5052:5052'
volumes:
mainnet_eth_plugeth_config_data: ./data/mainnet_eth_plugeth_config_data
mainnet_eth_plugeth_geth_1_data: ./data/mainnet_eth_plugeth_geth_1_data
mainnet_eth_plugeth_lighthouse_1_data: ./data/mainnet_eth_plugeth_lighthouse_1_data
```
In addition, a stack-wide port mapping "recipe" can be applied at the time the
`laconic-so deploy init` command is run, by supplying the desired recipe with the `--map-ports-to-host` option. The following recipes are supported:
| Recipe | Host Port Mapping |
|--------|-------------------|
| any-variable-random | Bind to 0.0.0.0 using a random port assigned at start time (default) |
| localhost-same | Bind to 127.0.0.1 using the same port number as exposed by the containers |
| any-same | Bind to 0.0.0.0 using the same port number as exposed by the containers |
| localhost-fixed-random | Bind to 127.0.0.1 using a random port number selected at the time the command is run (not checked for already in use)|
| any-fixed-random | Bind to 0.0.0.0 using a random port number selected at the time the command is run (not checked for already in use) |
## Data volumes
Container data volumes are bind-mounted to specified paths in the host filesystem.
The default setup (generated by `laconic-so deploy init`) places the volumes in the `./data` subdirectory of the deployment directory:
```
$ cat mainnet-eth-plugeth-spec.yml
stack: mainnet-eth-plugeth
ports:
mainnet-eth-plugeth-geth-1:
- '10.10.10.10:8545:8545'
- '40000'
mainnet-eth-plugeth-lighthouse-1:
- '10.10.10.10:5052:5052'
volumes:
mainnet_eth_plugeth_config_data: ./data/mainnet_eth_plugeth_config_data
mainnet_eth_plugeth_geth_1_data: ./data/mainnet_eth_plugeth_geth_1_data
mainnet_eth_plugeth_lighthouse_1_data: ./data/mainnet_eth_plugeth_lighthouse_1_data
```
A synced-up stack will consume around 900GB of data volume space:
```
$ sudo du -h mainnet-eth-plugeth-deployment/data/
150M mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon/freezer_db
25G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon/chain_db
16K mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon/network
368M mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon/logs
26G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data/beacon
26G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_lighthouse_1_data
8.0K mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_config_data
4.0K mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/keystore
527G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/chaindata/ancient/chain
527G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/chaindata/ancient
859G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/chaindata
4.8M mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/nodes
242M mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/ethash
669M mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth/triecache
860G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data/geth
860G mainnet-eth-plugeth-deployment/data/mainnet_eth_plugeth_geth_1_data
885G mainnet-eth-plugeth-deployment/data/
```
@@ -0,0 +1,32 @@
# Copyright © 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from secrets import token_hex
def init(ctx):
return None
def setup(ctx):
return None
def create(ctx, extra_args):
# Generate the JWT secret and save to its config file
secret = token_hex(32)
jwt_file_path = ctx.deployment_dir.joinpath("data", "mainnet_eth_plugeth_config_data", "jwtsecret")
with open(jwt_file_path, 'w+') as jwt_file:
jwt_file.write(secret)
@@ -0,0 +1,29 @@
version: "1.2"
name: mainnet-eth
description: "Ethereum Mainnet"
repos:
- git.vdb.to/cerc-io/plugeth@statediff
- git.vdb.to/cerc-io/plugeth-statediff
- git.vdb.to/cerc-io/lighthouse
- git.vdb.to/cerc-io/ipld-eth-db@v5
- git.vdb.to/cerc-io/ipld-eth-server@v5
- git.vdb.to/cerc-io/keycloak-reg-api
- git.vdb.to/cerc-io/keycloak-reg-ui
containers:
- cerc/plugeth-statediff
- cerc/plugeth
- cerc/plugeth-with-plugins
- cerc/lighthouse
- cerc/lighthouse-cli
- cerc/ipld-eth-db
- cerc/ipld-eth-server
- cerc/keycloak
- cerc/webapp-base
- cerc/keycloak-reg-api
- cerc/keycloak-reg-ui
pods:
- mainnet-eth-plugeth
- mainnet-eth-ipld-eth-db
- mainnet-eth-ipld-eth-server
- mainnet-eth-keycloak
- mainnet-eth-metrics
@@ -14,9 +14,9 @@
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from app.util import get_yaml
from app.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
from app.stack_state import State
from app.deploy_util import VolumeMapping, run_container_command
from app.deploy.deploy_types import DeployCommandContext, LaconicStackSetupCommand, DeploymentContext
from app.deploy.stack_state import State
from app.deploy.deploy_util import VolumeMapping, run_container_command
from app.command_types import CommandOptions
from enum import Enum
from pathlib import Path
@@ -13,3 +13,8 @@ pods:
path: gitea
pre_start_command: "run-this-first.sh"
post_start_command: "initialize-gitea.sh"
- name: act-runner
repository: cerc-io/hosting
path: act-runner
pre_start_command: "pre_start.sh"
post_start_command: "post_start.sh"
+3 -3
View File
@@ -14,9 +14,9 @@
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from app.util import get_yaml
from app.deploy_types import DeployCommandContext
from app.stack_state import State
from app.deploy_util import VolumeMapping, run_container_command
from app.deploy.deploy_types import DeployCommandContext
from app.deploy.stack_state import State
from app.deploy.deploy_util import VolumeMapping, run_container_command
from pathlib import Path
default_spec_file_content = """config:
+1 -1
View File
@@ -2,7 +2,7 @@ version: "1.0"
name: test
description: "A test stack"
repos:
- github.com/cerc-io/laconicd
- git.vdb.to/cerc-io/laconicd
- git.vdb.to/cerc-io/test-project@test-branch
containers:
- cerc/test-container
View File
View File
+67
View File
@@ -0,0 +1,67 @@
# Copyright © 2022, 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from python_on_whales import DockerClient, DockerException
from app.deploy.deployer import Deployer, DeployerException
class DockerDeployer(Deployer):
name: str = "compose"
def __init__(self, compose_files, compose_project_name, compose_env_file) -> None:
self.docker = DockerClient(compose_files=compose_files, compose_project_name=compose_project_name,
compose_env_file=compose_env_file)
def up(self, detach, services):
try:
return self.docker.compose.up(detach=detach, services=services)
except DockerException as e:
raise DeployerException(e)
def down(self, timeout, volumes):
try:
return self.docker.compose.down(timeout=timeout, volumes=volumes)
except DockerException as e:
raise DeployerException(e)
def ps(self):
try:
return self.docker.compose.ps()
except DockerException as e:
raise DeployerException(e)
def port(self, service, private_port):
try:
return self.docker.compose.port(service=service, private_port=private_port)
except DockerException as e:
raise DeployerException(e)
def execute(self, service, command, tty, envs):
try:
return self.docker.compose.execute(service=service, command=command, tty=tty, envs=envs)
except DockerException as e:
raise DeployerException(e)
def logs(self, services, tail, follow, stream):
try:
return self.docker.compose.logs(services=services, tail=tail, follow=follow, stream=stream)
except DockerException as e:
raise DeployerException(e)
def run(self, image, command, user, volumes, entrypoint=None):
try:
return self.docker.run(image=image, command=command, user=user, volumes=volumes, entrypoint=entrypoint)
except DockerException as e:
raise DeployerException(e)
+45 -39
View File
@@ -13,7 +13,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
# Deploys the system components using docker-compose
# Deploys the system components using a deployer (either docker-compose or k8s)
import hashlib
import copy
@@ -22,14 +22,15 @@ import sys
from dataclasses import dataclass
from importlib import resources
import subprocess
from python_on_whales import DockerClient, DockerException
import click
from pathlib import Path
from app.util import include_exclude_check, get_parsed_stack_config, global_options2, get_dev_root_path
from app.deploy_types import ClusterContext, DeployCommandContext
from app.deployment_create import create as deployment_create
from app.deployment_create import init as deployment_init
from app.deployment_create import setup as deployment_setup
from app.deploy.deployer import Deployer, DeployerException
from app.deploy.deployer_factory import getDeployer
from app.deploy.deploy_types import ClusterContext, DeployCommandContext
from app.deploy.deployment_create import create as deployment_create
from app.deploy.deployment_create import init as deployment_init
from app.deploy.deployment_create import setup as deployment_setup
@click.group()
@@ -37,8 +38,9 @@ from app.deployment_create import setup as deployment_setup
@click.option("--exclude", help="don\'t start these components")
@click.option("--env-file", help="env file to be used")
@click.option("--cluster", help="specify a non-default cluster name")
@click.option("--deploy-to", help="cluster system to deploy to (compose or k8s)")
@click.pass_context
def command(ctx, include, exclude, env_file, cluster):
def command(ctx, include, exclude, env_file, cluster, deploy_to):
'''deploy a stack'''
# Although in theory for some subcommands (e.g. deploy create) the stack can be inferred,
@@ -50,16 +52,20 @@ def command(ctx, include, exclude, env_file, cluster):
if ctx.parent.obj.debug:
print(f"ctx.parent.obj: {ctx.parent.obj}")
ctx.obj = create_deploy_context(global_options2(ctx), stack, include, exclude, cluster, env_file)
if deploy_to is None:
deploy_to = "compose"
ctx.obj = create_deploy_context(global_options2(ctx), stack, include, exclude, cluster, env_file, deploy_to)
# Subcommand is executed now, by the magic of click
def create_deploy_context(global_context, stack, include, exclude, cluster, env_file):
def create_deploy_context(global_context, stack, include, exclude, cluster, env_file, deployer):
cluster_context = _make_cluster_context(global_context, stack, include, exclude, cluster, env_file)
# See: https://gabrieldemarmiesse.github.io/python-on-whales/sub-commands/compose/
docker = DockerClient(compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster,
compose_env_file=cluster_context.env_file)
return DeployCommandContext(stack, cluster_context, docker)
deployer = getDeployer(deployer, compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster,
compose_env_file=cluster_context.env_file)
return DeployCommandContext(stack, cluster_context, deployer)
def up_operation(ctx, services_list, stay_attached=False):
@@ -74,10 +80,10 @@ def up_operation(ctx, services_list, stay_attached=False):
print(f"Running compose up with container_exec_env: {container_exec_env}, extra_args: {services_list}")
for pre_start_command in cluster_context.pre_start_commands:
_run_command(global_context, cluster_context.cluster, pre_start_command)
deploy_context.docker.compose.up(detach=not stay_attached, services=services_list)
deploy_context.deployer.up(detach=not stay_attached, services=services_list)
for post_start_command in cluster_context.post_start_commands:
_run_command(global_context, cluster_context.cluster, post_start_command)
_orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.docker, container_exec_env)
_orchestrate_cluster_config(global_context, cluster_context.config, deploy_context.deployer, container_exec_env)
def down_operation(ctx, delete_volumes, extra_args_list):
@@ -89,7 +95,7 @@ def down_operation(ctx, delete_volumes, extra_args_list):
if extra_args_list:
timeout_arg = extra_args_list[0]
# Specify shutdown timeout (default 10s) to give services enough time to shutdown gracefully
ctx.obj.docker.compose.down(timeout=timeout_arg, volumes=delete_volumes)
ctx.obj.deployer.down(timeout=timeout_arg, volumes=delete_volumes)
def ps_operation(ctx):
@@ -97,7 +103,7 @@ def ps_operation(ctx):
if not global_context.dry_run:
if global_context.verbose:
print("Running compose ps")
container_list = ctx.obj.docker.compose.ps()
container_list = ctx.obj.deployer.ps()
if len(container_list) > 0:
print("Running containers:")
for container in container_list:
@@ -128,7 +134,7 @@ def port_operation(ctx, extra_args):
exposed_port = extra_args_list[1]
if global_context.verbose:
print(f"Running compose port {service_name} {exposed_port}")
mapped_port_data = ctx.obj.docker.compose.port(service_name, exposed_port)
mapped_port_data = ctx.obj.deployer.port(service_name, exposed_port)
print(f"{mapped_port_data[0]}:{mapped_port_data[1]}")
@@ -145,8 +151,8 @@ def exec_operation(ctx, extra_args):
if global_context.verbose:
print(f"Running compose exec {service_name} {command_to_exec}")
try:
ctx.obj.docker.compose.execute(service_name, command_to_exec, envs=container_exec_env)
except DockerException:
ctx.obj.deployer.execute(service_name, command_to_exec, tty=True, envs=container_exec_env)
except DeployerException:
print("container command returned error exit status")
@@ -157,7 +163,7 @@ def logs_operation(ctx, tail: int, follow: bool, extra_args: str):
if global_context.verbose:
print("Running compose logs")
services_list = extra_args_list if extra_args_list is not None else []
logs_stream = ctx.obj.docker.compose.logs(services=services_list, tail=tail, follow=follow, stream=True)
logs_stream = ctx.obj.deployer.logs(services=services_list, tail=tail, follow=follow, stream=True)
for stream_type, stream_content in logs_stream:
print(stream_content.decode("utf-8"), end="")
@@ -214,11 +220,11 @@ def get_stack_status(ctx, stack):
ctx_copy.stack = stack
cluster_context = _make_cluster_context(ctx_copy, stack, None, None, None, None)
docker = DockerClient(compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster)
deployer = Deployer(compose_files=cluster_context.compose_files, compose_project_name=cluster_context.cluster)
# TODO: refactor to avoid duplicating this code above
if ctx.verbose:
print("Running compose ps")
container_list = docker.compose.ps()
container_list = deployer.ps()
if len(container_list) > 0:
if ctx.debug:
print(f"Container list from compose ps: {container_list}")
@@ -252,7 +258,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
deployment = True
else:
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
compose_dir = Path(__file__).absolute().parent.joinpath("data", "compose")
compose_dir = Path(__file__).absolute().parent.parent.joinpath("data", "compose")
if cluster is None:
# Create default unique, stable cluster name from confile file path and stack name if provided
@@ -301,7 +307,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_path}.yml")
else:
if deployment:
compose_file_name = os.path.join(compose_dir, "docker-compose.yml")
compose_file_name = os.path.join(compose_dir, f"docker-compose-{pod_name}.yml")
pod_pre_start_command = pod["pre_start_command"]
pod_post_start_command = pod["post_start_command"]
script_dir = compose_dir.parent.joinpath("pods", pod_name, "scripts")
@@ -311,7 +317,7 @@ def _make_cluster_context(ctx, stack, include, exclude, cluster, env_file):
post_start_commands.append(os.path.join(script_dir, pod_post_start_command))
else:
pod_root_dir = os.path.join(dev_root_path, pod_repository.split("/")[-1], pod["path"])
compose_file_name = os.path.join(pod_root_dir, "docker-compose.yml")
compose_file_name = os.path.join(pod_root_dir, f"docker-compose-{pod_name}.yml")
pod_pre_start_command = pod["pre_start_command"]
pod_post_start_command = pod["post_start_command"]
if pod_pre_start_command is not None:
@@ -359,7 +365,7 @@ def _run_command(ctx, cluster_name, command):
sys.exit(1)
def _orchestrate_cluster_config(ctx, cluster_config, docker, container_exec_env):
def _orchestrate_cluster_config(ctx, cluster_config, deployer, container_exec_env):
@dataclass
class ConfigDirective:
@@ -390,13 +396,13 @@ def _orchestrate_cluster_config(ctx, cluster_config, docker, container_exec_env)
# TODO: fix the script paths so they're consistent between containers
source_value = None
try:
source_value = docker.compose.execute(pd.source_container,
["sh", "-c",
"sh /docker-entrypoint-scripts.d/export-"
f"{pd.source_variable}.sh"],
tty=False,
envs=container_exec_env)
except DockerException as error:
source_value = deployer.execute(pd.source_container,
["sh", "-c",
"sh /docker-entrypoint-scripts.d/export-"
f"{pd.source_variable}.sh"],
tty=False,
envs=container_exec_env)
except DeployerException as error:
if ctx.debug:
print(f"Docker exception reading config source: {error}")
# If the script executed failed for some reason, we get:
@@ -411,12 +417,12 @@ def _orchestrate_cluster_config(ctx, cluster_config, docker, container_exec_env)
if source_value:
if ctx.debug:
print(f"fetched source value: {source_value}")
destination_output = docker.compose.execute(pd.destination_container,
["sh", "-c",
f"sh /scripts/import-{pd.destination_variable}.sh"
f" {source_value}"],
tty=False,
envs=container_exec_env)
destination_output = deployer.execute(pd.destination_container,
["sh", "-c",
f"sh /scripts/import-{pd.destination_variable}.sh"
f" {source_value}"],
tty=False,
envs=container_exec_env)
waiting_for_data = False
if ctx.debug:
print(f"destination output: {destination_output}")
@@ -16,8 +16,8 @@
from typing import List
from dataclasses import dataclass
from pathlib import Path
from python_on_whales import DockerClient
from app.command_types import CommandOptions
from app.deploy.deployer import Deployer
@dataclass
@@ -35,7 +35,7 @@ class ClusterContext:
class DeployCommandContext:
stack: str
cluster_context: ClusterContext
docker: DockerClient
deployer: Deployer
@dataclass
@@ -15,7 +15,7 @@
import os
from typing import List
from app.deploy_types import DeployCommandContext, VolumeMapping
from app.deploy.deploy_types import DeployCommandContext, VolumeMapping
from app.util import get_parsed_stack_config, get_yaml, get_compose_file_dir, get_pod_list
@@ -47,12 +47,12 @@ def _volumes_to_docker(mounts: List[VolumeMapping]):
def run_container_command(ctx: DeployCommandContext, service: str, command: str, mounts: List[VolumeMapping]):
docker = ctx.docker
deployer = ctx.deployer
container_image = _container_image_from_service(ctx.stack, service)
docker_volumes = _volumes_to_docker(mounts)
if ctx.cluster_context.options.debug:
print(f"Running this command in {service} container: {command}")
docker_output = docker.run(
docker_output = deployer.run(
container_image,
["-c", command], entrypoint="sh",
user=f"{os.getuid()}:{os.getgid()}",
+52
View File
@@ -0,0 +1,52 @@
# Copyright © 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from abc import ABC, abstractmethod
class Deployer(ABC):
@abstractmethod
def up(self, detach, services):
pass
@abstractmethod
def down(self, timeout, volumes):
pass
@abstractmethod
def ps(self):
pass
@abstractmethod
def port(self, service, private_port):
pass
@abstractmethod
def execute(self, service_name, command, tty, envs):
pass
@abstractmethod
def logs(self, services, tail, follow, stream):
pass
@abstractmethod
def run(self, image, command, user, volumes, entrypoint):
pass
class DeployerException(Exception):
def __init__(self, *args: object) -> None:
super().__init__(*args)
+26
View File
@@ -0,0 +1,26 @@
# Copyright © 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from app.deploy.k8s.deploy_k8s import K8sDeployer
from app.deploy.compose.deploy_docker import DockerDeployer
def getDeployer(type, compose_files, compose_project_name, compose_env_file):
if type == "compose" or type is None:
return DockerDeployer(compose_files, compose_project_name, compose_env_file)
elif type == "k8s":
return K8sDeployer(compose_files, compose_project_name, compose_env_file)
else:
print(f"ERROR: deploy-to {type} is not valid")
+26 -10
View File
@@ -14,20 +14,25 @@
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
import click
from dataclasses import dataclass
from pathlib import Path
import sys
from app.deploy import up_operation, down_operation, ps_operation, port_operation
from app.deploy import exec_operation, logs_operation, create_deploy_context
from app.deploy.deploy import up_operation, down_operation, ps_operation, port_operation
from app.deploy.deploy import exec_operation, logs_operation, create_deploy_context
from app.deploy.stack import Stack
from app.deploy.spec import Spec
@dataclass
class DeploymentContext:
dir: Path
spec: Spec
stack: Stack
def get_stack_file(self):
return self.dir.joinpath("stack.yml")
def get_spec_file(self):
return self.dir.joinpath("spec.yml")
def get_env_file(self):
return self.dir.joinpath("config.env")
@@ -35,12 +40,19 @@ class DeploymentContext:
def get_cluster_name(self):
return None
def init(self, dir):
self.dir = dir
self.stack = Stack()
self.stack.init_from_file(self.get_stack_file())
self.spec = Spec()
self.spec.init_from_file(self.get_spec_file())
@click.group()
@click.option("--dir", required=True, help="path to deployment directory")
@click.pass_context
def command(ctx, dir):
'''create a deployment'''
'''manage a deployment'''
# Check that --stack wasn't supplied
if ctx.parent.obj.stack:
@@ -55,14 +67,18 @@ def command(ctx, dir):
print(f"Error: supplied deployment directory path {dir} exists but is a file not a directory")
sys.exit(1)
# Store the deployment context for subcommands
ctx.obj = DeploymentContext(dir_path)
deployment_context = DeploymentContext()
deployment_context.init(dir_path)
ctx.obj = deployment_context
def make_deploy_context(ctx):
stack_file_path = ctx.obj.get_stack_file()
env_file = ctx.obj.get_env_file()
cluster_name = ctx.obj.get_cluster_name()
return create_deploy_context(ctx.parent.parent.obj, stack_file_path, None, None, cluster_name, env_file)
context: DeploymentContext = ctx.obj
stack_file_path = context.get_stack_file()
env_file = context.get_env_file()
cluster_name = context.get_cluster_name()
return create_deploy_context(ctx.parent.parent.obj, stack_file_path, None, None, cluster_name, env_file,
context.spec.obj["deploy-to"])
@command.command()
@@ -22,8 +22,8 @@ import random
from shutil import copy, copyfile, copytree
import sys
from app.util import (get_stack_file_path, get_parsed_deployment_spec, get_parsed_stack_config, global_options, get_yaml,
get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_path)
from app.deploy_types import DeploymentContext, DeployCommandContext, LaconicStackSetupCommand
get_pod_list, get_pod_file_path, pod_has_scripts, get_pod_script_paths, get_plugin_code_paths)
from app.deploy.deploy_types import DeploymentContext, DeployCommandContext, LaconicStackSetupCommand
def _make_default_deployment_dir():
@@ -106,9 +106,10 @@ def _fixup_pod_file(pod, spec, compose_dir):
pod["services"][container_name]["ports"] = container_ports
def _commands_plugin_path(ctx: DeployCommandContext):
plugin_path = get_plugin_code_path(ctx.stack)
return plugin_path.joinpath("deploy", "commands.py")
def _commands_plugin_paths(ctx: DeployCommandContext):
plugin_paths = get_plugin_code_paths(ctx.stack)
ret = [p.joinpath("deploy", "commands.py") for p in plugin_paths]
return ret
# See: https://stackoverflow.com/a/54625079/1701505
@@ -120,15 +121,23 @@ def call_stack_deploy_init(deploy_command_context):
# Link with the python file in the stack
# Call a function in it
# If no function found, return None
python_file_path = _commands_plugin_path(deploy_command_context)
if python_file_path.exists():
spec = util.spec_from_file_location("commands", python_file_path)
imported_stack = util.module_from_spec(spec)
spec.loader.exec_module(imported_stack)
if _has_method(imported_stack, "init"):
return imported_stack.init(deploy_command_context)
else:
return None
python_file_paths = _commands_plugin_paths(deploy_command_context)
ret = None
init_done = False
for python_file_path in python_file_paths:
if python_file_path.exists():
spec = util.spec_from_file_location("commands", python_file_path)
imported_stack = util.module_from_spec(spec)
spec.loader.exec_module(imported_stack)
if _has_method(imported_stack, "init"):
if not init_done:
ret = imported_stack.init(deploy_command_context)
init_done = True
else:
# TODO: remove this restriction
print(f"Skipping init() from plugin {python_file_path}. Only one init() is allowed.")
return ret
# TODO: fold this with function above
@@ -136,16 +145,14 @@ def call_stack_deploy_setup(deploy_command_context, parameters: LaconicStackSetu
# Link with the python file in the stack
# Call a function in it
# If no function found, return None
python_file_path = _commands_plugin_path(deploy_command_context)
print(f"Path: {python_file_path}")
if python_file_path.exists():
spec = util.spec_from_file_location("commands", python_file_path)
imported_stack = util.module_from_spec(spec)
spec.loader.exec_module(imported_stack)
if _has_method(imported_stack, "setup"):
return imported_stack.setup(deploy_command_context, parameters, extra_args)
else:
return None
python_file_paths = _commands_plugin_paths(deploy_command_context)
for python_file_path in python_file_paths:
if python_file_path.exists():
spec = util.spec_from_file_location("commands", python_file_path)
imported_stack = util.module_from_spec(spec)
spec.loader.exec_module(imported_stack)
if _has_method(imported_stack, "setup"):
imported_stack.setup(deploy_command_context, parameters, extra_args)
# TODO: fold this with function above
@@ -153,15 +160,14 @@ def call_stack_deploy_create(deployment_context, extra_args):
# Link with the python file in the stack
# Call a function in it
# If no function found, return None
python_file_path = _commands_plugin_path(deployment_context.command_context)
if python_file_path.exists():
spec = util.spec_from_file_location("commands", python_file_path)
imported_stack = util.module_from_spec(spec)
spec.loader.exec_module(imported_stack)
if _has_method(imported_stack, "create"):
return imported_stack.create(deployment_context, extra_args)
else:
return None
python_file_paths = _commands_plugin_paths(deployment_context.command_context)
for python_file_path in python_file_paths:
if python_file_path.exists():
spec = util.spec_from_file_location("commands", python_file_path)
imported_stack = util.module_from_spec(spec)
spec.loader.exec_module(imported_stack)
if _has_method(imported_stack, "create"):
imported_stack.create(deployment_context, extra_args)
# Inspect the pod yaml to find config files referenced in subdirectories
@@ -249,13 +255,13 @@ def init(ctx, config, output, map_ports_to_host):
stack = global_options(ctx).stack
debug = global_options(ctx).debug
default_spec_file_content = call_stack_deploy_init(ctx.obj)
spec_file_content = {"stack": stack}
spec_file_content = {"stack": stack, "deploy-to": ctx.obj.deployer.name}
if default_spec_file_content:
spec_file_content.update(default_spec_file_content)
config_variables = _parse_config_variables(config)
if config_variables:
# Implement merge, since update() overwrites
orig_config = spec_file_content["config"]
orig_config = spec_file_content.get("config", {})
new_config = config_variables["config"]
merged_config = {**new_config, **orig_config}
spec_file_content.update({"config": merged_config})
@@ -315,7 +321,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
sys.exit(1)
os.mkdir(deployment_dir)
# Copy spec file and the stack file into the deployment dir
copyfile(spec_file, os.path.join(deployment_dir, os.path.basename(spec_file)))
copyfile(spec_file, os.path.join(deployment_dir, "spec.yml"))
copyfile(stack_file, os.path.join(deployment_dir, os.path.basename(stack_file)))
# Copy any config varibles from the spec file into an env file suitable for compose
_write_config_file(spec_file, os.path.join(deployment_dir, "config.env"))
@@ -325,7 +331,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
os.mkdir(destination_compose_dir)
destination_pods_dir = os.path.join(deployment_dir, "pods")
os.mkdir(destination_pods_dir)
data_dir = Path(__file__).absolute().parent.joinpath("data")
data_dir = Path(__file__).absolute().parent.parent.joinpath("data")
yaml = get_yaml()
for pod in pods:
pod_file_path = get_pod_file_path(parsed_stack, pod)
@@ -336,7 +342,7 @@ def create(ctx, spec_file, deployment_dir, network_dir, initial_peers):
if global_options(ctx).debug:
print(f"extra config dirs: {extra_config_dirs}")
_fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir)
with open(os.path.join(destination_compose_dir, os.path.basename(pod_file_path)), "w") as output_file:
with open(os.path.join(destination_compose_dir, "docker-compose-%s.yml" % pod), "w") as output_file:
yaml.dump(parsed_pod_file, output_file)
# Copy the config files for the pod, if any
config_dirs = {pod}
View File
+84
View File
@@ -0,0 +1,84 @@
# Copyright © 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from kubernetes import client
from typing import Any, List, Set
from app.opts import opts
from app.util import get_yaml
class ClusterInfo:
parsed_pod_yaml_map: Any = {}
image_set: Set[str] = set()
app_name: str = "test-app"
deployment_name: str = "test-deployment"
def __init__(self) -> None:
pass
def int_from_pod_files(self, pod_files: List[str]):
for pod_file in pod_files:
with open(pod_file, "r") as pod_file_descriptor:
parsed_pod_file = get_yaml().load(pod_file_descriptor)
self.parsed_pod_yaml_map[pod_file] = parsed_pod_file
if opts.o.debug:
print(f"parsed_pod_yaml_map: {self.parsed_pod_yaml_map}")
# Find the set of images in the pods
for pod_name in self.parsed_pod_yaml_map:
pod = self.parsed_pod_yaml_map[pod_name]
services = pod["services"]
for service_name in services:
service_info = services[service_name]
image = service_info["image"]
self.image_set.add(image)
if opts.o.debug:
print(f"image_set: {self.image_set}")
def get_deployment(self):
containers = []
for pod_name in self.parsed_pod_yaml_map:
pod = self.parsed_pod_yaml_map[pod_name]
services = pod["services"]
for service_name in services:
container_name = service_name
service_info = services[service_name]
image = service_info["image"]
container = client.V1Container(
name=container_name,
image=image,
ports=[client.V1ContainerPort(container_port=80)],
resources=client.V1ResourceRequirements(
requests={"cpu": "100m", "memory": "200Mi"},
limits={"cpu": "500m", "memory": "500Mi"},
),
)
containers.append(container)
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"app": self.app_name}),
spec=client.V1PodSpec(containers=containers),
)
spec = client.V1DeploymentSpec(
replicas=1, template=template, selector={
"matchLabels":
{"app": self.app_name}})
deployment = client.V1Deployment(
api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=self.deployment_name),
spec=spec,
)
return deployment
+99
View File
@@ -0,0 +1,99 @@
# Copyright © 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from kubernetes import client, config
from app.deploy.deployer import Deployer
from app.deploy.k8s.helpers import create_cluster, destroy_cluster, load_images_into_kind
from app.deploy.k8s.helpers import pods_in_deployment, log_stream_from_string
from app.deploy.k8s.cluster_info import ClusterInfo
from app.opts import opts
class K8sDeployer(Deployer):
name: str = "k8s"
core_api: client.CoreV1Api
apps_api: client.AppsV1Api
kind_cluster_name: str
cluster_info : ClusterInfo
def __init__(self, compose_files, compose_project_name, compose_env_file) -> None:
if (opts.o.debug):
print(f"Compose files: {compose_files}")
print(f"Project name: {compose_project_name}")
print(f"Env file: {compose_env_file}")
self.kind_cluster_name = compose_project_name
self.cluster_info = ClusterInfo()
self.cluster_info.int_from_pod_files(compose_files)
def connect_api(self):
config.load_kube_config(context=f"kind-{self.kind_cluster_name}")
self.core_api = client.CoreV1Api()
self.apps_api = client.AppsV1Api()
def up(self, detach, services):
# Create the kind cluster
create_cluster(self.kind_cluster_name)
self.connect_api()
# Ensure the referenced containers are copied into kind
load_images_into_kind(self.kind_cluster_name, self.cluster_info.image_set)
# Process compose files into a Deployment
deployment = self.cluster_info.get_deployment()
# Create the k8s objects
resp = self.apps_api.create_namespaced_deployment(
body=deployment, namespace="default"
)
if opts.o.debug:
print("Deployment created.\n")
print(f"{resp.metadata.namespace} {resp.metadata.name} \
{resp.metadata.generation} {resp.spec.template.spec.containers[0].image}")
def down(self, timeout, volumes):
# Delete the k8s objects
# Destroy the kind cluster
destroy_cluster(self.kind_cluster_name)
def ps(self):
self.connect_api()
# Call whatever API we need to get the running container list
ret = self.core_api.list_pod_for_all_namespaces(watch=False)
if ret.items:
for i in ret.items:
print("%s\t%s\t%s" % (i.status.pod_ip, i.metadata.namespace, i.metadata.name))
ret = self.core_api.list_node(pretty=True, watch=False)
return []
def port(self, service, private_port):
# Since we handle the port mapping, need to figure out where this comes from
# Also look into whether it makes sense to get ports for k8s
pass
def execute(self, service_name, command, tty, envs):
# Call the API to execute a command in a running container
pass
def logs(self, services, tail, follow, stream):
self.connect_api()
pods = pods_in_deployment(self.core_api, "test-deployment")
if len(pods) > 1:
print("Warning: more than one pod in the deployment")
k8s_pod_name = pods[0]
log_data = self.core_api.read_namespaced_pod_log(k8s_pod_name, namespace="default", container="test")
return log_stream_from_string(log_data)
def run(self, image, command, user, volumes, entrypoint=None):
# We need to figure out how to do this -- check why we're being called first
pass
+57
View File
@@ -0,0 +1,57 @@
# Copyright © 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from kubernetes import client
import subprocess
from typing import Set
from app.opts import opts
def _run_command(command: str):
if opts.o.debug:
print(f"Running: {command}")
result = subprocess.run(command, shell=True)
if opts.o.debug:
print(f"Result: {result}")
def create_cluster(name: str):
_run_command(f"kind create cluster --name {name}")
def destroy_cluster(name: str):
_run_command(f"kind delete cluster --name {name}")
def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]):
for image in image_set:
_run_command(f"kind load docker-image {image} --name {kind_cluster_name}")
def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str):
pods = []
pod_response = core_api.list_namespaced_pod(namespace="default", label_selector="app=test-app")
if opts.o.debug:
print(f"pod_response: {pod_response}")
for pod_info in pod_response.items:
pod_name = pod_info.metadata.name
pods.append(pod_name)
return pods
def log_stream_from_string(s: str):
# Note response has to be UTF-8 encoded because the caller expects to decode it
yield ("ignore", s.encode())
+30
View File
@@ -0,0 +1,30 @@
# Copyright © 2022, 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from pathlib import Path
import typing
from app.util import get_yaml
class Spec:
obj: typing.Any
def __init__(self) -> None:
pass
def init_from_file(self, file_path: Path):
with file_path:
self.obj = get_yaml().load(open(file_path, "r"))
+30
View File
@@ -0,0 +1,30 @@
# Copyright © 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from pathlib import Path
import typing
from app.util import get_yaml
class Stack:
obj: typing.Any
def __init__(self) -> None:
pass
def init_from_file(self, file_path: Path):
with file_path:
self.obj = get_yaml().load(open(file_path, "r"))
+20
View File
@@ -0,0 +1,20 @@
# Copyright © 2023 Vulcanize
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http:#www.gnu.org/licenses/>.
from app.command_types import CommandOptions
class opts:
o: CommandOptions = None
View File
@@ -240,7 +240,7 @@ def command(ctx, include, exclude, git_ssh, check_only, pull, branches, branches
if stack:
# In order to be compatible with Python 3.8 we need to use this hack to get the path:
# See: https://stackoverflow.com/questions/25389095/python-get-path-of-root-project-structure
stack_file_path = Path(__file__).absolute().parent.joinpath("data", "stacks", stack, "stack.yml")
stack_file_path = Path(__file__).absolute().parent.parent.joinpath("data", "stacks", stack, "stack.yml")
with stack_file_path:
stack_config = yaml.safe_load(open(stack_file_path, "r"))
# TODO: syntax check the input here
+8 -8
View File
@@ -79,16 +79,16 @@ def get_pod_list(parsed_stack):
return result
def get_plugin_code_path(stack):
def get_plugin_code_paths(stack):
parsed_stack = get_parsed_stack_config(stack)
pods = parsed_stack["pods"]
# TODO: Hack
pod = pods[0]
if type(pod) is str:
result = get_stack_file_path(stack).parent
else:
pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"])
result = Path(os.path.join(pod_root_dir, "stack"))
result = []
for pod in pods:
if type(pod) is str:
result.append(get_stack_file_path(stack).parent)
else:
pod_root_dir = os.path.join(get_dev_root_path(None), pod["repository"].split("/")[-1], pod["path"])
result.append(Path(os.path.join(pod_root_dir, "stack")))
return result
+9 -6
View File
@@ -16,12 +16,13 @@
import click
from app.command_types import CommandOptions
from app import setup_repositories
from app import build_containers
from app import build_npms
from app import deploy
from app.repos import setup_repositories
from app.build import build_containers
from app.build import build_npms
from app.deploy import deploy
from app import version
from app import deployment
from app.deploy import deployment
from app import opts
from app import update
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
@@ -39,7 +40,9 @@ CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
@click.pass_context
def cli(ctx, stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error):
"""Laconic Stack Orchestrator"""
ctx.obj = CommandOptions(stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error)
command_options = CommandOptions(stack, quiet, verbose, dry_run, local_stack, debug, continue_on_error)
opts.opts.o = command_options
ctx.obj = command_options
cli.add_command(setup_repositories.command, "setup-repositories")
+6 -6
View File
@@ -2,10 +2,10 @@
The following tutorial explains the steps to run a laconicd fixturenet with CLI and web console that displays records in the registry. It is designed as an introduction to Stack Orchestrator and to showcase one component of the Laconic Stack. Prior to Stack Orchestrator, the following 4 repositories had to be cloned and setup manually:
- https://github.com/cerc-io/laconicd
- https://github.com/cerc-io/laconic-sdk
- https://github.com/cerc-io/laconic-registry-cli
- https://github.com/cerc-io/laconic-console
- https://git.vdb.to/cerc-io/laconicd
- https://git.vdb.to/cerc-io/laconic-sdk
- https://git.vdb.to/cerc-io/laconic-registry-cli
- https://git.vdb.to/cerc-io/laconic-console
Now, with Stack Orchestrator, it is a few quick commands. Additionally, the `docker` and `docker compose` integration on the back-end allows the stack to easily persist, facilitating workflows.
@@ -52,7 +52,7 @@ laconic-so version
1. Get the repositories
```
laconic-so --stack fixturenet-laconic-loaded setup-repositories --include github.com/cerc-io/laconicd,github.com/cerc-io/laconic-sdk,github.com/cerc-io/laconic-registry-cli,github.com/cerc-io/laconic-console
laconic-so --stack fixturenet-laconic-loaded setup-repositories --include git.vdb.to/cerc-io/laconicd,git.vdb.to/cerc-io/laconic-sdk,git.vdb.to/cerc-io/laconic-registry-cli,git.vdb.to/cerc-io/laconic-console
```
2. Set this environment variable to the Laconic self-hosted Gitea instance:
@@ -212,7 +212,7 @@ record:
3. Try out additional CLI commands
- these are documented [here](https://github.com/cerc-io/laconic-registry-cli#readme) and updates are forthcoming
- these are documented [here](https://git.vdb.to/cerc-io/laconic-registry-cli#readme) and updates are forthcoming
- e.g,:
```
+1
View File
@@ -8,3 +8,4 @@ ruamel.yaml>=0.17.32
pydantic==1.10.9
tomli==2.0.1
validators==0.22.0
kubernetes>=28.1.0
+44 -7
View File
@@ -24,21 +24,58 @@ echo "Deploying the cluster"
$TEST_TARGET_SO --stack $CERC_STACK_NAME deploy up
# Verify that the fixturenet is up and running
$TEST_TARGET_SO --stack $CERC_STACK_NAME deploy ps
$TEST_TARGET_SO --stack $CERC_STACK_NAME deploy exec fixturenet-eth-bootnode-lighthouse /scripts/status-internal.sh
initial_block_number=$($TEST_TARGET_SO --stack fixturenet-plugeth-tx deploy exec foundry "cast block-number")
# Check that the block number increases some time later
sleep 12
subsequent_block_number=$($TEST_TARGET_SO --stack $CERC_STACK_NAME deploy exec foundry "cast block-number")
# $TEST_TARGET_SO --stack $CERC_STACK_NAME deploy exec fixturenet-eth-bootnode-lighthouse /scripts/status-internal.sh
timeout=900 # 15 minutes
echo "$(date +"%Y-%m-%d %T"): Getting initial block number. Timeout set to $timeout seconds"
start_time=$(date +%s)
elapsed_time=0
initial_block_number=0
while [ "$initial_block_number" -eq 0 ] && [ $elapsed_time -lt $timeout ]; do
sleep 10
echo "$(date +"%Y-%m-%d %T"): Waiting for initial block..."
initial_block_number=$($TEST_TARGET_SO --stack $CERC_STACK_NAME deploy exec foundry "cast block-number")
current_time=$(date +%s)
elapsed_time=$((current_time - start_time))
done
subsequent_block_number=$initial_block_number
# if initial block was 0 after timeout, assume chain did not start successfully and skip finding subsequent block
if [[ $initial_block_number -gt 0 ]]; then
timeout=300
echo "$(date +"%Y-%m-%d %T"): Getting subsequent block number. Timeout set to $timeout seconds"
start_time=$(date +%s)
elapsed_time=0
# wait for 5 blocks or timeout
while [ "$subsequent_block_number" -le $((initial_block_number + 5)) ] && [ $elapsed_time -lt $timeout ]; do
sleep 10
echo "$(date +"%Y-%m-%d %T"): Waiting for five blocks or $timeout seconds..."
subsequent_block_number=$($TEST_TARGET_SO --stack $CERC_STACK_NAME deploy exec foundry "cast block-number")
current_time=$(date +%s)
elapsed_time=$((current_time - start_time))
done
fi
# will return 0 if either of the above loops timed out
block_number_difference=$((subsequent_block_number - initial_block_number))
echo "$(date +"%Y-%m-%d %T"): Results of block height queries:"
echo "Initial block height: $initial_block_number"
echo "Subsequent block height: $subsequent_block_number"
# Block height difference should be between 1 and some small number
if [[ $block_number_difference -gt 1 && $block_number_difference -lt 10 ]]; then
if [[ $block_number_difference -gt 1 && $block_number_difference -lt 100 ]]; then
echo "Test passed"
test_result=0
else
echo "Test failed: block numbers were ${initial_block_number} and ${subsequent_block_number}"
echo "Logs from stack:"
$TEST_TARGET_SO --stack $CERC_STACK_NAME deploy logs
test_result=1
fi
$TEST_TARGET_SO --stack $CERC_STACK_NAME deploy down
$TEST_TARGET_SO --stack $CERC_STACK_NAME deploy down --delete-volumes
echo "Removing cloned repositories"
rm -rf $CERC_REPO_BASE_DIR
exit $test_result
+39 -7
View File
@@ -24,13 +24,45 @@ echo "$(date +"%Y-%m-%d %T"): Stack started"
$TEST_TARGET_SO --stack fixturenet-eth deploy ps
# echo "$(date +"%Y-%m-%d %T"): Getting stack status"
# $TEST_TARGET_SO --stack fixturenet-eth deploy exec fixturenet-eth-bootnode-lighthouse /scripts/status-internal.sh
echo "$(date +"%Y-%m-%d %T"): Getting initial block number"
initial_block_number=$($TEST_TARGET_SO --stack fixturenet-eth deploy exec foundry "cast block-number")
# Check that the block number increases some time later
sleep 120
echo "$(date +"%Y-%m-%d %T"): Getting subsequent block number"
subsequent_block_number=$($TEST_TARGET_SO --stack fixturenet-eth deploy exec foundry "cast block-number")
timeout=900 # 15 minutes
echo "$(date +"%Y-%m-%d %T"): Getting initial block number. Timeout set to $timeout seconds"
start_time=$(date +%s)
elapsed_time=0
initial_block_number=0
while [ "$initial_block_number" -eq 0 ] && [ $elapsed_time -lt $timeout ]; do
sleep 10
echo "$(date +"%Y-%m-%d %T"): Waiting for initial block..."
initial_block_number=$($TEST_TARGET_SO --stack fixturenet-eth deploy exec foundry "cast block-number")
current_time=$(date +%s)
elapsed_time=$((current_time - start_time))
done
subsequent_block_number=$initial_block_number
# if initial block was 0 after timeout, assume chain did not start successfully and skip finding subsequent block
if [[ $initial_block_number -gt 0 ]]; then
timeout=300
echo "$(date +"%Y-%m-%d %T"): Getting subsequent block number. Timeout set to $timeout seconds"
start_time=$(date +%s)
elapsed_time=0
# wait for 5 blocks or timeout
while [ "$subsequent_block_number" -le $((initial_block_number + 5)) ] && [ $elapsed_time -lt $timeout ]; do
sleep 10
echo "$(date +"%Y-%m-%d %T"): Waiting for five blocks or $timeout seconds..."
subsequent_block_number=$($TEST_TARGET_SO --stack fixturenet-eth deploy exec foundry "cast block-number")
current_time=$(date +%s)
elapsed_time=$((current_time - start_time))
done
fi
# will return 0 if either of the above loops timed out
block_number_difference=$((subsequent_block_number - initial_block_number))
echo "$(date +"%Y-%m-%d %T"): Results of block height queries:"
echo "Initial block height: $initial_block_number"
echo "Subsequent block height: $subsequent_block_number"
# Block height difference should be between 1 and some small number
if [[ $block_number_difference -gt 1 && $block_number_difference -lt 100 ]]; then
echo "Test passed"
@@ -41,7 +73,7 @@ else
$TEST_TARGET_SO --stack fixturenet-eth deploy logs
test_result=1
fi
$TEST_TARGET_SO --stack fixturenet-eth deploy down
$TEST_TARGET_SO --stack fixturenet-eth deploy down --delete-volumes
echo "$(date +"%Y-%m-%d %T"): Removing cloned repositories"
rm -rf $CERC_REPO_BASE_DIR
echo "$(date +"%Y-%m-%d %T"): Test finished"
+84
View File
@@ -0,0 +1,84 @@
#!/usr/bin/env bash
set -e
if [ -n "$CERC_SCRIPT_DEBUG" ]; then
set -x
fi
echo "$(date +"%Y-%m-%d %T"): Running stack-orchestrator Laconicd fixturenet test"
env
cat /etc/hosts
# Bit of a hack, test the most recent package
TEST_TARGET_SO=$( ls -t1 ./package/laconic-so* | head -1 )
# Set a new unique repo dir
export CERC_REPO_BASE_DIR=$(mktemp -d $(pwd)/stack-orchestrator-fixturenet-laconicd-test.XXXXXXXXXX)
echo "$(date +"%Y-%m-%d %T"): Testing this package: $TEST_TARGET_SO"
echo "$(date +"%Y-%m-%d %T"): Test version command"
reported_version_string=$( $TEST_TARGET_SO version )
echo "$(date +"%Y-%m-%d %T"): Version reported is: ${reported_version_string}"
echo "$(date +"%Y-%m-%d %T"): Cloning laconicd repositories into: $CERC_REPO_BASE_DIR"
$TEST_TARGET_SO --stack fixturenet-laconicd setup-repositories
echo "$(date +"%Y-%m-%d %T"): Building containers"
$TEST_TARGET_SO --stack fixturenet-laconicd build-containers
echo "$(date +"%Y-%m-%d %T"): Starting stack"
$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd up
echo "$(date +"%Y-%m-%d %T"): Stack started"
# Verify that the fixturenet is up and running
$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd ps
timeout=900 # 15 minutes
echo "$(date +"%Y-%m-%d %T"): Getting initial block number. Timeout set to $timeout seconds"
start_time=$(date +%s)
elapsed_time=0
initial_block_number=null
while [ "$initial_block_number" == "null" ] && [ $elapsed_time -lt $timeout ]; do
sleep 10
echo "$(date +"%Y-%m-%d %T"): Waiting for initial block..."
initial_block_number=$(docker exec laconicd-laconicd-1 /usr/bin/laconicd status | jq -r .SyncInfo.latest_block_height)
current_time=$(date +%s)
elapsed_time=$((current_time - start_time))
done
subsequent_block_number=$initial_block_number
# if initial block was 0 after timeout, assume chain did not start successfully and skip finding subsequent block
if [[ $initial_block_number != "null" ]]; then
timeout=300
echo "$(date +"%Y-%m-%d %T"): Getting subsequent block number. Timeout set to $timeout seconds"
start_time=$(date +%s)
elapsed_time=0
# wait for 5 blocks or timeout
while [ "$subsequent_block_number" -le $((initial_block_number + 5)) ] && [ $elapsed_time -lt $timeout ]; do
sleep 10
echo "$(date +"%Y-%m-%d %T"): Waiting for five blocks or $timeout seconds..."
subsequent_block_number=$(docker exec laconicd-laconicd-1 /usr/bin/laconicd status | jq -r .SyncInfo.latest_block_height)
current_time=$(date +%s)
elapsed_time=$((current_time - start_time))
done
fi
# will return 0 if either of the above loops timed out
block_number_difference=$((subsequent_block_number - initial_block_number))
echo "$(date +"%Y-%m-%d %T"): Results of block height queries:"
echo "Initial block height: $initial_block_number"
echo "Subsequent block height: $subsequent_block_number"
# Block height difference should be between 1 and some small number
if [[ $block_number_difference -gt 1 && $block_number_difference -lt 100 ]]; then
echo "Test passed"
test_result=0
else
echo "Test failed: block numbers were ${initial_block_number} and ${subsequent_block_number}"
echo "Logs from stack:"
$TEST_TARGET_SO --stack fixturenet-laconicd deploy logs
test_result=1
fi
$TEST_TARGET_SO --stack fixturenet-laconicd deploy --cluster laconicd down --delete-volumes
echo "$(date +"%Y-%m-%d %T"): Removing cloned repositories"
rm -rf $CERC_REPO_BASE_DIR
echo "$(date +"%Y-%m-%d %T"): Test finished"
exit $test_result